apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-07 07:55:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into fix-wiki-sl-search-output

Browse source 
# Conflicts:
#	packages/cli/src/status-project.ts
This commit is contained in:
Luca Martial 2026-05-15 05:51:37 -07:00
commit de40d05c06
177 changed files with 17742 additions and 1412 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -1,3 +1,6 @@
# Playwright CLI session artifacts (snapshots, console logs, screenshots)
.playwright-cli/
# Python
__pycache__/
*.py[cod]

32
docs-site/components/logo.tsx
View file

@ -1,28 +1,36 @@
export function Logo() {
return (
<div className="flex items-center gap-2.5 group">
<div className="flex items-center gap-3.5 group">
<div className="relative flex items-center justify-center transition-transform duration-300 ease-out group-hover:rotate-[-4deg]">
<img
src="/brand/ktx-mascot.svg"
src="/ktx/brand/ktx-mascot.svg"
alt=""
aria-hidden="true"
className="h-14 w-14 object-contain block dark:hidden"
className="h-20 w-20 object-contain block dark:hidden"
/>
<img
src="/brand/ktx-mascot-dark.svg"
src="/ktx/brand/ktx-mascot-dark.svg"
alt=""
aria-hidden="true"
className="h-14 w-14 object-contain hidden dark:block"
className="h-20 w-20 object-contain hidden dark:block"
/>
</div>
<div className="flex flex-col items-start leading-none">
<span
className="text-[24px] font-semibold text-fd-foreground tracking-tight"
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
>
KTX
</span>
<span
className="mt-1 whitespace-nowrap text-[13px] font-medium text-fd-muted-foreground/80 tracking-tight"
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
>
by Kaelio
</span>
</div>
<span
className="text-[17px] font-semibold text-fd-foreground tracking-tight"
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
>
KTX
</span>
<span
className="text-[14px] font-medium text-fd-muted-foreground/80 tracking-tight border-l border-fd-border pl-2 ml-0.5"
className="text-[19px] font-medium text-fd-muted-foreground/80 tracking-tight border-l border-fd-border pl-3 ml-1"
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
>
Docs

52
docs-site/content/docs/ai-resources/index.mdx
View file

@ -1,22 +1,37 @@
---
title: AI Resources
description: Machine-readable docs and prompt recipes for coding assistants reading KTX documentation.
description: Machine-readable docs, retrieval paths, and prompt recipes for coding assistants using KTX documentation.
---
Use this section when a coding assistant, IDE agent, or automation system needs to understand the KTX documentation.
Use this section when a coding assistant, IDE agent, or automation system needs
to read, cite, or update KTX documentation. These resources are optimized for
retrieval: agents can fetch small Markdown pages, use the full corpus only when
needed, and copy prompts that point them at current setup and CLI behavior.
> **Documentation index**
>
> Start with [`/llms.txt`](/llms.txt) to discover the available docs. Use [`/llms-full.txt`](/llms-full.txt) when the assistant needs the complete docs corpus in one Markdown response.
> Start with [`/llms.txt`](/llms.txt) to discover the available docs. Use
> [`/llms-full.txt`](/llms-full.txt) when the assistant needs the complete docs
> corpus in one Markdown response.
## Choose the right path
## What agents can do
| Need | Recommended path |
|------|------------------|
| Find the right setup or CLI page | Fetch [`/llms.txt`](/llms.txt), then read the smallest matching `.md` page |
| Answer a setup question | Read [Agent Quickstart](/docs/ai-resources/agent-quickstart), then [Quickstart](/docs/getting-started/quickstart) or [ktx setup](/docs/cli-reference/ktx-setup) |
| Quote a command or flag | Read the matching [CLI Reference](/docs/cli-reference) page as Markdown |
| Update docs in this repo | Use [Agent Instructions](/docs/ai-resources/agent-instructions) and verify generated Markdown routes after editing |
| Reuse a prompt | Copy from [Prompt Recipes](/docs/ai-resources/prompt-recipes) |
## Section map
| Goal | Use this page |
|------|---------------|
| Tell a coding assistant how to approach KTX docs | [Agent Quickstart](/docs/ai-resources/agent-quickstart) |
| Fetch docs as Markdown instead of HTML | [Markdown Access](/docs/ai-resources/markdown-access) |
| Add lightweight instructions to an assistant prompt | [Agent Instructions](/docs/ai-resources/agent-instructions) |
| Copy prompts for common agent workflows | [Prompt Recipes](/docs/ai-resources/prompt-recipes) |
| Give an assistant a task-first route through the docs | [Agent Quickstart](/docs/ai-resources/agent-quickstart) |
| Fetch docs as Markdown instead of rendered HTML | [Markdown Access](/docs/ai-resources/markdown-access) |
| Add lightweight KTX docs guidance to a system prompt | [Agent Instructions](/docs/ai-resources/agent-instructions) |
| Copy prompts for setup, command lookup, and docs editing | [Prompt Recipes](/docs/ai-resources/prompt-recipes) |
## Available resources
@ -26,13 +41,24 @@ Use this section when a coding assistant, IDE agent, or automation system needs
| [`/llms-full.txt`](/llms-full.txt) | Complete docs corpus in one plain-text Markdown response |
| `/docs/<path>.md` | Per-page Markdown for any docs page |
| Page-level actions | Copy Markdown, view Markdown, or copy MDX from rendered docs pages |
| Prompt recipes | Reusable prompts for docs lookup, setup help, and docs editing |
| Prompt recipes | Reusable prompts for docs lookup, setup help, command discovery, and docs editing |
## Agent usage notes
When an assistant is unsure where to begin, use this order:
When an assistant is unsure where to begin, use this retrieval order:
1. Read [`/llms.txt`](/llms.txt).
2. Fetch the specific Markdown page for the task.
3. Use [Agent Quickstart](/docs/ai-resources/agent-quickstart) to choose the next command or page.
4. Use page-level copy actions when the user wants the exact Markdown or MDX source.
2. Fetch one or two specific Markdown pages for the task.
3. Use [Agent Quickstart](/docs/ai-resources/agent-quickstart) to choose the
next command, guide, or CLI reference page.
4. Use [`/llms-full.txt`](/llms-full.txt) only when the answer requires broad
context across setup, integrations, concepts, and CLI reference.
5. Use page-level copy actions when the user wants exact generated Markdown or
source MDX.
## Boundaries
AI Resources explain how agents consume the docs. To install KTX into an
agent client, use [Agent Clients](/docs/integrations/agent-clients). To set up a
project, use [Quickstart](/docs/getting-started/quickstart) or
[`ktx setup`](/docs/cli-reference/ktx-setup).

4
docs-site/content/docs/cli-reference/index.mdx
View file

@ -37,9 +37,7 @@ ktx
```
The public context-build entrypoint is `ktx ingest [connectionId]` or
`ktx ingest --all`. Legacy command shapes such as `ktx scan`, `ktx ingest run`,
`ktx ingest status`, `ktx ingest replay`, `ktx ingest watch`, and
`ktx setup status` are not part of the current public CLI.
`ktx ingest --all`.
## Global Options

2
docs-site/content/docs/community/contributing.mdx
View file

@ -77,7 +77,7 @@ python/
examples/ # Example projects and fixtures
scripts/ # Workspace scripts (benchmarks, verification, release)
docs/ # Documentation site (Fumadocs)
docs-site/ # Documentation site (Fumadocs)
```
All TypeScript packages are ESM (`"type": "module"`) and use `NodeNext` module resolution. The Python projects use `pyproject.toml` for dependency management.

54
docs-site/content/docs/community/index.mdx Normal file
View file

@ -0,0 +1,54 @@
---
title: Community
description: Contribute to KTX through code, docs, connectors, and examples.
---
KTX is an open-source context layer for database agents. The project welcomes
focused contributions that improve setup, integrations, CLI behavior,
documentation, connector coverage, and examples.
## Where to start
| Goal | Start here |
|------|------------|
| Prepare a local development checkout | [Contributing](/docs/community/contributing#development-setup) |
| Understand the workspace layout | [Repository structure](/docs/community/contributing#repository-structure) |
| Run verification before a pull request | [Running tests](/docs/community/contributing#running-tests) |
| Add a database connector | [Adding a connector](/docs/community/contributing#adding-a-connector) |
| Update docs for a user-visible CLI or setup change | [PR guidelines](/docs/community/contributing#pr-guidelines) |
## Contribution areas
| Area | Good first context |
|------|--------------------|
| CLI and setup | `packages/cli`, especially setup steps, command definitions, status checks, and smoke tests |
| Context engine | `packages/context`, including project config, ingest orchestration, and semantic search |
| Connectors | `packages/connector-*`, plus connector-specific tests and integration docs |
| Python semantic layer | `python/ktx-sl` for planning and SQL generation |
| Python daemon | `python/ktx-daemon` for the portable runtime API |
| Documentation | `docs-site/content/docs` for public docs and `docs-site/tests` for docs behavior |
## Development loop
```bash
pnpm install
uv sync --all-groups
pnpm run setup:dev
pnpm run link:dev
ktx-dev status
```
Use `ktx-dev` for local CLI testing after linking the development binary. Use
the published `ktx` command when you are testing the released package in a
separate analytics project.
## Before submitting
1. Keep the change focused on one behavior, connector, doc area, or workflow.
2. Run the smallest tests that cover the changed surface.
3. Run broader checks when changing shared exports, setup state, or generated files.
4. Update `docs-site/content/docs/` when user-visible setup, CLI, configuration, or integration behavior changes.
5. Do not commit local secrets, generated build output, virtualenvs, dependency directories, or local databases.
For complete contributor setup and verification commands, read
[Contributing](/docs/community/contributing).

2
docs-site/content/docs/community/meta.json
View file

@ -1,5 +1,5 @@
{
"title": "Community",
"defaultOpen": true,
"pages": ["contributing"]
"pages": ["index", "contributing"]
}

336
docs-site/content/docs/getting-started/quickstart.mdx
View file

@ -1,254 +1,286 @@
---
title: Quickstart
description: Set up KTX and build your first context in under 10 minutes.
description: Set up KTX, build local context, and connect your coding agent.
---
This guide walks you through `ktx setup` - an interactive wizard that configures your LLM provider, connects your database, optionally ingests from your existing tools, builds context, and installs agent integration.
This guide gets a local analytics project ready for KTX. You will install the
CLI, run the setup wizard, connect a database, build context, and install agent
rules that teach your coding assistant which KTX commands to run.
If you are a coding assistant trying to decide which KTX docs page to read, start with the [Agent Quickstart](/docs/ai-resources/agent-quickstart). This page is the human setup walkthrough.
If you are a coding assistant choosing a docs route, start with the
[Agent Quickstart](/docs/ai-resources/agent-quickstart). This page is the
human setup walkthrough.
## Workflow summary
## What setup does
Use this sequence when you are setting up KTX in an analytics project:
`ktx setup` is the main project workflow. It can create or resume `ktx.yaml`,
configure model and embedding providers, add database connections, add optional
context sources, build the first context artifacts, and install agent
integration.
1. `npm install -g @kaelio/ktx` - install the published KTX CLI from npm.
2. `ktx setup` - create or resume a KTX project.
When you run bare `ktx` in an interactive terminal outside a KTX project, the
CLI opens the same setup experience. Inside an existing project, `ktx setup`
resumes incomplete work or opens a menu for changing setup, connecting an
agent, checking status, or exploring a demo project.
The setup wizard is stateful. If it exits before completion, rerun `ktx setup` in the same project directory to resume from the first incomplete step.
## Install the CLI
## Install and run setup
Install the published [`@kaelio/ktx`](https://www.npmjs.com/package/@kaelio/ktx) CLI:
Install the published `@kaelio/ktx` package:
```bash
npm install -g @kaelio/ktx
```
Then run the setup wizard:
Then run setup from the analytics project directory:
```bash
ktx setup
```
The local checkout flow is only for contributors working on KTX itself. See [Contributing](/docs/community/contributing) for that setup.
The local checkout workflow is only for KTX contributors. See
[Contributing](/docs/community/contributing) for that path.
The wizard walks through six steps. You can go back at any point, and if you exit early, rerunning `ktx setup` resumes where you left off.
## Step 1: Choose the project
## Step 1: Configure LLM
In an interactive terminal, setup can create a new KTX project or resume the
nearest existing project. The main project file is `ktx.yaml`.
KTX uses an Anthropic model to enrich schema descriptions, generate semantic sources during ingestion, and reconcile metadata from your tools.
For scripted setup, pass the project directory explicitly:
The wizard asks how to find your API key:
```
◆ How should KTX find your Anthropic API key?
│ ○ Use ANTHROPIC_API_KEY from the environment
│ ○ Paste a key and save it as a local secret file
```bash
ktx setup --project-dir ./analytics
```
If you choose to paste a key, KTX saves it in `.ktx/secrets/anthropic-api-key` with local file permissions. Your `ktx.yaml` stores a `file:` reference, never the raw key.
If setup exits early, rerun `ktx setup` in the same directory. KTX tracks
completed setup steps and resumes from the remaining work.
Next, choose a model:
## Step 2: Configure the LLM
```
◆ Which Anthropic model should KTX use?
│ ○ Claude Sonnet 4.6 (recommended)
│ ○ Claude Opus 4.6
│ ○ Claude Haiku 4.5
│ ○ Enter a model ID manually
```
KTX uses a Claude model for ingest agents that turn schemas, SQL, BI metadata,
and documents into semantic-layer sources and wiki context.
KTX runs a health check to verify your key and model work before saving.
Setup supports two LLM provider paths:
## Step 2: Configure embeddings
| Provider | Use when | Credential model |
|----------|----------|------------------|
| Anthropic API | You have an Anthropic API key | `ANTHROPIC_API_KEY` or a local `file:` secret |
| Google Vertex AI for Anthropic Claude | Your organization runs Claude through Google Cloud | Application Default Credentials plus Vertex project and location |
KTX uses embeddings for semantic search over sources, wiki content, schema metadata, and relationship evidence.
For Anthropic API, setup can read the key from the environment or save a pasted
key to `.ktx/secrets/anthropic-api-key`. `ktx.yaml` stores an `env:` or `file:`
reference, not the raw key.
```
◆ Which embedding option should KTX use?
│ ○ Local sentence-transformers embeddings
│ ○ OpenAI embeddings (recommended)
```
For Vertex AI, setup uses Google Application Default Credentials. It can read
your active `gcloud` project, list visible projects, or accept explicit
`--vertex-project` and `--vertex-location` values.
**OpenAI embeddings** use `text-embedding-3-small` (1536 dimensions) and require an `OPENAI_API_KEY`.
Setup checks the selected model before saving. Anthropic API setup fetches live
Claude model choices when possible and falls back to bundled defaults if model
discovery is unavailable.
**Local embeddings** use `all-MiniLM-L6-v2` (384 dimensions) via the KTX managed Python runtime. No API key is needed. KTX can install and start the runtime during setup; to prepare it ahead of time, run:
## Step 3: Configure embeddings
KTX uses embeddings for semantic search over semantic-layer sources, wiki
context, schema metadata, and relationship evidence.
| Backend | Default model | Notes |
|---------|---------------|-------|
| OpenAI | `text-embedding-3-small` | Recommended for hosted embeddings. Requires an OpenAI API key. |
| Local sentence-transformers | `all-MiniLM-L6-v2` | Runs through the KTX-managed Python runtime. No hosted embedding key is required. |
OpenAI setup reads `OPENAI_API_KEY` or saves a local secret file. Local
sentence-transformers setup can install and start the managed runtime during
setup. To prepare that runtime before setup, run:
```bash
ktx dev runtime install --feature local-embeddings --yes
ktx dev runtime start --feature local-embeddings
```
## Step 3: Connect a database
## Step 4: Add a database
Select one or more databases for KTX to connect to. The wizard supports
SQLite, PostgreSQL, MySQL, ClickHouse, SQL Server, BigQuery, and Snowflake.
KTX needs at least one primary database connection before it can build database
context. The wizard supports SQLite, PostgreSQL, MySQL, ClickHouse, SQL Server,
BigQuery, and Snowflake.
For PostgreSQL, you can enter connection details field by field or paste a connection URL:
You can usually enter connection fields interactively or provide a URL. Secret
URLs can be stored as local files under `.ktx/secrets/` or referenced with
`env:NAME` in `ktx.yaml`.
```
◆ How do you want to connect to PostgreSQL?
│ ○ Enter connection details (host, port, database, user)
│ ○ Paste a connection URL
```
After saving a connection, setup tests it and builds fast schema context:
If your URL contains credentials, KTX saves it to `.ktx/secrets/` and writes a `file:` reference in `ktx.yaml`. You can also use `env:DATABASE_URL` to reference an environment variable.
After connecting, KTX automatically runs a connection test and builds fast
schema context:
```
Testing postgres-warehouse
```text
Testing warehouse
Connection test passed
Driver: PostgreSQL - Status: ok
Building schema context for postgres-warehouse
Building schema context for warehouse
Running fast database ingest
Schema context complete for postgres-warehouse
Changes: 42 new tables
Database ready
postgres-warehouse - PostgreSQL - schema context complete
warehouse - PostgreSQL - schema context complete
```
For PostgreSQL, Snowflake, and BigQuery, the wizard can enable query-history
ingest when the warehouse history feature is available. Query history is stored
under `connections.<id>.context.queryHistory` in `ktx.yaml`.
PostgreSQL, BigQuery, and Snowflake can also enable query-history ingest. Query
history helps KTX learn common query patterns, joins, service-account filters,
and warehouse-specific usage.
## Step 4: Add context sources
## Step 5: Add context sources
Context sources let KTX ingest metadata from your existing analytics tools. This step is optional - you can skip it and add sources later.
Context sources are optional, but they make the first context layer much richer.
Setup can add:
```
◆ Which context sources should KTX ingest?
│ ◻ dbt
│ ◻ MetricFlow
│ ◻ Metabase
│ ◻ Looker
│ ◻ LookML
│ ◻ Notion
```
| Source | Typical input | What KTX learns |
|--------|---------------|-----------------|
| dbt | Local project or Git repo | Models, columns, tests, descriptions, tags |
| MetricFlow | Local project or Git repo | Semantic models, metrics, dimensions, entities |
| LookML | Local files or Git repo | Views, explores, dimensions, measures, joins |
| Looker | API URL and credentials | Explores, looks, dashboards, model metadata |
| Metabase | API URL and key | Questions, dashboards, BI database mappings |
| Notion | Integration token and crawl settings | Business docs and knowledge pages |
For **dbt**, point KTX at a local path or git URL. KTX reads your `dbt_project.yml` and schema files to extract model metadata:
Setup maps BI and source metadata back to your primary warehouse connection so
generated context points at the right tables.
```
◆ dbt source location
│ ○ Local path
│ ○ Git URL
```
You can skip this step and add sources later by rerunning `ktx setup`.
For **Metabase** and **Looker**, you provide an API URL and credentials. KTX maps BI databases to your KTX primary source connections so it knows which warehouse tables the BI metadata refers to.
## Step 6: Build context
Context sources are saved to `ktx.yaml` and built during the next step.
The context build turns configured databases and sources into local artifacts
agents can read. It runs database ingest first, then source ingest and memory
updates.
## Step 5: Build context
Fast database ingest records deterministic schema grounding. Deep ingest adds
AI-enriched descriptions, embeddings, relationship evidence, and query-history
context when configured.
This is where KTX builds agent-ready context. It uses the database context
depth saved by setup and ingests metadata from any configured context sources.
When the build finishes, setup verifies that agent-ready context exists:
```
◆ Build KTX context for agents?
│ ○ Build context now (recommended)
│ ○ Leave context unbuilt and exit setup
```
Fast database context builds deterministic schema grounding. Deep database
context also generates AI descriptions, embeddings, and relationship evidence
when those capabilities are configured.
For a small database (under 50 tables), this can take a few minutes. Larger
warehouses can take longer. Context builds run in the foreground; press
<kbd>Ctrl+C</kbd> to stop the current run and rerun `ktx setup` or `ktx ingest`
when you are ready to try again.
When the build completes, KTX verifies that agent-ready context was produced:
```
```text
KTX context is ready for agents.
Databases:
postgres-warehouse: deep context complete
warehouse: deep context complete
Context sources:
dbt-main: memory update complete
dbt_main: memory update complete
Verification:
Agent context: ready
Semantic search: ready
```
## Step 6: Install agent integration
If a foreground build is interrupted, rerun `ktx setup` or build the same target
with `ktx ingest <connectionId>`.
The final step connects KTX to your coding agent. Choose how agents should access the project:
## Step 7: Install agent integration
```
◆ How should agents use this KTX project?
│ ○ CLI tools and skills
The final setup step installs project-local rules for your coding assistant.
Supported targets are Claude Code, Codex, Cursor, OpenCode, and universal
`.agents`.
You can also run this step later:
```bash
ktx setup --agents --target codex
```
Then select which agents to install for:
Claude Code and Codex also support global installs:
```
◆ Which agent targets should KTX install?
│ ◻ Claude Code
│ ◻ Codex
│ ◻ Cursor
│ ◻ OpenCode
│ ◻ Custom agent (.agents)
```bash
ktx setup --agents --target codex --global
```
**CLI mode** writes a skill file (e.g., `.claude/skills/ktx/SKILL.md`) that teaches the agent to call KTX commands directly.
**Custom agent** uses the universal `.agents` target for agents that can read project-local skills.
Agent rules are CLI-based. They point agents at the KTX CLI path that created
the file, so agents do not need a separate `ktx` binary in `PATH`. If the CLI
path changes after reinstalling or moving a checkout, rerun `ktx setup --agents`.
## Generated files
KTX writes project state as plain files so agents can inspect and edit changes in git.
KTX writes plain files so people and agents can inspect changes in git.
| Path | Created by | Purpose |
|------|------------|---------|
| `ktx.yaml` | `ktx setup` | Main project configuration: connections, LLM settings, embeddings, and context sources |
| `.ktx/secrets/*` | `ktx setup` when file-backed secrets are selected | Local secret files referenced from `ktx.yaml`; do not commit these |
| `semantic-layer/<connection-id>/*.yaml` | context build, ingestion, or direct file edits | Semantic source definitions agents use for SQL generation |
| `wiki/global/*.md` | ingestion, memory capture, or direct file edits | Shared business context and metric definitions |
| `wiki/user/<user-id>/*.md` | memory capture or direct file edits | User-scoped notes for one agent/user context |
| `.claude/skills/ktx/SKILL.md`, `.agents/skills/ktx/SKILL.md` | CLI-mode agent integration setup | Agent instructions for calling public `ktx` commands |
| Path | Purpose |
|------|---------|
| `ktx.yaml` | Project configuration for LLMs, embeddings, connections, context sources, and setup state |
| `.ktx/secrets/*` | Local secret files referenced from `ktx.yaml`; do not commit these |
| `.ktx/setup/*` | Local setup and context-build state |
| `.ktx/agents/install-manifest.json` | Manifest used to manage installed agent files |
| `semantic-layer/<connection-id>/*.yaml` | Semantic source definitions used for SQL generation |
| `wiki/global/*.md` | Shared business context and metric definitions |
| `wiki/user/<user-id>/*.md` | User-scoped notes and local context |
| `.claude/skills/ktx/SKILL.md` | Claude Code project skill |
| `.agents/skills/ktx/SKILL.md` | Codex or universal project skill |
| `.cursor/rules/ktx.mdc` | Cursor project rule |
| `.opencode/commands/ktx.md` | OpenCode project command |
## Verify it worked
## Verify setup
Check your project status:
Run:
```bash
ktx status
```
```
Example output:
```text
KTX project: /home/user/analytics
Project ready: yes
LLM ready: yes (claude-sonnet-4-6)
Embeddings ready: yes (text-embedding-3-small)
Databases configured: yes (postgres-warehouse)
Context sources configured: yes (dbt-main)
Databases configured: yes (warehouse)
Context sources configured: yes (dbt_main)
KTX context built: yes
Agent integration ready: yes (claude-code:project)
Agent integration ready: yes (codex:project)
```
Use JSON when an agent or script needs a structured readiness check:
```bash
ktx status --json
```
## Scripted setup example
Use non-interactive setup when creating repeatable fixtures or automation:
```bash
ktx setup \
--project-dir ./analytics \
--no-input \
--skip-llm \
--skip-embeddings \
--database postgres \
--new-database-connection-id warehouse \
--database-url env:DATABASE_URL \
--database-schema public
```
Then build context:
```bash
ktx ingest warehouse --fast
```
See [ktx setup](/docs/cli-reference/ktx-setup) for the full automation flag
surface.
## Common errors
| Error or symptom | Likely cause | Recovery |
|------------------|--------------|----------|
| `ktx: command not found` | The KTX package is not installed globally, or the shell cannot find the global binary | Run `npm install -g @kaelio/ktx` and open a new shell |
| LLM health check fails | Missing, invalid, or unauthorized Anthropic API key | Export `ANTHROPIC_API_KEY` or rerun `ktx setup` and choose the file-backed secret option |
| OpenAI embedding check fails | `OPENAI_API_KEY` is missing when OpenAI embeddings are selected | Export `OPENAI_API_KEY`, or rerun setup and choose local sentence-transformers embeddings |
| Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup |
| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection |
| `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need |
| Symptom | Likely cause | Recovery |
|---------|--------------|----------|
| `ktx: command not found` | The global package is not installed or your shell cannot find it | Reinstall `@kaelio/ktx` and open a new shell |
| Setup resumes the wrong project | `KTX_PROJECT_DIR` or the nearest `ktx.yaml` points somewhere else | Pass `--project-dir <path>` |
| Anthropic health check fails | API key, model id, or access is invalid | Fix `ANTHROPIC_API_KEY` or rerun setup with a different key or model |
| Vertex AI health check fails | Vertex API, Claude access, project, location, or IAM permissions are missing | Check the project, location, Application Default Credentials, and Vertex AI permissions |
| OpenAI embeddings fail | `OPENAI_API_KEY` is missing or invalid | Export the key or choose local sentence-transformers embeddings |
| Local embeddings fail | Managed Python runtime cannot install or start | Run `ktx dev runtime status`, then install the local embeddings runtime |
| Database test fails | Credentials, network access, database, warehouse, or schema is wrong | Test the same values with the database's native client, then rerun setup |
| Context is not built | Setup saved configuration but skipped or interrupted the build | Run `ktx setup` or `ktx ingest --all` |
| Agent integration is incomplete | Setup skipped the agents step or installed a different target | Run `ktx setup --agents --target <target>` |
## Next steps
- **Build more context** - learn about [database ingest](/docs/guides/building-context), relationship detection, and source ingestion workflows in the Building Context guide.
- **Refine your semantic layer** - the [Writing Context](/docs/guides/writing-context) guide covers source YAML, measures, joins, and wiki pages.
- **Understand the architecture** - read [The Context Layer](/docs/concepts/the-context-layer) to learn why a context layer is more than a semantic layer.
- **Connect more agents** - see the [Agent Clients](/docs/integrations/agent-clients) integration page for per-tool setup details.
- Build and refresh context with [Building Context](/docs/guides/building-context).
- Edit semantic sources and wiki pages with [Writing Context](/docs/guides/writing-context).
- Connect more tools with [Agent Clients](/docs/integrations/agent-clients).
- Read [The Context Layer](/docs/concepts/the-context-layer) to understand the architecture.

282
docs-site/content/docs/guides/building-context.mdx
View file

@ -1,171 +1,195 @@
---
title: Building Context
description: Build database and source context from configured KTX connections.
description: Build and refresh KTX context from databases, source tools, query history, and text.
---
Building context reads your configured connections and writes local context that
agents can use. Database connections produce schema context, and source
connections such as dbt, Looker, Metabase, and Notion produce semantic sources
and wiki pages.
Building context turns configured connections into local semantic-layer sources
and wiki pages. Agents use those files to understand your schema, business
definitions, metric logic, joins, and known caveats before they write SQL.
Use this guide after `ktx setup` has created `ktx.yaml` and at least one
database or context-source connection.
## The build loop
Most projects use this loop:
1. Check readiness with `ktx status`.
2. Build one connection with `ktx ingest <connectionId>`, or build everything
with `ktx ingest --all`.
3. Search or inspect the generated files under `semantic-layer/` and `wiki/`.
4. Edit source YAML or Markdown when business logic needs refinement.
5. Validate and query representative sources before handing the context to an
agent.
`ktx ingest --all` runs database connections first, then context-source
connections. That order lets dbt, BI, Notion, and text ingest attach context to
known warehouse tables.
## Database ingest
Database ingest connects to your warehouse and extracts structural metadata.
KTX stores the results locally so agents can understand your schema without
querying the database directly.
### Running database ingest
Database ingest connects to a configured warehouse and records local schema
context. It gives agents table, column, type, constraint, and row-count
grounding without requiring them to inspect the database directly.
```bash
ktx ingest <connection-id>
```
This runs a fast schema ingest by default. You can choose the depth with public
flags:
| Flag | What it does |
|------|-------------|
| `--fast` | Tables, columns, types, constraints, and row counts |
| `--deep` | Fast ingest plus AI-enriched database context |
```bash
# Build one connection quickly
ktx ingest my-postgres --fast
# Build AI-enriched database context
ktx ingest my-postgres --deep
# Build one configured database connection
ktx ingest warehouse
# Build all configured connections
ktx ingest --all
```
### Checking results
Depth controls how much context KTX builds:
Every ingest prints a summary and writes local artifacts. Use `ktx status`
after ingest to review project readiness and follow-up setup work:
| Flag | Best for | What it does |
|------|----------|--------------|
| `--fast` | First setup, quick refreshes, CI smoke checks | Deterministic schema ingest with tables, columns, types, constraints, and row counts |
| `--deep` | Agent-ready context for real analysis | Fast ingest plus AI-enriched descriptions, embeddings, relationship evidence, and optional query history |
Examples:
```bash
ktx status
ktx ingest warehouse --fast
ktx ingest warehouse --deep
ktx ingest --all --deep
```
### Relationship detection
Deep ingest needs LLM and embedding readiness. If those providers are not
configured, run `ktx setup` or use `--fast`.
Many databases lack declared foreign keys. KTX infers relationships by scoring column pairs across seven signals - name similarity, type compatibility, value overlap, embedding similarity, profile uniqueness, null rate, and structural priors. The weighted score determines each candidate's status:
## Query history
| Score range | Status | Meaning |
|-------------|--------|---------|
| &ge; 0.85 | `accepted` | High confidence - applied automatically |
| 0.55 &ndash; 0.84 | `review` | Plausible - needs human review |
| &lt; 0.55 | `rejected` | Low confidence - not applied |
PostgreSQL, BigQuery, and Snowflake can add query-history context. This helps
KTX learn common joins, filters, service-account patterns, redaction rules, and
usage-heavy query templates.
Deep database ingest can include relationship evidence where the connector can
provide it. Relationship review and calibration subcommands are not part of the
current public CLI surface.
## Ingestion
Ingestion pulls semantic context from your existing analytics tools - dbt projects, Looker models, Metabase questions, and more - and writes it into your KTX project as semantic sources and wiki pages.
### How it works
Each ingest run follows this flow:
1. An **adapter** extracts metadata from your tool (dbt manifest, LookML files, Metabase API, etc.)
2. An **LLM agent** reconciles the extracted metadata with your existing context - it merges intelligently rather than overwriting
3. **Semantic sources** (YAML) and **wiki pages** (Markdown) are written to your project directory
### Running an ingest
Enable it during setup, store it under `connections.<id>.context.queryHistory`,
or request it for one run:
```bash
ktx ingest my-dbt-source
ktx ingest warehouse --deep --query-history
ktx ingest warehouse --query-history-window-days 30
```
Useful output flags:
Use `--no-query-history` when you want to skip a stored query-history setting
for one run.
## Relationship evidence
Many databases do not declare all foreign keys. KTX can score relationship
candidates using signals such as name similarity, type compatibility, value
overlap, embedding similarity, uniqueness, null rate, and structural priors.
The public CLI does not expose separate relationship review subcommands.
Relationship evidence is built as part of deep database ingest when the
connector and readiness checks support it.
## Context-source ingest
Context-source connections pull business metadata from tools your team already
uses. The current public `ktx ingest` command is connection-centric: pass one
configured connection id, or pass `--all`.
```bash
# Build one source connection
ktx ingest dbt_main
# Build every configured database and source connection
ktx ingest --all
```
Supported source types:
| Driver | Typical source | Output |
|--------|----------------|--------|
| `dbt` | dbt project or Git repo | Semantic sources with model, column, test, tag, and description metadata |
| `metricflow` | MetricFlow project or Git repo | Metrics, dimensions, entities, and semantic joins |
| `lookml` | LookML files or Git repo | Views, explores, dimensions, measures, and joins |
| `looker` | Looker API | Explores, looks, dashboards, and model metadata |
| `metabase` | Metabase API | Questions, dashboards, table metadata, and mappings |
| `notion` | Notion API | Wiki pages and business knowledge |
Source ingest extracts metadata, reconciles it with existing local context, and
writes semantic-layer YAML plus wiki Markdown. It merges rather than blindly
overwriting local edits.
## Text ingest
Use `ktx ingest text` for notes, Markdown files, runbooks, Slack exports, or
other free-form knowledge that should become searchable KTX memory.
```bash
# Capture a Markdown file
ktx ingest text docs/revenue-notes.md --connection-id warehouse
# Capture one stdin item
printf "Refunds are excluded from net revenue." | ktx ingest text -
# Capture direct text
ktx ingest text --text "ARR excludes one-time implementation fees."
```
Useful flags:
| Flag | Description |
|------|-------------|
| `--json` | Output as JSON |
| `--plain` | Plain text output |
| `--connection-id <connectionId>` | Attach the captured memory to a KTX connection |
| `--user-id <id>` | Attribute capture to a user scope, default `local-cli` |
| `--json` | Print structured output |
| `--fail-fast` | Stop after the first failed text item |
Foreground context builds do not detach into background control sessions. If a
run is interrupted, rerun `ktx ingest <connection-id>` or `ktx ingest --all`.
Text ingest is a good fit for small, high-signal documents. For system-specific
connectors such as Notion, dbt, or Metabase, prefer configured source ingest so
KTX can preserve source metadata.
### Supported context sources
## Output and artifacts
| Driver | Source | What gets ingested |
|--------|--------|--------------------|
| `dbt` | dbt project | Model definitions, column descriptions, tests, tags |
| `metricflow` | MetricFlow semantic models | Metrics, dimensions, entities, semantic joins |
| `lookml` | LookML files | Views, explores, dimensions, measures, joins |
| `looker` | Looker API | Explores, looks, dashboard metadata |
| `metabase` | Metabase API | Questions, dashboards, table metadata |
| `notion` | Notion API | Database pages, knowledge articles |
Every ingest run prints a summary. Use `--json` when an agent or script needs a
structured plan and per-target results.
Query history is a database connection facet. Enable it with
`connections.<id>.context.queryHistory` or pass `--query-history` for a current
run. See [Context Sources](/docs/integrations/context-sources) for
driver-specific setup and auth configuration.
### What gets generated
A typical dbt ingest produces semantic sources and wiki pages in your project:
**Semantic source** (`semantic-layer/my-postgres/orders.yaml`):
```yaml title="semantic-layer/my-postgres/orders.yaml"
name: orders
table: public.orders
grain:
- order_id
columns:
- name: order_id
type: string
description: Unique order identifier
- name: customer_id
type: string
description: Foreign key to customers table
- name: order_date
type: time
role: time
description: Date the order was placed
- name: total_amount
type: number
description: Total order value in USD
measures:
- name: total_revenue
expr: SUM(total_amount)
description: Sum of all order values
- name: order_count
expr: COUNT(DISTINCT order_id)
description: Number of distinct orders
joins:
- to: customers
on: orders.customer_id = customers.customer_id
relationship: many_to_one
```bash
ktx ingest --all --json
```
**Wiki page** (`wiki/global/order-status-definitions.md`):
Typical generated files:
```markdown
---
summary: Business definitions for order status values
tags: [orders, definitions]
sl_refs: [orders]
---
| Path | Created by | Purpose |
|------|------------|---------|
| `semantic-layer/<connection-id>/*.yaml` | Database and source ingest | Queryable semantic source definitions |
| `wiki/global/*.md` | Source, text, and memory ingest | Shared business definitions and notes |
| `wiki/user/<user-id>/*.md` | Text and memory ingest | User-scoped context |
| `.ktx/setup/context-build.json` | Setup context build | Resume and readiness state for setup |
## Order Statuses
Ingest sessions also record transcripts with tool calls, LLM responses, and
write decisions. Inspect them when you need to debug why a source or wiki page
was written a certain way.
- **pending**: Order placed but not yet processed
- **confirmed**: Payment received, awaiting fulfillment
- **shipped**: Order dispatched to carrier
- **delivered**: Order received by customer
- **cancelled**: Order cancelled before shipment
## Example: first full refresh
Orders in "pending" status for more than 48 hours are flagged for review.
After interactive setup:
```bash
ktx status
ktx ingest --all --deep
ktx status
```
### Ingest transcripts
Then inspect what changed:
Every ingest session records a full transcript: tool calls, LLM responses, and
write decisions. Inspect the stored transcript files when you need to debug why
a source was written a certain way.
```bash
git status --short
ktx sl list --json
ktx wiki search "revenue" --json --limit 10
```
## Common errors
| Symptom | Likely cause | Recovery |
|---------|--------------|----------|
| Connection not configured | The connection id is missing from `ktx.yaml` | Add it with `ktx setup` |
| Deep readiness is missing | LLM or embeddings are not setup-ready | Run `ktx setup`, or rerun with `--fast` |
| Query history is unsupported | The selected database driver does not expose query history | Run schema ingest without query-history flags |
| No target selected | You omitted both a connection id and `--all` | Run `ktx ingest <connectionId>` or `ktx ingest --all` |
| Source flags have no effect | Depth and query-history flags were supplied for a source connector | Use those flags only for database connections |
| Text ingest stops early | `--fail-fast` stopped on the first failed item | Fix the item or rerun without `--fail-fast` |

166
docs-site/content/docs/guides/serving-agents.mdx
View file

@ -1,59 +1,167 @@
---
title: Serving Agents
description: Expose your context to Claude Code, Cursor, Codex, and other coding agents.
description: Expose KTX context to Claude Code, Codex, Cursor, OpenCode, and custom agents.
---
Once you've built and refined your context, expose it to coding agents through
the public KTX CLI. Claude Code, Cursor, Codex, OpenCode, and custom agent
workflows can call the same commands you use at a terminal.
KTX serves agents through the public CLI and project-local instruction files.
Agents do not need a separate server. They read the generated rules, call KTX
commands, inspect local context files, and use JSON output when they need
structured results.
## CLI Commands
## Recommended setup
KTX public commands support JSON output for the context reads that agents use
most often. Use `--project-dir` when the agent is not already running inside the
KTX project directory.
### Available commands
Run the agent install step from a KTX project:
```bash
ktx setup --agents
```
Or install a specific target:
```bash
ktx setup --agents --target codex
```
Supported targets:
| Target | Generated project file |
|--------|------------------------|
| Claude Code | `.claude/skills/ktx/SKILL.md` |
| Codex | `.agents/skills/ktx/SKILL.md` |
| Cursor | `.cursor/rules/ktx.mdc` |
| OpenCode | `.opencode/commands/ktx.md` |
| Universal `.agents` | `.agents/skills/ktx/SKILL.md` |
Claude Code and Codex also support global installs:
```bash
ktx setup --agents --target claude-code --global
ktx setup --agents --target codex --global
```
KTX records installed files in `.ktx/agents/install-manifest.json`. Rerun
`ktx setup --agents` after moving a checkout or reinstalling the CLI so the
generated instructions point at the current CLI path.
## Agent command set
All supported agent clients use the same command surface. Use `--project-dir`
when the agent is running outside the KTX project directory.
### Readiness
```bash
# Check setup and context readiness
ktx status --json
```
**Semantic layer:**
Agents should run this before relying on context. It reports project, LLM,
embedding, database, context-source, context-build, and agent-integration
readiness.
### Semantic layer discovery
```bash
# List sources
ktx sl list --json
ktx sl list --json --connection-id my-postgres
ktx sl search "revenue" --json
ktx sl list --connection-id warehouse --json
ktx sl search "revenue" --json --limit 10
```
# Run a query from a JSON file
ktx sl query --json \
--connection-id my-postgres \
--query-file query.json \
Agents use these commands to discover source names, connection ids, measures,
dimensions, and likely files to inspect.
### Semantic-layer validation and queries
```bash
ktx sl validate orders --connection-id warehouse
```
Compile SQL before executing:
```bash
ktx sl query \
--connection-id warehouse \
--measure orders.total_revenue \
--dimension orders.created_date \
--format sql
```
Execute only when the task calls for live data:
```bash
ktx sl query \
--connection-id warehouse \
--measure orders.total_revenue \
--dimension orders.status \
--execute \
--max-rows 100
```
**Wiki:**
For complex calls, agents can write a JSON query object and pass it with
`--query-file`.
### Wiki context
```bash
# Search wiki pages
ktx wiki list --json
ktx wiki search "revenue recognition" --json --limit 10
```
## Setting Up Your Agent
Agents should search wiki context when a question depends on business
definitions, metric caveats, process rules, or terms that are not obvious from
schema names.
The fastest way to connect an agent is through the setup wizard:
### Context refresh
Agents can refresh context when the user asks them to:
```bash
ktx setup
ktx ingest warehouse --fast
ktx ingest --all
ktx ingest text docs/revenue-notes.md --connection-id warehouse
```
The agents step auto-detects installed tools and generates the right
configuration. For manual setup or per-tool details, see the
[Agent Clients](/docs/integrations/agent-clients) integration page.
Use `--deep` only when LLM and embedding setup is ready and the user expects an
AI-enriched refresh.
After configuration, the agent can immediately call KTX commands to list
sources, search wiki pages, and query your semantic layer.
## Good agent behavior
Agents should:
- Run `ktx status --json` before using KTX context.
- Use `ktx sl search` and `ktx wiki search` before writing SQL from memory.
- Inspect the relevant YAML or Markdown files after search returns candidates.
- Compile SQL with `ktx sl query --format sql` before executing.
- Use `--max-rows` whenever executing a live query.
- Validate edited semantic sources with `ktx sl validate`.
- Keep generated context changes reviewable in git.
Agents should not assume a background server, ORPC route, frontend app, or
external migration system exists. KTX is a local context layer with a CLI and
plain project files.
## Manual setup
Manual setup is useful for custom agents that can read project-local
instructions but are not yet a named target.
1. Install the universal target:
```bash
ktx setup --agents --target universal
```
2. Configure the agent to read `.agents/skills/ktx/SKILL.md`.
3. Open the agent in the KTX project directory.
4. Ask it to run `ktx status --json` and summarize readiness.
For per-client notes, see [Agent Clients](/docs/integrations/agent-clients).
## Troubleshooting
| Symptom | Likely cause | Recovery |
|---------|--------------|----------|
| Agent says KTX is unavailable | Agent did not load the generated instruction file | Rerun `ktx setup --agents --target <target>` and restart the agent session |
| Agent command cannot find the project | Agent is running outside the KTX directory | Add `--project-dir <path>` or open the agent in the project root |
| Generated rules point at a missing CLI path | CLI was moved, rebuilt, or reinstalled | Rerun `ktx setup --agents` |
| Agent cannot find a metric | Context is missing or stale | Run `ktx sl search`, inspect source YAML, then refresh with `ktx ingest` if needed |
| Agent query returns too many rows | The command executed without a result cap | Require `--max-rows` for executed queries |

424
docs-site/content/docs/guides/writing-context.mdx
View file

@ -1,295 +1,341 @@
---
title: Writing Context
description: Write and refine semantic sources and wiki pages.
description: Edit semantic sources and wiki pages so agents use your business logic.
---
After building context through scanning and ingestion, you'll want to refine it - edit semantic sources to match your business logic, add wiki pages that capture tribal knowledge, and query your data through the semantic layer to verify everything works.
KTX context is meant to be edited. Ingest gives you a grounded first draft, then
you refine source YAML and wiki Markdown until agents can answer data questions
with the same definitions your team uses.
## Agent workflow summary
Use this guide when you are adding measures, fixing joins, documenting business
rules, or reviewing context changes made by an agent.
Agents should refine context in this order:
## Editing workflow
1. `ktx sl list --json` - discover available sources and connection ids.
2. `ktx sl search <query> --json` - find source candidates for a concept.
3. Edit the source YAML directly in `semantic-layer/<connection-id>/`.
4. `ktx sl validate <source> --connection-id <id>` - verify columns, joins, and table references.
5. `ktx sl query ... --format sql` - compile a representative query without executing it.
6. `ktx wiki search ...` - check business context captured by ingest or memory.
Use this order for most context changes:
## Semantic Sources
1. Discover existing context.
Semantic sources are YAML files that describe your tables, columns, measures, and joins. They're the core of the context layer - the structured definitions that agents use to generate correct SQL.
```bash
ktx sl list --json
ktx sl search "revenue" --json
ktx wiki search "revenue recognition" --json --limit 10
```
### Listing sources
2. Edit the smallest relevant files under `semantic-layer/<connection-id>/` or
`wiki/`.
3. Validate semantic source changes.
```bash
# List all sources across connections
ktx sl list
```bash
ktx sl validate orders --connection-id warehouse
```
# List sources for a specific connection
ktx sl list --connection-id my-postgres
4. Compile a representative query before executing it.
# Output as JSON
ktx sl list --json
```bash
ktx sl query \
--connection-id warehouse \
--measure orders.total_revenue \
--dimension orders.created_date \
--format sql
```
5. Search again using likely user wording to confirm the new context is
discoverable.
## Semantic sources
Semantic sources are YAML files that describe queryable entities. A source is
usually a table, but it can also point at a custom SQL expression. Sources
define the vocabulary agents use for measures, dimensions, segments, joins, and
grain-aware query planning.
Source files live at:
```text
semantic-layer/<connection-id>/<source-name>.yaml
```
### Searching sources
```bash
ktx sl search "revenue" --connection-id my-postgres --json
```
Search returns ranked source summaries. To inspect or edit a source, open the
YAML file under `semantic-layer/<connection-id>/`.
### The source schema
A semantic source defines a single queryable entity - usually a table or a SQL expression. Here's a fully annotated example:
### Minimal source
```yaml
name: orders
description: Customer orders with line-item totals
table: public.orders # or use `sql:` for a custom SQL expression
description: Customer orders with booked revenue.
table: public.orders
grain:
- order_id # columns that uniquely identify a row
- order_id
columns:
- name: order_id
type: string
description: Unique order identifier.
- name: order_date
type: time
role: time
description: Date the order was placed.
- name: total_amount
type: number
description: Booked order value in USD.
measures:
- name: total_revenue
expr: SUM(total_amount)
description: Sum of booked order value before refunds.
```
### Full source shape
```yaml
name: orders
description: Customer orders with line-item totals.
table: public.orders
grain:
- order_id
columns:
- name: order_id
type: string # string | number | time | boolean
description: Unique order identifier
type: string
description: Unique order identifier.
- name: order_date
type: time
role: time # marks this as the default time dimension
description: Date the order was placed
role: time
description: Date the order was placed.
- name: status
type: string
visibility: public # public (default) | internal | hidden
description: Current order status
visibility: public
description: Current order status.
- name: _etl_loaded_at
type: time
visibility: hidden # hidden columns are excluded from agent queries
description: Internal ETL timestamp
visibility: hidden
description: Internal load timestamp.
- name: total_amount
type: number
description: Order total in USD
description: Order total in USD.
measures:
- name: total_revenue
expr: SUM(total_amount)
description: Sum of all order values
description: Sum of all order values.
- name: order_count
expr: COUNT(DISTINCT order_id)
description: Number of distinct orders
description: Number of distinct orders.
- name: avg_order_value
expr: AVG(total_amount)
description: Average order value
description: Average booked order value.
- name: high_value_revenue
expr: SUM(total_amount)
filter: total_amount > 100
description: Revenue from orders over $100
description: Revenue from orders over $100.
segments:
- name: us_orders
expr: country = 'US'
description: Orders from US customers
- name: completed_orders
expr: status = 'completed'
description: Orders that completed fulfillment.
joins:
- to: customers
on: orders.customer_id = customers.customer_id
relationship: many_to_one # many_to_one | one_to_many | one_to_one
relationship: many_to_one
- to: order_items
on: orders.order_id = order_items.order_id
relationship: one_to_many
alias: items # optional alias for the joined source
alias: items
```
Key fields:
### Source fields
| Field | Required | Description |
|-------|----------|-------------|
| `name` | Yes | Source identifier (lowercase, underscores) |
| `table` or `sql` | Yes | Database table or custom SQL expression (exactly one) |
| `grain` | Yes | Columns that define row uniqueness |
| `columns` | No | Column definitions with type, role, visibility |
| `measures` | No | Aggregation expressions (SUM, COUNT, AVG, etc.) |
| `joins` | No | Relationships to other sources |
| `segments` | No | Named filter conditions |
| `inherits_columns_from` | No | Inherit column metadata from a manifest entry |
| `name` | Yes | Source identifier. Use lowercase words and underscores. |
| `table` or `sql` | Yes | Database table or custom SQL expression. Use exactly one. |
| `grain` | Yes | Columns that uniquely identify a row at the source grain. |
| `columns` | No | Column definitions with type, role, visibility, and descriptions. |
| `measures` | No | Aggregation expressions such as `SUM`, `COUNT`, and `AVG`. |
| `segments` | No | Named predicates agents can reuse. |
| `joins` | No | Relationships to other semantic sources. |
| `inherits_columns_from` | No | Inherit column metadata from a manifest entry. |
Source component fields:
### Component fields
| Component | Field | Required | Description |
|-----------|-------|----------|-------------|
| Column | `name` | Yes | Column identifier as used in SQL expressions |
| Column | `type` | Yes | Agent-facing type: `string`, `number`, `time`, or `boolean` |
| Column | `role` | No | Special role such as `time` for default time dimensions |
| Column | `visibility` | No | `public`, `internal`, or `hidden` |
| Column | `description` | Strongly recommended | Human-readable business meaning |
| Measure | `name` | Yes | Queryable metric name |
| Measure | `expr` | Yes | SQL aggregation expression at the source grain |
| Measure | `filter` | No | SQL predicate applied only to this measure |
| Measure | `description` | Strongly recommended | Definition agents can cite and compare |
| Segment | `name` | Yes | Reusable filter name |
| Segment | `expr` | Yes | SQL predicate for the segment |
| Join | `to` | Yes | Target semantic source name |
| Join | `on` | Yes | SQL join condition using source names or aliases |
| Join | `relationship` | Yes | `many_to_one`, `one_to_many`, or `one_to_one` |
| Join | `alias` | No | Query alias for repeated or clearer joins |
| Column | `name` | Yes | Column identifier used in SQL expressions. |
| Column | `type` | Yes | Agent-facing type: `string`, `number`, `time`, or `boolean`. |
| Column | `role` | No | Special role such as `time` for default time dimensions. |
| Column | `visibility` | No | `public`, `internal`, or `hidden`. |
| Column | `description` | Strongly recommended | Business meaning and usage notes. |
| Measure | `name` | Yes | Queryable metric name. |
| Measure | `expr` | Yes | SQL aggregation expression at the source grain. |
| Measure | `filter` | No | SQL predicate applied only to this measure. |
| Measure | `description` | Strongly recommended | Definition agents can cite and compare. |
| Segment | `name` | Yes | Reusable filter name. |
| Segment | `expr` | Yes | SQL predicate for the segment. |
| Join | `to` | Yes | Target semantic source name. |
| Join | `on` | Yes | SQL join condition using source names or aliases. |
| Join | `relationship` | Yes | `many_to_one`, `one_to_many`, or `one_to_one`. |
| Join | `alias` | No | Query alias for repeated or clearer joins. |
Column visibility controls what agents see:
### Visibility
| Visibility | Behavior |
|------------|----------|
| `public` | Included in agent queries and listings (default) |
| `internal` | Available for joins and measures but not shown to agents |
| `hidden` | Excluded entirely - useful for ETL columns |
| Visibility | Agent behavior |
|------------|----------------|
| `public` | Included in listings and available for agent queries. |
| `internal` | Available for joins and measures, but not highlighted to agents. |
| `hidden` | Excluded from agent-facing context. Use for ETL fields and sensitive internals. |
### Editing a source
## Measures
Edit source files directly. They live at
`semantic-layer/<connection-id>/<source-name>.yaml` in your project directory.
Good measures have precise names, SQL expressions at the correct grain, and
descriptions that say what is included and excluded.
### Validating sources
Validation checks a source definition against the actual database schema:
```bash
ktx sl validate orders --connection-id my-postgres
```yaml
measures:
- name: net_revenue
expr: SUM(total_amount - refunded_amount)
filter: status = 'completed'
description: Completed order revenue after refunds, excluding cancelled orders.
```
This catches mismatches - columns that don't exist in the table, type mismatches, invalid join targets - before an agent tries to use the source.
Prefer one canonical measure plus wiki synonyms over several nearly identical
measures. If your team uses multiple definitions, document the distinction in a
wiki page and link it with `sl_refs`.
### Querying
## Joins and grain
The semantic layer compiles your measures and dimensions into SQL, optionally executing it against the database:
`grain` and `relationship` prevent agents from producing double-counted SQL.
State the row grain even when it seems obvious.
```yaml
grain:
- order_id
joins:
- to: customers
on: orders.customer_id = customers.customer_id
relationship: many_to_one
```
Use `many_to_one` for dimensions such as customer, account, product, or plan.
Use `one_to_many` only when the target can fan out the source rows, such as
orders to order items.
## Validate and query
Validation checks source YAML against the live database schema:
```bash
ktx sl validate orders --connection-id warehouse
```
It catches missing columns, invalid join targets, and table-reference problems
before an agent relies on the source.
Compile a query to inspect generated SQL:
```bash
# Compile a query to SQL
ktx sl query \
--connection-id my-postgres \
--measure total_revenue \
--measure order_count \
--dimension "order_date" \
--filter "status = 'completed'" \
--order-by order_date:desc \
--connection-id warehouse \
--measure orders.total_revenue \
--dimension orders.order_date \
--filter "orders.status = 'completed'" \
--order-by orders.order_date:desc \
--limit 10 \
--format sql
```
This outputs the compiled SQL without executing it. To run the query:
Execute only when you need live rows:
```bash
# Execute and return results
ktx sl query \
--connection-id my-postgres \
--measure total_revenue \
--dimension "order_date" \
--connection-id warehouse \
--measure orders.total_revenue \
--dimension orders.status \
--execute \
--max-rows 100
```
Query flags:
## Wiki pages
| Flag | Description |
|------|-------------|
| `--measure <name>` | Measure to query (repeatable, at least one required) |
| `--dimension <name>` | Dimension to group by (repeatable) |
| `--filter <expr>` | Filter expression (repeatable) |
| `--segment <name>` | Named segment to apply (repeatable) |
| `--order-by <field[:dir]>` | Sort field, optionally with `:asc` or `:desc` (repeatable) |
| `--limit <n>` | Maximum rows in the compiled query |
| `--format <mode>` | Output format: `json` (default) or `sql` |
| `--execute` | Execute the query against the database |
| `--max-rows <n>` | Maximum rows to return when executing |
| `--include-empty` | Include empty/null rows in results |
Wiki pages capture business context that does not belong in a single source
file: metric policies, dashboard caveats, company vocabulary, data freshness,
known issues, and source-of-truth notes.
The query planner is grain-aware - it understands the cardinality of joins and avoids chasm traps (double-counting caused by many-to-many fan-outs). When you query measures that span multiple sources, KTX generates sub-queries at the correct grain before joining.
Wiki files live under:
### Workflow: edit and validate a source
1. Open `semantic-layer/my-postgres/orders.yaml`.
2. Edit the file to add columns, measures, joins, or descriptions.
3. `ktx sl validate orders --connection-id my-postgres` - check the definition against the live schema.
4. `ktx sl query --connection-id my-postgres --measure total_revenue --dimension order_date --format sql` - compile a representative query.
If validation fails, fix the YAML before asking an agent to use the source. Common validation failures are missing columns, invalid join targets, and measure expressions that reference fields outside the source.
## Wiki Pages
Wiki pages are Markdown files that capture business context - definitions, rules, gotchas, and anything an agent needs to understand beyond what the schema tells it.
### What they are
When an agent asks "what counts as an active user?" or "why do revenue numbers differ between the dashboard and the SQL query?", the answer isn't in the schema. It's tribal knowledge that lives in Slack threads, Notion pages, or someone's head. Wiki pages make that context searchable and available to agents.
### Organization
Wiki pages are organized by scope:
```
```text
wiki/
├── global/ # Cross-cutting definitions
│ ├── order-status-definitions.md
│ ├── revenue-recognition-rules.md
│ └── data-freshness-sla.md
└── user/
└── local/ # User-scoped context
├── schema-conventions.md
└── known-data-issues.md
global/
user/<user-id>/
```
- **Global pages** apply across all connections - business definitions, metric standards, company terminology.
- **User-scoped pages** are private to a user ID - personal notes, local gotchas, or context you do not want shared globally.
Use global pages for shared business rules. Use user-scoped pages for local
notes, personal conventions, or context that should not be shared broadly.
### Editing pages
### Wiki page example
Create and edit wiki pages directly as Markdown files in the `wiki/`
directory. Ingest and memory capture also create these pages automatically.
```markdown
---
summary: Revenue recognition rules for finance reporting.
tags: [revenue, finance, reporting]
sl_refs: [orders]
external_refs:
- type: notion
id: finance-revenue-policy
---
Wiki page fields:
## Recognized Revenue
Recognized revenue includes completed orders after refunds. It excludes
cancelled orders, test orders, implementation fees, and tax.
Finance reporting uses order completion date, not invoice creation date.
```
Useful frontmatter:
| Field | Required | Description |
|-------|----------|-------------|
| Key | Yes | Stable page identifier used as the Markdown filename |
| Summary | Yes | Short text shown in search results |
| Content | Yes | Full Markdown business context |
| Scope | No | `global` for shared context or `user` for user-scoped notes |
| Tags | No | Search and organization labels |
| External refs | No | Links or identifiers for source-of-truth systems |
| Semantic-layer refs | No | Source names the page explains or constrains |
| `summary` | Yes | Short text shown in search results. |
| `tags` | No | Business terms and synonyms that improve search. |
| `sl_refs` | No | Semantic source names the page explains or constrains. |
| `external_refs` | No | Source-of-truth system links or ids. |
### Listing pages
## Add searchable business context
1. Search first.
```bash
ktx wiki search "active customer definition" --json --limit 10
```
2. If no page covers the rule, create or edit a Markdown file under
`wiki/global/`.
3. Write a compact `summary` with the wording users are likely to ask.
4. Add tags for synonyms and related business areas.
5. Add `sl_refs` for relevant semantic sources.
6. Search again with a user-like phrase.
## Review context changes
Before accepting agent-written context:
```bash
ktx wiki list
git diff -- semantic-layer wiki
ktx sl validate orders --connection-id warehouse
ktx sl search "revenue" --json
ktx wiki search "revenue recognition" --json --limit 10
```
### Searching
```bash
ktx wiki search "revenue recognition"
```
Search uses both full-text matching and semantic similarity - it finds relevant pages even when the exact terms don't match. Agents call this automatically when they need business context to answer a question.
### Workflow: add searchable business context
1. Search first: `ktx wiki search "order status definitions"`.
2. If no page already covers the rule, create or edit a Markdown file under `wiki/global/`.
3. Include concise frontmatter; agents see the summary before loading full content.
4. Add `tags` values for the business area and `sl_refs` values for related semantic sources.
5. Search again with the user's likely wording to confirm the page is discoverable.
Check that definitions are specific, hidden columns stay hidden, joins have
explicit relationships, and measures compile into the expected SQL.
## Common errors
| Error or symptom | Likely cause | Recovery |
|------------------|--------------|----------|
| `ktx sl validate` reports a missing column | YAML references a column that is absent from the scanned table | Run a fresh scan or update the YAML to match the warehouse schema |
| Query compilation double-counts a measure | Join relationship or grain is missing or wrong | Add `grain` and explicit `relationship` values, then validate and recompile |
| Agent cannot find a metric | Measure name or description does not match business terminology | Add a measure description and a wiki page with common synonyms |
| Wiki search misses a page | Summary and tags do not include likely user wording | Rewrite the summary and add relevant tags, then search again |
| Semantic-layer changes are hard to review | The YAML edit is too large or unfocused | Split the change into smaller source-file edits, then review the git diff |
| Symptom | Likely cause | Recovery |
|---------|--------------|----------|
| `ktx sl validate` reports a missing column | YAML references a column absent from the scanned table | Refresh database context or update the YAML |
| Query compilation double-counts a measure | `grain` or join `relationship` is missing or wrong | Add explicit grain and relationship values, then recompile |
| Agent cannot find a metric | Measure name and description do not match business terminology | Add a clearer measure description and a wiki page with synonyms |
| Wiki search misses a page | Summary, tags, or content do not match user wording | Rewrite the summary and add likely synonyms |
| Context diff is hard to review | One edit changed too many concepts | Split the change into focused source and wiki edits |

65
docs-site/content/docs/integrations/agent-clients.mdx
View file

@ -7,7 +7,46 @@ KTX integrates with coding agents through CLI skills and command files. These
files teach agents to call public `ktx` commands directly from the terminal for
semantic-layer context and wiki knowledge.
Run `ktx setup` and select your agent targets, or configure manually using the snippets below.
Run `ktx setup` and select your agent targets, or configure manually using the
snippets below. Setup pins generated skill files to the KTX CLI path that
created them, so agents do not need `ktx` on `PATH`.
## Install with setup
```bash
ktx setup --agents
```
Use `--target` for one target:
```bash
ktx setup --agents --target codex
```
Use `--global` only with `claude-code` or `codex`:
```bash
ktx setup --agents --target claude-code --global
ktx setup --agents --target codex --global
```
KTX records installed files in `.ktx/agents/install-manifest.json`. That
manifest lets status checks report agent readiness and lets future cleanup
remove only files KTX installed.
## Generated files
| Target | Project-scoped files | Global files |
|--------|----------------------|--------------|
| Claude Code | `.claude/skills/ktx/SKILL.md`, `.claude/rules/ktx.md` | `~/.claude/skills/ktx/SKILL.md`, `~/.claude/rules/ktx.md` |
| Codex | `.agents/skills/ktx/SKILL.md`, `.codex/instructions/ktx.md` | `$CODEX_HOME/skills/ktx/SKILL.md`, `$CODEX_HOME/instructions/ktx.md` |
| Cursor | `.cursor/rules/ktx.mdc` | Not supported |
| OpenCode | `.opencode/commands/ktx.md` | Not supported |
| Universal `.agents` | `.agents/skills/ktx/SKILL.md` | Not supported |
Skill files list pinned `ktx` commands. Rule files tell the agent when KTX is
appropriate, such as data schemas, metrics, dimensions, database structure, and
SQL questions.
## Claude Code
@ -15,11 +54,12 @@ Run `ktx setup` and select your agent targets, or configure manually using the s
During setup, select **Claude Code** from the agent targets. KTX writes:
| Mode | File |
|------|------|
| CLI skills | `.claude/skills/ktx/SKILL.md` |
| Scope | Files |
|-------|-------|
| Project | `.claude/skills/ktx/SKILL.md`, `.claude/rules/ktx.md` |
| Global | `~/.claude/skills/ktx/SKILL.md`, `~/.claude/rules/ktx.md` |
Both project-scoped and global installations are supported. Global installs write to `~/.claude/skills/ktx/SKILL.md`.
Both project-scoped and global installations are supported.
### Manual CLI skills configuration
@ -42,6 +82,7 @@ Available commands:
### Workflow tips
- Claude Code discovers skills automatically from `.claude/skills/`.
- Claude rules in `.claude/rules/` tell Claude when KTX should be used.
- Global installation makes KTX available in all projects without per-project setup.
- Keep generated skills committed only when your team wants project-local agent instructions in git.
@ -76,11 +117,13 @@ Create `.cursor/rules/ktx.mdc` with the same content structure as the Claude Cod
During setup, select **Codex** from the agent targets. KTX writes:
| Mode | File |
|------|------|
| CLI skills | `.agents/skills/ktx/SKILL.md` |
| Scope | Files |
|-------|-------|
| Project | `.agents/skills/ktx/SKILL.md`, `.codex/instructions/ktx.md` |
| Global | `$CODEX_HOME/skills/ktx/SKILL.md`, `$CODEX_HOME/instructions/ktx.md` |
Both project-scoped and global installations are supported. Global installs write to `$CODEX_HOME/skills/ktx/SKILL.md` (defaults to `~/.codex/skills/ktx/SKILL.md`).
Both project-scoped and global installations are supported. `CODEX_HOME`
defaults to `~/.codex`.
### Manual CLI skills configuration
@ -90,6 +133,7 @@ Create `.agents/skills/ktx/SKILL.md` with the same content structure as Claude C
- Set `CODEX_HOME` to customize the global installation directory.
- Codex shares the `.agents/` directory structure with the universal format.
- Codex instructions in `.codex/instructions/` tell Codex when KTX should be used.
- Global installation makes KTX available across all Codex sessions.
---
@ -143,4 +187,5 @@ All supported agent clients call the same KTX CLI commands:
|---|---|---|---|---|
| CLI skills | Yes | Yes (.mdc) | Yes | Yes |
| Global install | Yes | No | Yes | No |
| Config location | `.claude/skills/ktx/SKILL.md` | `.cursor/rules/ktx.mdc` | `.agents/skills/ktx/SKILL.md` | `.opencode/commands/ktx.md` |
| Rule or instruction file | `.claude/rules/ktx.md` | `.cursor/rules/ktx.mdc` | `.codex/instructions/ktx.md` | `.opencode/commands/ktx.md` |
| Skill file | `.claude/skills/ktx/SKILL.md` | Not separate | `.agents/skills/ktx/SKILL.md` | Not separate |

70
docs-site/content/docs/integrations/index.mdx Normal file
View file

@ -0,0 +1,70 @@
---
title: Integrations
description: Connect KTX to warehouses, analytics tools, and coding agents.
---
KTX integrations bring trusted context into an analytics project and make that
context available to coding agents through the CLI. Start with `ktx setup` when
you want the guided flow, then use the integration reference pages for exact
configuration fields, generated files, and manual setup.
## Integration types
| Type | What it connects | Start here |
|------|------------------|------------|
| Primary sources | Warehouses and databases that KTX scans for schemas, constraints, row counts, and optional query history | [Primary Sources](/docs/integrations/primary-sources) |
| Context sources | Existing analytics and knowledge tools such as dbt, MetricFlow, LookML, Metabase, Looker, and Notion | [Context Sources](/docs/integrations/context-sources) |
| Agent clients | Claude Code, Codex, Cursor, OpenCode, and universal `.agents` consumers | [Agent Clients](/docs/integrations/agent-clients) |
## Recommended setup flow
Use this order for a new project:
1. Run `ktx setup` from the analytics project directory.
2. Configure an LLM backend and embeddings so KTX can enrich and search context.
3. Add at least one primary source connection.
4. Add optional context sources that describe the same warehouse or business domain.
5. Build context during setup, or run `ktx ingest <connectionId>` later.
6. Install agent integration with `ktx setup --agents` when the context is ready.
For repeatable setup, pass `--project-dir`, `--no-input`, and the relevant
automation flags documented in [`ktx setup`](/docs/cli-reference/ktx-setup).
## What setup writes
| Path | Purpose |
|------|---------|
| `ktx.yaml` | Main project configuration for providers, embeddings, connections, source mappings, query history, and setup state |
| `.ktx/secrets/*` | Local file-backed secrets when you choose file references during setup |
| `semantic-layer/<connection-id>/` | YAML semantic sources generated by database and source ingestion |
| `wiki/` | Markdown business context, definitions, and ingested knowledge |
| `.ktx/agents/install-manifest.json` | Manifest of agent integration files installed by `ktx setup --agents` |
| Agent client files | Skills, rules, or commands that teach agents when and how to call KTX |
## Common commands
```bash
# Start or resume the guided flow
ktx setup
# Add or refresh every configured integration
ktx ingest --all
# Refresh one configured warehouse, source, or knowledge integration
ktx ingest warehouse
# Install one project-scoped agent target
ktx setup --agents --target codex
# Check whether integrations are ready
ktx status
```
## Choosing docs
Read [Primary Sources](/docs/integrations/primary-sources) when you need
database driver fields, authentication formats, query history support, or
warehouse-specific notes. Read [Context Sources](/docs/integrations/context-sources)
when you need source adapter fields, repository authentication, BI tool mapping,
or Notion crawl options. Read [Agent Clients](/docs/integrations/agent-clients)
when you need generated file locations or manual agent configuration.

2
docs-site/content/docs/integrations/meta.json
View file

@ -1,5 +1,5 @@
{
"title": "Integrations",
"defaultOpen": true,
"pages": ["primary-sources", "context-sources", "agent-clients"]
"pages": ["index", "primary-sources", "context-sources", "agent-clients"]
}

5
docs-site/middleware.ts
View file

@ -12,12 +12,13 @@ export function middleware(request: NextRequest) {
}
const { pathname } = request.nextUrl;
if (!pathname.startsWith("/docs/") || pathname.endsWith(".md")) {
const docsIndex = pathname.indexOf("/docs/");
if (docsIndex < 0 || pathname.endsWith(".md")) {
return NextResponse.next();
}
const rewriteUrl = request.nextUrl.clone();
rewriteUrl.pathname = `/llms.mdx${pathname}`;
rewriteUrl.pathname = `${pathname.slice(0, docsIndex)}/llms.mdx${pathname.slice(docsIndex)}`;
return NextResponse.rewrite(rewriteUrl);
}

19
docs-site/next.config.mjs
View file

@ -4,6 +4,7 @@ const withMDX = createMDX();
/** @type {import('next').NextConfig} */
const config = {
basePath: "/ktx",
async rewrites() {
return [
{
@ -12,6 +13,24 @@ const config = {
},
];
},
async redirects() {
return [
{
source: "/:path*",
has: [{ type: "host", value: "docs.ktx.sh" }],
destination: "https://docs.kaelio.com/ktx/:path*",
permanent: true,
basePath: false,
},
{
source: "/:path*",
has: [{ type: "host", value: "ktx.sh" }],
destination: "https://docs.kaelio.com/ktx/:path*",
permanent: true,
basePath: false,
},
];
},
};
export default withMDX(config);

808
docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md Normal file
View file

@ -0,0 +1,808 @@
# Connection Driver Discriminated Union Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Replace the loose `connectionSchema` in `packages/context/src/project/config.ts` with a Zod 4 discriminated union keyed on `driver`, so that every driver's documented connection fields — including the `mappings` block — appear in the JSON schema emitted by `ktx dev schema`.
**Architecture:** Add a new module `packages/context/src/project/driver-schemas.ts` that defines one `z.looseObject({ driver: z.literal('x'), ... })` per supported driver and combines them with `z.discriminatedUnion('driver', [...])`. Reuse the existing Metabase/Looker/LookML mapping shapes from `mappings-yaml-schema.ts` by exporting them. Wire the union into `config.ts`. Each per-driver shape stays `looseObject` so today's existing yaml configs with extra fields keep parsing.
**Tech Stack:** TypeScript (Node 22+, ESM, `NodeNext`), Zod 4 (`^4.4.3`), Vitest, pnpm workspace.
---
## File Structure
**Create:**
- `packages/context/src/project/driver-schemas.ts` — per-driver Zod schemas + the discriminated union and exported types.
- `packages/context/src/project/driver-schemas.test.ts` — unit tests for each driver schema and the union.
**Modify:**
- `packages/context/src/project/mappings-yaml-schema.ts` — export the three mapping shapes (`metabaseMappingsSchema`, `lookerMappingsSchema`, `lookmlMappingsSchema`) with `.describe()` annotations and a small description on each field so they surface meaningfully in JSON Schema.
- `packages/context/src/project/config.ts:209-214` — replace `connectionSchema` with the discriminated union imported from `driver-schemas.ts`. Update `KtxProjectConnectionConfig` (line `272`) to be `z.infer<typeof connectionSchema>` — still works because `connectionSchema` is the union name we keep.
- `packages/context/src/project/index.ts` — re-export `KtxConnectionConfig` per-driver type aliases if useful (optional; only if tests need them).
- `packages/context/src/project/config.test.ts` — add a test that the JSON schema now describes `mappings` for metabase/looker/lookml.
**No changes needed:**
- `packages/context/src/project/mappings-yaml-schema.ts` parsing helpers (`parseMetabaseMappingBootstrap`, etc.) keep working because `KtxProjectConnectionConfig` still has loose-object semantics per driver.
- Doc files in `docs-site/` already show the `mappings` blocks correctly.
---
## Drivers In Scope
The discriminated union enumerates the drivers actually used in code, fixtures, and docs (no `fake`/test-only driver — none exist in fixtures, verified via `grep "driver:\s*fake"`).
Warehouse drivers (read `driver`, `url`; nothing else schema-modeled — kept `looseObject` so warehouse-specific overrides like `historicSql`/`context.queryHistory` pass through):
- `postgres`, `postgresql` (separate literals; KTX normalizes `postgresql` → `postgres` at runtime, but ktx.yaml accepts both)
- `mysql`
- `snowflake`
- `bigquery`
- `sqlite`
- `clickhouse`
- `sqlserver`
Context-source drivers (model documented fields):
- `metabase` — `api_url`, `api_key`, `api_key_ref`, `network_proxy`/`networkProxy`, `mappings` (metabaseMappingsSchema).
- `looker` — `base_url`, `client_id`, `client_secret`, `client_secret_ref`, `mappings` (lookerMappingsSchema).
- `lookml` — `repoUrl` (camelCase intentional — matches code at `setup-sources.ts:1466`), `branch`, `path`, `auth_token_ref`, `mappings` (lookmlMappingsSchema).
- `notion` — `auth_token`, `auth_token_ref`, `crawl_mode` (`'selected_roots' | 'all_accessible'`), `root_page_ids`, `root_database_ids`, `root_data_source_ids`, `max_pages_per_run`, `max_knowledge_creates_per_run`, `max_knowledge_updates_per_run`.
- `dbt` — `source_dir`, `repo_url`, `branch`, `path`, `auth_token_ref`, `profiles_path`, `target`, `project_name`.
- `metricflow` — `metricflow` (nested object: `repoUrl`, `branch`, `path`, `auth_token_ref`).
Why not strict-object: existing warehouse connections may carry `historicSql` / `context.queryHistory` blocks and other driver-tunable fields not modeled here. `looseObject` preserves the current pass-through behavior while still surfacing the documented fields in JSON Schema.
---
## Task 1: Export and describe mapping shapes
Make the three existing mapping schemas reusable and documented.
**Files:**
- Modify: `packages/context/src/project/mappings-yaml-schema.ts:4-31`
- Test: `packages/context/src/project/mappings-yaml-schema.test.ts` (no behavior change — existing tests must still pass)
- [ ] **Step 1: Add a failing test that imports the new exports**
Append to `packages/context/src/project/mappings-yaml-schema.test.ts` (inside the existing `describe` block):
```typescript
import {
metabaseMappingsSchema,
lookerMappingsSchema,
lookmlMappingsSchema,
} from './mappings-yaml-schema.js';
// ...inside describe(...)
it('exports mapping shapes that parse documented examples', () => {
expect(metabaseMappingsSchema.parse({ databaseMappings: { '1': 'wh' } })).toMatchObject({
databaseMappings: { '1': 'wh' },
syncMode: 'ALL',
});
expect(lookerMappingsSchema.parse({ connectionMappings: { x: 'wh' } })).toEqual({
connectionMappings: { x: 'wh' },
});
expect(lookmlMappingsSchema.parse({ expectedLookerConnectionName: 'x' })).toEqual({
expectedLookerConnectionName: 'x',
});
});
```
- [ ] **Step 2: Run test to verify it fails**
Run: `pnpm --filter @ktx/context exec vitest run src/project/mappings-yaml-schema.test.ts`
Expected: FAIL with `metabaseMappingsSchema is not exported` or equivalent module-resolution error.
- [ ] **Step 3: Add `export` and `.describe()` to the three schemas**
In `packages/context/src/project/mappings-yaml-schema.ts`, change the three internal `const` declarations:
```typescript
export const metabaseMappingsSchema = z
.object({
databaseMappings: z
.record(z.string(), stringTargetSchema)
.default({})
.describe('Map of Metabase database ID (positive integer string) to KTX connection ID. Use null to explicitly unmap.'),
syncEnabled: z
.record(z.string(), z.boolean())
.default({})
.describe('Per-Metabase-database sync toggle, keyed by Metabase database ID string.'),
syncMode: metabaseSyncModeSchema
.default('ALL')
.describe('Sync scope: ALL ingests every mapped DB; ONLY restricts to syncEnabled=true; EXCEPT excludes syncEnabled=true.'),
selections: metabaseSelectionsSchema
.default({ collections: [], items: [] })
.describe('Optional Metabase collection and item IDs to scope ingest.'),
defaultTagNames: z
.array(z.string().min(1))
.default([])
.describe('Default tag names applied to ingested Metabase artifacts.'),
})
.describe('Metabase database-to-warehouse mapping and sync configuration.');
export const lookerMappingsSchema = z
.object({
connectionMappings: z
.record(z.string().min(1), stringTargetSchema)
.default({})
.describe('Map of Looker connection name to KTX connection ID. Use null to explicitly unmap.'),
})
.describe('Looker connection-to-warehouse mapping configuration.');
export const lookmlMappingsSchema = z
.object({
expectedLookerConnectionName: z
.string()
.min(1)
.nullable()
.default(null)
.describe('Looker connection name that LookML models must declare; mismatches block sl_write_source at ingest time.'),
})
.describe('LookML connection-name expectation for ingest gating.');
```
Leave `metabaseSyncModeSchema`, `metabaseSelectionsSchema`, `stringTargetSchema`, and `positiveIntegerValueSchema` private (no need to export). Leave all parsing helpers (`parseMetabaseMappingBootstrap` etc.) unchanged — they keep working because `.describe()` does not change runtime behavior.
- [ ] **Step 4: Run test to verify it passes and existing tests still pass**
Run: `pnpm --filter @ktx/context exec vitest run src/project/mappings-yaml-schema.test.ts`
Expected: PASS for all tests including the new one.
- [ ] **Step 5: Type-check the package**
Run: `pnpm --filter @ktx/context run type-check`
Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add packages/context/src/project/mappings-yaml-schema.ts packages/context/src/project/mappings-yaml-schema.test.ts
git commit -m "refactor(context): export and describe mapping shape schemas"
```
---
## Task 2: Create the driver-schemas module — warehouse drivers
Add the new module with the seven warehouse driver schemas first. Smaller surface, easier to validate.
**Files:**
- Create: `packages/context/src/project/driver-schemas.ts`
- Test: `packages/context/src/project/driver-schemas.test.ts`
- [ ] **Step 1: Write failing tests for warehouse driver schemas**
Create `packages/context/src/project/driver-schemas.test.ts`:
```typescript
import { describe, expect, it } from 'vitest';
import { connectionConfigSchema } from './driver-schemas.js';
describe('connectionConfigSchema (driver discriminated union)', () => {
it.each([
['postgres', 'postgres://user:pass@host:5432/db'], // pragma: allowlist secret
['postgresql', 'postgresql://user:pass@host:5432/db'], // pragma: allowlist secret
['mysql', 'mysql://user:pass@host:3306/db'], // pragma: allowlist secret
['snowflake', 'snowflake://account/db'],
['bigquery', 'bigquery://project/dataset'],
['sqlite', 'sqlite:///tmp/db.sqlite'],
['clickhouse', 'clickhouse://host:8123/db'],
['sqlserver', 'sqlserver://host:1433;database=db'],
])('parses %s warehouse connection', (driver, url) => {
expect(connectionConfigSchema.parse({ driver, url })).toMatchObject({ driver, url });
});
it('preserves unknown warehouse fields via looseObject passthrough', () => {
const parsed = connectionConfigSchema.parse({
driver: 'postgres',
url: 'postgres://x',
historicSql: { enabled: true },
context: { queryHistory: { enabled: false } },
});
expect(parsed).toMatchObject({
driver: 'postgres',
historicSql: { enabled: true },
context: { queryHistory: { enabled: false } },
});
});
it('rejects an unknown driver', () => {
expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow();
});
});
```
- [ ] **Step 2: Run tests to verify they fail**
Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts`
Expected: FAIL — `driver-schemas.js` not found.
- [ ] **Step 3: Create `driver-schemas.ts` with warehouse drivers only**
Create `packages/context/src/project/driver-schemas.ts`:
```typescript
import * as z from 'zod';
const warehouseDrivers = [
'postgres',
'postgresql',
'mysql',
'snowflake',
'bigquery',
'sqlite',
'clickhouse',
'sqlserver',
] as const;
function warehouseConnectionSchema(driver: (typeof warehouseDrivers)[number]) {
return z
.looseObject({
driver: z.literal(driver),
url: z
.string()
.min(1)
.optional()
.describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'),
})
.describe(`${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`);
}
export const connectionConfigSchema = z.discriminatedUnion(
'driver',
warehouseDrivers.map(warehouseConnectionSchema),
);
export type KtxConnectionConfig = z.infer<typeof connectionConfigSchema>;
```
- [ ] **Step 4: Run tests to verify they pass**
Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts`
Expected: PASS for all eight warehouse drivers + passthrough + unknown-driver rejection.
- [ ] **Step 5: Type-check**
Run: `pnpm --filter @ktx/context run type-check`
Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts
git commit -m "feat(context): add driver-schemas module with warehouse drivers"
```
---
## Task 3: Add Metabase, Looker, LookML driver schemas (the mapping-bearing ones)
These are the most important drivers — they're why we're doing this refactor.
**Files:**
- Modify: `packages/context/src/project/driver-schemas.ts`
- Modify: `packages/context/src/project/driver-schemas.test.ts`
- [ ] **Step 1: Write failing tests**
Append to `packages/context/src/project/driver-schemas.test.ts`:
```typescript
describe('connectionConfigSchema — context source drivers with mappings', () => {
it('parses a metabase connection with mappings', () => {
const parsed = connectionConfigSchema.parse({
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '3': 'prod-warehouse' },
syncEnabled: { '3': true },
syncMode: 'ONLY',
},
});
expect(parsed).toMatchObject({
driver: 'metabase',
api_url: 'https://metabase.example.com',
mappings: {
databaseMappings: { '3': 'prod-warehouse' },
syncMode: 'ONLY',
},
});
});
it('parses a looker connection with connectionMappings', () => {
const parsed = connectionConfigSchema.parse({
driver: 'looker',
base_url: 'https://looker.example.com',
client_id: 'abc',
client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret
mappings: { connectionMappings: { bigquery_prod: 'wh' } },
});
expect(parsed.mappings).toEqual({ connectionMappings: { bigquery_prod: 'wh' } });
});
it('parses a lookml connection with expectedLookerConnectionName', () => {
const parsed = connectionConfigSchema.parse({
driver: 'lookml',
repoUrl: 'https://github.com/acme/looker.git',
branch: 'main',
mappings: { expectedLookerConnectionName: 'bigquery_prod' },
});
expect(parsed.mappings).toEqual({ expectedLookerConnectionName: 'bigquery_prod' });
});
it('rejects metabase mapping with non-integer database key', () => {
expect(() =>
connectionConfigSchema.parse({
driver: 'metabase',
api_url: 'https://x',
mappings: { databaseMappings: { 'abc': 'wh' } },
}),
).toThrow();
});
});
```
- [ ] **Step 2: Run tests to verify they fail**
Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts`
Expected: FAIL — `driver: 'metabase'` is not in the discriminated union.
- [ ] **Step 3: Extend `driver-schemas.ts` with metabase/looker/lookml schemas**
Edit `packages/context/src/project/driver-schemas.ts` — add imports and the three new schemas, and include them in the union:
```typescript
import * as z from 'zod';
import {
lookerMappingsSchema,
lookmlMappingsSchema,
metabaseMappingsSchema,
} from './mappings-yaml-schema.js';
// ... (warehouseDrivers + warehouseConnectionSchema stay as-is) ...
const positiveIntKeyMessage = (field: string) =>
`${field} keys must be positive-integer strings (e.g. "1", "42")`;
const positiveIntKeyRegex = /^[1-9]\d*$/;
const metabaseMappingsStrictSchema = metabaseMappingsSchema.superRefine((value, ctx) => {
for (const key of Object.keys(value.databaseMappings ?? {})) {
if (!positiveIntKeyRegex.test(key)) {
ctx.addIssue({ code: 'custom', path: ['databaseMappings', key], message: positiveIntKeyMessage('databaseMappings') });
}
}
for (const key of Object.keys(value.syncEnabled ?? {})) {
if (!positiveIntKeyRegex.test(key)) {
ctx.addIssue({ code: 'custom', path: ['syncEnabled', key], message: positiveIntKeyMessage('syncEnabled') });
}
}
});
const metabaseConnectionSchema = z
.looseObject({
driver: z.literal('metabase'),
api_url: z.string().url().describe('Metabase instance API URL (e.g. https://metabase.example.com).'),
api_key: z.string().min(1).optional().describe('Literal Metabase API key. Prefer api_key_ref for safety.'),
api_key_ref: z
.string()
.min(1)
.optional()
.describe('Reference to Metabase API key (e.g. env:METABASE_API_KEY or file:/path).'),
network_proxy: z
.looseObject({})
.optional()
.describe('Optional network proxy configuration (snake_case form).'),
networkProxy: z
.looseObject({})
.optional()
.describe('Optional network proxy configuration (camelCase form).'),
mappings: metabaseMappingsStrictSchema.optional().describe('Metabase database-to-warehouse mappings and sync configuration.'),
})
.describe('Metabase context-source connection.');
const lookerConnectionSchema = z
.looseObject({
driver: z.literal('looker'),
base_url: z.string().url().describe('Looker instance base URL (e.g. https://looker.example.com).'),
client_id: z.string().min(1).describe('Looker OAuth client ID.'),
client_secret: z.string().min(1).optional().describe('Literal Looker OAuth client secret. Prefer client_secret_ref.'),
client_secret_ref: z
.string()
.min(1)
.optional()
.describe('Reference to Looker OAuth client secret (e.g. env:LOOKER_CLIENT_SECRET).'),
mappings: lookerMappingsSchema.optional().describe('Looker connection-name to KTX warehouse mappings.'),
})
.describe('Looker context-source connection.');
const lookmlConnectionSchema = z
.looseObject({
driver: z.literal('lookml'),
repoUrl: z
.string()
.min(1)
.describe('Git URL of the LookML project (https, ssh, or file:). Field is camelCase by convention.'),
branch: z.string().min(1).optional().describe('Git branch (default "main" downstream).'),
path: z.string().optional().describe('Subdirectory within the repo when the LookML project lives in a monorepo.'),
auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos (e.g. env:GITHUB_TOKEN).'),
mappings: lookmlMappingsSchema.optional().describe('LookML expected-connection mapping for ingest gating.'),
})
.describe('LookML context-source connection.');
export const connectionConfigSchema = z.discriminatedUnion(
'driver',
[
...warehouseDrivers.map(warehouseConnectionSchema),
metabaseConnectionSchema,
lookerConnectionSchema,
lookmlConnectionSchema,
],
);
```
Important: the existing `parseMetabaseMappingBootstrap` in `mappings-yaml-schema.ts` already enforces positive-integer keys via `assertPositiveIntegerKeys`. Adding `metabaseMappingsStrictSchema` here gives the same guarantee at the top-level config parse, so a malformed ktx.yaml fails fast at `parseKtxProjectConfig` time rather than at ingest time.
- [ ] **Step 4: Run tests to verify they pass**
Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts`
Expected: PASS.
- [ ] **Step 5: Type-check**
Run: `pnpm --filter @ktx/context run type-check`
Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts
git commit -m "feat(context): add metabase, looker, lookml driver schemas with mappings"
```
---
## Task 4: Add Notion, dbt, MetricFlow driver schemas
The remaining context-source drivers; no `mappings` for these, but plenty of driver-specific fields.
**Files:**
- Modify: `packages/context/src/project/driver-schemas.ts`
- Modify: `packages/context/src/project/driver-schemas.test.ts`
- [ ] **Step 1: Write failing tests**
Append to `packages/context/src/project/driver-schemas.test.ts`:
```typescript
describe('connectionConfigSchema — notion / dbt / metricflow', () => {
it('parses a notion connection with selected_roots crawl', () => {
const parsed = connectionConfigSchema.parse({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['abc', 'def'],
max_pages_per_run: 500,
});
expect(parsed).toMatchObject({
driver: 'notion',
crawl_mode: 'selected_roots',
root_page_ids: ['abc', 'def'],
max_pages_per_run: 500,
});
});
it('rejects notion with unknown crawl_mode', () => {
expect(() =>
connectionConfigSchema.parse({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'everything',
}),
).toThrow();
});
it('parses a dbt connection from a local source_dir', () => {
const parsed = connectionConfigSchema.parse({
driver: 'dbt',
source_dir: '/tmp/dbt-project',
target: 'dev',
});
expect(parsed).toMatchObject({ driver: 'dbt', source_dir: '/tmp/dbt-project', target: 'dev' });
});
it('parses a metricflow connection with nested config', () => {
const parsed = connectionConfigSchema.parse({
driver: 'metricflow',
metricflow: {
repoUrl: 'https://github.com/acme/sl.git',
branch: 'main',
},
});
expect(parsed).toMatchObject({
driver: 'metricflow',
metricflow: { repoUrl: 'https://github.com/acme/sl.git' },
});
});
});
```
- [ ] **Step 2: Run tests to verify they fail**
Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts`
Expected: FAIL — `driver: 'notion'` etc. not in union.
- [ ] **Step 3: Extend `driver-schemas.ts`**
Add to `packages/context/src/project/driver-schemas.ts` before the final `connectionConfigSchema` export:
```typescript
const notionConnectionSchema = z
.looseObject({
driver: z.literal('notion'),
auth_token: z.string().min(1).optional().describe('Literal Notion integration token. Prefer auth_token_ref.'),
auth_token_ref: z
.string()
.min(1)
.optional()
.describe('Reference to Notion integration token (e.g. env:NOTION_TOKEN).'),
crawl_mode: z
.enum(['selected_roots', 'all_accessible'])
.optional()
.describe('Crawl scope. "selected_roots" requires at least one of root_page_ids, root_database_ids, root_data_source_ids.'),
root_page_ids: z.array(z.string().min(1)).optional().describe('Notion page IDs to crawl when crawl_mode is selected_roots.'),
root_database_ids: z.array(z.string().min(1)).optional().describe('Notion database IDs to crawl when crawl_mode is selected_roots.'),
root_data_source_ids: z
.array(z.string().min(1))
.optional()
.describe('Notion data source IDs to crawl when crawl_mode is selected_roots.'),
max_pages_per_run: z
.number()
.int()
.min(1)
.max(10000)
.optional()
.describe('Maximum Notion pages fetched in a single ingest run.'),
max_knowledge_creates_per_run: z
.number()
.int()
.min(0)
.max(25)
.optional()
.describe('Maximum new wiki pages created per run.'),
max_knowledge_updates_per_run: z
.number()
.int()
.min(0)
.max(100)
.optional()
.describe('Maximum existing wiki pages updated per run.'),
})
.describe('Notion context-source connection.');
const dbtConnectionSchema = z
.looseObject({
driver: z.literal('dbt'),
source_dir: z.string().min(1).optional().describe('Absolute or project-relative path to a local dbt project.'),
repo_url: z.string().min(1).optional().describe('Git URL of the dbt project (https, ssh, or file:).'),
branch: z.string().min(1).optional().describe('Git branch when using repo_url.'),
path: z.string().optional().describe('Subdirectory within the repo when the dbt project lives in a monorepo.'),
auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'),
profiles_path: z.string().optional().describe('Override path to dbt profiles.yml.'),
target: z.string().min(1).optional().describe('dbt target name (e.g. dev, prod).'),
project_name: z.string().min(1).optional().describe('Override auto-detected dbt project name.'),
})
.describe('dbt context-source connection.');
const metricflowConnectionSchema = z
.looseObject({
driver: z.literal('metricflow'),
metricflow: z
.looseObject({
repoUrl: z.string().min(1).describe('Git URL of the MetricFlow / SL project.'),
branch: z.string().min(1).optional().describe('Git branch (default "main").'),
path: z.string().optional().describe('Subdirectory within the repo when the SL config lives in a monorepo.'),
auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'),
})
.describe('Nested MetricFlow configuration block.'),
})
.describe('MetricFlow / SL context-source connection.');
```
Then update the final union:
```typescript
export const connectionConfigSchema = z.discriminatedUnion(
'driver',
[
...warehouseDrivers.map(warehouseConnectionSchema),
metabaseConnectionSchema,
lookerConnectionSchema,
lookmlConnectionSchema,
notionConnectionSchema,
dbtConnectionSchema,
metricflowConnectionSchema,
],
);
```
- [ ] **Step 4: Run tests to verify they pass**
Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts`
Expected: PASS.
- [ ] **Step 5: Type-check**
Run: `pnpm --filter @ktx/context run type-check`
Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts
git commit -m "feat(context): add notion, dbt, metricflow driver schemas"
```
---
## Task 5: Wire the discriminated union into `config.ts`
Now switch the top-level `connectionSchema` to the new union. This is the change that flips JSON-schema output.
**Files:**
- Modify: `packages/context/src/project/config.ts:209-214, 272`
- Test: `packages/context/src/project/config.test.ts` — add a JSON-schema assertion.
- [ ] **Step 1: Write a failing test for the JSON schema output**
Append to `packages/context/src/project/config.test.ts`:
```typescript
import { generateKtxProjectConfigJsonSchema } from './config.js';
describe('generateKtxProjectConfigJsonSchema', () => {
it('emits the metabase mappings shape under connections', () => {
const schema = generateKtxProjectConfigJsonSchema();
const serialized = JSON.stringify(schema);
expect(serialized).toContain('databaseMappings');
expect(serialized).toContain('connectionMappings');
expect(serialized).toContain('expectedLookerConnectionName');
});
});
```
- [ ] **Step 2: Run test to verify it fails**
Run: `pnpm --filter @ktx/context exec vitest run src/project/config.test.ts`
Expected: FAIL — the strings are not in the emitted schema yet because `connectionSchema` is still loose.
- [ ] **Step 3: Replace `connectionSchema` in `config.ts`**
In `packages/context/src/project/config.ts`, delete lines `209-214`:
```typescript
const connectionSchema = z
.looseObject({
driver: z.string().min(1).optional().describe('Connector driver identifier (e.g. "postgres", "bigquery", "snowflake").'),
url: z.string().optional().describe('Connection URL or DSN. Format depends on the driver; may contain environment-variable references.'),
})
.describe('A single database/connector connection entry. Additional driver-specific fields are accepted and passed through.');
```
Replace with an import + re-bind at the top of the file (after the existing imports):
```typescript
import { connectionConfigSchema } from './driver-schemas.js';
const connectionSchema = connectionConfigSchema;
```
(Re-binding to the local name `connectionSchema` keeps the rest of the file unchanged, including the export of `KtxProjectConnectionConfig` at line `272`.)
- [ ] **Step 4: Run the new test plus existing config tests**
Run: `pnpm --filter @ktx/context exec vitest run src/project/`
Expected: PASS for all tests.
If any existing test fails (e.g. a fixture used an undocumented driver string), update the fixture or expand the union — do not loosen the union.
- [ ] **Step 5: Run the full context test suite to catch downstream regressions**
Run: `pnpm --filter @ktx/context run test`
Expected: PASS.
- [ ] **Step 6: Type-check the workspace**
Run: `pnpm run type-check`
Expected: PASS. `KtxProjectConnectionConfig` is now a union; any consumer that destructured fields not present on every driver branch will surface here.
If type-check fails in a consumer, the fix is usually `if (connection.driver === 'metabase')` style narrowing — or, for code that already does this dynamically (e.g. `String(connection.driver).toLowerCase() === 'metabase'`), an explicit cast at the call site is acceptable. Do not add `as any`; prefer narrowing.
- [ ] **Step 7: Commit**
```bash
git add packages/context/src/project/config.ts packages/context/src/project/config.test.ts
git commit -m "refactor(context): make connectionSchema a driver-discriminated union"
```
---
## Task 6: Verify the user-visible result and CLI smoke
Confirm the original bug is fixed and the CLI behavior is unchanged.
**Files:** none modified in this task.
- [ ] **Step 1: Build the CLI**
Run: `pnpm run build`
Expected: PASS.
- [ ] **Step 2: Confirm `ktx dev schema | rg -i mapping` now returns hits**
Run: `node scripts/run-ktx.mjs -- dev schema | rg -i mapping`
Expected: multiple lines, including the `databaseMappings`, `connectionMappings`, `expectedLookerConnectionName` keys and their descriptions.
- [ ] **Step 3: Run the CLI smoke**
Run: `pnpm --filter @ktx/cli run smoke`
Expected: PASS.
- [ ] **Step 4: Run the broader workspace test suite**
Run: `pnpm run test 2>&1 | tee /tmp/ktx-test-output.log`
Expected: PASS. Inspect `/tmp/ktx-test-output.log` if anything fails.
- [ ] **Step 5: Run pre-commit on changed files**
Run: `pnpm run check`
Expected: PASS.
- [ ] **Step 6: Knip dead-code sweep (in case we introduced unused exports)**
Run: `pnpm run dead-code`
Expected: PASS — or, if Knip flags `KtxConnectionConfig` as unused, decide whether to export it from `packages/context/src/project/index.ts` (preferred — it documents intent) or drop the export.
If exporting: add to `packages/context/src/project/index.ts`:
```typescript
export type { KtxConnectionConfig } from './driver-schemas.js';
```
- [ ] **Step 7: Final commit if any docs / index changes were made**
```bash
git status --short
# If only docs/index were touched in step 6:
git add packages/context/src/project/index.ts
git commit -m "chore(context): re-export KtxConnectionConfig from project package"
```
---
## Self-Review
**1. Spec coverage:** Original request was "I need to be able to see full schema" with chosen approach option 1 (discriminated union). Task 5 step 2 verifies that `ktx dev schema | rg -i mapping` now returns hits. Task 6 step 2 is the explicit end-to-end check. All catalogued drivers (warehouse + metabase + looker + lookml + notion + dbt + metricflow) have a schema and a test. ✅
**2. Placeholder scan:** No "TBD", "add validation", "similar to Task N", or skipped code. Every step has the actual code or command. ✅
**3. Type consistency:**
- `connectionConfigSchema` is defined in Task 2 and extended (not renamed) in Tasks 3–4. ✅
- `KtxConnectionConfig` (new type) appears only in `driver-schemas.ts` and the optional re-export in Task 6. `KtxProjectConnectionConfig` (existing type at `config.ts:272`) keeps its name. ✅
- `metabaseMappingsSchema`, `lookerMappingsSchema`, `lookmlMappingsSchema` — Task 1 exports them; Task 3 imports them by the same names. ✅
- `metabaseMappingsStrictSchema` is defined and used in Task 3 only. ✅
- The `warehouseDrivers` array and `warehouseConnectionSchema` helper are introduced in Task 2 and reused unchanged in Task 4's union extension. ✅
---
## Execution Handoff
Plan complete and saved to `docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md`. Two execution options:
**1. Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration.
**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints.
Which approach?

939
docs/superpowers/plans/2026-05-14-research-agent-mcp-dictionary-search.md Normal file
View file

@ -0,0 +1,939 @@
# Research Agent MCP Dictionary Search Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Add the MCP-shaped `dictionary_search` tool so external research agents can resolve user-mentioned literal values to profile-sampled warehouse columns.
**Architecture:** Reuse the existing relationship-profile dictionary extraction as the source of truth, add a focused local dictionary-search service that reports coverage and non-authoritative misses per connection, then register the service through the MCP context tool surface and local project ports. The service re-reads the latest profile artifact on each call instead of keeping a long-lived cache, so scan freshness is correct for the MCP daemon v1.
**Tech Stack:** TypeScript, Vitest, Zod, KTX local file store, relationship-profile artifacts, KTX MCP context ports.
---
## Current Audit
Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md`
Implemented v1 slices:
- `docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md` is implemented. Current source has sqlglot read-only validation in `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`, `SqlAnalysisPort.validateReadOnly()` in `packages/context/src/sql-analysis/ports.ts`, MCP `sql_execution` registration in `packages/context/src/mcp/context-tools.ts`, and local connector execution gated by validation in `packages/context/src/mcp/local-project-ports.ts`.
- `docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md` is implemented. Current source has `packages/context/src/scan/entity-details.ts`, MCP `entity_details` registration in `packages/context/src/mcp/context-tools.ts`, and local project wiring in `packages/context/src/mcp/local-project-ports.ts`.
V1-blocking gaps remaining against the original spec:
- `dictionary_search` is not registered on the MCP surface and `KtxMcpContextPorts` has no dictionary-search port.
- `discover_data` is not registered on the MCP surface and the unified ranked result shape is not implemented.
- The ingest-side warehouse-verification tools still use `connectionName` / `targets` / `rowLimit` contracts and have not been fully converged with shared MCP-shaped services.
- `ktx mcp start|stop|status|logs` and the HTTP Streamable MCP daemon do not exist.
- `ktx setup-agents` does not install MCP client config entries or the `ktx-research` skill.
This plan covers only the next focused blocker: MCP `dictionary_search`. Later plans still need to cover `discover_data`, ingest contract convergence, the HTTP daemon, and setup-agent/research-skill installation.
Non-blocking or explicitly out-of-scope gaps:
- Python code execution over MCP.
- Stdio MCP transport.
- OS-level auto-start.
- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing.
- Streaming SQL results.
## File Structure
Create:
- `packages/context/src/sl/dictionary-search.ts`
- Reads the latest `relationship-profile.json` per searched connection.
- Uses `loadLatestSlDictionaryEntries()` for dictionary entries.
- Returns spec-shaped `searched` coverage records, matches, and per-value miss reasons.
- Re-reads artifacts per call rather than caching, satisfying MCP freshness for v1.
- `packages/context/src/sl/dictionary-search.test.ts`
- Covers matches, non-authoritative misses, missing profile artifacts, no candidate columns, case-insensitive substring matching, and connection scoping.
Modify:
- `packages/context/src/sl/index.ts`
- Export the new service and response types.
- `packages/context/src/mcp/types.ts`
- Add `KtxDictionarySearchMcpPort` and include `dictionarySearch` in `KtxMcpContextPorts`.
- `packages/context/src/mcp/context-tools.ts`
- Add the `dictionary_search` Zod schema and registration.
- `packages/context/src/mcp/server.test.ts`
- Assert MCP registration and structured output for `dictionary_search`.
- `packages/context/src/mcp/local-project-ports.ts`
- Wire local project dictionary search to the new service.
- `packages/context/src/mcp/local-project-ports.test.ts`
- Cover local-port `dictionary_search` success and missing-profile behavior.
- `packages/context/src/mcp/index.ts`
- Export the new MCP port type if it is not already covered by existing barrel exports.
## Task 1: Add The Dictionary Search Service
**Files:**
- Create: `packages/context/src/sl/dictionary-search.test.ts`
- Create: `packages/context/src/sl/dictionary-search.ts`
- Modify: `packages/context/src/sl/index.ts`
- [ ] **Step 1: Write failing service tests**
Create `packages/context/src/sl/dictionary-search.test.ts`:
```typescript
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { initKtxProject, type KtxLocalProject } from '../project/index.js';
import { createKtxDictionarySearchService } from './dictionary-search.js';
describe('createKtxDictionarySearchService', () => {
let tempDir: string;
let project: KtxLocalProject;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-dictionary-search-'));
project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL' };
project.config.connections.billing = { driver: 'postgres', url: 'env:BILLING_DATABASE_URL' };
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
async function seedProfile(input: {
connectionId: string;
syncId: string;
columns: Record<string, unknown>;
}): Promise<void> {
await project.fileStore.writeFile(
`raw-sources/${input.connectionId}/live-database/${input.syncId}/enrichment/relationship-profile.json`,
`${JSON.stringify(
{
connectionId: input.connectionId,
driver: 'postgres',
sqlAvailable: true,
queryCount: 4,
tables: [],
columns: input.columns,
warnings: [],
},
null,
2,
)}\n`,
'ktx',
'ktx@example.com',
'Seed relationship profile',
);
}
it('returns matches and non-authoritative misses across configured connections', async () => {
await seedProfile({
connectionId: 'warehouse',
syncId: 'sync-1',
columns: {
'orders.status': {
table: { catalog: null, db: 'public', name: 'orders' },
column: 'status',
nativeType: 'text',
normalizedType: 'string',
distinctCount: 3,
sampleValues: ['paid', 'refunded', 'pending'],
},
},
});
await seedProfile({
connectionId: 'billing',
syncId: 'sync-2',
columns: {
'customers.name': {
table: { catalog: null, db: 'public', name: 'customers' },
column: 'name',
nativeType: 'text',
normalizedType: 'string',
distinctCount: 4,
sampleValues: ['Acme Corp', 'Globex'],
},
},
});
const service = createKtxDictionarySearchService(project);
await expect(service.search({ values: ['PAID', 'missing'] })).resolves.toEqual({
searched: [
{
connectionId: 'billing',
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 1,
syncId: 'sync-2',
profiledAt: null,
},
status: 'ready',
},
{
connectionId: 'warehouse',
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 1,
syncId: 'sync-1',
profiledAt: null,
},
status: 'ready',
},
],
results: [
{
value: 'PAID',
matches: [
{
connectionId: 'warehouse',
sourceName: 'orders',
columnName: 'status',
matchedValue: 'paid',
cardinality: 3,
},
],
misses: [{ connectionId: 'billing', reason: 'value_not_in_sample' }],
},
{
value: 'missing',
matches: [],
misses: [
{ connectionId: 'billing', reason: 'value_not_in_sample' },
{ connectionId: 'warehouse', reason: 'value_not_in_sample' },
],
},
],
});
});
it('distinguishes missing profile artifacts from profiles with no candidate columns', async () => {
await seedProfile({
connectionId: 'billing',
syncId: 'sync-empty',
columns: {
'events.id': {
table: { catalog: null, db: 'public', name: 'events' },
column: 'id',
nativeType: 'integer',
normalizedType: 'integer',
distinctCount: 100,
sampleValues: [1, 2, 3],
},
},
});
const service = createKtxDictionarySearchService(project);
await expect(service.search({ values: ['Acme'] })).resolves.toEqual({
searched: [
{
connectionId: 'billing',
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 0,
syncId: 'sync-empty',
profiledAt: null,
},
status: 'no_candidate_columns',
},
{
connectionId: 'warehouse',
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 0,
syncId: null,
profiledAt: null,
},
status: 'no_profile_artifact',
},
],
results: [
{
value: 'Acme',
matches: [],
misses: [
{ connectionId: 'billing', reason: 'no_candidate_columns' },
{ connectionId: 'warehouse', reason: 'no_profile_artifact' },
],
},
],
});
});
it('scopes search to the requested connection', async () => {
await seedProfile({
connectionId: 'warehouse',
syncId: 'sync-1',
columns: {
'orders.status': {
table: { catalog: null, db: 'public', name: 'orders' },
column: 'status',
nativeType: 'text',
normalizedType: 'string',
distinctCount: 3,
sampleValues: ['paid'],
},
},
});
await seedProfile({
connectionId: 'billing',
syncId: 'sync-2',
columns: {
'invoices.status': {
table: { catalog: null, db: 'public', name: 'invoices' },
column: 'status',
nativeType: 'text',
normalizedType: 'string',
distinctCount: 2,
sampleValues: ['paid'],
},
},
});
const service = createKtxDictionarySearchService(project);
await expect(service.search({ connectionId: 'billing', values: ['paid'] })).resolves.toMatchObject({
searched: [{ connectionId: 'billing', status: 'ready' }],
results: [
{
value: 'paid',
matches: [{ connectionId: 'billing', sourceName: 'invoices', columnName: 'status', matchedValue: 'paid' }],
misses: [],
},
],
});
});
});
```
- [ ] **Step 2: Run service tests to verify they fail**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/sl/dictionary-search.test.ts
```
Expected: FAIL with `Cannot find module './dictionary-search.js'`.
- [ ] **Step 3: Implement the dictionary search service**
Create `packages/context/src/sl/dictionary-search.ts`:
```typescript
import type { KtxLocalProject } from '../project/index.js';
import { loadLatestSlDictionaryEntries, type SlDictionaryEntry } from './sl-dictionary-profile.js';
export type KtxDictionarySearchStatus = 'ready' | 'no_profile_artifact' | 'no_candidate_columns';
export type KtxDictionarySearchMissReason = 'no_profile_artifact' | 'no_candidate_columns' | 'value_not_in_sample';
export interface KtxDictionarySearchInput {
values: string[];
connectionId?: string;
}
export interface KtxDictionarySearchCoverage {
sampledRows: number | null;
valuesPerColumn: number | null;
profiledColumns: number;
syncId: string | null;
profiledAt: string | null;
}
export interface KtxDictionarySearchSearchedConnection {
connectionId: string;
coverage: KtxDictionarySearchCoverage;
status: KtxDictionarySearchStatus;
}
export interface KtxDictionarySearchMatch {
connectionId: string;
sourceName: string;
columnName: string;
matchedValue: string;
cardinality: number | null;
}
export interface KtxDictionarySearchMiss {
connectionId: string;
reason: KtxDictionarySearchMissReason;
}
export interface KtxDictionarySearchValueResult {
value: string;
matches: KtxDictionarySearchMatch[];
misses: KtxDictionarySearchMiss[];
}
export interface KtxDictionarySearchResponse {
searched: KtxDictionarySearchSearchedConnection[];
results: KtxDictionarySearchValueResult[];
}
interface RelationshipProfileArtifact {
connectionId?: string;
profileSampleRows?: unknown;
sampleValuesPerColumn?: unknown;
profiledAt?: unknown;
extractedAt?: unknown;
}
function uniqueSorted(values: Iterable<string>): string[] {
return [...new Set([...values].filter((value) => value.trim().length > 0))].sort((left, right) =>
left.localeCompare(right),
);
}
function latestProfileSyncId(path: string): string | null {
const parts = path.split('/');
return parts.at(-3) ?? null;
}
function optionalNumber(value: unknown): number | null {
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
function optionalString(value: unknown): string | null {
return typeof value === 'string' && value.trim().length > 0 ? value : null;
}
async function latestProfilePath(project: KtxLocalProject, connectionId: string): Promise<string | null> {
const root = `raw-sources/${connectionId}/live-database`;
let files: string[];
try {
files = (await project.fileStore.listFiles(root)).files;
} catch {
return null;
}
return files
.filter((path) => path.endsWith('/enrichment/relationship-profile.json'))
.sort((left, right) => left.localeCompare(right))
.at(-1) ?? null;
}
async function readProfile(project: KtxLocalProject, path: string): Promise<RelationshipProfileArtifact> {
const raw = await project.fileStore.readFile(path);
const parsed = JSON.parse(raw.content) as unknown;
return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)
? (parsed as RelationshipProfileArtifact)
: {};
}
function profiledColumnCount(entries: readonly SlDictionaryEntry[]): number {
return new Set(entries.map((entry) => `${entry.sourceName}\u001f${entry.columnName}`)).size;
}
async function searchedConnection(
project: KtxLocalProject,
connectionId: string,
entries: readonly SlDictionaryEntry[],
): Promise<KtxDictionarySearchSearchedConnection> {
const path = await latestProfilePath(project, connectionId);
if (!path) {
return {
connectionId,
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 0,
syncId: null,
profiledAt: null,
},
status: 'no_profile_artifact',
};
}
const profile = await readProfile(project, path);
const count = profiledColumnCount(entries);
return {
connectionId,
coverage: {
sampledRows: optionalNumber(profile.profileSampleRows),
valuesPerColumn: optionalNumber(profile.sampleValuesPerColumn),
profiledColumns: count,
syncId: latestProfileSyncId(path),
profiledAt: optionalString(profile.profiledAt) ?? optionalString(profile.extractedAt),
},
status: count > 0 ? 'ready' : 'no_candidate_columns',
};
}
function entryMatchesValue(entry: SlDictionaryEntry, value: string): boolean {
return entry.value.toLowerCase().includes(value.toLowerCase());
}
function toMatch(entry: SlDictionaryEntry): KtxDictionarySearchMatch {
return {
connectionId: entry.connectionId,
sourceName: entry.sourceName,
columnName: entry.columnName,
matchedValue: entry.value,
cardinality: entry.cardinality,
};
}
function sortMatches(matches: KtxDictionarySearchMatch[]): KtxDictionarySearchMatch[] {
return matches.sort(
(left, right) =>
left.connectionId.localeCompare(right.connectionId) ||
left.sourceName.localeCompare(right.sourceName) ||
left.columnName.localeCompare(right.columnName) ||
left.matchedValue.localeCompare(right.matchedValue),
);
}
function missReason(status: KtxDictionarySearchStatus): KtxDictionarySearchMissReason {
return status === 'ready' ? 'value_not_in_sample' : status;
}
export function createKtxDictionarySearchService(project: KtxLocalProject) {
return {
async search(input: KtxDictionarySearchInput): Promise<KtxDictionarySearchResponse> {
const connectionIds = input.connectionId ? [input.connectionId] : uniqueSorted(Object.keys(project.config.connections));
const entries = await loadLatestSlDictionaryEntries(project, connectionIds);
const entriesByConnection = new Map<string, SlDictionaryEntry[]>();
for (const connectionId of connectionIds) {
entriesByConnection.set(
connectionId,
entries.filter((entry) => entry.connectionId === connectionId),
);
}
const searched = (
await Promise.all(
connectionIds.map((connectionId) =>
searchedConnection(project, connectionId, entriesByConnection.get(connectionId) ?? []),
),
)
).sort((left, right) => left.connectionId.localeCompare(right.connectionId));
const searchedByConnection = new Map(searched.map((connection) => [connection.connectionId, connection]));
return {
searched,
results: input.values.map((value) => {
const matches = sortMatches(entries.filter((entry) => entryMatchesValue(entry, value)).map(toMatch));
const matchedConnections = new Set(matches.map((match) => match.connectionId));
return {
value,
matches,
misses: searched
.filter((connection) => !matchedConnections.has(connection.connectionId))
.map((connection) => ({
connectionId: connection.connectionId,
reason: missReason(searchedByConnection.get(connection.connectionId)?.status ?? 'no_profile_artifact'),
})),
};
}),
};
},
};
}
```
- [ ] **Step 4: Export the service**
In `packages/context/src/sl/index.ts`, add:
```typescript
export {
createKtxDictionarySearchService,
} from './dictionary-search.js';
export type {
KtxDictionarySearchCoverage,
KtxDictionarySearchInput,
KtxDictionarySearchMatch,
KtxDictionarySearchMiss,
KtxDictionarySearchMissReason,
KtxDictionarySearchResponse,
KtxDictionarySearchSearchedConnection,
KtxDictionarySearchStatus,
KtxDictionarySearchValueResult,
} from './dictionary-search.js';
```
- [ ] **Step 5: Run service tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/sl/dictionary-search.test.ts
```
Expected: PASS.
- [ ] **Step 6: Commit the service slice**
Run:
```bash
git add packages/context/src/sl/dictionary-search.ts packages/context/src/sl/dictionary-search.test.ts packages/context/src/sl/index.ts
git commit -m "feat(context): add dictionary search service"
```
## Task 2: Register The MCP `dictionary_search` Tool
**Files:**
- Modify: `packages/context/src/mcp/types.ts`
- Modify: `packages/context/src/mcp/context-tools.ts`
- Modify: `packages/context/src/mcp/server.test.ts`
- Modify: `packages/context/src/mcp/index.ts`
- [ ] **Step 1: Add MCP port types**
In `packages/context/src/mcp/types.ts`, extend the imports:
```typescript
import type { KtxDictionarySearchInput, KtxDictionarySearchResponse } from '../sl/index.js';
```
Add this interface near the other MCP port interfaces:
```typescript
export interface KtxDictionarySearchMcpPort {
search(input: KtxDictionarySearchInput): Promise<KtxDictionarySearchResponse>;
}
```
Add the new optional port to `KtxMcpContextPorts`:
```typescript
export interface KtxMcpContextPorts {
connections?: KtxConnectionsMcpPort;
knowledge?: KtxKnowledgeMcpPort;
semanticLayer?: KtxSemanticLayerMcpPort;
entityDetails?: KtxEntityDetailsMcpPort;
dictionarySearch?: KtxDictionarySearchMcpPort;
sqlExecution?: KtxSqlExecutionMcpPort;
ingest?: KtxIngestMcpPort;
scan?: KtxScanMcpPort;
}
```
- [ ] **Step 2: Write failing MCP registration test**
In `packages/context/src/mcp/server.test.ts`, update the type import list to include:
```typescript
KtxDictionarySearchMcpPort,
```
Add this test after the `entity_details` registration test:
```typescript
it('registers dictionary_search when the host provides a dictionary-search port', async () => {
const fake = makeFakeServer();
const dictionarySearch: KtxDictionarySearchMcpPort = {
search: vi.fn<KtxDictionarySearchMcpPort['search']>().mockResolvedValue({
searched: [
{
connectionId: 'warehouse',
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 1,
syncId: 'sync-1',
profiledAt: null,
},
status: 'ready',
},
],
results: [
{
value: 'paid',
matches: [
{
connectionId: 'warehouse',
sourceName: 'orders',
columnName: 'status',
matchedValue: 'paid',
cardinality: 3,
},
],
misses: [],
},
],
}),
};
createKtxMcpServer({
server: fake.server,
userContext: { userId: 'local-user' },
contextTools: { dictionarySearch },
});
expect(fake.tools.map((tool) => tool.name)).toEqual(['dictionary_search']);
await expect(
getTool(fake.tools, 'dictionary_search').handler({
connectionId: 'warehouse',
values: ['paid'],
}),
).resolves.toMatchObject({
structuredContent: {
searched: [{ connectionId: 'warehouse', status: 'ready' }],
results: [
{
value: 'paid',
matches: [{ connectionId: 'warehouse', sourceName: 'orders', columnName: 'status' }],
misses: [],
},
],
},
});
expect(dictionarySearch.search).toHaveBeenCalledWith({
connectionId: 'warehouse',
values: ['paid'],
});
});
```
- [ ] **Step 3: Run failing MCP registration test**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "dictionary_search"
```
Expected: FAIL because `dictionary_search` is not registered.
- [ ] **Step 4: Add the MCP schema and registration**
In `packages/context/src/mcp/context-tools.ts`, add the input schema near the other research schemas:
```typescript
const dictionarySearchSchema = z.object({
values: z.array(z.string().min(1)).min(1).max(20),
connectionId: connectionIdSchema.optional(),
});
```
Add this registration block after `entity_details` and before `sql_execution`:
```typescript
if (ports.dictionarySearch) {
const dictionarySearch = ports.dictionarySearch;
registerParsedTool(
server,
'dictionary_search',
{
title: 'Dictionary Search',
description:
'Search profile-sampled warehouse values and report matching connection/source/column locations plus non-authoritative miss reasons.',
inputSchema: dictionarySearchSchema.shape,
},
dictionarySearchSchema,
async (input) => jsonToolResult(await dictionarySearch.search(input)),
);
}
```
- [ ] **Step 5: Confirm MCP barrel exports**
Open `packages/context/src/mcp/index.ts`. If it exports from `./types.js`, no change is needed. If it lists named type exports, add `KtxDictionarySearchMcpPort` to that list.
- [ ] **Step 6: Run MCP registration test**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "dictionary_search"
```
Expected: PASS.
- [ ] **Step 7: Commit MCP registration**
Run:
```bash
git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/index.ts
git commit -m "feat(context): register MCP dictionary search tool"
```
## Task 3: Wire Local Project MCP Ports
**Files:**
- Modify: `packages/context/src/mcp/local-project-ports.ts`
- Modify: `packages/context/src/mcp/local-project-ports.test.ts`
- [ ] **Step 1: Write failing local-port tests**
In `packages/context/src/mcp/local-project-ports.test.ts`, add this test after the entity-details local-port tests:
```typescript
it('exposes local dictionary search through MCP ports', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
};
await project.fileStore.writeFile(
'raw-sources/warehouse/live-database/sync-1/enrichment/relationship-profile.json',
`${JSON.stringify(
{
connectionId: 'warehouse',
driver: 'postgres',
sqlAvailable: true,
queryCount: 4,
tables: [],
columns: {
'orders.status': {
table: { catalog: null, db: 'public', name: 'orders' },
column: 'status',
nativeType: 'text',
normalizedType: 'string',
distinctCount: 2,
sampleValues: ['paid', 'refunded'],
},
},
warnings: [],
},
null,
2,
)}\n`,
'ktx',
'ktx@example.com',
'Seed dictionary profile',
);
const ports = createLocalProjectMcpContextPorts(project);
await expect(ports.dictionarySearch?.search({ values: ['paid'] })).resolves.toMatchObject({
searched: [{ connectionId: 'warehouse', status: 'ready' }],
results: [
{
value: 'paid',
matches: [{ connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', matchedValue: 'paid' }],
misses: [],
},
],
});
});
it('reports missing local dictionary profiles through MCP ports', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
};
const ports = createLocalProjectMcpContextPorts(project);
await expect(ports.dictionarySearch?.search({ values: ['paid'] })).resolves.toEqual({
searched: [
{
connectionId: 'warehouse',
coverage: {
sampledRows: null,
valuesPerColumn: null,
profiledColumns: 0,
syncId: null,
profiledAt: null,
},
status: 'no_profile_artifact',
},
],
results: [
{
value: 'paid',
matches: [],
misses: [{ connectionId: 'warehouse', reason: 'no_profile_artifact' }],
},
],
});
});
```
- [ ] **Step 2: Run failing local-port tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "dictionary"
```
Expected: FAIL because `ports.dictionarySearch` is undefined.
- [ ] **Step 3: Wire the local port**
In `packages/context/src/mcp/local-project-ports.ts`, update the SL import block to include:
```typescript
createKtxDictionarySearchService,
```
Add this port to the `ports` object returned by `createLocalProjectMcpContextPorts()` near `entityDetails`:
```typescript
dictionarySearch: {
async search(input) {
return createKtxDictionarySearchService(project).search(input);
},
},
```
- [ ] **Step 4: Run local-port tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "dictionary"
```
Expected: PASS.
- [ ] **Step 5: Commit local-port wiring**
Run:
```bash
git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts
git commit -m "feat(context): expose local MCP dictionary search"
```
## Task 4: Final Verification
**Files:**
- Verify all files changed in Tasks 1-3.
- [ ] **Step 1: Run focused tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/sl/dictionary-search.test.ts src/mcp/server.test.ts src/mcp/local-project-ports.test.ts
```
Expected: PASS for dictionary-search service, MCP registration, and local-port coverage.
- [ ] **Step 2: Run context type-check**
Run:
```bash
pnpm --filter @ktx/context run type-check
```
Expected: PASS.
- [ ] **Step 3: Inspect diff**
Run:
```bash
git status --short
git diff --stat HEAD
```
Expected: only the dictionary-search service, MCP type/registration, tests, and exports changed.
- [ ] **Step 4: Commit verification note if needed**
If the previous tasks already committed all source changes, do not create an empty commit. If a small follow-up fix was required during verification, commit only those files:
```bash
git add packages/context/src/sl/dictionary-search.ts packages/context/src/sl/dictionary-search.test.ts packages/context/src/sl/index.ts packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/index.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts
git commit -m "test(context): cover MCP dictionary search"
```

1315
docs/superpowers/plans/2026-05-14-research-agent-mcp-discover-data.md Normal file
View file

File diff suppressed because it is too large Load diff

1175
docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md Normal file
View file

File diff suppressed because it is too large Load diff

1561
docs/superpowers/plans/2026-05-14-research-agent-mcp-http-daemon.md Normal file
View file

File diff suppressed because it is too large Load diff

804
docs/superpowers/plans/2026-05-14-research-agent-mcp-ingest-contract-convergence.md Normal file
View file

@ -0,0 +1,804 @@
# Research Agent MCP Ingest Contract Convergence Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Finish the v1 research-agent MCP spec by converging the existing ingest warehouse-verification tools on `connectionId` terminology and a shared raw-schema catalog service.
**Architecture:** Move the existing warehouse catalog reader out of the ingest-only tool folder into `packages/context/src/scan/warehouse-catalog.ts`, rename its public contract from `connectionName` to `connectionId`, and make the ingest adapters consume that shared service. Keep the ingest tools' ingest-specific output shape (`markdown` plus `structured`) and their existing `targets` / `rowLimit` controls; the v1 blocker is the divergent connection parameter and stale prompt guidance, not changing ingest output into the MCP pure-structured shape.
**Tech Stack:** TypeScript, Zod, Vitest, existing KTX local file-store scan artifacts, existing ingest BaseTool framework.
---
## Audit Summary
Implemented and no longer v1-blocking:
- MCP `sql_execution`, `entity_details`, `dictionary_search`, and `discover_data` are registered in `packages/context/src/mcp/context-tools.ts` and wired through local project ports.
- `sql_execution` is parser-gated through the Python sqlglot validator before reaching local scan connectors.
- The HTTP-only `ktx mcp` daemon exists with Streamable HTTP `POST`, `GET`, and `DELETE` handling, session tracking, host/origin checks, token checks for `/mcp`, lifecycle state, and CLI commands.
- `ktx setup-agents` installs the `ktx-research` skill, writes Claude/Cursor JSON MCP config entries, and prints Codex/opencode snippets.
Remaining v1 blocker:
- The ingest warehouse-verification tools still expose and teach `connectionName` while the spec requires `connectionId` across `warehouse-verification/*.tool.ts`, `WarehouseCatalogService`, callers, tests, and prompt assets.
Non-blocking follow-ups not covered here:
- `ktx mcp status` does not print `startedAt` as a separate line, although the state file records it.
- `ktx setup-agents` writes safe `${KTX_MCP_TOKEN}` references for shared project configs, but it does not offer the spec's optional skip prompt when token auth is active.
- `discover_data` sample-value snippets use ASCII `" - samples: "` instead of the spec prose's middle-dot separator.
## File Structure
- Move: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` to `packages/context/src/scan/warehouse-catalog.ts`
- Shared live-database scan catalog reader, display resolver, raw schema search, and table detail source of truth.
- Modify: `packages/context/src/scan/index.ts`
- Export the shared warehouse catalog service and public types.
- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`
- Accept `connectionId`, call shared catalog service, and emit connectionId-shaped markdown and structured output.
- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`
- Accept optional `connectionId`, search raw schema via shared catalog service, and teach follow-up calls with `connectionId`.
- Modify: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`
- Accept `connectionId`, keep `rowLimit`, and pass `connectionId` to `SlConnectionCatalogPort.executeQuery`.
- Modify tests:
- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`
- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`
- `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts`
- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`
- Rename the service test file to `packages/context/src/scan/warehouse-catalog.test.ts`.
- Modify prompt assets:
- `packages/context/skills/_shared/identifier-verification.md`
- `packages/context/skills/dbt_ingest/SKILL.md`
- `packages/context/skills/historic_sql_patterns/SKILL.md`
- `packages/context/skills/historic_sql_table_digest/SKILL.md`
- `packages/context/skills/live_database_ingest/SKILL.md`
- `packages/context/skills/looker_ingest/SKILL.md`
- `packages/context/skills/lookml_ingest/SKILL.md`
- `packages/context/skills/metabase_ingest/SKILL.md`
- `packages/context/skills/metricflow_ingest/SKILL.md`
- `packages/context/skills/notion_synthesize/SKILL.md`
- `packages/context/skills/sl_capture/SKILL.md`
- `packages/context/skills/wiki_capture/SKILL.md`
- Preserve Looker/LookML prose where `connectionName` refers to a Looker runtime field, not a KTX tool parameter.
## Task 1: Add Failing Contract Tests
**Files:**
- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`
- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`
- Modify: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts`
- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`
- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts`
- [ ] **Step 1: Add entity_details input-contract coverage**
Add this test inside the existing `describe('EntityDetailsTool', ...)` block:
```typescript
it('uses connectionId as the public input field', async () => {
expect(
tool.parseInput({
connectionId: 'warehouse',
targets: [{ display: 'public.orders' }],
}),
).toEqual({
connectionId: 'warehouse',
targets: [{ display: 'public.orders' }],
});
expect(() =>
tool.parseInput({
connectionName: 'warehouse',
targets: [{ display: 'public.orders' }],
}),
).toThrow();
});
```
Update the existing `tool.call(...)` inputs in the same test file from `connectionName` to `connectionId`. For example:
```typescript
const result = await tool.call({ connectionId: 'warehouse', targets: [{ display: 'public.orders' }] }, context);
```
- [ ] **Step 2: Add sql_execution input-contract coverage**
Add this test inside `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts`:
```typescript
it('uses connectionId as the public input field', () => {
expect(
tool.parseInput({
connectionId: 'warehouse',
sql: 'select 1',
rowLimit: 5,
}),
).toEqual({
connectionId: 'warehouse',
sql: 'select 1',
rowLimit: 5,
});
expect(() =>
tool.parseInput({
connectionName: 'warehouse',
sql: 'select 1',
rowLimit: 5,
}),
).toThrow();
});
```
Update the existing `tool.call(...)` inputs in the same test file from `connectionName` to `connectionId`.
- [ ] **Step 3: Add discover_data input and hint coverage**
Update the existing discover tests so the first case calls:
```typescript
const result = await tool.call({ query: 'orders', connectionId: 'warehouse', limit: 5 }, context);
```
Change the routing-hint assertions to:
```typescript
expect(result.markdown).toContain('use `entity_details({connectionId, targets: [{display}]})`');
```
In the multi-connection test, use a `connectionId` hit field and assert the follow-up call is connectionId-shaped:
```typescript
catalog.searchByName.mockImplementation(async (connectionId: string, query: string) => [
{
kind: 'table',
connectionId,
ref: { catalog: null, db: 'public', name: `${connectionId}_${query}` },
display: `public.${connectionId}_${query}`,
matchedOn: 'name',
},
]);
const result = await tool.call({ query: 'orders', limit: 10 }, multiConnectionContext);
expect(catalog.searchByName).toHaveBeenCalledWith('analytics', 'orders', 10);
expect(catalog.searchByName).toHaveBeenCalledWith('warehouse', 'orders', 10);
expect(result.markdown).toContain('connectionId=analytics');
expect(result.markdown).toContain('connectionId=warehouse');
expect(result.markdown).toContain(
'entity_details({connectionId: "analytics", targets: [{display: "public.analytics_orders"}]})',
);
expect(result.structured.raw?.hits.map((hit) => hit.connectionId)).toEqual(['analytics', 'warehouse']);
```
Add a parse contract test:
```typescript
it('uses connectionId as the optional connection filter', () => {
expect(tool.parseInput({ query: 'orders', connectionId: 'warehouse', limit: 5 })).toEqual({
query: 'orders',
connectionId: 'warehouse',
limit: 5,
});
expect(() => tool.parseInput({ query: 'orders', connectionName: 'warehouse', limit: 5 })).toThrow();
});
```
- [ ] **Step 4: Add shared catalog output coverage**
Rename `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` to `packages/context/src/scan/warehouse-catalog.test.ts`.
Update the import to:
```typescript
import { WarehouseCatalogService } from './warehouse-catalog.js';
```
Update the main detail assertion to use `connectionId`:
```typescript
const detail = await catalog.getTable({ connectionId: 'warehouse', catalog: null, db: 'public', name: 'orders' });
expect(detail).toMatchObject({
connectionId: 'warehouse',
display: 'public.orders',
});
expect(detail).not.toHaveProperty('connectionName');
```
Add raw hit coverage:
```typescript
const hits = await catalog.searchByName('warehouse', 'orders', 5);
expect(hits[0]).toMatchObject({
kind: 'table',
connectionId: 'warehouse',
display: 'public.orders',
});
expect(hits[0]).not.toHaveProperty('connectionName');
```
- [ ] **Step 5: Update prompt-asset test expectations first**
In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, change the identifier verification expectations to:
```typescript
expect(shared).toContain('sql_execution({connectionId, sql: "SELECT DISTINCT');
expect(shared).toContain('sql_execution({connectionId, sql: "SELECT 1 FROM');
expect(shared).not.toContain('entity_details({connectionName');
expect(shared).not.toContain('sql_execution({connectionName');
```
- [ ] **Step 6: Run focused tests and verify they fail**
Run:
```bash
pnpm --filter @ktx/context exec vitest run \
src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \
src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \
src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \
src/scan/warehouse-catalog.test.ts \
src/ingest/ingest-runtime-assets.test.ts
```
Expected: FAIL because schemas still require `connectionName`, the catalog service still returns `connectionName`, and the prompt asset still contains old tool-call examples.
## Task 2: Move And Rename The Shared Warehouse Catalog Service
**Files:**
- Move: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` to `packages/context/src/scan/warehouse-catalog.ts`
- Modify: `packages/context/src/scan/index.ts`
- Delete: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`
- [ ] **Step 1: Move the service into the scan package**
Run:
```bash
git mv packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts packages/context/src/scan/warehouse-catalog.ts
```
- [ ] **Step 2: Fix imports for the new location**
In `packages/context/src/scan/warehouse-catalog.ts`, change the imports at the top to:
```typescript
import { getDialectForDriver } from '../connections/index.js';
import type { KtxFileStorePort } from '../core/index.js';
import type {
KtxConnectionDriver,
KtxSchemaColumn,
KtxSchemaForeignKey,
KtxSchemaTable,
KtxTableRef,
} from './types.js';
```
- [ ] **Step 3: Rename public catalog fields and method parameters**
In `packages/context/src/scan/warehouse-catalog.ts`, rename the service's public contract to this shape:
```typescript
export interface TableDetail {
connectionId: string;
catalog: string | null;
db: string | null;
name: string;
display: string;
kind: string;
comment: string | null;
description: string | null;
rowCount: number | null;
columns: WarehouseColumnDetail[];
foreignKeys: KtxSchemaForeignKey[];
}
export type RawSchemaHit =
| {
kind: 'table';
connectionId: string;
ref: KtxTableRef;
display: string;
matchedOn: 'name' | 'db' | 'comment' | 'description';
}
| {
kind: 'column';
connectionId: string;
ref: KtxTableRef & { column: string };
display: string;
matchedOn: 'name' | 'comment' | 'description';
};
interface ConnectionCatalog {
connectionId: string;
syncId: string;
driver: CatalogDriver;
tables: KtxSchemaTable[];
profile: RelationshipProfileArtifact | null;
}
```
Update the method signatures to:
```typescript
async hasScan(connectionId: string): Promise<boolean>
async getLatestSyncId(connectionId: string): Promise<string | null>
async listTables(connectionId: string): Promise<KtxTableRef[]>
async getTable(ref: { connectionId: string } & KtxTableRef): Promise<TableDetail | null>
async resolveDisplay(connectionId: string, display: string): Promise<{ resolved: KtxTableRef | null; candidates: KtxTableRef[]; dialect: string }>
async resolveDisplayTarget(connectionId: string, display: string): Promise<DisplayTargetResolution>
async searchByName(connectionId: string, query: string, limit: number): Promise<RawSchemaHit[]>
private loadCatalog(connectionId: string): Promise<ConnectionCatalog | null>
private async readCatalog(connectionId: string): Promise<ConnectionCatalog | null>
```
Within those methods, use `connectionId` for the cache key, raw artifact root, returned `TableDetail.connectionId`, and returned `RawSchemaHit.connectionId`.
- [ ] **Step 4: Export the shared service**
Add these exports to `packages/context/src/scan/index.ts` near the existing entity-details exports:
```typescript
export type {
DisplayTargetResolution,
RawSchemaHit,
TableDetail,
WarehouseCatalogServiceDeps,
} from './warehouse-catalog.js';
export { WarehouseCatalogService } from './warehouse-catalog.js';
```
- [ ] **Step 5: Run the catalog test**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/scan/warehouse-catalog.test.ts
```
Expected: PASS.
- [ ] **Step 6: Commit the shared catalog move**
Run:
```bash
git add packages/context/src/scan/warehouse-catalog.ts packages/context/src/scan/warehouse-catalog.test.ts packages/context/src/scan/index.ts packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts
git commit -m "refactor(context): share warehouse catalog service"
```
## Task 3: Rename Ingest Warehouse-Verification Tool Inputs
**Files:**
- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`
- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`
- Modify: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`
- Modify: `packages/context/src/ingest/tools/warehouse-verification/index.ts`
- [ ] **Step 1: Update imports from the shared scan service**
In `entity-details.tool.ts`, use:
```typescript
import { WarehouseCatalogService, type TableDetail } from '../../../scan/warehouse-catalog.js';
```
In `discover-data.tool.ts`, use:
```typescript
import { WarehouseCatalogService, type RawSchemaHit } from '../../../scan/warehouse-catalog.js';
```
In `index.ts`, use:
```typescript
import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js';
```
- [ ] **Step 2: Rename entity_details input and calls**
In `entity-details.tool.ts`, update the schema:
```typescript
const entityDetailsInputSchema = z.object({
connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
targets: z.array(targetSchema).min(1).max(50),
});
```
Update `resolveTarget`:
```typescript
async function resolveTarget(
catalog: WarehouseCatalogService,
connectionId: string,
target: EntityDetailsTarget,
): Promise<{ resolved: (KtxTableRef & { column?: string }) | null; candidates: KtxTableRef[] }> {
if ('display' in target) {
return catalog.resolveDisplayTarget(connectionId, target.display);
}
const candidateResolution = await catalog.resolveDisplayTarget(connectionId, targetLabel(target));
return {
resolved: {
catalog: target.catalog,
db: target.db,
name: target.name,
column: target.column,
},
candidates: candidateResolution.candidates,
};
}
```
Update the start of `call`:
```typescript
async call(input: EntityDetailsInput, context: ToolContext): Promise<ToolOutput<EntityDetailsStructured>> {
const allowed = allowedConnectionNames(context);
if (allowed && !allowed.has(input.connectionId)) {
return {
markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`,
structured: { resolved: [], missing: [], scanAvailable: false },
};
}
const catalog = this.catalogFactory(context);
const scanAvailable = await catalog.hasScan(input.connectionId);
if (!scanAvailable) {
return {
markdown: `No live-database scan available for connection "${input.connectionId}"; run \`ktx scan\` first.`,
structured: { resolved: [], missing: [], scanAvailable: false },
};
}
```
Update the table lookup:
```typescript
const resolution = await resolveTarget(catalog, input.connectionId, target);
const detail = await catalog.getTable({ connectionId: input.connectionId, ...resolution.resolved });
```
- [ ] **Step 3: Rename sql_execution input and calls**
In `sql-execution.tool.ts`, update the schema:
```typescript
const sqlExecutionInputSchema = z.object({
connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
sql: z.string().min(1),
rowLimit: z.number().int().positive().max(1000).optional().default(100),
});
```
Update the allowed-connection guard:
```typescript
const allowed = context.session?.allowedConnectionNames;
if (allowed && !allowed.has(input.connectionId)) {
return {
markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`,
structured: {
headers: [],
rows: [],
rowCount: 0,
truncated: false,
sql: input.sql,
wrappedSql: '',
error: 'connection_not_allowed',
},
};
}
```
Update execution:
```typescript
const result = await this.connections.executeQuery(input.connectionId, wrappedSql);
```
- [ ] **Step 4: Rename discover_data input, raw hits, and routing hints**
In `discover-data.tool.ts`, update the schema:
```typescript
const discoverDataInputSchema = z.object({
query: z.string().optional(),
connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(),
limit: z.number().int().positive().max(50).optional().default(10),
sourceName: z.string().optional(),
});
```
Update the out-of-scope check:
```typescript
if (input.connectionId && allowed && !allowed.has(input.connectionId)) {
return {
markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`,
structured: { wiki: null, sl: null, raw: null },
};
}
```
Update the source inspect mode:
```typescript
const sl = await this.deps.slDiscoverTool.call(
{ sourceName: input.sourceName, connectionId: input.connectionId },
context,
);
```
Update the SL discover call:
```typescript
const slResult = await this.deps.slDiscoverTool.call(
{ query: query || undefined, connectionId: input.connectionId },
context,
);
```
Update the raw search loop and hints:
```typescript
const connections = input.connectionId ? [input.connectionId] : [...(allowed ?? [])].sort();
const rawHits: RawSchemaHit[] = [];
for (const connectionId of connections) {
rawHits.push(...(await catalog.searchByName(connectionId, query, limit)));
}
if (rawHits.length > 0) {
parts.push(
'## Raw Warehouse Schema',
'> use `entity_details({connectionId, targets: [{display}]})` for full DDL + sample values',
);
parts.push(
rawHits
.slice(0, limit)
.map(
(hit) =>
`- ${hit.kind}: ${hit.display} [connectionId=${hit.connectionId}] (matched on ${hit.matchedOn}) - ` +
`follow up with \`entity_details({connectionId: "${hit.connectionId}", targets: [{display: "${hit.display}"}]})\``,
)
.join('\n'),
);
raw = { hits: rawHits.slice(0, limit) };
}
```
- [ ] **Step 5: Run focused tool tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run \
src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \
src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \
src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts
```
Expected: PASS.
- [ ] **Step 6: Commit the ingest tool contract rename**
Run:
```bash
git add packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts packages/context/src/ingest/tools/warehouse-verification/index.ts packages/context/src/ingest/tools/warehouse-verification/*.test.ts
git commit -m "refactor(context): use connectionId in warehouse verification tools"
```
## Task 4: Update Prompt Assets And Runtime Tests
**Files:**
- Modify: `packages/context/skills/_shared/identifier-verification.md`
- Modify: `packages/context/skills/dbt_ingest/SKILL.md`
- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md`
- Modify: `packages/context/skills/historic_sql_table_digest/SKILL.md`
- Modify: `packages/context/skills/live_database_ingest/SKILL.md`
- Modify: `packages/context/skills/looker_ingest/SKILL.md`
- Modify: `packages/context/skills/lookml_ingest/SKILL.md`
- Modify: `packages/context/skills/metabase_ingest/SKILL.md`
- Modify: `packages/context/skills/metricflow_ingest/SKILL.md`
- Modify: `packages/context/skills/notion_synthesize/SKILL.md`
- Modify: `packages/context/skills/sl_capture/SKILL.md`
- Modify: `packages/context/skills/wiki_capture/SKILL.md`
- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts`
- [ ] **Step 1: Update the shared identifier verification protocol**
Replace the tool-call examples in `packages/context/skills/_shared/identifier-verification.md` with:
```markdown
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
```
- [ ] **Step 2: Update copied skill assets**
In the listed `packages/context/skills/*/SKILL.md` files, replace only KTX tool-call examples:
```text
entity_details({connectionName, targets:
```
with:
```text
entity_details({connectionId, targets:
```
Replace:
```text
sql_execution({connectionName, sql:
```
with:
```text
sql_execution({connectionId, sql:
```
Replace concrete KTX tool-call examples like:
```text
sql_execution({connectionName: "warehouse", sql:
```
with:
```text
sql_execution({connectionId: "warehouse", sql:
```
In `packages/context/skills/sl_capture/SKILL.md`, replace the JSON field inside the example object:
```yaml
connectionName: "warehouse",
```
with:
```yaml
connectionId: "warehouse",
```
Do not change `packages/context/skills/looker_ingest/SKILL.md` text that defines Looker runtime `connectionName`, and do not change LookML parser docs where `connectionName` names a LookML model property.
- [ ] **Step 3: Update runtime asset tests**
In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, ensure the identifier test asserts the new examples:
```typescript
expect(shared).toContain('sql_execution({connectionId, sql: "SELECT DISTINCT');
expect(shared).toContain('sql_execution({connectionId, sql: "SELECT 1 FROM');
expect(shared).not.toContain('entity_details({connectionName');
expect(shared).not.toContain('sql_execution({connectionName');
```
- [ ] **Step 4: Run prompt asset checks**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/ingest/ingest-runtime-assets.test.ts
```
Expected: PASS.
- [ ] **Step 5: Verify stale tool-call examples are gone**
Run:
```bash
rg -n "entity_details\\(\\{connectionName|sql_execution\\(\\{connectionName|connectionName=" packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts
```
Expected: no output. If this reports Looker/LookML prose that is not a KTX tool-call example, narrow the regex and keep the Looker/LookML prose unchanged.
- [ ] **Step 6: Commit prompt asset updates**
Run:
```bash
git add packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts
git commit -m "docs(context): update ingest verification prompts for connectionId"
```
## Task 5: Final Verification
**Files:**
- Verify all files changed in Tasks 1-4.
- [ ] **Step 1: Run focused research-agent ingest tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run \
src/scan/warehouse-catalog.test.ts \
src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \
src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \
src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \
src/ingest/ingest-runtime-assets.test.ts
```
Expected: PASS.
- [ ] **Step 2: Run context type-check**
Run:
```bash
pnpm --filter @ktx/context run type-check
```
Expected: PASS.
- [ ] **Step 3: Run dead-code checks after TypeScript changes**
Run:
```bash
pnpm run dead-code
```
Expected: PASS. If Knip reports unrelated pre-existing findings, record the exact unrelated findings in the implementation handoff and do not add broad ignores.
- [ ] **Step 4: Verify the v1-blocking old contract is gone**
Run:
```bash
rg -n "connectionName" packages/context/src/ingest/tools/warehouse-verification packages/context/src/scan/warehouse-catalog.ts packages/context/src/scan/warehouse-catalog.test.ts
```
Expected: no output.
Run:
```bash
rg -n "entity_details\\(\\{connectionName|sql_execution\\(\\{connectionName|connectionName=" packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts
```
Expected: no output.
- [ ] **Step 5: Inspect git status**
Run:
```bash
git status --short
```
Expected: only the intended scan catalog move, warehouse-verification tools/tests, prompt assets, and ingest runtime asset test changes are present.
- [ ] **Step 6: Commit final fixes if verification required any**
If Steps 1-5 required follow-up edits, commit those edits:
```bash
git add packages/context/src packages/context/skills
git commit -m "test(context): verify warehouse verification connectionId contract"
```
If `git status --short` is empty after the earlier task commits, skip this commit.
## Self-Review
- Spec coverage: This plan covers the remaining v1 requirement that ingest-side warehouse verification uses `connectionId` and shares the raw-schema catalog service instead of preserving a divergent `connectionName` contract.
- Placeholder scan: The plan contains no deferred-work marker phrases.
- Type consistency: The plan uses `connectionId` consistently in public tool inputs, `TableDetail`, `RawSchemaHit`, `WarehouseCatalogService` method parameters, tests, and prompt assets.

938
docs/superpowers/plans/2026-05-14-research-agent-mcp-setup-agents.md Normal file
View file

@ -0,0 +1,938 @@
# Research Agent MCP Setup Agents Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make `ktx setup-agents` install the `ktx-research` skill and configure or print MCP client entries that point agents at the local `ktx mcp` HTTP endpoint.
**Architecture:** Keep `packages/cli/src/setup-agents.ts` as the setup orchestration point. Add a small MCP-client config planner/writer in the same module, backed by `.ktx/mcp.json` when present, and install the research skill from a copied runtime asset so source checkouts and published CLI builds use the same `SKILL.md`.
**Tech Stack:** TypeScript, Vitest, Node fs/path APIs, Commander setup options, KTX MCP daemon state, JSON config writers.
---
## Current Audit
Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md`
Implemented v1 slices confirmed in current source:
- MCP `sql_execution`, `entity_details`, `dictionary_search`, and `discover_data` are registered in `packages/context/src/mcp/context-tools.ts`.
- Local project MCP ports wire all four tools in `packages/context/src/mcp/local-project-ports.ts`.
- Parser-backed SQL validation exists in `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` and is exposed through `POST /sql/validate-read-only`.
- `ktx mcp start|stop|status|logs` exists in `packages/cli/src/commands/mcp-commands.ts`, with HTTP hosting in `packages/cli/src/mcp-http-server.ts` and daemon state in `packages/cli/src/managed-mcp-daemon.ts`.
- Targeted verification passed:
- `pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts src/search/discover.test.ts src/scan/entity-details.test.ts src/sl/dictionary-search.test.ts`
- `pnpm --filter @ktx/cli exec vitest run src/mcp-http-server.test.ts src/managed-mcp-daemon.test.ts src/commands/mcp-commands.test.ts src/setup-agents.test.ts`
V1-blocking gaps remaining against the original spec:
- `ktx setup-agents` still installs only the existing `ktx` agent files; it does not install `ktx-research`.
- `ktx setup-agents` does not write Claude Code or Cursor MCP JSON config entries.
- `ktx setup-agents` does not print Codex or opencode copy-paste snippets.
- `ktx setup-agents --remove` cannot remove written MCP JSON keys because none are written or tracked.
- The ingest-side warehouse-verification tools still use `connectionName`, `targets`, and `rowLimit`, and `WarehouseCatalogService` still exposes connection-name terminology. That is a separate v1-blocking subsystem and is not mixed into this setup-agent plan.
Non-blocking or explicitly out-of-scope gaps:
- Python code execution over MCP.
- Stdio MCP transport.
- OS-level auto-start.
- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing.
- Streaming SQL results.
## File Structure
Create:
- `packages/cli/src/skills/research/SKILL.md`
- Canonical research skill body from the spec.
- Copied into `dist/skills/research/SKILL.md` during `@ktx/cli` build.
- `packages/cli/scripts/copy-runtime-assets.mjs`
- Copies `src/skills` into `dist/skills` after TypeScript compilation.
Modify:
- `packages/cli/package.json`
- Append the runtime asset copy step to the `build` script.
- `packages/cli/src/setup-agents.ts`
- Add `local` agent scope for Claude Code's per-project private config path.
- Add `research-skill` file entries in `plannedKtxAgentFiles()`.
- Read the research skill asset when writing research-skill entries.
- Add MCP endpoint resolution from `.ktx/mcp.json`, falling back to `http://localhost:7878/mcp`.
- Add JSON writers for Claude Code and Cursor MCP entries.
- Add printed snippets for Codex and opencode.
- Track written JSON keys in the install manifest.
- Print the daemon-start hint when the daemon is not currently running.
- `packages/cli/src/setup-agents.test.ts`
- Cover research skill install paths, MCP JSON writers, snippets, manifest removal, token handling, and no literal-token rendering.
- `packages/cli/src/commands/setup-commands.ts`
- Add `--local` for Claude Code local-scope setup.
- Reject `--local` with non-Claude targets and reject `--local --global`.
- `packages/cli/src/setup.ts`
- No behavior change beyond accepting `KtxAgentScope` with the new `local` value.
- `packages/cli/src/cli-program.ts`
- Keep the default bare setup `agentScope: 'project'`; no code change needed unless TypeScript requires the widened scope type in nearby annotations.
## Task 1: Add The Research Skill Runtime Asset
**Files:**
- Create: `packages/cli/src/skills/research/SKILL.md`
- Create: `packages/cli/scripts/copy-runtime-assets.mjs`
- Modify: `packages/cli/package.json`
- Modify: `packages/cli/src/setup-agents.test.ts`
- Modify: `packages/cli/src/setup-agents.ts`
- [ ] **Step 1: Write the failing research-skill install tests**
In `packages/cli/src/setup-agents.test.ts`, update the first test to expect `ktx-research` entries. Replace the project-scoped assertions with:
```typescript
it('plans project-scoped CLI and research files for every target', () => {
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-code', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' },
{ kind: 'file', path: join(tempDir, '.claude/rules/ktx.md'), role: 'rule' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'codex', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md'), role: 'skill' },
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' },
{ kind: 'file', path: join(tempDir, '.codex/instructions/ktx.md'), role: 'rule' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'cursor', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.cursor/rules/ktx.mdc') },
{ kind: 'file', path: join(tempDir, '.cursor/rules/ktx-research.mdc'), role: 'research-skill' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'opencode', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.opencode/commands/ktx.md') },
{ kind: 'file', path: join(tempDir, '.opencode/commands/ktx-research.md'), role: 'research-skill' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'universal', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md') },
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' },
]);
});
```
Add this test after `installs target files, writes a manifest, and marks agents complete`:
```typescript
it('installs the research skill from the runtime asset', async () => {
const io = makeIo();
await expect(
runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'universal',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
),
).resolves.toMatchObject({ status: 'ready' });
const researchSkill = await readFile(join(tempDir, '.agents/skills/ktx-research/SKILL.md'), 'utf-8');
expect(researchSkill).toContain('name: ktx-research');
expect(researchSkill).toContain('Always run `discover_data` before writing SQL.');
expect(researchSkill).toContain('Treat a `dictionary_search` miss as non-authoritative.');
});
```
- [ ] **Step 2: Run tests to verify they fail**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts
```
Expected: FAIL because `plannedKtxAgentFiles()` does not return `ktx-research` entries and the installed research skill file does not exist.
- [ ] **Step 3: Add the research skill asset**
Create `packages/cli/src/skills/research/SKILL.md`:
```markdown
---
name: ktx-research
description: Use when answering a question that needs data from a KTX-connected database - investigating, analyzing, "how many", "show me", "what's the breakdown of", finding records by value, exploring tables, comparing periods, or any data-investigation request. Triggers even when the user does not say "research"; if the answer requires querying a configured KTX connection, this skill applies.
---
# KTX Research Workflow
You have access to KTX MCP tools for investigating data. Follow this workflow.
<workflow>
1. **Discover** - call `discover_data` first to see what exists across wiki, semantic-layer sources, and raw tables. Returns refs only.
2. **Inspect top hits in parallel** - for each promising ref:
- `kind: 'wiki'` -> `wiki_read`
- `kind: 'sl_source'`, `kind: 'sl_measure'`, or `kind: 'sl_dimension'` -> `sl_read_source`
- `kind: 'table'` or `kind: 'column'` -> `entity_details`
3. **Resolve literals** - if the user named a value such as "Acme Corp" or "status=shipped", call `dictionary_search` to find which column holds it.
4. **Query** -
- Prefer `sl_query` when the semantic layer covers the question.
- Use `sql_execution` only for questions the semantic layer does not cover.
5. **Capture learnings** - at the end of the turn, call `memory_capture` so future turns benefit. Skip when the answer carries no durable knowledge.
</workflow>
<rules>
- Always run `discover_data` before writing SQL. Do not guess table names.
- Prefer the semantic layer over raw SQL when both can answer the question; measures are the source of truth.
- Read entity details before writing SQL against an unfamiliar table. Do not assume column names.
- Treat `sql_execution` as read-only. Writes are rejected by the server.
- Validate value mentions with `dictionary_search` instead of guessing case or spelling. Treat a `dictionary_search` miss as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent.
</rules>
<examples>
**Input:** "How many orders did Acme Corp place last month?"
**Workflow:**
1. `dictionary_search({ values: ["Acme Corp"] })` finds `customers.name`.
2. `discover_data({ query: "orders customer monthly" })` finds an orders semantic-layer source.
3. `sl_read_source({ connectionId: "warehouse", sourceName: "orders_facts" })` confirms the source grain, measures, and dimensions.
4. `sl_query({ connectionId: "warehouse", measures: ["order_count"], filters: ["customer_name = 'Acme Corp'"] })` answers through the semantic layer.
5. `memory_capture({ userMessage, assistantMessage })` captures the durable finding.
---
**Input:** "What columns does the events table have?"
**Workflow:**
1. `discover_data({ query: "events table" })` returns a `table` ref.
2. `entity_details({ connectionId: "warehouse", entities: [{ table: "analytics.events" }] })` returns columns, types, and foreign keys.
3. Answer directly. No query is needed.
</examples>
```
- [ ] **Step 4: Copy skill assets during CLI build**
Create `packages/cli/scripts/copy-runtime-assets.mjs`:
```javascript
import { cp, mkdir, rm } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
const packageRoot = fileURLToPath(new URL('..', import.meta.url));
const skillsSource = join(packageRoot, 'src', 'skills');
const skillsTarget = join(packageRoot, 'dist', 'skills');
await rm(skillsTarget, { recursive: true, force: true });
await mkdir(dirname(skillsTarget), { recursive: true });
await cp(skillsSource, skillsTarget, { recursive: true });
```
Modify `packages/cli/package.json`:
```json
"build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs"
```
- [ ] **Step 5: Add research-skill install entries and content loading**
In `packages/cli/src/setup-agents.ts`, update the manifest entry role type:
```typescript
| { kind: 'file'; path: string; role?: 'skill' | 'rule' | 'research-skill' }
```
Add this helper near `ktxCliLauncher()`:
```typescript
async function readResearchSkillContent(): Promise<string> {
const path = fileURLToPath(new URL('./skills/research/SKILL.md', import.meta.url));
const content = await readFile(path, 'utf-8');
return content.endsWith('\n') ? content : `${content}\n`;
}
```
Update `plannedKtxAgentFiles()` so every supported project target includes the `ktx-research` entry shown in Step 1. For global targets, return:
```typescript
if (input.scope === 'global') {
if (input.target === 'claude-code') {
const home = process.env.HOME ?? '';
return [
{ kind: 'file', path: join(home, '.claude/skills/ktx/SKILL.md'), role: 'skill' as const },
{ kind: 'file', path: join(home, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' as const },
{ kind: 'file', path: join(home, '.claude/rules/ktx.md'), role: 'rule' as const },
];
}
if (input.target === 'codex') {
const codexHome = process.env.CODEX_HOME ?? join(process.env.HOME ?? '', '.codex');
return [
{ kind: 'file', path: join(codexHome, 'skills/ktx/SKILL.md'), role: 'skill' as const },
{ kind: 'file', path: join(codexHome, 'skills/ktx-research/SKILL.md'), role: 'research-skill' as const },
{ kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const },
];
}
if (input.target === 'cursor' || input.target === 'opencode') {
return [];
}
throw new Error(`Global ${input.target} installation is not supported; omit --global.`);
}
```
In `installTarget()`, switch the file content selection to:
```typescript
const content =
entry.role === 'rule'
? ruleInstructionContent({ projectDir: input.projectDir })
: entry.role === 'research-skill'
? await readResearchSkillContent()
: cliInstructionContent({ projectDir: input.projectDir, launcher });
```
- [ ] **Step 6: Run tests to verify the research skill passes**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts
```
Expected: PASS for the research skill install tests. MCP config tests are added in the next task and will fail until implemented.
- [ ] **Step 7: Commit**
```bash
git add packages/cli/src/skills/research/SKILL.md packages/cli/scripts/copy-runtime-assets.mjs packages/cli/package.json packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts
git commit -m "feat(cli): install KTX research skill"
```
## Task 2: Add MCP Client Config Planning And Rendering
**Files:**
- Modify: `packages/cli/src/setup-agents.test.ts`
- Modify: `packages/cli/src/setup-agents.ts`
- [ ] **Step 1: Write failing MCP config planner tests**
In `packages/cli/src/setup-agents.test.ts`, add these tests before `removes only manifest-listed files`:
```typescript
it('writes Claude Code project MCP config and tracks the json key', async () => {
const io = makeIo();
await expect(
runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'claude-code',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
),
).resolves.toMatchObject({ status: 'ready' });
const mcpJson = JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8')) as {
mcpServers: { ktx: { type: string; url: string; headers?: Record<string, string> } };
};
expect(mcpJson.mcpServers.ktx).toEqual({ type: 'http', url: 'http://localhost:7878/mcp' });
expect(await readKtxAgentInstallManifest(tempDir)).toMatchObject({
entries: expect.arrayContaining([{ kind: 'json-key', path: join(tempDir, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] }]),
});
expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.');
});
it('writes Cursor project MCP config', async () => {
const io = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'cursor',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
);
const cursorJson = JSON.parse(await readFile(join(tempDir, '.cursor/mcp.json'), 'utf-8')) as {
mcpServers: { ktx: { url: string; headers?: Record<string, string> } };
};
expect(cursorJson.mcpServers.ktx).toEqual({ url: 'http://localhost:7878/mcp' });
});
it('prints Codex and opencode snippets without mutating printed-only config files', async () => {
const codexIo = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'codex',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
codexIo.io,
);
expect(codexIo.stdout()).toContain('[mcp_servers.ktx]');
expect(codexIo.stdout()).toContain('url = "http://localhost:7878/mcp"');
const opencodeIo = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'opencode',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
opencodeIo.io,
);
expect(opencodeIo.stdout()).toContain('"mcp"');
expect(opencodeIo.stdout()).toContain('"type": "remote"');
await expect(readFile(join(tempDir, 'opencode.json'), 'utf-8')).rejects.toThrow();
});
it('uses MCP daemon state for port and token metadata without rendering literal tokens', async () => {
await mkdir(join(tempDir, '.ktx'), { recursive: true });
await writeFile(
join(tempDir, '.ktx/mcp.json'),
`${JSON.stringify(
{
schemaVersion: 1,
pid: 999999,
host: '127.0.0.1',
port: 8787,
tokenAuth: true,
projectDir: tempDir,
startedAt: '2026-05-14T00:00:00.000Z',
logPath: join(tempDir, '.ktx/logs/mcp.log'),
},
null,
2,
)}\n`,
'utf-8',
);
const io = makeIo();
const previousToken = process.env.KTX_MCP_TOKEN;
process.env.KTX_MCP_TOKEN = 'secret-token';
try {
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'claude-code',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
);
const rendered = JSON.stringify(JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8')));
expect(rendered).toContain('http://127.0.0.1:8787/mcp');
expect(rendered).toContain('Bearer ${KTX_MCP_TOKEN}');
expect(rendered).not.toContain('secret-token');
expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.');
} finally {
process.env.KTX_MCP_TOKEN = previousToken;
}
});
```
- [ ] **Step 2: Run tests to verify they fail**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts
```
Expected: FAIL because no MCP config writer or snippet renderer exists.
- [ ] **Step 3: Add JSON helpers and MCP endpoint resolution**
In `packages/cli/src/setup-agents.ts`, add `existsSync` and `readKtxMcpDaemonStatus` imports:
```typescript
import { existsSync } from 'node:fs';
import { readKtxMcpDaemonStatus } from './managed-mcp-daemon.js';
```
Add these types and helpers after `type InstallEntry`:
```typescript
interface KtxMcpEndpointInfo {
url: string;
tokenAuth: boolean;
running: boolean;
}
interface KtxMcpClientInstallResult {
entries: InstallEntry[];
snippets: string[];
notices: string[];
}
async function readJsonObject(path: string): Promise<Record<string, unknown>> {
if (!existsSync(path)) return {};
const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error(`Expected JSON object in ${path}`);
}
return parsed as Record<string, unknown>;
}
function objectAtPath(root: Record<string, unknown>, jsonPath: string[]): Record<string, unknown> {
let cursor = root;
for (const segment of jsonPath) {
const current = cursor[segment];
if (!current || typeof current !== 'object' || Array.isArray(current)) {
cursor[segment] = {};
}
cursor = cursor[segment] as Record<string, unknown>;
}
return cursor;
}
async function writeJsonKey(path: string, jsonPath: string[], value: unknown): Promise<void> {
const root = await readJsonObject(path);
const parent = objectAtPath(root, jsonPath.slice(0, -1));
parent[jsonPath.at(-1) as string] = value;
await mkdir(dirname(path), { recursive: true });
await writeFile(path, `${JSON.stringify(root, null, 2)}\n`, 'utf-8');
}
async function resolveMcpEndpoint(projectDir: string): Promise<KtxMcpEndpointInfo> {
const status = await readKtxMcpDaemonStatus({ projectDir }).catch(() => null);
if (status?.kind === 'running') {
return {
url: status.url,
tokenAuth: status.state.tokenAuth,
running: true,
};
}
if (status?.kind === 'stale' && status.state) {
return {
url: `http://${status.state.host}:${status.state.port}/mcp`,
tokenAuth: status.state.tokenAuth || Boolean(process.env.KTX_MCP_TOKEN),
running: false,
};
}
return {
url: 'http://localhost:7878/mcp',
tokenAuth: Boolean(process.env.KTX_MCP_TOKEN),
running: false,
};
}
```
- [ ] **Step 4: Add MCP entry renderers**
Add these helpers after `resolveMcpEndpoint()`:
```typescript
function tokenHeaders(endpoint: KtxMcpEndpointInfo): Record<string, string> | undefined {
return endpoint.tokenAuth ? { Authorization: 'Bearer ${KTX_MCP_TOKEN}' } : undefined;
}
function claudeMcpEntry(endpoint: KtxMcpEndpointInfo): Record<string, unknown> {
return {
type: 'http',
url: endpoint.url,
...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}),
};
}
function cursorMcpEntry(endpoint: KtxMcpEndpointInfo): Record<string, unknown> {
return {
url: endpoint.url,
...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}),
};
}
function codexSnippet(endpoint: KtxMcpEndpointInfo): string {
if (endpoint.tokenAuth) {
return [
'Codex MCP config does not currently document HTTP headers.',
'Run KTX on loopback without token auth for Codex, or configure headers after Codex documents support.',
].join('\n');
}
return [`[mcp_servers.ktx]`, `url = "${endpoint.url}"`].join('\n');
}
function opencodeSnippet(endpoint: KtxMcpEndpointInfo): string {
return JSON.stringify(
{
mcp: {
ktx: {
type: 'remote',
url: endpoint.url,
enabled: true,
...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}),
},
},
},
null,
2,
);
}
function claudeConfigPath(projectDir: string, scope: KtxAgentScope): { path: string; jsonPath: string[] } {
const home = process.env.HOME ?? '';
if (scope === 'global') {
return { path: join(home, '.claude.json'), jsonPath: ['mcpServers', 'ktx'] };
}
if (scope === 'local') {
return { path: join(home, '.claude.json'), jsonPath: ['projects', resolve(projectDir), 'mcpServers', 'ktx'] };
}
return { path: join(resolve(projectDir), '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] };
}
function cursorConfigPath(projectDir: string, scope: KtxAgentScope): { path: string; jsonPath: string[] } {
const home = process.env.HOME ?? '';
return {
path: scope === 'global' ? join(home, '.cursor/mcp.json') : join(resolve(projectDir), '.cursor/mcp.json'),
jsonPath: ['mcpServers', 'ktx'],
};
}
```
- [ ] **Step 5: Add the MCP client install planner**
Add this function after the snippet helpers:
```typescript
async function installMcpClientConfig(input: {
projectDir: string;
target: KtxAgentTarget;
scope: KtxAgentScope;
}): Promise<KtxMcpClientInstallResult> {
const endpoint = await resolveMcpEndpoint(input.projectDir);
const entries: InstallEntry[] = [];
const snippets: string[] = [];
const notices: string[] = [];
if (!endpoint.running) {
notices.push('Run `ktx mcp start` to enable the configured KTX MCP server.');
}
if (input.target === 'claude-code') {
const config = claudeConfigPath(input.projectDir, input.scope);
await writeJsonKey(config.path, config.jsonPath, claudeMcpEntry(endpoint));
entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath });
} else if (input.target === 'cursor') {
const config = cursorConfigPath(input.projectDir, input.scope);
await writeJsonKey(config.path, config.jsonPath, cursorMcpEntry(endpoint));
entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath });
} else if (input.target === 'codex') {
snippets.push(`Codex MCP snippet for ~/.codex/config.toml:\n${codexSnippet(endpoint)}`);
} else if (input.target === 'opencode') {
const path =
input.scope === 'global' ? '~/.config/opencode/opencode.json' : `${relative(input.projectDir, join(input.projectDir, 'opencode.json'))}`;
snippets.push(`opencode MCP snippet for ${path}:\n${opencodeSnippet(endpoint)}`);
}
return { entries, snippets, notices };
}
```
- [ ] **Step 6: Call the MCP planner during setup**
Keep `installTarget()` responsible only for writing agent files and returning those file entries.
In `runKtxSetupAgentsStep()`, replace the current install loop:
```typescript
const entries: InstallEntry[] = [];
for (const install of installs) entries.push(...(await installTarget({ projectDir: args.projectDir, ...install })));
```
with:
```typescript
const entries: InstallEntry[] = [];
const snippets: string[] = [];
const notices = new Set<string>();
for (const install of installs) {
entries.push(...(await installTarget({ projectDir: args.projectDir, ...install })));
const mcpResult = await installMcpClientConfig({ projectDir: args.projectDir, target: install.target, scope: install.scope });
entries.push(...mcpResult.entries);
for (const snippet of mcpResult.snippets) snippets.push(snippet);
for (const notice of mcpResult.notices) notices.add(notice);
}
```
After the install summary write:
```typescript
for (const snippet of snippets) {
io.stdout.write(`\n${snippet}\n`);
}
for (const notice of notices) {
io.stdout.write(`\n${notice}\n`);
}
```
- [ ] **Step 7: Run tests to verify MCP config passes**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts
```
Expected: PASS for research-skill and MCP config tests.
- [ ] **Step 8: Commit**
```bash
git add packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts
git commit -m "feat(cli): configure MCP clients in setup agents"
```
## Task 3: Add Claude Local Scope
**Files:**
- Modify: `packages/cli/src/commands/setup-commands.ts`
- Modify: `packages/cli/src/setup-agents.ts`
- Modify: `packages/cli/src/setup-agents.test.ts`
- Modify: `packages/cli/src/setup.test.ts`
- Modify: `packages/cli/src/index.test.ts`
- [ ] **Step 1: Write failing local-scope tests**
Add this test to `packages/cli/src/setup-agents.test.ts`:
```typescript
it('writes Claude Code local MCP config under the project key in ~/.claude.json', async () => {
const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-'));
const previousHome = process.env.HOME;
process.env.HOME = home;
try {
const io = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'claude-code',
scope: 'local',
mode: 'cli',
skipAgents: false,
},
io.io,
);
const config = JSON.parse(await readFile(join(home, '.claude.json'), 'utf-8')) as {
projects: Record<string, { mcpServers: { ktx: { type: string; url: string } } }>;
};
expect(config.projects[tempDir].mcpServers.ktx).toEqual({ type: 'http', url: 'http://localhost:7878/mcp' });
} finally {
process.env.HOME = previousHome;
await rm(home, { recursive: true, force: true });
}
});
```
Add these command-level tests after the existing `dispatches setup agent flags` test in `packages/cli/src/index.test.ts`:
```typescript
it('rejects --local with non-Claude targets', async () => {
const setup = vi.fn(async () => 0);
const setupIo = makeIo();
await expect(
runKtxCli(
['--project-dir', tempDir, 'setup', '--agents', '--target', 'cursor', '--local', '--no-input'],
setupIo.io,
{ setup },
),
).resolves.toBe(0);
expect(setupIo.stderr()).toContain('--local is only supported with --target claude-code');
expect(setup).not.toHaveBeenCalled();
});
it('rejects --local and --global together', async () => {
const setup = vi.fn(async () => 0);
const setupIo = makeIo();
await expect(
runKtxCli(
['--project-dir', tempDir, 'setup', '--agents', '--target', 'claude-code', '--local', '--global', '--no-input'],
setupIo.io,
{ setup },
),
).resolves.toBe(0);
expect(setupIo.stderr()).toContain('Choose only one agent scope: --local or --global.');
expect(setup).not.toHaveBeenCalled();
});
```
- [ ] **Step 2: Run tests to verify they fail**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts src/index.test.ts
```
Expected: FAIL because `KtxAgentScope` does not include `local` and the setup command has no `--local` option.
- [ ] **Step 3: Add the local scope type and command option**
In `packages/cli/src/setup-agents.ts`, change:
```typescript
export type KtxAgentScope = 'project' | 'global';
```
to:
```typescript
export type KtxAgentScope = 'project' | 'global' | 'local';
```
In `packages/cli/src/commands/setup-commands.ts`, add `local` to `isOnlyAgentOptions()`:
```typescript
'local',
```
Add the command option after `--global`:
```typescript
.option('--local', 'Install Claude Code MCP config into the private per-project ~/.claude.json scope', false)
```
In the setup action before `const mode = ...`, add:
```typescript
if (options.local && options.global) {
context.io.stderr.write('Choose only one agent scope: --local or --global.\n');
context.setExitCode(1);
return;
}
if (options.local && options.target && options.target !== 'claude-code') {
context.io.stderr.write('--local is only supported with --target claude-code.\n');
context.setExitCode(1);
return;
}
```
Replace:
```typescript
const resolvedAgentScope = options.global ? 'global' : 'project';
```
with:
```typescript
const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project';
```
- [ ] **Step 4: Run local-scope tests**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts src/index.test.ts
```
Expected: PASS for the new local-scope coverage.
- [ ] **Step 5: Commit**
```bash
git add packages/cli/src/commands/setup-commands.ts packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts
git commit -m "feat(cli): support Claude local MCP setup scope"
```
## Task 4: Final Verification
**Files:**
- Verify all files changed in Tasks 1-3.
- [ ] **Step 1: Run focused CLI tests**
Run:
```bash
pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts src/commands/mcp-commands.test.ts src/mcp-http-server.test.ts src/managed-mcp-daemon.test.ts
```
Expected: all selected test files pass.
- [ ] **Step 2: Run CLI type-check**
Run:
```bash
pnpm --filter @ktx/cli run type-check
```
Expected: TypeScript completes with no errors.
- [ ] **Step 3: Run CLI build**
Run:
```bash
pnpm --filter @ktx/cli run build
```
Expected: build succeeds and `packages/cli/dist/skills/research/SKILL.md` exists.
- [ ] **Step 4: Run dead-code check for the changed TypeScript surface**
Run:
```bash
pnpm run dead-code
```
Expected: Biome and Knip complete with no new findings from the setup-agent changes.
- [ ] **Step 5: Inspect git status**
Run:
```bash
git status --short
```
Expected: only intended setup-agent, skill asset, package script, and test files are modified.
## Self-Review
Spec coverage:
- Covers `ktx-research` skill installation paths for Claude Code, Codex, Cursor, opencode, and universal project targets.
- Covers Claude Code and Cursor JSON MCP writers.
- Covers Codex and opencode printed snippets.
- Covers token handling with `${KTX_MCP_TOKEN}` and no literal token rendering.
- Covers `.ktx/mcp.json` port selection and daemon-start hint.
- Covers manifest tracking for written JSON keys and removal through existing `json-key` cleanup.
Known v1 gap not covered by this plan:
- Ingest warehouse-verification contract convergence from `connectionName` to `connectionId`, shared service extraction, and caller/test updates remains v1-blocking and needs its own focused plan after this setup-agent slice lands.

999
docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md Normal file
View file

@ -0,0 +1,999 @@
# Research Agent MCP SQL Execution Foundation Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Add the parser-backed safety prerequisite and MCP `sql_execution` surface needed before the research-agent MCP tools can safely execute warehouse SQL.
**Architecture:** Keep connector `executeReadOnly()` as the execution path, but make the MCP adapter require a sqlglot-backed validator before calling any connector. Extend the existing Python SQL-analysis daemon with a read-only validation endpoint, expose it through the TypeScript SQL-analysis port, then register an MCP `sql_execution` tool only when the host provides that validator and a local scan connector factory.
**Tech Stack:** TypeScript, Vitest, Zod, Python, pytest, FastAPI, sqlglot, KTX MCP context ports, KTX scan connectors.
---
## Audit Summary
Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md`
Implemented plans that overlap with the spec:
- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` is implemented for the existing in-process MCP semantic runtime. Current evidence: `packages/context/src/mcp/context-tools.ts` registers `connection_*`, `wiki_*`, `sl_*`, `ingest_*`, and `scan_*` tools, and `packages/context/src/mcp/local-project-ports.ts` provides local ports for those surfaces.
- `docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md` plus its May 12 and May 13 closure plans are implemented for ingest-only warehouse verification. Current evidence: `packages/context/src/ingest/tools/warehouse-verification/{discover-data,entity-details,sql-execution,warehouse-catalog.service}.ts` exist and are wired for ingest agents.
V1-blocking gaps remaining against the original spec:
- The public MCP research tools are not registered. `KtxMcpContextPorts` has no `discover`, `entityDetails`, `dictionarySearch`, or `sqlExecution` ports.
- The existing ingest `discover_data`, `entity_details`, and `sql_execution` tools use `connectionName`, `targets`, and `rowLimit`, and return markdown plus structured output. The spec requires MCP-shaped `connectionId`, `entities` / `maxRows`, and pure structured outputs.
- `sql_execution` cannot be safely exposed yet: `packages/context/src/connections/read-only-sql.ts` still uses first-token regex checks. The spec requires a sqlglot/AST-backed guard or connector-side read-only session before MCP registration.
- `packages/context/src/scan/entity-details.ts`, `packages/context/src/sl/dictionary-search.ts`, and `packages/context/src/search/discover.ts` do not exist.
- `WarehouseCatalogService` caches by connection only and does not invalidate when latest scan artifact identity advances.
- `dictionary_search` has no MCP service, no coverage metadata, and no per-connection miss reasons.
- `discover_data` has no unified ranked MCP result shape with `summary`, `snippet`, `matchedOn`, `kind`, `tableRef`, and RRF fusion across wiki, SL, and raw schema.
- `ktx mcp start|stop|status|logs` does not exist, and no HTTP Streamable MCP daemon exists.
- `ktx setup-agents` installs only the existing `ktx` CLI skill/rules; it does not install `ktx-research` or MCP client config entries/snippets.
Non-blocking or explicitly out-of-scope gaps:
- Python code execution over MCP.
- Stdio MCP transport.
- OS-level auto-start.
- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing.
- Streaming SQL results.
- Full DDL-style ingest `entity_details` markdown formatting and hard write-time validation in ingest writer tools.
This plan covers the first prerequisite blocker: parser-backed SQL validation and MCP `sql_execution`. The remaining v1-blocking tool, daemon, and setup-agent work stays visible for subsequent plans.
## File Structure
Create no new files.
Modify these files:
- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`: add a sqlglot-backed read-only SQL validator.
- `python/ktx-daemon/src/ktx_daemon/app.py`: expose `POST /sql/validate-read-only`.
- `python/ktx-daemon/tests/test_sql_analysis.py`: cover accepted SELECT/WITH and rejected CTE-DML, multi-statement, command, pragma, and parse-error payloads.
- `python/ktx-daemon/tests/test_app.py`: cover the new HTTP endpoint.
- `packages/context/src/sql-analysis/ports.ts`: add `validateReadOnly()` to `SqlAnalysisPort`.
- `packages/context/src/sql-analysis/http-sql-analysis-port.ts`: call `/sql/validate-read-only` and map its response.
- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`: cover request and response mapping.
- `packages/context/src/mcp/types.ts`: add `KtxSqlExecutionMcpPort` and `sqlExecution` to `KtxMcpContextPorts`.
- `packages/context/src/mcp/context-tools.ts`: add the MCP `sql_execution` schema and registration.
- `packages/context/src/mcp/server.test.ts`: assert MCP registration and structured output for `sql_execution`.
- `packages/context/src/mcp/local-project-ports.ts`: expose local project SQL execution only when both `SqlAnalysisPort.validateReadOnly()` and a local scan connector factory are available.
- `packages/context/src/mcp/local-project-ports.test.ts`: cover validator success and validator rejection.
### Task 1: Add sqlglot Read-Only Validation
**Files:**
- Modify: `python/ktx-daemon/tests/test_sql_analysis.py`
- Modify: `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`
- Modify: `python/ktx-daemon/tests/test_app.py`
- Modify: `python/ktx-daemon/src/ktx_daemon/app.py`
- [ ] **Step 1: Write failing sqlglot validator tests**
In `python/ktx-daemon/tests/test_sql_analysis.py`, update the import block to include the new request model and function:
```python
from ktx_daemon.sql_analysis import (
AnalyzeSqlBatchItem,
AnalyzeSqlBatchRequest,
ValidateReadOnlySqlRequest,
_columns_from_nodes,
analyze_sql_batch_response,
validate_read_only_sql_response,
)
```
Add these tests after `test_columns_from_nodes_ignores_non_expression_clause_values`:
```python
def test_validate_read_only_sql_accepts_select_and_with_queries() -> None:
select_response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(
dialect="postgres",
sql="select id, status from public.orders where status = 'paid'",
)
)
with_response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(
dialect="postgres",
sql=(
"with paid as (select * from public.orders where status = 'paid') "
"select count(*) from paid"
),
)
)
assert select_response.ok is True
assert select_response.error is None
assert with_response.ok is True
assert with_response.error is None
def test_validate_read_only_sql_rejects_cte_dml() -> None:
response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(
dialect="postgres",
sql="with x as (insert into audit.events values (1) returning *) select * from x",
)
)
assert response.ok is False
assert response.error == "SQL contains read/write operation: Insert"
def test_validate_read_only_sql_rejects_multi_statement_payloads() -> None:
response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(
dialect="postgres",
sql="select * from public.orders; delete from public.orders",
)
)
assert response.ok is False
assert response.error == "Only one SQL statement can be executed."
def test_validate_read_only_sql_rejects_commands_and_pragmas() -> None:
command_response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(dialect="postgres", sql="call refresh_stats()")
)
pragma_response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(dialect="sqlite", sql="pragma table_info(users)")
)
assert command_response.ok is False
assert command_response.error == "SQL contains read/write operation: Command"
assert pragma_response.ok is False
assert pragma_response.error == "SQL contains read/write operation: Pragma"
def test_validate_read_only_sql_reports_parse_errors() -> None:
response = validate_read_only_sql_response(
ValidateReadOnlySqlRequest(dialect="postgres", sql="select * from where")
)
assert response.ok is False
assert response.error is not None
assert "Invalid expression" in response.error
```
- [ ] **Step 2: Run failing Python validator tests**
Run:
```bash
source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py -q
```
Expected: FAIL with an import error for `ValidateReadOnlySqlRequest` or `validate_read_only_sql_response`.
- [ ] **Step 3: Implement the sqlglot validator**
In `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`, add this model after `AnalyzeSqlBatchResponse`:
```python
class ValidateReadOnlySqlRequest(BaseModel):
dialect: str
sql: str
class ValidateReadOnlySqlResponse(BaseModel):
ok: bool
error: str | None = None
```
Add this constant after the model definitions:
```python
_READ_ONLY_ROOT_TYPES = (exp.Select, exp.Union)
_READ_WRITE_NODE_TYPES = (
exp.Alter,
exp.Analyze,
exp.Cache,
exp.Command,
exp.Commit,
exp.Copy,
exp.Create,
exp.Delete,
exp.Describe,
exp.Drop,
exp.Execute,
exp.Grant,
exp.Insert,
exp.Merge,
exp.Pragma,
exp.Refresh,
exp.Revoke,
exp.Rollback,
exp.Set,
exp.Show,
exp.Transaction,
exp.TruncateTable,
exp.Uncache,
exp.Update,
exp.Use,
)
```
Add this function after `_analyze_payload`:
```python
def validate_read_only_sql_response(
request: ValidateReadOnlySqlRequest,
) -> ValidateReadOnlySqlResponse:
try:
statements = sqlglot.parse(request.sql, read=request.dialect)
except sqlglot.errors.SqlglotError as exc:
return ValidateReadOnlySqlResponse(ok=False, error=str(exc))
if len(statements) != 1:
return ValidateReadOnlySqlResponse(
ok=False,
error="Only one SQL statement can be executed.",
)
tree = statements[0]
if tree is None:
return ValidateReadOnlySqlResponse(ok=False, error="SQL did not parse to a statement.")
if not isinstance(tree, _READ_ONLY_ROOT_TYPES):
return ValidateReadOnlySqlResponse(
ok=False,
error=f"SQL contains read/write operation: {type(tree).__name__}",
)
for node in tree.walk():
if isinstance(node, _READ_WRITE_NODE_TYPES):
return ValidateReadOnlySqlResponse(
ok=False,
error=f"SQL contains read/write operation: {type(node).__name__}",
)
return ValidateReadOnlySqlResponse(ok=True, error=None)
```
- [ ] **Step 4: Run Python validator tests**
Run:
```bash
source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py -q
```
Expected: PASS.
- [ ] **Step 5: Write failing HTTP endpoint test**
In `python/ktx-daemon/tests/test_app.py`, add this test after `test_sql_parse_table_identifier_endpoint`:
```python
def test_sql_validate_read_only_endpoint() -> None:
client = TestClient(create_app())
ok_response = client.post(
"/sql/validate-read-only",
json={"dialect": "postgres", "sql": "select * from public.orders"},
)
bad_response = client.post(
"/sql/validate-read-only",
json={
"dialect": "postgres",
"sql": "with x as (insert into audit.events values (1) returning *) select * from x",
},
)
assert ok_response.status_code == 200
assert ok_response.json() == {"ok": True, "error": None}
assert bad_response.status_code == 200
assert bad_response.json() == {
"ok": False,
"error": "SQL contains read/write operation: Insert",
}
```
- [ ] **Step 6: Run failing HTTP endpoint test**
Run:
```bash
source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py -q -k validate_read_only
```
Expected: FAIL with HTTP 404 for `/sql/validate-read-only`.
- [ ] **Step 7: Register the HTTP endpoint**
In `python/ktx-daemon/src/ktx_daemon/app.py`, update the SQL-analysis import to include the new symbols:
```python
from ktx_daemon.sql_analysis import (
AnalyzeSqlBatchRequest,
AnalyzeSqlBatchResponse,
ValidateReadOnlySqlRequest,
ValidateReadOnlySqlResponse,
analyze_sql_batch_response,
validate_read_only_sql_response,
)
```
Add this endpoint immediately before the existing `@app.post("/sql/analyze-batch", ...)` route:
```python
@app.post("/sql/validate-read-only", response_model=ValidateReadOnlySqlResponse)
async def sql_validate_read_only(
request: ValidateReadOnlySqlRequest,
) -> ValidateReadOnlySqlResponse:
try:
return validate_read_only_sql_response(request)
except Exception as error:
logger.exception("SQL read-only validation failed: %s", error)
raise HTTPException(
status_code=500,
detail=f"SQL read-only validation failed: {error}",
) from error
```
- [ ] **Step 8: Run Python HTTP endpoint test**
Run:
```bash
source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py -q -k validate_read_only
```
Expected: PASS.
- [ ] **Step 9: Commit Python validator**
Run:
```bash
git add python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py
git commit -m "feat(daemon): validate read-only SQL with sqlglot"
```
### Task 2: Expose Read-Only Validation Through the TypeScript SQL-Analysis Port
**Files:**
- Modify: `packages/context/src/sql-analysis/ports.ts`
- Modify: `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`
- Modify: `packages/context/src/sql-analysis/http-sql-analysis-port.ts`
- [ ] **Step 1: Add the port contract**
In `packages/context/src/sql-analysis/ports.ts`, add this interface after `SqlAnalysisBatchResult`:
```typescript
export interface SqlReadOnlyValidationResult {
ok: boolean;
error?: string | null;
}
```
Update `SqlAnalysisPort` to include the new method:
```typescript
export interface SqlAnalysisPort {
analyzeForFingerprint(sql: string, dialect: SqlAnalysisDialect): Promise<SqlAnalysisFingerprintResult>;
analyzeBatch(
items: SqlAnalysisBatchItem[],
dialect: SqlAnalysisDialect,
): Promise<Map<string, SqlAnalysisBatchResult>>;
validateReadOnly(sql: string, dialect: SqlAnalysisDialect): Promise<SqlReadOnlyValidationResult>;
}
```
- [ ] **Step 2: Write failing HTTP port tests**
In `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`, add this test inside the existing `describe('createHttpSqlAnalysisPort', ...)` block:
```typescript
it('maps read-only SQL validation responses', async () => {
const requests: Array<{ path: string; payload: Record<string, unknown> }> = [];
const port = createHttpSqlAnalysisPort({
baseUrl: 'http://127.0.0.1:8765',
requestJson: async (path, payload) => {
requests.push({ path, payload });
return { ok: false, error: 'SQL contains read/write operation: Insert' };
},
});
await expect(port.validateReadOnly('with x as (insert into t values (1)) select * from x', 'postgres')).resolves.toEqual({
ok: false,
error: 'SQL contains read/write operation: Insert',
});
expect(requests).toEqual([
{
path: '/sql/validate-read-only',
payload: {
dialect: 'postgres',
sql: 'with x as (insert into t values (1)) select * from x',
},
},
]);
});
```
Add this test after it:
```typescript
it('rejects malformed read-only validation responses', async () => {
const port = createHttpSqlAnalysisPort({
baseUrl: 'http://127.0.0.1:8765',
requestJson: async () => ({ ok: 'yes' }),
});
await expect(port.validateReadOnly('select 1', 'postgres')).rejects.toThrow(
'sql analysis response is missing boolean field ok',
);
});
```
- [ ] **Step 3: Run failing HTTP port tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts
```
Expected: FAIL because `validateReadOnly` is not implemented.
- [ ] **Step 4: Implement HTTP response mapping**
In `packages/context/src/sql-analysis/http-sql-analysis-port.ts`, update the type import to include `SqlReadOnlyValidationResult`:
```typescript
SqlReadOnlyValidationResult,
```
Add this helper after `requiredStringArray`:
```typescript
function requiredBoolean(raw: Record<string, unknown>, field: string): boolean {
const value = raw[field];
if (typeof value !== 'boolean') {
throw new Error(`sql analysis response is missing boolean field ${field}`);
}
return value;
}
```
Add this mapper after `mapBatchResponse`:
```typescript
function mapReadOnlyValidation(raw: Record<string, unknown>): SqlReadOnlyValidationResult {
const error = optionalString(raw, 'error');
return {
ok: requiredBoolean(raw, 'ok'),
...(error !== undefined ? { error } : {}),
};
}
```
Add this method to the object returned by `createHttpSqlAnalysisPort`:
```typescript
async validateReadOnly(sql: string, dialect: SqlAnalysisDialect) {
const raw = await requestJson('/sql/validate-read-only', {
dialect,
sql,
});
return mapReadOnlyValidation(raw);
},
```
- [ ] **Step 5: Run HTTP port tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts
```
Expected: PASS.
- [ ] **Step 6: Commit TypeScript SQL-analysis port**
Run:
```bash
git add packages/context/src/sql-analysis/ports.ts packages/context/src/sql-analysis/http-sql-analysis-port.ts packages/context/src/sql-analysis/http-sql-analysis-port.test.ts
git commit -m "feat(context): expose read-only SQL validation port"
```
### Task 3: Register the MCP `sql_execution` Tool Contract
**Files:**
- Modify: `packages/context/src/mcp/types.ts`
- Modify: `packages/context/src/mcp/context-tools.ts`
- Modify: `packages/context/src/mcp/server.test.ts`
- [ ] **Step 1: Add the MCP SQL execution port types**
In `packages/context/src/mcp/types.ts`, add these interfaces immediately before `KtxMcpContextPorts`:
```typescript
export interface KtxSqlExecutionResponse {
headers: string[];
headerTypes?: string[];
rows: unknown[][];
rowCount: number;
}
export interface KtxSqlExecutionMcpPort {
execute(input: { connectionId: string; sql: string; maxRows: number }): Promise<KtxSqlExecutionResponse>;
}
```
Then add the new optional port to `KtxMcpContextPorts`:
```typescript
sqlExecution?: KtxSqlExecutionMcpPort;
```
- [ ] **Step 2: Write failing MCP registration test**
In `packages/context/src/mcp/server.test.ts`, update the type import from `./types.js` to include `KtxSqlExecutionMcpPort`.
Add this test in `describe('createKtxMcpServer', ...)` after the existing connection-list registration test:
```typescript
it('registers parser-gated sql_execution when the host provides a SQL execution port', async () => {
const fake = makeFakeServer();
const sqlExecution: KtxSqlExecutionMcpPort = {
execute: vi.fn<KtxSqlExecutionMcpPort['execute']>().mockResolvedValue({
headers: ['status', 'count'],
headerTypes: ['text', 'bigint'],
rows: [['paid', 42]],
rowCount: 1,
}),
};
createKtxMcpServer({
server: fake.server,
userContext: { userId: 'local-user' },
contextTools: {
sqlExecution,
},
});
expect(fake.tools.map((tool) => tool.name)).toEqual(['sql_execution']);
await expect(
getTool(fake.tools, 'sql_execution').handler({
connectionId: 'warehouse',
sql: 'select status, count(*) from public.orders group by status',
maxRows: 50,
}),
).resolves.toEqual({
content: [
{
type: 'text',
text: JSON.stringify(
{
headers: ['status', 'count'],
headerTypes: ['text', 'bigint'],
rows: [['paid', 42]],
rowCount: 1,
},
null,
2,
),
},
],
structuredContent: {
headers: ['status', 'count'],
headerTypes: ['text', 'bigint'],
rows: [['paid', 42]],
rowCount: 1,
},
});
expect(sqlExecution.execute).toHaveBeenCalledWith({
connectionId: 'warehouse',
sql: 'select status, count(*) from public.orders group by status',
maxRows: 50,
});
});
```
- [ ] **Step 3: Run failing MCP registration test**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t sql_execution
```
Expected: FAIL because `sql_execution` is not registered.
- [ ] **Step 4: Add the MCP schema and registration**
In `packages/context/src/mcp/context-tools.ts`, add this schema after `scanArtifactReadSchema`:
```typescript
const sqlExecutionSchema = z.object({
connectionId: connectionIdSchema,
sql: z.string().min(1),
maxRows: z.number().int().min(1).max(10_000).default(1000).optional(),
});
```
Add this registration block in `registerKtxContextTools`, after the semantic-layer block and before the ingest block:
```typescript
if (ports.sqlExecution) {
const sqlExecution = ports.sqlExecution;
registerParsedTool(
server,
'sql_execution',
{
title: 'SQL Execution',
description:
'Execute one parser-validated read-only SQL query against a configured KTX connection and return structured rows.',
inputSchema: sqlExecutionSchema.shape,
},
sqlExecutionSchema,
async (input) => {
try {
return jsonToolResult(
await sqlExecution.execute({
connectionId: input.connectionId,
sql: input.sql,
maxRows: input.maxRows ?? 1000,
}),
);
} catch (error) {
return jsonErrorToolResult(error instanceof Error ? error.message : String(error));
}
},
);
}
```
- [ ] **Step 5: Run MCP registration test**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t sql_execution
```
Expected: PASS.
- [ ] **Step 6: Commit MCP tool contract**
Run:
```bash
git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts
git commit -m "feat(context): register MCP sql execution tool"
```
### Task 4: Implement Local Project SQL Execution With Parser Validation
**Files:**
- Modify: `packages/context/src/mcp/local-project-ports.ts`
- Modify: `packages/context/src/mcp/local-project-ports.test.ts`
- [ ] **Step 1: Write failing local-port success test**
In `packages/context/src/mcp/local-project-ports.test.ts`, update the imports from `../scan/index.js` to include `type KtxQueryResult`.
Replace the existing `testConnector` helper with this version so tests can opt into read-only SQL:
```typescript
function testConnector(
snapshot = testSnapshot(),
queryResult?: KtxQueryResult,
): KtxScanConnector {
return {
id: `test:${snapshot.connectionId}`,
driver: snapshot.driver,
capabilities: createKtxConnectorCapabilities({ readOnlySql: queryResult !== undefined }),
introspect: vi.fn(async () => snapshot),
executeReadOnly: queryResult === undefined ? undefined : vi.fn(async () => queryResult),
cleanup: vi.fn(async () => {}),
};
}
```
Add this test after `tests a local project connection through the native scan connector factory`:
```typescript
it('executes MCP SQL only after parser-backed validation passes', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
};
const connector = testConnector(testSnapshot(), {
headers: ['id'],
headerTypes: ['integer'],
rows: [[1]],
totalRows: 1,
rowCount: 1,
});
const createConnector = vi.fn(async () => connector);
const sqlAnalysis = {
analyzeForFingerprint: vi.fn(),
analyzeBatch: vi.fn(),
validateReadOnly: vi.fn(async () => ({ ok: true, error: null })),
};
const ports = createLocalProjectMcpContextPorts(project, {
sqlAnalysis,
localScan: {
createConnector,
},
});
await expect(
ports.sqlExecution?.execute({
connectionId: 'warehouse',
sql: 'select id from public.orders',
maxRows: 5,
}),
).resolves.toEqual({
headers: ['id'],
headerTypes: ['integer'],
rows: [[1]],
rowCount: 1,
});
expect(sqlAnalysis.validateReadOnly).toHaveBeenCalledWith('select id from public.orders', 'postgres');
expect(createConnector).toHaveBeenCalledWith('warehouse');
expect(connector.executeReadOnly).toHaveBeenCalledWith(
{
connectionId: 'warehouse',
sql: 'select id from public.orders',
maxRows: 5,
},
{ runId: 'mcp-sql-execution' },
);
expect(connector.cleanup).toHaveBeenCalled();
});
```
- [ ] **Step 2: Write failing local-port rejection test**
Add this test after the success test:
```typescript
it('rejects MCP SQL before connector execution when parser validation fails', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
};
const connector = testConnector(testSnapshot(), {
headers: ['id'],
rows: [[1]],
totalRows: 1,
rowCount: 1,
});
const sqlAnalysis = {
analyzeForFingerprint: vi.fn(),
analyzeBatch: vi.fn(),
validateReadOnly: vi.fn(async () => ({
ok: false,
error: 'SQL contains read/write operation: Insert',
})),
};
const ports = createLocalProjectMcpContextPorts(project, {
sqlAnalysis,
localScan: {
createConnector: vi.fn(async () => connector),
},
});
await expect(
ports.sqlExecution?.execute({
connectionId: 'warehouse',
sql: 'with x as (insert into t values (1) returning *) select * from x',
maxRows: 1000,
}),
).rejects.toThrow('SQL contains read/write operation: Insert');
expect(connector.executeReadOnly).not.toHaveBeenCalled();
});
```
- [ ] **Step 3: Run failing local-port tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "MCP SQL"
```
Expected: FAIL because `CreateLocalProjectMcpContextPortsOptions` has no `sqlAnalysis` option and no `sqlExecution` port.
- [ ] **Step 4: Add SQL-analysis option and helper imports**
In `packages/context/src/mcp/local-project-ports.ts`, add this import with the other context imports:
```typescript
import type { SqlAnalysisDialect, SqlAnalysisPort } from '../sql-analysis/index.js';
```
Add `sqlAnalysis` to `CreateLocalProjectMcpContextPortsOptions`:
```typescript
sqlAnalysis?: SqlAnalysisPort;
```
Add this helper near `dialectForDriver`:
```typescript
function sqlAnalysisDialectForDriver(driver: string | undefined): SqlAnalysisDialect {
return dialectForDriver(driver) as SqlAnalysisDialect;
}
```
- [ ] **Step 5: Implement the local SQL execution port**
In `packages/context/src/mcp/local-project-ports.ts`, add this function before `createLocalProjectMcpContextPorts`:
```typescript
async function executeValidatedReadOnlySql(
project: KtxLocalProject,
options: CreateLocalProjectMcpContextPortsOptions,
input: { connectionId: string; sql: string; maxRows: number },
): Promise<{ headers: string[]; headerTypes?: string[]; rows: unknown[][]; rowCount: number }> {
const connectionId = assertSafeConnectionId(input.connectionId);
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
if (!options.sqlAnalysis) {
throw new Error('sql_execution requires parser-backed SQL validation.');
}
const validation = await options.sqlAnalysis.validateReadOnly(
input.sql,
sqlAnalysisDialectForDriver(connection.driver),
);
if (!validation.ok) {
throw new Error(validation.error ?? 'SQL is not read-only.');
}
const createConnector = options.localScan?.createConnector;
if (!createConnector) {
throw new Error('sql_execution requires a local scan connector factory.');
}
let connector: KtxScanConnector | null = null;
try {
connector = await createConnector(connectionId);
if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) {
throw new Error(`Connection "${connectionId}" does not support read-only SQL execution.`);
}
const result = await connector.executeReadOnly(
{
connectionId,
sql: input.sql,
maxRows: input.maxRows,
},
{ runId: 'mcp-sql-execution' },
);
return {
headers: result.headers,
...(result.headerTypes ? { headerTypes: result.headerTypes } : {}),
rows: result.rows,
rowCount: result.rowCount ?? result.rows.length,
};
} finally {
await cleanupConnector(connector);
}
}
```
In `createLocalProjectMcpContextPorts`, add this conditional block immediately after the initial `ports` object is created and before the existing `if (options.localIngest)` block:
```typescript
if (options.sqlAnalysis && options.localScan?.createConnector) {
ports.sqlExecution = {
async execute(input) {
return executeValidatedReadOnlySql(project, options, input);
},
};
}
```
- [ ] **Step 6: Run local-port tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "MCP SQL"
```
Expected: PASS.
- [ ] **Step 7: Commit local MCP SQL execution**
Run:
```bash
git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts
git commit -m "feat(context): execute MCP SQL through validated connector path"
```
### Task 5: Verification
**Files:**
- Verify: all modified files from Tasks 1-4
- [ ] **Step 1: Run Python SQL-analysis and app tests**
Run:
```bash
source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -q
```
Expected: PASS.
- [ ] **Step 2: Run focused TypeScript tests**
Run:
```bash
pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts src/mcp/server.test.ts src/mcp/local-project-ports.test.ts
```
Expected: PASS.
- [ ] **Step 3: Run type-check**
Run:
```bash
pnpm --filter @ktx/context run type-check
```
Expected: PASS.
- [ ] **Step 4: Run Python pre-commit on changed Python files**
Run:
```bash
source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py
```
Expected: PASS. If the repository has no usable pre-commit configuration in the active environment, record the exact error and keep the pytest results above as the closest Python verification.
- [ ] **Step 5: Confirm the remaining v1 blockers are unchanged**
Run:
```bash
test -e packages/context/src/scan/entity-details.ts; printf 'entity-details:%s\n' "$?"
test -e packages/context/src/sl/dictionary-search.ts; printf 'dictionary-search:%s\n' "$?"
test -e packages/context/src/search/discover.ts; printf 'discover:%s\n' "$?"
test -e packages/cli/src/commands/mcp-commands.ts; printf 'mcp-commands:%s\n' "$?"
test -e packages/cli/src/skills/research/SKILL.md; printf 'research-skill:%s\n' "$?"
```
Expected:
```text
entity-details:1
dictionary-search:1
discover:1
mcp-commands:1
research-skill:1
```
These `1` exit-code markers confirm this plan landed only the SQL execution foundation and did not silently claim the remaining research-tool, daemon, or setup-agent v1 work.
- [ ] **Step 6: Commit verification notes if any test docs changed**
Run:
```bash
git status --short
```
Expected: no uncommitted source changes after the task commits. If verification required a small documentation note, commit only that note with:
```bash
git add docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md
git commit -m "docs: record research MCP SQL execution plan"
```

931
docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md Normal file
View file

@ -0,0 +1,931 @@
# Research Agent MCP Tools Design
**Date:** 2026-05-14
**Author:** Andrey Avtomonov
**Status:** Design — pending implementation plan
## Background
KTX positions itself as a standalone context layer for database agents.
External agents — Claude Code, Cursor, Codex, opencode — should be able to
connect to a local KTX instance via MCP and perform research against
configured data connections.
The existing MCP surface (`packages/context/src/mcp/context-tools.ts`) already
exposes strong **context** primitives: wiki search/read/write, semantic-layer
list/read/write/validate/query, ingest and scan run management, memory
capture. What it is missing is the **active investigation** primitives a
research agent needs:
- The agent cannot run raw SQL against a connection. `sl_query` only covers
semantic-layer-defined queries.
- The agent cannot inspect raw table or column metadata for tables that are
not yet modeled in the semantic layer.
- The agent cannot find which column holds a literal value mentioned by the
user (e.g., "Acme Corp").
- The agent must call multiple separate search tools (`wiki_search`,
`sl_list_sources`) and reconcile results manually instead of getting a
unified ranked discovery view.
The Kaelio research agent (reference implementation at
`/Users/andrey/conductor/workspaces/kaelio-main2/douala/server/src/cores/research-execution.core.ts`)
addresses these gaps with tools named `sql_execution`, `entity_details`,
`dictionary_search`, and `discover_data`, used in a discovery → inspection →
query loop. The corresponding KTX infrastructure already exists in pieces:
- `KtxScanConnector.executeReadOnly` on every connector
(`packages/connector-postgres/src/connector.ts:447` and siblings) — read-only
SQL execution with `assertReadOnlySql` and `limitSqlForExecution`.
- `KtxSchemaSnapshot` from scan reports — full table/column/FK metadata.
- `SlDictionaryEntry` extraction over relationship-profiling artifacts
(`packages/context/src/sl/sl-dictionary-profile.ts`).
- Hybrid search core with Reciprocal Rank Fusion
(`packages/context/src/search/{hybrid-search-core,rrf}.ts`).
This design exposes those primitives as four new MCP tools, adds a research
skill to guide external agents, and introduces an HTTP-only `ktx mcp` daemon
to host the MCP server.
## Goals
- Expose four new MCP tools that turn KTX into a research-capable context
layer for any MCP-compatible client: `discover_data`, `entity_details`,
`dictionary_search`, `sql_execution`.
- Ship a `ktx-research` skill installable via `ktx setup-agents`, describing
the discover → inspect → query → capture workflow for external agents.
- Provide a `ktx mcp` CLI subtree that runs the MCP server over HTTP on
localhost, with the same lifecycle pattern as the existing managed Python
daemon (`packages/cli/src/managed-python-daemon.ts`).
- Make `ktx setup-agents` install MCP client configuration for the configured
targets pointing at the local HTTP endpoint. v1 splits this by client: for
claude-code and cursor (JSON config), `setup-agents` writes the entry
directly; for codex (TOML) and opencode (different JSON wrapper),
`setup-agents` prints a copy-pasteable snippet rather than writing the file.
See the client matrix below for full per-target behavior.
- Reuse existing infrastructure (connector `executeReadOnly`, schema
snapshots, dictionary profile, hybrid search + RRF) rather than building
parallel implementations.
## Non-goals
- This spec does not build an agent loop inside KTX. The system prompt, step
budget, tool dispatch, and methodology tracking remain in the external
client. KTX is a context provider, not an agent runner.
- This spec does not expose Python code execution. The `ktx-daemon`
`/code/execute` endpoint exists but is not surfaced via MCP. That is a
separate design with its own sandboxing and security considerations.
- This spec does not ship widget rendering, chart creation, or scheduled
report execution. Those are presentation concerns the external client owns.
- This spec does not implement stdio MCP transport. HTTP-only.
- This spec does not implement OS-level auto-start (launchd, systemd user
units). `ktx mcp start` must be run explicitly.
- This spec does not implement remote network exposure beyond loopback. Token
auth and non-`127.0.0.1` binding are supported but TLS, audit logging, and
multi-tenant isolation are out of scope for v1.
## Tool inventory
Four new MCP tools, registered in `packages/context/src/mcp/context-tools.ts`
alongside the existing tools.
### Relationship to existing warehouse-verification tools
KTX already ships ingest-side implementations of `sql_execution`,
`entity_details`, and `discover_data` at
`packages/context/src/ingest/tools/warehouse-verification/{sql-execution,entity-details,discover-data}.tool.ts`,
backed by `warehouse-catalog.service.ts`. Their contracts differ from the
MCP shapes proposed below in three concrete ways:
- They currently take `connectionName` (slug-shaped); this spec renames
them to `connectionId` in the same change (see below).
- They take `targets` (a discriminated `display` vs. `{catalog,db,name}`
union) and `rowLimit`, not `entities` / `maxRows`.
- They return `{ markdown, structured }` with scan availability, candidate
matches, and ingest-session-allowed-connection scoping, not the
MCP-shaped pure-structured outputs in this spec.
To avoid two divergent contracts for the same primitives, the MCP tools
**must be implemented by extracting the shared logic out of
`warehouse-verification/*` and into reusable services**
(e.g., `WarehouseCatalogService` as the source of truth for table/column
resolution and discovery, plus a shared read-only SQL executor that wraps
`assertReadOnlySql`/`limitSqlForExecution`). The ingest tools and the new
MCP tools then become thin adapters around those services with their own
input/output shapes appropriate to each surface.
KTX has no public users yet, so the same change that introduces the MCP
tools renames the ingest-side `connectionName` parameter to `connectionId`
across `warehouse-verification/*.tool.ts`, `warehouse-catalog.service.ts`,
and any callers. `connectionId` matches the rest of the in-process MCP
surface (`sl_query`, `sl_list_sources`, `scan_trigger`, etc.) and the new
MCP tool inputs. The ingest tools and the new MCP tools then share both
the service layer and the parameter name; only their input/output shapes
differ (markdown+structured for the ingest surface, pure structured for
the MCP surface).
### discover_data
Unified ranked search across wiki, semantic-layer sources/measures/dimensions,
and raw schema tables/columns. Returns refs only with a uniform shape; the
agent dereferences top hits using the existing `wiki_read`, `sl_read_source`,
or `entity_details` tools.
**Input schema:**
```typescript
{
query: z.string().min(1),
connectionId: z.string().optional(), // omit → all connections
kinds: z.array(z.enum([
'wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column',
])).optional(), // omit → all kinds
limit: z.number().int().min(1).max(50).default(15).optional(),
}
```
**Output:** array of refs, each:
```typescript
{
kind: 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column',
id: string, // stable id: wiki key, source name, or driver-qualified table/column display string
score: number, // RRF fused score, 0-1 range
summary: string | null, // one-line description; null when no source field is populated
snippet: string | null, // short context snippet, ≤200 chars; null when nothing meaningful to show
matchedOn: // why this result matched (powers the snippet for non-description kinds)
| 'name' | 'display' | 'description' | 'comment' | 'expr' | 'sample_value' | 'body',
connectionId?: string, // present for non-wiki kinds
tableRef?: { // present for kind 'table' and 'column'
catalog: string | null,
db: string | null,
name: string,
},
columnName?: string, // present for kind 'column'
}
```
The structured `tableRef` mirrors the live `KtxSchemaTable` identity
(`packages/context/src/scan/types.ts:74-83`) so callers can pass refs into
`entity_details` without losing `catalog`/`db` qualification on drivers
that need it (BigQuery `project.dataset.table`, Snowflake/SQL Server
`database.schema.table`).
#### `summary` and `snippet` provenance per kind
Both fields are derived from existing source data, never invented or
LLM-generated. The resolver is pure and deterministic per kind. When no
source field exists for a given kind, the field is `null`; agents must
not assume a missing snippet means "no context" — they should dereference
the ref via `wiki_read`, `sl_read_source`, or `entity_details` to get
authoritative content.
| Kind | `summary` source | `snippet` source |
|---|---|---|
| `wiki` | `WikiFrontmatter.summary` (`packages/context/src/wiki/types.ts:15`) — populated at write time | Up to 200 chars from the wiki body around the match position; falls back to first 200 chars of body when `matchedOn === 'name'`/`'display'` |
| `sl_source` | `resolveDescription(source.descriptions, priority)` (`packages/context/src/sl/descriptions.ts:16-34`) over the `user|ai|dbt|db` priority chain (`packages/context/src/sl/types.ts:5`) | When `matchedOn === 'description'`/`'body'`: a window of the resolved description; otherwise the source's `name` + first 1–2 measure or dimension names as context |
| `sl_measure` | `measure.description` (`packages/context/src/sl/types.ts:37`) | `measure.expr` truncated to 200 chars — the calculation is the most informative one-line context for a measure |
| `sl_dimension` | `resolveDescription(column.descriptions, priority)` (same precedence as `sl_source`); when empty, fall back to `null` | `${column.name} (${column.type})` formatted exactly like the existing inline rendering in `sl-search.service.ts:29-41` |
| `table` | `firstDescription(table.descriptions)` then `table.comment` (precedence already used by `warehouse-catalog.service.ts:286-287`); `null` when both are empty | When `matchedOn === 'description'`/`'comment'`: a window of that string; when `matchedOn === 'name'`/`'display'`: a comma-joined list of up to 5 of the table's column names |
| `column` | `resolveDescription(column.descriptions)` then `column.comment` (`warehouse-catalog.service.ts:228-245`); `null` when both are empty | When `matchedOn === 'description'`/`'comment'`: that text; when `matchedOn === 'sample_value'`: `${column.nativeType} · samples: <up to 5 sampleValues>` formatted from `column.sampleValues` (`warehouse-catalog.service.ts:18-23`); otherwise `${column.nativeType}` |
The `matchedOn` field is the same concept as the existing
`RawSchemaHit.matchedOn` in `warehouse-catalog.service.ts:40-54`,
extended to the wiki and SL kinds. Snippets always come from a single
already-stored field; the resolver never concatenates across sources or
invents bridging text. Length cap is enforced at the producer side (≤200
chars after a single-pass slice; no ellipsis appended — clients render
one if they want).
**Implementation:** new module `packages/context/src/search/discover.ts`.
Composes three sub-searches in parallel:
1. Wiki search via the existing wiki search backend.
2. SL search over sources/measures/dimensions using existing
`sl-sources-index` (or a new lightweight index if needed for measure
granularity).
3. Raw schema search over tables and columns from `KtxSchemaSnapshot`,
indexed at scan time and stored alongside other scan artifacts.
Results from each sub-search are fused with `packages/context/src/search/rrf.ts`
using equal weights. The `kinds` filter constrains which sub-searches run.
### entity_details
Read structured metadata for one or more raw tables (and optionally specific
columns) from the latest scan snapshot. The raw-data equivalent of
`sl_read_source`.
**Input schema:**
```typescript
{
connectionId: z.string().min(1),
entities: z.array(z.object({
// table accepts either a driver-display string ("project.dataset.table",
// "schema.name", "db.schema.name") or a structured ref. The resolver
// returns a structured error when the input is ambiguous across multiple
// schemas/catalogs.
table: z.union([
z.string().min(1),
z.object({
catalog: z.string().nullable(),
db: z.string().nullable(),
name: z.string().min(1),
}),
]),
columns: z.array(z.string()).optional(), // omit → all columns
})).min(1).max(20),
}
```
**Output:** for each entity, a structured record:
```typescript
{
connectionId: string,
tableRef: { // structured identity, lossless on every driver
catalog: string | null, // BigQuery project, Snowflake/SQL Server database
db: string | null, // schema/dataset
name: string,
},
display: string, // driver-formatted display string
// (e.g. "project.dataset.table", "schema.name")
kind: 'table' | 'view' | 'external' | 'event_stream', // matches KtxSchemaTableKind
comment: string | null,
estimatedRows: number | null,
columns: Array<{
name: string,
nativeType: string,
normalizedType: string,
dimensionType: 'time' | 'string' | 'number' | 'boolean',
nullable: boolean,
primaryKey: boolean,
comment: string | null,
}>,
foreignKeys: Array<{
fromColumn: string,
toCatalog: string | null, // qualified FK target, preserves cross-db FKs
toDb: string | null,
toTable: string,
toColumn: string,
constraintName: string | null,
}>,
snapshot: { // freshness metadata, present on every response
syncId: string, // latest scan/sync identifier
extractedAt: string, // ISO-8601 UTC of the snapshot
scanRunId: string | null, // scan run id if available
},
}
```
Output fields mirror `KtxSchemaTable` / `KtxSchemaColumn` /
`KtxSchemaForeignKey` from `packages/context/src/scan/types.ts:51-82`. The
full `KtxSchemaTableKind` set is preserved so BigQuery `external` tables
and warehouses with event-stream sources are not silently coerced. FK
target qualification (`toCatalog`/`toDb`) carries through so agents can
write valid SQL for cross-schema or cross-database references without
re-resolving.
If `columns` is provided, only the requested columns appear in the `columns`
array (PKs and FKs still report on the full table).
**Implementation:** new module `packages/context/src/scan/entity-details.ts`.
Reads `KtxSchemaSnapshot` from the same store the existing `scan_*` tools
read. No new infrastructure. If the requested table is not in the latest
snapshot, the tool returns a structured error with a suggestion to run
`ktx ingest <connectionId>`.
**Cache freshness.** Today `WarehouseCatalogService` caches `ConnectionCatalog`
per connection name with no invalidation
(`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts:248-249`,
`:404-411`). For an ingest tool that runs inside a single short-lived ingest
session that is acceptable, but the MCP daemon is long-lived and serves
clients across multiple `scan_trigger` / `ktx ingest` runs. The MCP adapter
**must** key its cache on the latest scan artifact identity (the `syncId`
derived from the artifact path, or the artifact file mtime) and re-read when
that identity advances. The same rule applies to the shared services backing
`discover_data` and `dictionary_search`. The implementation plan must
either:
1. Extend `WarehouseCatalogService` (and equivalent dictionary/discover
services) to invalidate cached entries when the underlying artifact
identity advances, or
2. Wrap those services in an MCP-adapter cache layer that performs the
identity check before returning cached values.
### dictionary_search
Find which connection, source, and column **profile-sampled** a given literal
value (or substring) such as "Acme Corp" or "shipped". Backed by the existing
`SlDictionaryEntry` extraction over relationship-profiling artifacts.
**Authoritativeness.** The dictionary index is built from *sampled* values
captured during relationship profiling — by default 5 values per column,
drawn from a sample of up to 10,000 rows
(`packages/context/src/scan/relationship-profiling.ts:409-410`,
`packages/context/src/sl/sl-dictionary-profile.ts:70`). A hit confirms a
column contains the value; a miss is **not** proof that the value is absent
from the column or warehouse — the value may simply have been outside the
profile sample. The tool must surface this distinction in its output and the
research skill must teach agents not to treat a miss as exhaustive.
**Input schema:**
```typescript
{
values: z.array(z.string().min(1)).min(1).max(20),
connectionId: z.string().optional(), // omit → all connections
}
```
**Output:** for each input value, the list of matching entries plus
per-connection provenance. Coverage and miss reasons are connection-scoped
because `loadLatestSlDictionaryEntries` iterates each connection's profile
artifact independently
(`packages/context/src/sl/sl-dictionary-profile.ts:96-112`); a single
all-connections call can mix `no_profile_artifact` (one connection never
ran an enriched scan), `value_not_in_sample` (another connection ran but
the literal was outside the sample), and matches in the same response.
```typescript
{
// The set of connections actually searched on this call. When the input
// omits connectionId this is every configured connection; otherwise it
// contains the single requested connection.
searched: Array<{
connectionId: string,
coverage: {
sampledRows: number | null, // profileSampleRows used at profile time
valuesPerColumn: number | null, // sampleValuesPerColumn used at profile time
profiledColumns: number, // count of columns in the dictionary index for this connection
syncId: string | null, // identifier of the profile artifact (null when missing)
profiledAt: string | null, // ISO-8601 UTC of the profile artifact (null when missing)
},
// Per-connection status, independent of any specific input value:
// ready — profile present with profiled columns
// no_profile_artifact — enriched scan never ran for this connection
// no_candidate_columns — profile present but no columns profile-eligible
status: 'ready' | 'no_profile_artifact' | 'no_candidate_columns',
}>,
results: Array<{
value: string, // input value
matches: Array<{
connectionId: string,
sourceName: string,
columnName: string,
matchedValue: string, // actual value found (may differ in case)
cardinality: number | null, // column cardinality if known
}>,
// Per-connection miss reasons for this value, present when that
// connection produced no match. Connections that matched do not appear
// in `misses`. For ready connections with no match, the reason is
// 'value_not_in_sample' (non-authoritative miss). For unready
// connections, the reason mirrors their `status` above.
misses: Array<{
connectionId: string,
reason:
| 'no_profile_artifact'
| 'no_candidate_columns'
| 'value_not_in_sample',
}>,
}>,
}
```
**Matching semantics:** case-insensitive substring match against the
profile-sampled values. Misses are never authoritative — they only state
that the value was not in the captured sample for the listed connection.
`misses[].reason` distinguishes "no enriched scan has run on this
connection" (`no_profile_artifact`), "enriched scan ran but no columns
were profile-eligible" (`no_candidate_columns`), and "scan ran but value
was not in the sample" (`value_not_in_sample`). The research skill must
direct agents to follow up a `value_not_in_sample` miss with
`sql_execution` against the most plausible columns, not to conclude the
value is absent.
**Cache freshness:** the dictionary index is keyed on the profile artifact
identity (the `syncId` derived from its path or the artifact mtime). When
that identity advances, the daemon re-reads the artifact on next call. See
the `entity_details` cache-freshness note above for the shared rule.
**Implementation:** new module `packages/context/src/sl/dictionary-search.ts`.
Loads `SlDictionaryEntry` records via the existing extraction code path,
builds a per-connection in-memory index on first call, caches it for the
lifetime of the MCP daemon. Invalidated on next ingest run (the daemon
watches `.ktx/db.sqlite` for changes, or simply re-reads on each call when
the artifact mtime advances).
### sql_execution
Execute a read-only SQL query against a configured connection and return the
result. The fallback path for questions the semantic layer does not cover.
**Input schema:**
```typescript
{
connectionId: z.string().min(1),
sql: z.string().min(1),
maxRows: z.number().int().min(1).max(10_000).default(1000).optional(),
}
```
**Output:**
```typescript
{
headers: string[],
headerTypes?: string[], // driver-mapped type names, one per header; optional
rows: Array<Array<unknown>>,
rowCount: number,
}
```
`headerTypes` is optional because not every connector exposes per-column
type metadata. The current contract makes it optional
(`KtxQueryResult.headerTypes` in `packages/context/src/scan/types.ts:272-277`),
and the SQLite connector currently omits it
(`packages/connector-sqlite/src/connector.ts:237-240`, `:301-308`). When a
connector returns header types, the MCP adapter passes them through
verbatim. When a connector does not, the MCP adapter omits the field rather
than fabricating values.
**Implementation:** delegates to `KtxScanConnector.executeReadOnly` on the
matching connector. The connector calls `assertReadOnlySql` and
`limitSqlForExecution` (`packages/context/src/connections/read-only-sql.ts`).
**Read-only enforcement is lexical, not parser-backed.** The current guard
inspects the first token with regex: it accepts queries whose first non-space
token is `SELECT` or `WITH`, and rejects queries whose first non-space token
matches a fixed mutating-verb list. Implications:
- A CTE that nests a data-modifying statement (e.g., `WITH x AS (INSERT ...
RETURNING *) SELECT ...`, valid in Postgres) passes the first-token check
and would reach the connector.
- Dialect-specific read/write constructs and procedure calls that do not
start with a listed verb are not caught.
Because `sql_execution` exposes this boundary to external MCP clients, the
tool **must not** be enabled until one of the following holds:
1. The guard is upgraded to a sqlglot/AST-based read-only check that
inspects every statement and CTE node, with explicit tests for CTE-DML,
`CALL`, `DO`, vendor pragmas, and multi-statement payloads; or
2. Connector-side execution forces a read-only transaction / session (e.g.,
`SET TRANSACTION READ ONLY` for Postgres, `READ ONLY` connection for
MySQL, equivalent for each connector), so the guard is defense-in-depth
rather than the sole boundary.
The implementation plan that follows this spec is required to choose and
land one of those before registering `sql_execution` in the MCP surface.
Errors from `assertReadOnlySql` (whichever implementation) are returned as
structured tool errors so the agent can correct the query and retry.
## Tool naming convention
Match the existing KTX MCP convention (no prefix): `discover_data`,
`entity_details`, `dictionary_search`, `sql_execution`. The existing tools
(`wiki_search`, `sl_list_sources`, `scan_trigger`, `memory_capture`) all use
unprefixed snake_case; the new tools follow suit.
## Connection model
- `sql_execution` and `entity_details` require `connectionId` — these tools
cannot operate without a target.
- `discover_data` and `dictionary_search` make `connectionId` optional. Omit
it to search across all configured connections; provide it to scope. This
matches the existing pattern for `sl_list_sources({ connectionId? })`.
- All tools are project-locked: the MCP daemon runs in one KTX project dir;
to operate on a different project, restart the daemon with a different
`--project-dir` or `cwd`.
## MCP daemon: `ktx mcp`
A new CLI subtree in `packages/cli/src/commands/mcp-commands.ts`, wired into
`cli-program.ts` alongside `setup`, `connection`, `ingest`, `wiki`, `sl`,
`status`, `dev`.
### Commands
```bash
ktx mcp start [--port <n>] [--host <h>] [--token <t>] [--foreground] \
[--allowed-host <h>...] [--allowed-origin <o>...]
ktx mcp stop
ktx mcp status
ktx mcp logs [--follow]
```
`--allowed-host` and `--allowed-origin` are repeatable. They extend (not
replace) the defaults defined in the security model below.
### `ktx mcp start`
Starts a long-lived HTTP MCP server bound to the configured host and port,
serving every tool registered by `createKtxMcpServer`. The server stays alive
until `ktx mcp stop` is invoked or the process is terminated.
- Default `--host` is `127.0.0.1`. Any value other than `127.0.0.1` or
`localhost` **requires** `--token` (or `KTX_MCP_TOKEN` in the environment);
the command refuses otherwise.
- Default `--port` is 7878. If the port is in use, the command exits with an
error explaining how to choose another. Allocated port is persisted to
`.ktx/mcp.json` for subsequent `status`, `stop`, `logs`, and
`setup-agents` calls.
- `--foreground` runs the server in the foreground and pipes all logs to
stdout, for debugging. Default is background.
- Background runs detach via the same pattern as the managed Python daemon
(`packages/cli/src/managed-python-daemon.ts`): spawn a detached child,
write `pid`, `port`, `startedAt` to `.ktx/mcp.json`, return immediately
with the URL the user should configure in their client.
- Logs go to `.ktx/logs/mcp.log` (matches existing log layout).
### `ktx mcp stop`
Reads `.ktx/mcp.json` for the daemon PID, sends SIGTERM, waits up to 10
seconds for graceful exit, then SIGKILLs if still running. Removes the state
file on success.
### `ktx mcp status`
Reads `.ktx/mcp.json`, checks the process is alive, hits the server's
`/health` endpoint, and reports:
- Running / stopped / stale (state file present but process not alive)
- Port, host, started-at, pid
- Whether token auth is enabled
- Configured project dir
### `ktx mcp logs`
Tails or follows `.ktx/logs/mcp.log`. Standard `--follow` flag.
### Lifecycle
Manual: the user runs `ktx mcp start` after each reboot or whenever they
want the server running. No auto-start on other `ktx` commands (matches the
explicit pattern established by the daemon model).
### Transport
HTTP-only via `StreamableHTTPServerTransport` from
`@modelcontextprotocol/sdk/server/streamableHttp.js`.
The `/mcp` endpoint must implement the full Streamable HTTP contract, not
just `POST`:
- `POST /mcp` — JSON-RPC requests (and the `initialize` handshake when no
session exists). On the first `initialize` post, the server allocates a
session id and returns it in the `Mcp-Session-Id` response header.
- `GET /mcp` — opens an SSE stream for server-initiated messages on an
existing session. Requires a valid `Mcp-Session-Id` header.
- `DELETE /mcp` — explicit session termination by the client. Requires a
valid `Mcp-Session-Id` header; the server must drop the session and any
associated SSE streams.
**Session model.** v1 ships **stateful** sessions: the server generates a
session id with `randomUUID()` on `initialize`, stores the transport in an
in-memory map keyed by session id, reuses it on subsequent
`POST`/`GET`/`DELETE` calls that carry the same `Mcp-Session-Id`, and
removes it on `DELETE` or transport close. Requests that carry an unknown
session id are rejected with HTTP 404 so the client knows to re-initialize.
Health: `GET /health` returns `{ status: 'ok', projectDir, port }` for
liveness checks. `/health` is separate from `/mcp` and is not subject to
session-id requirements (but is subject to host/origin validation; see
below).
### Security model
- `127.0.0.1` binding is the default and requires no token auth (loopback
only). Even on loopback, the server enforces **Host and Origin header
validation** on every `/mcp` and `/health` request to defend against
browser-driven DNS-rebinding attacks (the same defense the MCP SDK
exposes in `createMcpExpressApp` / `createMcpHonoApp`).
- **Host validation** compares the incoming `Host` header to the allowed-host
list after normalizing: lowercase, strip any port, strip surrounding
brackets from IPv6 literals (`[::1]:7878` → `::1`). Comparison is exact
on the normalized host string. The default allowed-host list is
`['localhost', '127.0.0.1', '::1']`. `--allowed-host` values are appended
after the same normalization.
- **Origin validation** compares the full browser `Origin` header (scheme +
host + port) to the allowed-origin list. The default allowed-origin list
is empty: any request that carries an `Origin` header is rejected unless
an explicit `--allowed-origin` entry matches. Non-browser clients that
do not send an `Origin` header (Claude Code, Cursor, Codex, opencode
HTTP transports) are accepted regardless of `Origin`. Each
`--allowed-origin` value must be a full origin string
(e.g., `http://localhost:7878`); KTX validates the format at startup.
- Non-loopback binding requires `--token <t>` or `KTX_MCP_TOKEN`. The
server checks `Authorization: Bearer <t>` on **every** `/mcp` method —
`POST`, `GET` (SSE), and `DELETE` — and rejects with HTTP 401 otherwise.
Token enforcement is independent of the session check; both must pass.
When `--host` is non-loopback, the allowed-host list expands to include
the normalized bound host plus any user-supplied `--allowed-host`
values.
- TLS is out of scope. For remote access, document running KTX behind a
reverse proxy (Caddy, nginx) that terminates TLS.
## Client config installation via `ktx setup-agents`
`ktx setup-agents` extends its existing per-target file installation
(`plannedKtxAgentFiles` in `packages/cli/src/setup-agents.ts:64`) to also
write MCP server entries.
The per-client config matrix is **not uniform**. Each client has its own
file location, scope semantics, and entry shape; `setup-agents` must
produce the correct shape per target rather than emit one JSON blob.
| Target | Scope | MCP config path | Writer behavior |
|---|---|---|---|
| claude-code | user (global) | `~/.claude.json` → root `mcpServers.ktx` | write JSON |
| claude-code | local (per-project, private) | `~/.claude.json` → `projects[<absProjectPath>].mcpServers.ktx` | write JSON |
| claude-code | project (shared, checked in) | `<projectDir>/.mcp.json` → `mcpServers.ktx` | write JSON |
| cursor | global | `~/.cursor/mcp.json` → `mcpServers.ktx` | write JSON |
| cursor | project | `<projectDir>/.cursor/mcp.json` → `mcpServers.ktx` | write JSON |
| codex | user (global) | `~/.codex/config.toml` → `[mcp_servers.ktx]` (TOML) | print instructions; do not auto-write in v1 |
| opencode | user (global) | `~/.config/opencode/opencode.json` → `mcp.ktx` | print instructions; do not auto-write in v1 |
| opencode | project | `<projectDir>/opencode.json` → `mcp.ktx` | print instructions; do not auto-write in v1 |
The shared global `~/.claude.json` and per-project `~/.claude.json` →
`projects[...]` scope are both supported because Claude Code's "user" vs.
"local" scopes write to different sub-trees of the same file; `setup-agents`
must select the scope explicitly per invocation.
Codex and opencode entries are **printed as copy-pasteable snippets** in v1
because their config formats (TOML for codex, a different JSON wrapper for
opencode) diverge enough from the JSON writers above that mixing them into
the same writer codepath risks silently producing invalid files. This is a
deliberate v1 scoping decision, not a permanent limitation.
#### Entry shapes by target
Claude Code (HTTP):
```jsonc
{
"mcpServers": {
"ktx": {
"type": "http",
"url": "http://localhost:7878/mcp"
// when token auth is active, env-var expansion only:
// "headers": { "Authorization": "Bearer ${KTX_MCP_TOKEN}" }
}
}
}
```
Cursor (HTTP, project `.cursor/mcp.json` or global `~/.cursor/mcp.json`):
```jsonc
{
"mcpServers": {
"ktx": {
"url": "http://localhost:7878/mcp"
// when token auth is active, env-var expansion only:
// "headers": { "Authorization": "Bearer ${KTX_MCP_TOKEN}" }
}
}
}
```
Codex (printed snippet, `~/.codex/config.toml`):
```toml
[mcp_servers.ktx]
url = "http://localhost:7878/mcp"
# Codex MCP config does not currently document a headers field; if token
# auth is active, instruct the user to either run KTX on loopback without a
# token or wait for codex header support before enabling.
```
opencode (printed snippet, `opencode.json`):
```jsonc
{
"mcp": {
"ktx": {
"type": "remote",
"url": "http://localhost:7878/mcp",
"enabled": true
// when token auth is active, env-var expansion only:
// "headers": { "Authorization": "Bearer ${KTX_MCP_TOKEN}" }
}
}
}
```
#### Token handling per client
When `--token` / `KTX_MCP_TOKEN` is active, `setup-agents` writes the bearer
token **only via environment-variable reference** (`Bearer ${KTX_MCP_TOKEN}`),
never as a literal token value. Claude Code, Cursor, and opencode all
support environment-variable expansion inside `headers` values; the
written entry references `${KTX_MCP_TOKEN}` and the user is responsible
for exporting it in the shell that launches the MCP client.
Rules:
- **No literal-token writes, anywhere.** Even the user-scope (private)
Claude Code / Cursor config receives env-var references, not the raw
token. This keeps the same writer codepath for every scope and avoids a
branch that materializes secrets.
- **Project-scope (shared, checked-in) configs are gated.** When a token is
active and the user requests a shared scope — `<projectDir>/.mcp.json`
for Claude Code, `<projectDir>/.cursor/mcp.json` for Cursor — `setup-agents`
prints a warning and offers a choice: (a) write the entry with the
`${KTX_MCP_TOKEN}` reference (the file is safe to commit; readers must
export the variable locally), or (b) skip the shared entry and rely on a
user-scope entry instead. The default is (a).
- **Verify header support per client before writing.** The matrix below
reflects the current state of each client's MCP config docs:
- claude-code: supports `headers` with `${VAR}` expansion on HTTP entries.
- cursor: supports `headers` with `${VAR}` expansion on HTTP entries.
- opencode: supports `headers` with `${VAR}` expansion on remote MCP
entries.
- codex: **not currently supported** in published config docs. When a
token is active and the user selects codex, `setup-agents` prints a
warning and skips the codex entry rather than writing an entry that
codex will silently ignore. The recommended workaround is to bind KTX
to loopback without a token for codex users.
- **Implementation acceptance test.** Setup-agents writer tests must assert
that no rendered output contains the literal token string for any
scope/target combination — only the `${KTX_MCP_TOKEN}` reference.
Port is read from `.ktx/mcp.json` if present, falling back to 7878. The
install manifest (`agentInstallManifestPath`,
`packages/cli/src/setup-agents.ts:60`) tracks each **written** entry so
`ktx setup-agents --remove` can roll back cleanly. The current manifest
entry kinds are `file` and `json-key`
(`packages/cli/src/setup-agents.ts:42-50`); the MCP client writers for
claude-code and cursor add `json-key` entries for their respective config
files. Printed-only snippets for codex and opencode are **not** tracked in
the manifest, and `--remove` does not attempt to mutate user-written
files for those targets; the printed instructions tell the user how to
remove the entry by hand.
If the daemon is not running when `setup-agents` runs, the command prints a
follow-up hint: "Run `ktx mcp start` to enable the configured KTX MCP
server." It does **not** auto-start the daemon (matches the manual
lifecycle decision).
## Research skill
A new skill source file at `packages/cli/src/skills/research/SKILL.md`,
installed by `ktx setup-agents` to all configured targets. The skill is
separate from the existing setup skill (different triggers: "work in a KTX
project" vs. "answer a data question") and lives in its own per-target
folder so global vs. project scope and removal stay clean.
`plannedKtxAgentFiles` in `packages/cli/src/setup-agents.ts:64` is extended
to return both the existing `ktx` entries and new `ktx-research` entries:
| Target | Scope | Path |
|---|---|---|
| claude-code | global | `~/.claude/skills/ktx-research/SKILL.md` |
| claude-code | project | `.claude/skills/ktx-research/SKILL.md` |
| codex | global | `${CODEX_HOME}/skills/ktx-research/SKILL.md` |
| codex | project | `.agents/skills/ktx-research/SKILL.md` |
| cursor | project | `.cursor/rules/ktx-research.mdc` |
| opencode | project | `.opencode/commands/ktx-research.md` |
| universal | project | `.agents/skills/ktx-research/SKILL.md` |
The skill body is identical across targets; only the wrapper format and
file path differ to match each target's convention.
### Skill content
```markdown
---
name: ktx-research
description: Use when answering a question that needs data from a KTX-connected database — investigating, analyzing, "how many", "show me", "what's the breakdown of", finding records by value, exploring tables, comparing periods, or any data-investigation request. Triggers even when the user does not say "research"; if the answer requires querying a configured KTX connection, this skill applies.
---
# KTX Research Workflow
You have access to KTX MCP tools for investigating data. Follow this workflow.
<workflow>
1. **Discover** — call `discover_data(query)` first to see what exists across wiki, semantic-layer sources, and raw tables. Returns refs only.
2. **Inspect top hits in parallel** — for each promising ref:
- `kind: 'wiki'` → `wiki_read(key)`
- `kind: 'sl_source'` / `'sl_measure'` / `'sl_dimension'` → `sl_read_source(connectionId, sourceName)`
- `kind: 'table'` / `'column'` → `entity_details(connectionId, entities)`
3. **Resolve literals** — if the user named a value (e.g., "Acme Corp", "status=shipped"), call `dictionary_search(values)` to find which column holds it.
4. **Query** —
- Prefer `sl_query` when the semantic layer covers the question (joins, measures pre-defined).
- Use `sql_execution` only for things the semantic layer doesn't cover.
5. **Capture learnings** — at the end of the turn, call `memory_capture(userMessage, assistantMessage)` so future turns benefit. Skip when the answer carries no durable knowledge (e.g., the user only asked for schema info).
</workflow>
<rules>
- Always run `discover_data` before writing SQL. Do not guess table names.
- Prefer the semantic layer over raw SQL when both can answer the question — measures are the source of truth.
- Read entity details before writing SQL against an unfamiliar table; do not assume column names.
- Treat `sql_execution` as read-only. Writes are rejected by the server.
- Validate value mentions with `dictionary_search` instead of guessing case/spelling — but treat a `dictionary_search` *miss* as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent.
</rules>
<examples>
**Input:** "How many orders did Acme Corp place last month?"
**Output workflow:**
1. `dictionary_search(["Acme Corp"])` → finds `customers.name`
2. `discover_data("orders customer monthly")` → finds `orders_facts` SL source
3. `sl_read_source("warehouse", "orders_facts")` → confirms measure `order_count`, dim `customer_name`, dim `ordered_at`
4. `sl_query({ measures: ["order_count"], filters: ["customer_name = 'Acme Corp'", "ordered_at >= date_trunc('month', now() - interval '1 month')"], dimensions: [{ field: "ordered_at", granularity: "month" }] })`
5. `memory_capture(userMessage, assistantMessage)`
---
**Input:** "What columns does the events table have?"
**Output workflow:**
1. `discover_data("events table")` → top hit `kind: 'table', id: 'analytics.events'`
2. `entity_details("warehouse", [{ table: "analytics.events" }])` → returns columns, types, FKs
3. Answer directly. (No query needed; no `memory_capture` since no durable learning.)
</examples>
```
## Files
### New
- `packages/context/src/scan/entity-details.ts` — derives entity-detail
records from `KtxSchemaSnapshot`, sharing resolution logic with
`warehouse-verification/warehouse-catalog.service.ts` (refactored or
imported, not duplicated).
- `packages/context/src/sl/dictionary-search.ts` — builds and queries the
dictionary index over relationship-profiling artifacts.
- `packages/context/src/search/discover.ts` — composes wiki, SL, and raw
schema searches; fuses results via `rrf.ts`. Reuses the same wiki/SL/raw
search building blocks as `warehouse-verification/discover-data.tool.ts`.
- `packages/cli/src/commands/mcp-commands.ts` — `ktx mcp start|stop|status|logs`.
- `packages/cli/src/managed-mcp-daemon.ts` — daemon lifecycle (spawn,
pidfile, log management), mirroring `managed-python-daemon.ts`.
- `packages/cli/src/skills/research/SKILL.md` — research workflow skill.
- Tests for each new module following existing patterns
(`*.test.ts` siblings), including coverage of the per-client config
writer/printer matrix.
### Modified
- `packages/context/src/mcp/context-tools.ts` — register the four new tools
with their Zod schemas.
- `packages/context/src/mcp/server.ts` — extend `KtxMcpContextPorts` with the
new ports (`sqlExecution`, `entityDetails`, `dictionarySearch`, `discover`).
- `packages/context/src/mcp/types.ts` — add the new port interface
definitions.
- `packages/cli/src/cli-program.ts` — register the `mcp` command subtree.
- `packages/cli/src/setup-agents.ts` — install the research skill and write
MCP client config entries to each configured target.
## Testing strategy
- Unit tests for each new module (`entity-details.ts`,
`dictionary-search.ts`, `discover.ts`) using existing fixture patterns.
- MCP-level integration test in `packages/context/src/mcp/server.test.ts`
that registers a fake server, invokes each tool, and asserts the
responses.
- CLI integration test for `ktx mcp start|stop|status` lifecycle following
the pattern in `managed-python-daemon.test.ts`.
- Setup-agents tests verifying behavior per target: claude-code and cursor
writers add the correct JSON entry and a corresponding `json-key`
manifest entry that `--remove` cleans up; codex and opencode targets
produce printed snippet output and do not mutate any user config file
or add manifest entries in v1.
- Verification commands per CLAUDE.md: `pnpm --filter @ktx/context run test`
and `pnpm --filter @ktx/cli run test` for the affected packages, plus
`pnpm run type-check`.
## Out of scope / follow-ups
- **Python code execution via MCP.** The daemon's `/code/execute` endpoint
exists; surfacing it via MCP is a separate design with sandbox/security
considerations.
- **Stdio MCP transport.** HTTP-only for now. Stdio can be added later as an
additional transport mode without changing the tool surface.
- **OS-level auto-start.** Manual `ktx mcp start` only. Adding launchd /
systemd unit installation is a UX polish for a later release.
- **TLS in the daemon itself.** Reverse proxy is the documented path. Native
TLS support if/when demand emerges.
- **Multi-project / project-switching MCP.** One daemon per project. A
cross-project model would require per-call `projectDir` arguments or a
`set_project_dir` tool and is deferred.
- **Audit logging, rate limiting, per-tool authorization.** Not in scope for
v1; the security boundary is loopback or bearer token.
## Open trade-offs
- **`dictionary_search` requires `--deep` (enriched) scan to have run.** The
relationship-profiling artifact that powers the dictionary index is only
produced by enriched scans. The tool reports this distinctly when missing,
but the dependency is real: without enriched scan, the tool returns
empty.
- **`entity_details` reads from the latest snapshot, not live.** If the
database schema changes after the last scan, the tool will reflect the
scan state, not reality. Surfacing this clearly in the tool's response
(snapshot timestamp) is part of the implementation.
- **No streaming for `sql_execution`.** Large results are capped at
`maxRows` (default 1000, max 10k). The tool returns the full result set
in one response. Streaming partial results is left for a later iteration
if real workloads demand it.

4
packages/cli/package.json
View file

@ -26,7 +26,8 @@
],
"scripts": {
"assets:demo": "node scripts/build-demo-assets.mjs",
"build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node ../../scripts/prepare-cli-bin.mjs",
"build": "tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs",
"clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true }); fs.rmSync('node_modules/.cache/tsc.tsbuildinfo', { force: true })\"",
"docs:commands": "pnpm run build && node dist/print-command-tree.js",
"smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000",
"test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts",
@ -45,6 +46,7 @@
"@ktx/connector-sqlserver": "workspace:*",
"@ktx/context": "workspace:*",
"@ktx/llm": "workspace:*",
"@modelcontextprotocol/sdk": "^1.29.0",
"commander": "14.0.3",
"ink": "^7.0.2",
"react": "^19.2.6",

11
packages/cli/scripts/copy-runtime-assets.mjs Normal file
View file

@ -0,0 +1,11 @@
import { cp, mkdir, rm } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
const packageRoot = fileURLToPath(new URL('..', import.meta.url));
const skillsSource = join(packageRoot, 'src', 'skills');
const skillsTarget = join(packageRoot, 'dist', 'skills');
await rm(skillsTarget, { recursive: true, force: true });
await mkdir(dirname(skillsTarget), { recursive: true });
await cp(skillsSource, skillsTarget, { recursive: true });

61
packages/cli/src/cli-program.ts
View file

@ -5,6 +5,7 @@ import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js';
import { registerConnectionCommands } from './commands/connection-commands.js';
import { registerIngestCommands } from './commands/ingest-commands.js';
import { registerWikiCommands } from './commands/knowledge-commands.js';
import { registerMcpCommands } from './commands/mcp-commands.js';
import { registerSetupCommands } from './commands/setup-commands.js';
import { registerSlCommands } from './commands/sl-commands.js';
import { registerStatusCommands } from './commands/status-commands.js';
@ -55,9 +56,11 @@ type CommandPathNode = CommandWithGlobalOptions & {
parent?: CommandPathNode | null;
};
const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status']);
const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'mcp']);
const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx dev init']);
const COMMANDS_WITH_OWN_MISSING_PROJECT_HANDLING = new Set(['status']);
const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']);
const GLOBAL_OPTIONS_WITHOUT_VALUE = new Set(['--debug', '--help', '-h', '--version', '-v']);
class KtxProjectMissingAbortError extends Error {
readonly isKtxProjectMissingAbort = true;
@ -72,24 +75,6 @@ function isKtxProjectMissingAbortError(error: unknown): error is KtxProjectMissi
(typeof error === 'object' && error !== null && (error as { isKtxProjectMissingAbort?: unknown }).isKtxProjectMissingAbort === true)
);
}
const REMOVED_COMMAND_PATHS = new Set([
'scan',
'wiki read',
'wiki write',
]);
const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']);
const OPTIONS_WITH_VALUE = new Set([
'--project-dir',
'--query-history-window-days',
'--user-id',
'--limit',
'--format',
'--connection-id',
'--source-name',
'--query-file',
'--max-rows',
]);
export interface CommandWithGlobalOptions {
opts: () => object;
optsWithGlobals?: () => object;
@ -336,43 +321,32 @@ function formatCliError(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function commandPathFromArgv(argv: string[]): string[] {
const path: string[] = [];
for (let index = 0; index < argv.length && path.length < 2; index += 1) {
function firstTopLevelCommandToken(argv: string[]): string | null {
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === undefined) {
continue;
}
if (arg === '--') {
break;
return null;
}
if ((path.length === 0 ? GLOBAL_OPTIONS_WITH_VALUE : OPTIONS_WITH_VALUE).has(arg)) {
if (GLOBAL_OPTIONS_WITH_VALUE.has(arg)) {
index += 1;
continue;
}
const optionsWithValue = path.length === 0 ? GLOBAL_OPTIONS_WITH_VALUE : OPTIONS_WITH_VALUE;
if ([...optionsWithValue].some((option) => arg.startsWith(`${option}=`))) {
if ([...GLOBAL_OPTIONS_WITH_VALUE].some((option) => arg.startsWith(`${option}=`))) {
continue;
}
if (path.length === 0 && arg === '--debug') {
if (GLOBAL_OPTIONS_WITHOUT_VALUE.has(arg) || arg.startsWith('-')) {
continue;
}
if (arg.startsWith('-')) {
continue;
}
path.push(arg);
return arg;
}
return path;
return null;
}
function removedCommandName(argv: string[]): string | null {
const path = commandPathFromArgv(argv);
if (path.length === 0) {
return null;
}
const pathKey = path.join(' ');
return REMOVED_COMMAND_PATHS.has(pathKey) ? path.at(-1) ?? null : null;
function isKnownTopLevelCommand(program: Command, commandName: string): boolean {
return program.commands.some((command) => command.name() === commandName || command.aliases().includes(commandName));
}
async function runBareInteractiveCommand(
@ -439,6 +413,7 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
registerWikiCommands(program, context);
registerSlCommands(program, context);
registerStatusCommands(program, context);
registerMcpCommands(program, context);
registerDevCommands(program, context);
return program;
@ -489,9 +464,9 @@ export async function runCommanderKtxCli(
return 0;
}
const removedCommand = removedCommandName(argv);
if (removedCommand) {
io.stderr.write(`error: unknown command '${removedCommand}'\n`);
const topLevelCommand = firstTopLevelCommandToken(argv);
if (topLevelCommand && !isKnownTopLevelCommand(program, topLevelCommand)) {
io.stderr.write(`error: unknown command '${topLevelCommand}'\n`);
return 1;
}

6
packages/cli/src/cli-runtime.ts
View file

@ -34,6 +34,12 @@ export interface KtxCliDeps {
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;
knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise<number>;
sl?: (args: KtxSlArgs, io: KtxCliIo) => Promise<number>;
mcp?: {
startDaemon?: typeof import('./managed-mcp-daemon.js').startKtxMcpDaemon;
stopDaemon?: typeof import('./managed-mcp-daemon.js').stopKtxMcpDaemon;
readStatus?: typeof import('./managed-mcp-daemon.js').readKtxMcpDaemonStatus;
runServer?: typeof import('./mcp-http-server.js').runKtxMcpHttpServer;
};
}
export function getKtxCliPackageInfo(): KtxCliPackageInfo {

57
packages/cli/src/commands/mcp-commands.test.ts Normal file
View file

@ -0,0 +1,57 @@
import { Command } from '@commander-js/extra-typings';
import { describe, expect, it, vi } from 'vitest';
import type { KtxCliCommandContext } from '../cli-program.js';
import { registerMcpCommands } from './mcp-commands.js';
function makeContext(overrides: Partial<KtxCliCommandContext> = {}): KtxCliCommandContext {
let exitCode = 0;
return {
io: {
stdout: { write: vi.fn() },
stderr: { write: vi.fn() },
},
deps: {},
packageInfo: { name: '@ktx/cli', version: '0.0.0-test', contextPackageName: '@ktx/context' },
setExitCode: (code) => {
exitCode = code;
},
runInit: vi.fn(),
writeDebug: vi.fn(),
...overrides,
get exitCode() {
return exitCode;
},
} as KtxCliCommandContext;
}
describe('registerMcpCommands', () => {
it('registers the public mcp lifecycle commands', () => {
const program = new Command().exitOverride();
registerMcpCommands(program, makeContext());
const mcp = program.commands.find((command) => command.name() === 'mcp');
expect(mcp?.commands.map((command) => command.name()).sort()).toEqual([
'logs',
'serve-internal',
'start',
'status',
'stop',
]);
expect(
(mcp?.commands.find((command) => command.name() === 'serve-internal') as { _hidden?: boolean } | undefined)
?._hidden,
).toBe(true);
});
it('rejects non-loopback start without token before spawning', async () => {
const program = new Command().exitOverride();
const startDaemon = vi.fn();
const context = makeContext({ deps: { mcp: { startDaemon } } });
registerMcpCommands(program, context);
await expect(program.parseAsync(['mcp', 'start', '--host', '0.0.0.0'], { from: 'user' })).rejects.toThrow(
'Binding KTX MCP to 0.0.0.0 requires --token or KTX_MCP_TOKEN',
);
expect(startDaemon).not.toHaveBeenCalled();
});
});

136
packages/cli/src/commands/mcp-commands.ts Normal file
View file

@ -0,0 +1,136 @@
import { spawn } from 'node:child_process';
import { readFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { Command } from '@commander-js/extra-typings';
import type { KtxCliCommandContext } from '../cli-program.js';
import {
collectOption,
parsePositiveIntegerOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import {
mcpDaemonLayout,
readKtxMcpDaemonStatus,
startKtxMcpDaemon,
stopKtxMcpDaemon,
} from '../managed-mcp-daemon.js';
import { buildMcpSecurityConfig, runKtxMcpHttpServer } from '../mcp-http-server.js';
function tokenFromOption(value: string | undefined): string | undefined {
return value ?? process.env.KTX_MCP_TOKEN;
}
function binPath(): string {
return fileURLToPath(new URL('../bin.js', import.meta.url));
}
export function registerMcpCommands(program: Command, context: KtxCliCommandContext): void {
const mcp = program.command('mcp').description('Run the KTX MCP HTTP server');
mcp
.command('start')
.description('Start the KTX MCP HTTP server')
.option('--host <host>', 'Host to bind', '127.0.0.1')
.option('--port <n>', 'Port to bind', parsePositiveIntegerOption, 7878)
.option('--token <token>', 'Bearer token required for non-loopback binding')
.option('--foreground', 'Run in the foreground', false)
.option('--allowed-host <host>', 'Additional allowed Host header', collectOption, [])
.option('--allowed-origin <origin>', 'Allowed browser Origin header', collectOption, [])
.action(async (options, command) => {
const projectDir = resolveCommandProjectDir(command);
const token = tokenFromOption(options.token);
buildMcpSecurityConfig({
host: options.host,
port: options.port,
token,
allowedHosts: options.allowedHost,
allowedOrigins: options.allowedOrigin,
});
if (options.foreground) {
await (context.deps.mcp?.runServer ?? runKtxMcpHttpServer)({
projectDir,
cliVersion: context.packageInfo.version,
host: options.host,
port: options.port,
token,
allowedHosts: options.allowedHost,
allowedOrigins: options.allowedOrigin,
io: context.io,
});
context.io.stdout.write(`KTX MCP server listening at http://${options.host}:${options.port}/mcp\n`);
return;
}
const result = await (context.deps.mcp?.startDaemon ?? startKtxMcpDaemon)({
projectDir,
cliVersion: context.packageInfo.version,
host: options.host,
port: options.port,
token,
allowedHosts: options.allowedHost,
allowedOrigins: options.allowedOrigin,
binPath: binPath(),
});
context.io.stdout.write(`KTX MCP daemon started: ${result.url}\n`);
});
mcp
.command('stop')
.description('Stop the KTX MCP daemon')
.action(async (_options, command) => {
const result = await (context.deps.mcp?.stopDaemon ?? stopKtxMcpDaemon)({
projectDir: resolveCommandProjectDir(command),
});
context.io.stdout.write(result.status === 'stopped' ? 'KTX MCP daemon stopped.\n' : 'KTX MCP daemon is not running.\n');
});
mcp
.command('status')
.description('Show KTX MCP daemon status')
.action(async (_options, command) => {
const status = await (context.deps.mcp?.readStatus ?? readKtxMcpDaemonStatus)({
projectDir: resolveCommandProjectDir(command),
});
context.io.stdout.write(`${status.detail}\n`);
if (status.kind === 'running') {
context.io.stdout.write(`URL: ${status.url}\n`);
context.io.stdout.write(`PID: ${status.state.pid}\n`);
context.io.stdout.write(`Token auth: ${status.state.tokenAuth ? 'enabled' : 'disabled'}\n`);
context.io.stdout.write(`Project: ${status.state.projectDir}\n`);
}
});
mcp
.command('logs')
.description('Print the KTX MCP daemon log')
.option('--follow', 'Follow log output', false)
.action(async (options, command) => {
const logPath = mcpDaemonLayout(resolveCommandProjectDir(command)).logPath;
if (options.follow) {
const child = spawn('tail', ['-f', logPath], { stdio: ['ignore', 'pipe', 'pipe'] });
child.stdout?.on('data', (chunk: Buffer) => context.io.stdout.write(chunk.toString('utf8')));
child.stderr?.on('data', (chunk: Buffer) => context.io.stderr.write(chunk.toString('utf8')));
await new Promise((resolve) => child.on('close', resolve));
return;
}
context.io.stdout.write(await readFile(logPath, 'utf8'));
});
mcp
.command('serve-internal', { hidden: true })
.option('--host <host>', 'Host to bind', '127.0.0.1')
.requiredOption('--port <n>', 'Port to bind', parsePositiveIntegerOption)
.option('--allowed-host <host>', 'Additional allowed Host header', collectOption, [])
.option('--allowed-origin <origin>', 'Allowed browser Origin header', collectOption, [])
.action(async (options, command) => {
await (context.deps.mcp?.runServer ?? runKtxMcpHttpServer)({
projectDir: resolveCommandProjectDir(command),
cliVersion: context.packageInfo.version,
host: options.host,
port: options.port,
token: process.env.KTX_MCP_TOKEN,
allowedHosts: options.allowedHost,
allowedOrigins: options.allowedOrigin,
io: context.io,
});
});
}

15
packages/cli/src/commands/setup-commands.ts
View file

@ -90,6 +90,7 @@ function shouldShowSetupEntryMenu(
agents?: boolean;
target?: string;
global?: boolean;
local?: boolean;
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
@ -163,6 +164,7 @@ function shouldShowSetupEntryMenu(
'agents',
'target',
'global',
'local',
'skipAgents',
'yes',
'input',
@ -223,6 +225,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
]),
)
.option('--global', 'Install agent integration into the global target scope', false)
.option('--local', 'Install Claude Code MCP config into the private per-project ~/.claude.json scope', false)
.addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false))
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
.option('--no-input', 'Disable interactive terminal input')
@ -392,9 +395,19 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
context.setExitCode(1);
return;
}
if (options.local && options.global) {
context.io.stderr.write('Choose only one agent scope: --local or --global.\n');
context.setExitCode(1);
return;
}
if (options.local && options.target && options.target !== 'claude-code') {
context.io.stderr.write('--local is only supported with --target claude-code.\n');
context.setExitCode(1);
return;
}
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
const resolvedAgentScope = options.global ? 'global' : 'project';
const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project';
await runSetupArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),

12
packages/cli/src/connection.test.ts
View file

@ -489,15 +489,17 @@ describe('runKtxConnection', () => {
it('rejects unknown drivers with a helpful error', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });
await writeConnections(projectDir, {
mystery: { driver: 'duckdb' },
});
await writeFile(
join(projectDir, 'ktx.yaml'),
'connections:\n mystery:\n driver: duckdb\n',
'utf-8',
);
const io = makeIo();
await expect(
runKtxConnection({ command: 'test', projectDir, connectionId: 'mystery' }, io.io),
).resolves.toBe(1);
expect(io.stderr()).toContain('uses driver "duckdb"');
expect(io.stderr()).toContain('Supported:');
expect(io.stderr()).toContain('connections.mystery.driver');
expect(io.stderr()).toContain('postgres');
});
});

13
packages/cli/src/doctor.test.ts
View file

@ -64,6 +64,11 @@ describe('formatDoctorReport', () => {
expect(output).toContain('Node 22+ · pnpm 10.20+');
expect(output).not.toContain('v22.16.0');
expect(output).toContain('Everything ready.');
expect(output).toContain('ktx status --json');
expect(output).toContain('ktx sl list');
expect(output).toContain('ktx wiki list');
expect(output).not.toContain('ktx scan');
expect(output).not.toContain('ktx sl ask');
});
it('shows the underlying detail for a single-check group on the group line', () => {
@ -462,6 +467,7 @@ describe('runKtxDoctor', () => {
it('includes Postgres query-history readiness in project doctor output', async () => {
process.env.ANTHROPIC_API_KEY = 'test-key'; // pragma: allowlist secret
process.env.OPENAI_API_KEY = 'test-key'; // pragma: allowlist secret
process.env.WAREHOUSE_DATABASE_URL = 'postgresql://reader@example.test/warehouse';
await writeFile(
join(tempDir, 'ktx.yaml'),
[
@ -516,8 +522,14 @@ describe('runKtxDoctor', () => {
expect(out).toContain('pg_stat_statements ready (PostgreSQL 16.4)');
expect(out).toContain('info: pg_stat_statements.max is 1000');
expect(out).not.toContain('Update the Postgres parameter group or config');
expect(out).toContain('ktx status --json');
expect(out).toContain('ktx sl list');
expect(out).toContain('ktx wiki list');
expect(out).not.toContain('ktx scan');
expect(out).not.toContain('ktx sl ask');
delete process.env.ANTHROPIC_API_KEY;
delete process.env.OPENAI_API_KEY;
delete process.env.WAREHOUSE_DATABASE_URL;
});
it('returns blocked verdict when LLM is not configured', async () => {
@ -543,6 +555,7 @@ describe('runKtxDoctor', () => {
).resolves.toBe(1);
expect(testIo.stdout()).toContain('no LLM configured');
expect(testIo.stdout()).not.toContain('ktx ask');
expect(testIo.stdout()).toContain('ktx setup');
});

3
packages/cli/src/doctor.ts
View file

@ -5,6 +5,7 @@ import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import type { KtxConfigIssue } from '@ktx/context/project';
import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js';
import type { BuildProjectStatusOptions } from './status-project.js';
const execFileAsync = promisify(execFile);
@ -287,7 +288,7 @@ interface RenderOptions {
command?: 'setup' | 'project';
}
const NEXT_STEPS_PROJECT = ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'];
const NEXT_STEPS_PROJECT = KTX_NEXT_STEP_DIRECT_COMMANDS.map((step) => step.command);
export function formatDoctorReport(report: DoctorReport, options: Partial<RenderOptions> = {}): string {
const opts: RenderOptions = {

33
packages/cli/src/index.test.ts
View file

@ -440,6 +440,7 @@ describe('runKtxCli', () => {
expect(stdout).toContain('--agents');
expect(stdout).toContain('--target <target>');
expect(stdout).toContain('--global');
expect(stdout).toContain('--local');
expect(stdout).toContain('--yes');
expect(stdout).toContain('--no-input');
expect(stdout).toContain('Global Options:');
@ -1286,6 +1287,38 @@ describe('runKtxCli', () => {
);
});
it('rejects --local with non-Claude targets', async () => {
const setup = vi.fn(async () => 0);
const setupIo = makeIo();
await expect(
runKtxCli(
['--project-dir', tempDir, 'setup', '--agents', '--target', 'cursor', '--local', '--no-input'],
setupIo.io,
{ setup },
),
).resolves.toBe(1);
expect(setupIo.stderr()).toContain('--local is only supported with --target claude-code');
expect(setup).not.toHaveBeenCalled();
});
it('rejects --local and --global together', async () => {
const setup = vi.fn(async () => 0);
const setupIo = makeIo();
await expect(
runKtxCli(
['--project-dir', tempDir, 'setup', '--agents', '--target', 'claude-code', '--local', '--global', '--no-input'],
setupIo.io,
{ setup },
),
).resolves.toBe(1);
expect(setupIo.stderr()).toContain('Choose only one agent scope: --local or --global.');
expect(setup).not.toHaveBeenCalled();
});
it('rejects source-path with source-git-url', async () => {
const setup = vi.fn(async () => 0);
const testIo = makeIo();

1
packages/cli/src/ingest.test-utils.ts
View file

@ -109,6 +109,7 @@ export async function writeWarehouseConfig(projectDir: string): Promise<void> {
'connections:',
' prod-metabase:',
' driver: metabase',
' api_url: https://metabase.example.test',
' warehouse_a:',
' driver: postgres',
'ingest:',

4
packages/cli/src/io/print-list.ts
View file

@ -43,13 +43,13 @@ export interface PrintListArgs<Row> {
io: KtxCliIo;
}
export interface KtxJsonResultEnvelope<T> {
interface KtxJsonResultEnvelope<T> {
kind: string;
data: T;
meta?: Record<string, unknown>;
}
export function writeJsonResult<T>(io: KtxCliIo, envelope: KtxJsonResultEnvelope<T>): void {
function writeJsonResult<T>(io: KtxCliIo, envelope: KtxJsonResultEnvelope<T>): void {
io.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`);
}

143
packages/cli/src/knowledge.test.ts
View file

@ -1,8 +1,9 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject } from '@ktx/context/project';
import { initKtxProject, loadKtxProject } from '@ktx/context/project';
import type { KtxEmbeddingPort } from '@ktx/context';
import { writeLocalKnowledgePage } from '@ktx/context/wiki';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { runKtxKnowledge } from './knowledge.js';
@ -40,6 +41,28 @@ class FakeEmbeddingPort implements KtxEmbeddingPort {
}
}
interface WikiPageFixture {
key?: string;
summary?: string;
content?: string;
tags?: string[];
slRefs?: string[];
}
async function seedWikiPage(projectDir: string, fixture: WikiPageFixture = {}): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeLocalKnowledgePage(project, {
key: fixture.key ?? 'metrics-revenue',
scope: 'GLOBAL',
userId: 'local',
summary: fixture.summary ?? 'Revenue',
content: fixture.content ?? 'Revenue is paid order value.',
tags: fixture.tags ?? ['finance'],
refs: [],
slRefs: fixture.slRefs ?? ['orders'],
});
}
describe('runKtxKnowledge', () => {
let tempDir: string;
@ -51,36 +74,10 @@ describe('runKtxKnowledge', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('writes, reads, lists, and searches wiki pages', async () => {
it('lists and searches wiki pages', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });
const writeIo = makeIo();
await expect(
runKtxKnowledge(
{
command: 'write',
projectDir,
key: 'metrics-revenue',
scope: 'GLOBAL',
userId: 'local',
summary: 'Revenue',
content: 'Revenue is paid order value.',
tags: ['finance'],
refs: [],
slRefs: ['orders'],
},
writeIo.io,
),
).resolves.toBe(0);
expect(writeIo.stdout()).toContain('Wrote wiki/global/metrics-revenue.md');
const readIo = makeIo();
await expect(
runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local' }, readIo.io),
).resolves.toBe(0);
expect(readIo.stdout()).toContain('# metrics-revenue');
expect(readIo.stdout()).toContain('Revenue is paid order value.');
await seedWikiPage(projectDir);
const listIo = makeIo();
await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local' }, listIo.io)).resolves.toBe(0);
@ -93,27 +90,10 @@ describe('runKtxKnowledge', () => {
expect(searchIo.stdout()).toContain('metrics-revenue');
});
it('prints wiki list, search, and read as public JSON envelopes', async () => {
it('prints wiki list and search as public JSON envelopes', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });
await expect(
runKtxKnowledge(
{
command: 'write',
projectDir,
key: 'metrics-revenue',
scope: 'GLOBAL',
userId: 'local',
summary: 'Revenue',
content: 'Revenue is paid order value.',
tags: ['finance'],
refs: [],
slRefs: ['orders'],
},
makeIo().io,
),
).resolves.toBe(0);
await seedWikiPage(projectDir);
const listIo = makeIo();
await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local', json: true }, listIo.io)).resolves.toBe(
@ -137,48 +117,6 @@ describe('runKtxKnowledge', () => {
data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] },
meta: { command: 'wiki search' },
});
const readIo = makeIo();
await expect(
runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local', json: true }, readIo.io),
).resolves.toBe(0);
expect(JSON.parse(readIo.stdout())).toMatchObject({
kind: 'wiki.page',
data: {
key: 'metrics-revenue',
summary: 'Revenue',
content: 'Revenue is paid order value.',
},
});
});
it('rejects slash-delimited write keys with a flat-key suggestion', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });
const writeIo = makeIo();
await expect(
runKtxKnowledge(
{
command: 'write',
projectDir,
key: 'orbit/company-overview',
scope: 'GLOBAL',
userId: 'local',
summary: 'Orbit',
content: 'Orbit overview.',
tags: [],
refs: [],
slRefs: [],
},
writeIo.io,
),
).resolves.toBe(1);
expect(writeIo.stderr()).toContain(
'Invalid wiki key "orbit/company-overview". Wiki keys must be flat; use "orbit-company-overview".',
);
expect(writeIo.stdout()).toBe('');
});
it('explains empty search results for a project without wiki pages', async () => {
@ -198,24 +136,13 @@ describe('runKtxKnowledge', () => {
it('uses configured embeddings for semantic wiki search', async () => {
const projectDir = join(tempDir, 'semantic-project');
await initKtxProject({ projectDir });
await expect(
runKtxKnowledge(
{
command: 'write',
projectDir,
key: 'active-contract-arr-open-tickets',
scope: 'GLOBAL',
userId: 'local',
summary: 'Active Contract ARR Ranked by Open Support Ticket Count',
content: 'Accounts ranked by annual recurring contract value and support ticket load.',
tags: ['historic-sql'],
refs: [],
slRefs: [],
},
makeIo().io,
),
).resolves.toBe(0);
await seedWikiPage(projectDir, {
key: 'active-contract-arr-open-tickets',
summary: 'Active Contract ARR Ranked by Open Support Ticket Count',
content: 'Accounts ranked by annual recurring contract value and support ticket load.',
tags: ['historic-sql'],
slRefs: [],
});
const searchIo = makeIo();
await expect(

49
packages/cli/src/knowledge.ts
View file

@ -5,20 +5,16 @@ import {
} from '@ktx/context';
import { loadKtxProject } from '@ktx/context/project';
import {
type LocalKnowledgeScope,
type LocalKnowledgeSearchResult,
type LocalKnowledgeSummary,
listLocalKnowledgePages,
readLocalKnowledgePage,
searchLocalKnowledgePages,
writeLocalKnowledgePage,
} from '@ktx/context/wiki';
import { resolveOutputMode } from './io/mode.js';
import { printList, type PrintListColumn, writeJsonResult } from './io/print-list.js';
import { printList, type PrintListColumn } from './io/print-list.js';
export type KtxKnowledgeArgs =
| { command: 'list'; projectDir: string; userId: string; output?: string; json?: boolean }
| { command: 'read'; projectDir: string; key: string; userId: string; json?: boolean }
| {
command: 'search';
projectDir: string;
@ -27,18 +23,6 @@ export type KtxKnowledgeArgs =
output?: string;
json?: boolean;
limit?: number;
}
| {
command: 'write';
projectDir: string;
key: string;
scope: LocalKnowledgeScope;
userId: string;
summary: string;
content: string;
tags: string[];
refs: string[];
slRefs: string[];
};
type KtxKnowledgeIo = import('./cli-runtime.js').KtxCliIo;
@ -104,25 +88,6 @@ export async function runKtxKnowledge(
});
return 0;
}
if (args.command === 'read') {
const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId });
if (!page) {
throw new Error(`Wiki page "${args.key}" was not found`);
}
if (args.json) {
writeJsonResult(io, {
kind: 'wiki.page',
data: page,
meta: { command: 'wiki read' },
});
return 0;
}
io.stdout.write(`# ${page.key}\n\n`);
io.stdout.write(`Scope: ${page.scope}\n`);
io.stdout.write(`Summary: ${page.summary}\n\n`);
io.stdout.write(`${page.content}\n`);
return 0;
}
if (args.command === 'search') {
const results = await searchLocalKnowledgePages(project, {
query: args.query,
@ -153,18 +118,6 @@ export async function runKtxKnowledge(
});
return 0;
}
const write = await writeLocalKnowledgePage(project, {
key: args.key,
scope: args.scope,
userId: args.userId,
summary: args.summary,
content: args.content,
tags: args.tags,
refs: args.refs,
slRefs: args.slRefs,
});
io.stdout.write(`Wrote ${write.path}\n`);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);

3
packages/cli/src/local-adapters.test.ts
View file

@ -18,6 +18,9 @@ function sqlAnalysisStub() {
async analyzeBatch() {
return new Map();
},
async validateReadOnly() {
return { ok: true };
},
};
}

14
packages/cli/src/local-scan-connectors.test.ts
View file

@ -92,7 +92,7 @@ describe('createKtxCliScanConnector', () => {
expect(bigQueryMock.constructorInputs[0]).not.toHaveProperty('maxBytesBilled');
});
it('throws for structural daemon-only fallback configs', async () => {
it('rejects daemon-only fallback driver configs at config parse time', async () => {
await initKtxProject({ projectDir: tempDir });
await writeFile(
join(tempDir, 'ktx.yaml'),
@ -105,14 +105,13 @@ describe('createKtxCliScanConnector', () => {
].join('\n'),
'utf-8',
);
const project = await loadKtxProject({ projectDir: tempDir });
await expect(createKtxCliScanConnector(project, 'warehouse')).rejects.toThrow(
'Connection "warehouse" uses driver "duckdb", which has no native standalone KTX scan connector',
await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow(
/connections\.warehouse\.driver:.*Invalid discriminator value/,
);
});
it('throws a clear error when the connection block has no driver field', async () => {
it('rejects connection blocks with no driver field at config parse time', async () => {
await initKtxProject({ projectDir: tempDir });
await writeFile(
join(tempDir, 'ktx.yaml'),
@ -125,10 +124,9 @@ describe('createKtxCliScanConnector', () => {
].join('\n'),
'utf-8',
);
const project = await loadKtxProject({ projectDir: tempDir });
await expect(createKtxCliScanConnector(project, 'warehouse')).rejects.toThrow(
'Connection "warehouse" has no `driver` field in ktx.yaml',
await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow(
/connections\.warehouse\.driver:.*Invalid discriminator value/,
);
});
});

133
packages/cli/src/managed-mcp-daemon.test.ts Normal file
View file

@ -0,0 +1,133 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
mcpDaemonLayout,
readKtxMcpDaemonStatus,
startKtxMcpDaemon,
stopKtxMcpDaemon,
type KtxMcpDaemonChild,
type KtxMcpDaemonState,
} from './managed-mcp-daemon.js';
function child(pid = 4242): KtxMcpDaemonChild {
return { pid, unref: vi.fn() };
}
function state(projectDir: string, overrides: Partial<KtxMcpDaemonState> = {}): KtxMcpDaemonState {
return {
schemaVersion: 1,
pid: 4242,
host: '127.0.0.1',
port: 7878,
tokenAuth: false,
projectDir,
startedAt: '2026-05-14T00:00:00.000Z',
logPath: join(projectDir, '.ktx/logs/mcp.log'),
...overrides,
};
}
describe('managed MCP daemon lifecycle', () => {
let tempDir: string;
let projectDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-mcp-daemon-'));
projectDir = join(tempDir, 'project');
await mkdir(projectDir, { recursive: true });
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('uses the spec state and log paths', () => {
expect(mcpDaemonLayout(projectDir)).toEqual({
statePath: join(projectDir, '.ktx/mcp.json'),
logPath: join(projectDir, '.ktx/logs/mcp.log'),
});
});
it('starts a detached child and writes state without the token value', async () => {
const spawnDaemon = vi.fn(() => child(5555));
await startKtxMcpDaemon({
projectDir,
cliVersion: '0.0.0-test',
host: '0.0.0.0',
port: 7879,
token: 'secret-token',
allowedHosts: ['mcp.example.test'],
allowedOrigins: ['https://mcp.example.test'],
binPath: '/repo/packages/cli/dist/bin.js',
spawnDaemon,
processAlive: vi.fn(() => false),
portAvailable: vi.fn(async () => true),
now: () => new Date('2026-05-14T00:00:00.000Z'),
});
expect(spawnDaemon).toHaveBeenCalledWith(
process.execPath,
[
'/repo/packages/cli/dist/bin.js',
'--project-dir',
projectDir,
'mcp',
'serve-internal',
'--host',
'0.0.0.0',
'--port',
'7879',
'--allowed-host',
'mcp.example.test',
'--allowed-origin',
'https://mcp.example.test',
],
expect.objectContaining({
detached: true,
env: expect.objectContaining({ KTX_MCP_TOKEN: 'secret-token' }),
}),
);
expect(JSON.stringify(JSON.parse(await readFile(join(projectDir, '.ktx/mcp.json'), 'utf8')))).not.toContain(
'secret-token',
);
});
it('reports running when the process is alive and health passes', async () => {
await mkdir(join(projectDir, '.ktx'), { recursive: true });
await writeFile(join(projectDir, '.ktx/mcp.json'), `${JSON.stringify(state(projectDir), null, 2)}\n`);
const status = await readKtxMcpDaemonStatus({
projectDir,
processAlive: vi.fn(() => true),
fetchHealth: vi.fn(async () => ({ ok: true, body: { status: 'ok', projectDir, port: 7878 } })),
});
expect(status.kind).toBe('running');
if (status.kind !== 'running') {
throw new Error(`Expected running status, received ${status.kind}`);
}
expect(status.url).toBe('http://127.0.0.1:7878/mcp');
});
it('stops a recorded daemon and removes state', async () => {
await mkdir(join(projectDir, '.ktx'), { recursive: true });
await writeFile(join(projectDir, '.ktx/mcp.json'), `${JSON.stringify(state(projectDir), null, 2)}\n`);
const alive = new Set([4242]);
const killProcess = vi.fn((pid: number) => alive.delete(pid));
await expect(
stopKtxMcpDaemon({
projectDir,
processAlive: vi.fn((pid) => alive.has(pid)),
killProcess,
stopGraceMs: 1,
pollIntervalMs: 1,
}),
).resolves.toEqual({ status: 'stopped' });
expect(killProcess).toHaveBeenCalledWith(4242, 'SIGTERM');
await expect(readFile(join(projectDir, '.ktx/mcp.json'), 'utf8')).rejects.toThrow();
});
});

238
packages/cli/src/managed-mcp-daemon.ts Normal file
View file

@ -0,0 +1,238 @@
import { spawn } from 'node:child_process';
import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises';
import { createServer } from 'node:net';
import { dirname, join } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { z } from 'zod';
export interface KtxMcpDaemonState {
schemaVersion: 1;
pid: number;
host: string;
port: number;
tokenAuth: boolean;
projectDir: string;
startedAt: string;
logPath: string;
}
export interface KtxMcpDaemonChild {
pid?: number;
unref(): void;
}
export type KtxMcpDaemonStatus =
| { kind: 'stopped'; detail: string }
| { kind: 'running'; detail: string; state: KtxMcpDaemonState; url: string }
| { kind: 'stale'; detail: string; state?: KtxMcpDaemonState };
const stateSchema = z.object({
schemaVersion: z.literal(1),
pid: z.number().int().positive(),
host: z.string().min(1),
port: z.number().int().min(1).max(65535),
tokenAuth: z.boolean(),
projectDir: z.string().min(1),
startedAt: z.string().min(1),
logPath: z.string().min(1),
});
export function mcpDaemonLayout(projectDir: string): { statePath: string; logPath: string } {
return {
statePath: join(projectDir, '.ktx/mcp.json'),
logPath: join(projectDir, '.ktx/logs/mcp.log'),
};
}
function defaultProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function defaultKillProcess(pid: number, signal: NodeJS.Signals): void {
try {
process.kill(pid, signal);
} catch (error) {
if ((error as { code?: unknown }).code !== 'ESRCH') {
throw error;
}
}
}
async function readState(projectDir: string): Promise<KtxMcpDaemonState | undefined> {
try {
return stateSchema.parse(JSON.parse(await readFile(mcpDaemonLayout(projectDir).statePath, 'utf8')) as unknown);
} catch (error) {
if ((error as { code?: unknown }).code === 'ENOENT') {
return undefined;
}
throw error;
}
}
async function writeState(projectDir: string, state: KtxMcpDaemonState): Promise<void> {
const { statePath } = mcpDaemonLayout(projectDir);
await mkdir(dirname(statePath), { recursive: true });
await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
}
async function defaultPortAvailable(host: string, port: number): Promise<boolean> {
return await new Promise((resolve) => {
const server = createServer();
server.once('error', () => resolve(false));
server.listen(port, host, () => server.close(() => resolve(true)));
});
}
function defaultSpawnDaemon(
command: string,
args: string[],
options: { detached: boolean; stdio: ['ignore', number, number]; env: NodeJS.ProcessEnv },
): KtxMcpDaemonChild {
return spawn(command, args, options);
}
async function defaultFetchHealth(state: KtxMcpDaemonState): Promise<{ ok: boolean; body: unknown; detail?: string }> {
try {
const response = await fetch(`http://${state.host}:${state.port}/health`, {
headers: { host: `${state.host}:${state.port}` },
});
const body = await response.json();
return { ok: response.ok, body, detail: response.ok ? undefined : `HTTP ${response.status}` };
} catch (error) {
return { ok: false, body: null, detail: error instanceof Error ? error.message : String(error) };
}
}
export async function startKtxMcpDaemon(options: {
projectDir: string;
cliVersion: string;
host: string;
port: number;
token?: string;
allowedHosts: string[];
allowedOrigins: string[];
binPath: string;
processAlive?: (pid: number) => boolean;
portAvailable?: (host: string, port: number) => Promise<boolean>;
spawnDaemon?: typeof defaultSpawnDaemon;
now?: () => Date;
}): Promise<{ status: 'started'; state: KtxMcpDaemonState; url: string }> {
const existing = await readState(options.projectDir).catch(() => undefined);
const processAlive = options.processAlive ?? defaultProcessAlive;
if (existing && processAlive(existing.pid)) {
throw new Error(`KTX MCP daemon is already recorded at http://${existing.host}:${existing.port}/mcp`);
}
const portAvailable = options.portAvailable ?? defaultPortAvailable;
if (!(await portAvailable(options.host, options.port))) {
throw new Error(`Port ${options.port} is already in use. Choose another port with --port <n>.`);
}
const { logPath } = mcpDaemonLayout(options.projectDir);
await mkdir(dirname(logPath), { recursive: true });
const log = await open(logPath, 'a');
try {
const args = [
options.binPath,
'--project-dir',
options.projectDir,
'mcp',
'serve-internal',
'--host',
options.host,
'--port',
String(options.port),
...options.allowedHosts.flatMap((host) => ['--allowed-host', host]),
...options.allowedOrigins.flatMap((origin) => ['--allowed-origin', origin]),
];
const child = (options.spawnDaemon ?? defaultSpawnDaemon)(process.execPath, args, {
detached: true,
stdio: ['ignore', log.fd, log.fd],
env: {
...process.env,
KTX_CLI_VERSION: options.cliVersion,
...(options.token ? { KTX_MCP_TOKEN: options.token } : {}),
},
});
if (!child.pid) {
throw new Error('Failed to start KTX MCP daemon: child process pid was not available.');
}
child.unref();
const state: KtxMcpDaemonState = {
schemaVersion: 1,
pid: child.pid,
host: options.host,
port: options.port,
tokenAuth: Boolean(options.token),
projectDir: options.projectDir,
startedAt: (options.now ?? (() => new Date()))().toISOString(),
logPath,
};
await writeState(options.projectDir, state);
return { status: 'started', state, url: `http://${state.host}:${state.port}/mcp` };
} finally {
await log.close();
}
}
export async function readKtxMcpDaemonStatus(options: {
projectDir: string;
processAlive?: (pid: number) => boolean;
fetchHealth?: (state: KtxMcpDaemonState) => Promise<{ ok: boolean; body: unknown; detail?: string }>;
}): Promise<KtxMcpDaemonStatus> {
let state: KtxMcpDaemonState | undefined;
try {
state = await readState(options.projectDir);
} catch (error) {
return { kind: 'stale', detail: `MCP daemon state is invalid: ${error instanceof Error ? error.message : String(error)}` };
}
if (!state) {
return { kind: 'stopped', detail: `No MCP daemon state at ${mcpDaemonLayout(options.projectDir).statePath}` };
}
const processAlive = options.processAlive ?? defaultProcessAlive;
if (!processAlive(state.pid)) {
return { kind: 'stale', detail: `MCP daemon process ${state.pid} is not running`, state };
}
const health = await (options.fetchHealth ?? defaultFetchHealth)(state);
if (!health.ok) {
return { kind: 'stale', detail: health.detail ?? 'MCP daemon health check failed', state };
}
return {
kind: 'running',
detail: `KTX MCP daemon running at http://${state.host}:${state.port}/mcp`,
state,
url: `http://${state.host}:${state.port}/mcp`,
};
}
export async function stopKtxMcpDaemon(options: {
projectDir: string;
processAlive?: (pid: number) => boolean;
killProcess?: (pid: number, signal: NodeJS.Signals) => void;
stopGraceMs?: number;
pollIntervalMs?: number;
}): Promise<{ status: 'stopped' | 'already-stopped' }> {
const state = await readState(options.projectDir);
const { statePath } = mcpDaemonLayout(options.projectDir);
if (!state) {
return { status: 'already-stopped' };
}
const processAlive = options.processAlive ?? defaultProcessAlive;
const killProcess = options.killProcess ?? defaultKillProcess;
if (processAlive(state.pid)) {
killProcess(state.pid, 'SIGTERM');
const deadline = Date.now() + (options.stopGraceMs ?? 10_000);
while (Date.now() <= deadline && processAlive(state.pid)) {
await delay(options.pollIntervalMs ?? 100);
}
if (processAlive(state.pid)) {
killProcess(state.pid, 'SIGKILL');
}
}
await rm(statePath, { force: true });
return { status: 'stopped' };
}

274
packages/cli/src/mcp-http-server.test.ts Normal file
View file

@ -0,0 +1,274 @@
import { request } from 'node:http';
import type { AddressInfo } from 'node:net';
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { describe, expect, it } from 'vitest';
import {
buildMcpSecurityConfig,
isMcpRequestAuthorized,
normalizeHostHeader,
runKtxMcpHttpServer,
} from './mcp-http-server.js';
describe('normalizeHostHeader', () => {
it('normalizes host headers before allow-list comparison', () => {
expect(normalizeHostHeader('LOCALHOST:7878')).toBe('localhost');
expect(normalizeHostHeader('127.0.0.1:7878')).toBe('127.0.0.1');
expect(normalizeHostHeader('[::1]:7878')).toBe('::1');
expect(normalizeHostHeader(' Example.COM ')).toBe('example.com');
});
});
describe('buildMcpSecurityConfig', () => {
it('allows loopback hosts without a token', () => {
const config = buildMcpSecurityConfig({
host: '127.0.0.1',
port: 7878,
token: undefined,
allowedHosts: [],
allowedOrigins: [],
});
expect(config.token).toBeUndefined();
expect(config.allowedHosts).toEqual(['localhost', '127.0.0.1', '::1']);
});
it('requires a token for non-loopback binding', () => {
expect(() =>
buildMcpSecurityConfig({
host: '0.0.0.0',
port: 7878,
token: undefined,
allowedHosts: [],
allowedOrigins: [],
}),
).toThrow('Binding KTX MCP to 0.0.0.0 requires --token or KTX_MCP_TOKEN');
});
it('validates allowed origins as full origins', () => {
expect(() =>
buildMcpSecurityConfig({
host: '127.0.0.1',
port: 7878,
token: undefined,
allowedHosts: [],
allowedOrigins: ['localhost:7878'],
}),
).toThrow('Allowed origin must be a full origin URL');
});
});
describe('isMcpRequestAuthorized', () => {
const config = buildMcpSecurityConfig({
host: '0.0.0.0',
port: 7878,
token: 'secret-token',
allowedHosts: ['mcp.example.test'],
allowedOrigins: ['https://mcp.example.test'],
});
it('accepts a valid host, origin, and bearer token', () => {
expect(
isMcpRequestAuthorized(
{
path: '/mcp',
headers: {
host: 'mcp.example.test:7878',
origin: 'https://mcp.example.test',
authorization: 'Bearer secret-token',
},
},
config,
),
).toEqual({ ok: true });
});
it('rejects bad host headers before MCP handling', () => {
expect(
isMcpRequestAuthorized(
{ path: '/health', headers: { host: 'evil.example.test' } },
config,
),
).toEqual({ ok: false, status: 403, message: 'Host header is not allowed for KTX MCP.' });
});
it('rejects browser origins unless explicitly allowed', () => {
expect(
isMcpRequestAuthorized(
{
path: '/health',
headers: { host: 'mcp.example.test', origin: 'https://evil.example.test' },
},
config,
),
).toEqual({ ok: false, status: 403, message: 'Origin header is not allowed for KTX MCP.' });
});
it('requires bearer auth on /mcp when token auth is enabled', () => {
expect(
isMcpRequestAuthorized(
{ path: '/mcp', headers: { host: 'mcp.example.test', authorization: 'Bearer wrong' } },
config,
),
).toEqual({ ok: false, status: 401, message: 'Missing or invalid KTX MCP bearer token.' });
});
it('does not require bearer auth on /health', () => {
expect(isMcpRequestAuthorized({ path: '/health', headers: { host: 'mcp.example.test' } }, config)).toEqual({
ok: true,
});
});
});
function postJson(port: number, path: string, body: unknown, headers: Record<string, string> = {}) {
return new Promise<{ status: number; headers: Record<string, string | string[] | undefined>; body: string }>(
(resolve, reject) => {
const payload = JSON.stringify(body);
const req = request(
{
host: '127.0.0.1',
port,
path,
method: 'POST',
headers: {
host: `127.0.0.1:${port}`,
accept: 'application/json, text/event-stream',
'content-type': 'application/json',
'content-length': Buffer.byteLength(payload),
...headers,
},
},
(res) => {
const chunks: Buffer[] = [];
res.on('data', (chunk: Buffer) => chunks.push(chunk));
res.on('end', () =>
resolve({
status: res.statusCode ?? 0,
headers: res.headers,
body: Buffer.concat(chunks).toString('utf8'),
}),
);
},
);
req.on('error', reject);
req.end(payload);
},
);
}
function get(port: number, path: string, headers: Record<string, string> = {}) {
return new Promise<{ status: number; headers: Record<string, string | string[] | undefined>; body: string }>(
(resolve, reject) => {
const req = request(
{
host: '127.0.0.1',
port,
path,
method: 'GET',
headers: { host: `127.0.0.1:${port}`, ...headers },
},
(res) => {
const chunks: Buffer[] = [];
res.on('data', (chunk: Buffer) => chunks.push(chunk));
res.on('end', () =>
resolve({
status: res.statusCode ?? 0,
headers: res.headers,
body: Buffer.concat(chunks).toString('utf8'),
}),
);
},
);
req.on('error', reject);
req.end();
},
);
}
function createTestMcpServer() {
return () => {
const server = new McpServer({ name: 'ktx-test', version: '0.0.0-test' });
server.registerTool('ping', { inputSchema: {} }, async () => ({
content: [{ type: 'text', text: 'pong' }],
}));
return server;
};
}
describe('runKtxMcpHttpServer', () => {
it('serves /health with project metadata', async () => {
const handle = await runKtxMcpHttpServer({
projectDir: '/tmp/ktx-project',
host: '127.0.0.1',
port: 0,
allowedHosts: [],
allowedOrigins: [],
createMcpServer: createTestMcpServer(),
});
try {
const port = (handle.server.address() as AddressInfo).port;
const response = await get(port, '/health');
expect(response.status).toBe(200);
expect(JSON.parse(response.body)).toEqual({
status: 'ok',
projectDir: '/tmp/ktx-project',
port,
});
} finally {
await handle.close();
}
});
it('allocates a stateful MCP session on initialize', async () => {
const handle = await runKtxMcpHttpServer({
projectDir: '/tmp/ktx-project',
host: '127.0.0.1',
port: 0,
allowedHosts: [],
allowedOrigins: [],
createMcpServer: createTestMcpServer(),
});
try {
const port = (handle.server.address() as AddressInfo).port;
const response = await postJson(port, '/mcp', {
jsonrpc: '2.0',
id: 1,
method: 'initialize',
params: {
protocolVersion: '2025-06-18',
capabilities: {},
clientInfo: { name: 'vitest', version: '0.0.0' },
},
});
expect(response.status).toBe(200);
expect(response.headers['mcp-session-id']).toBeTruthy();
} finally {
await handle.close();
}
});
it('rejects unknown session ids with 404', async () => {
const handle = await runKtxMcpHttpServer({
projectDir: '/tmp/ktx-project',
host: '127.0.0.1',
port: 0,
allowedHosts: [],
allowedOrigins: [],
createMcpServer: createTestMcpServer(),
});
try {
const port = (handle.server.address() as AddressInfo).port;
const response = await postJson(
port,
'/mcp',
{ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} },
{ 'mcp-session-id': 'missing-session' },
);
expect(response.status).toBe(404);
expect(response.body).toContain('Unknown MCP session');
} finally {
await handle.close();
}
});
});

340
packages/cli/src/mcp-http-server.ts Normal file
View file

@ -0,0 +1,340 @@
import { randomUUID } from 'node:crypto';
import { createServer, type IncomingHttpHeaders, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
import { createDefaultKtxMcpServer, createLocalProjectMcpContextPorts } from '@ktx/context/mcp';
import { createLocalProjectMemoryCapture } from '@ktx/context/memory';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
import type { KtxCliIo } from './cli-runtime.js';
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { createManagedPythonSemanticLayerComputePort } from './managed-python-command.js';
import { createManagedDaemonSqlAnalysisPort } from './managed-python-http.js';
const DEFAULT_ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1'] as const;
export interface McpSecurityConfigInput {
host: string;
port: number;
token?: string;
allowedHosts: string[];
allowedOrigins: string[];
}
export interface McpSecurityConfig {
host: string;
port: number;
token?: string;
allowedHosts: string[];
allowedOrigins: string[];
}
export type McpAuthorizationResult =
| { ok: true }
| { ok: false; status: 401 | 403; message: string };
function isLoopbackHost(host: string): boolean {
const normalized = normalizeHostHeader(host);
return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '::1';
}
export function normalizeHostHeader(value: string): string {
const trimmed = value.trim().toLowerCase();
if (trimmed.startsWith('[')) {
const close = trimmed.indexOf(']');
return close >= 0 ? trimmed.slice(1, close) : trimmed.replace(/^\[/, '');
}
const colon = trimmed.lastIndexOf(':');
if (colon > -1 && trimmed.indexOf(':') === colon) {
return trimmed.slice(0, colon);
}
return trimmed;
}
function fullOrigin(value: string): string {
let parsed: URL;
try {
parsed = new URL(value);
} catch {
throw new Error(`Allowed origin must be a full origin URL: ${value}`);
}
if (!parsed.protocol || !parsed.host || parsed.pathname !== '/' || parsed.search || parsed.hash) {
throw new Error(`Allowed origin must be a full origin URL: ${value}`);
}
return parsed.origin;
}
export function buildMcpSecurityConfig(input: McpSecurityConfigInput): McpSecurityConfig {
if (!isLoopbackHost(input.host) && !input.token) {
throw new Error(`Binding KTX MCP to ${input.host} requires --token or KTX_MCP_TOKEN`);
}
const allowedHostSet = new Set<string>(DEFAULT_ALLOWED_HOSTS);
if (!isLoopbackHost(input.host)) {
allowedHostSet.add(normalizeHostHeader(input.host));
}
for (const host of input.allowedHosts) {
allowedHostSet.add(normalizeHostHeader(host));
}
return {
host: input.host,
port: input.port,
...(input.token ? { token: input.token } : {}),
allowedHosts: [...allowedHostSet],
allowedOrigins: input.allowedOrigins.map(fullOrigin),
};
}
function headerValue(headers: IncomingHttpHeaders | Record<string, string | undefined>, name: string): string | undefined {
const value = headers[name.toLowerCase()];
return Array.isArray(value) ? value[0] : value;
}
export function isMcpRequestAuthorized(
request: { path: string; headers: IncomingHttpHeaders | Record<string, string | undefined> },
config: McpSecurityConfig,
): McpAuthorizationResult {
const host = headerValue(request.headers, 'host');
if (!host || !config.allowedHosts.includes(normalizeHostHeader(host))) {
return { ok: false, status: 403, message: 'Host header is not allowed for KTX MCP.' };
}
const origin = headerValue(request.headers, 'origin');
if (origin && !config.allowedOrigins.includes(origin)) {
return { ok: false, status: 403, message: 'Origin header is not allowed for KTX MCP.' };
}
if (request.path === '/mcp' && config.token) {
const auth = headerValue(request.headers, 'authorization');
if (auth !== `Bearer ${config.token}`) {
return { ok: false, status: 401, message: 'Missing or invalid KTX MCP bearer token.' };
}
}
return { ok: true };
}
export interface KtxMcpHttpServerHandle {
server: Server;
close(): Promise<void>;
}
export interface RunKtxMcpHttpServerOptions extends McpSecurityConfigInput {
projectDir: string;
cliVersion?: string;
io?: KtxCliIo;
createMcpServer?: () => McpServer;
loadProject?: typeof loadKtxProject;
}
function noopIo(): KtxCliIo {
return {
stdout: { write() {} },
stderr: { write() {} },
};
}
function writeJson(res: ServerResponse, status: number, body: object): void {
const payload = `${JSON.stringify(body)}\n`;
res.writeHead(status, {
'content-type': 'application/json',
'content-length': Buffer.byteLength(payload),
});
res.end(payload);
}
function writeText(res: ServerResponse, status: number, body: string): void {
res.writeHead(status, { 'content-type': 'text/plain; charset=utf-8' });
res.end(body);
}
function requestPath(req: IncomingMessage): string {
const url = new URL(req.url ?? '/', 'http://127.0.0.1');
return url.pathname;
}
async function readJsonBody(req: IncomingMessage): Promise<unknown> {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
const raw = Buffer.concat(chunks).toString('utf8');
return raw.trim().length === 0 ? undefined : (JSON.parse(raw) as unknown);
}
async function defaultMcpServerFactory(input: {
project: KtxLocalProject;
projectDir: string;
cliVersion: string;
io?: KtxCliIo;
}): Promise<() => McpServer> {
const io = input.io ?? noopIo();
const queryExecutor = createKtxCliIngestQueryExecutor(input.project);
const semanticLayerCompute = await createManagedPythonSemanticLayerComputePort({
cliVersion: input.cliVersion,
installPolicy: 'auto',
io,
});
const sqlAnalysis = createManagedDaemonSqlAnalysisPort({
cliVersion: input.cliVersion,
projectDir: input.projectDir,
installPolicy: 'auto',
io,
});
const contextTools = createLocalProjectMcpContextPorts(input.project, {
semanticLayerCompute,
queryExecutor,
sqlAnalysis,
localScan: {
createConnector: async (connectionId) => createKtxCliScanConnector(input.project, connectionId),
},
localIngest: {
semanticLayerCompute,
queryExecutor,
},
});
let memoryCapture: ReturnType<typeof createLocalProjectMemoryCapture> | undefined;
try {
memoryCapture = createLocalProjectMemoryCapture(input.project, { semanticLayerCompute, queryExecutor });
} catch (error) {
input.io?.stderr.write(`KTX MCP memory_capture disabled: ${error instanceof Error ? error.message : String(error)}\n`);
}
return () =>
createDefaultKtxMcpServer({
name: 'ktx',
version: input.cliVersion,
userContext: { userId: 'local' },
contextTools,
memoryCapture,
});
}
function listenerPort(server: Server, fallback: number): number {
const address = server.address();
return typeof address === 'object' && address ? address.port : fallback;
}
function transportAllowedHosts(config: McpSecurityConfig, server: Server): string[] {
const port = listenerPort(server, config.port);
const hosts = new Set<string>(config.allowedHosts);
for (const host of config.allowedHosts) {
hosts.add(`${host}:${port}`);
if (config.port !== 0 && config.port !== port) {
hosts.add(`${host}:${config.port}`);
}
}
return [...hosts];
}
export async function runKtxMcpHttpServer(options: RunKtxMcpHttpServerOptions): Promise<KtxMcpHttpServerHandle> {
const config = buildMcpSecurityConfig(options);
const project =
options.createMcpServer === undefined
? await (options.loadProject ?? loadKtxProject)({ projectDir: options.projectDir })
: undefined;
const createMcpServer =
options.createMcpServer ??
(await defaultMcpServerFactory({
project: project!,
projectDir: options.projectDir,
cliVersion: options.cliVersion ?? '0.0.0-private',
io: options.io,
}));
const sessions = new Map<string, StreamableHTTPServerTransport>();
async function newTransport(): Promise<StreamableHTTPServerTransport> {
let transport: StreamableHTTPServerTransport;
transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => randomUUID(),
onsessioninitialized: (sessionId) => {
sessions.set(sessionId, transport);
},
onsessionclosed: (sessionId) => {
sessions.delete(sessionId);
},
allowedHosts: transportAllowedHosts(config, server),
allowedOrigins: config.allowedOrigins,
enableDnsRebindingProtection: true,
});
transport.onclose = () => {
if (transport.sessionId) {
sessions.delete(transport.sessionId);
}
};
await createMcpServer().connect(transport);
return transport;
}
const server = createServer(async (req, res) => {
const path = requestPath(req);
const auth = isMcpRequestAuthorized({ path, headers: req.headers }, config);
if (!auth.ok) {
writeText(res, auth.status, auth.message);
return;
}
if (path === '/health' && req.method === 'GET') {
const port = listenerPort(server, config.port);
writeJson(res, 200, { status: 'ok', projectDir: options.projectDir, port });
return;
}
if (path !== '/mcp' || !['POST', 'GET', 'DELETE'].includes(req.method ?? '')) {
writeText(res, 404, 'Not found');
return;
}
const sessionId = req.headers['mcp-session-id'];
const normalizedSessionId = Array.isArray(sessionId) ? sessionId[0] : sessionId;
if (req.method === 'POST') {
let body: unknown;
try {
body = await readJsonBody(req);
} catch (error) {
writeText(res, 400, `Invalid JSON body: ${error instanceof Error ? error.message : String(error)}`);
return;
}
const existing = normalizedSessionId ? sessions.get(normalizedSessionId) : undefined;
if (existing) {
await existing.handleRequest(req, res, body);
return;
}
if (normalizedSessionId) {
writeText(res, 404, `Unknown MCP session: ${normalizedSessionId}`);
return;
}
if (!isInitializeRequest(body)) {
writeText(res, 400, 'MCP initialize request is required before session traffic.');
return;
}
await (await newTransport()).handleRequest(req, res, body);
return;
}
if (!normalizedSessionId || !sessions.has(normalizedSessionId)) {
writeText(res, 404, normalizedSessionId ? `Unknown MCP session: ${normalizedSessionId}` : 'Missing MCP session id.');
return;
}
await sessions.get(normalizedSessionId)!.handleRequest(req, res);
});
await new Promise<void>((resolve, reject) => {
server.once('error', reject);
server.listen(config.port, config.host, () => {
server.off('error', reject);
resolve();
});
});
return {
server,
async close() {
for (const transport of sessions.values()) {
await transport.close();
}
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
},
};
}

4
packages/cli/src/print-command-tree.test.ts
View file

@ -12,11 +12,12 @@ describe('renderKtxCommandTree', () => {
.filter((line) => /^ {2}[├└]── \S/.test(line))
.map((line) => line.replace(/^ {2}[├└]── /, '').trim().split(' ')[0]);
for (const expected of ['setup', 'connection', 'ingest', 'sl', 'dev']) {
for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'dev']) {
expect(topLevel).toContain(expected);
}
expect(output).toContain('│ └── test [connectionId]');
expect(output).toContain('│ ├── status Show KTX MCP daemon status');
expect(output).not.toContain('│ ├── add');
expect(output).not.toContain('│ ├── remove');
expect(output).not.toContain('│ ├── map');
@ -24,7 +25,6 @@ describe('renderKtxCommandTree', () => {
expect(output).not.toContain('│ ├── metabase');
expect(output).not.toContain('│ ├── notion');
expect(output).not.toContain('scan <connectionId>');
expect(output).not.toContain('│ ├── status');
expect(output).not.toContain('│ ├── replay');
expect(output).not.toContain('│ └── replay');
expect(output).not.toContain('│ ├── run');

4
packages/cli/src/public-ingest.test.ts
View file

@ -85,7 +85,7 @@ describe('buildPublicIngestPlan', () => {
it('plans warehouse connections as scan targets and source connections as source ingest targets', () => {
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
prod_metabase: { driver: 'metabase' },
prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' },
docs: { driver: 'notion' },
});
@ -745,7 +745,7 @@ describe('runKtxPublicIngest', () => {
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
prod_metabase: { driver: 'metabase' },
prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' },
});
const runScan = vi.fn(async () => 1);
const runIngest = vi.fn(async () => 0);

69
packages/cli/src/scan.ts
View file

@ -133,6 +133,50 @@ function warningLine(warning: KtxScanWarning): string {
return `${warning.code}: ${location}${warning.message}`;
}
function groupWarningsByCode(warnings: readonly KtxScanWarning[]): Map<string, KtxScanWarning[]> {
const groups = new Map<string, KtxScanWarning[]>();
for (const warning of warnings) {
const list = groups.get(warning.code);
if (list) {
list.push(warning);
} else {
groups.set(warning.code, [warning]);
}
}
return groups;
}
function describeWarningGroup(code: string, count: number): string {
switch (code) {
case 'sampling_failed':
return `${count} ${plural(count, 'table')} could not be sampled (retries exhausted); descriptions used metadata-only fallback or were skipped.`;
case 'description_fallback_used':
return `${count} ${plural(count, 'table')} got an AI description from column metadata only (no sample rows available).`;
case 'enrichment_failed':
return `${count} ${plural(count, 'table/column')} could not be enriched.`;
case 'connector_capability_missing':
return `${count} ${plural(count, 'table')} affected by missing connector capability.`;
case 'statistics_failed':
return `${count} statistics ${plural(count, 'lookup')} failed.`;
case 'llm_unavailable':
return 'LLM provider unavailable; AI enrichment was skipped.';
case 'embedding_unavailable':
return 'Embedding provider unavailable; embeddings were skipped.';
case 'relationship_validation_failed':
return `${count} relationship ${plural(count, 'validation')} could not run.`;
case 'relationship_llm_invalid_reference':
return `${count} LLM-proposed ${plural(count, 'relationship')} referenced unknown columns.`;
case 'relationship_llm_proposal_failed':
return `${count} LLM relationship ${plural(count, 'proposal')} failed.`;
case 'scan_enrichment_backend_not_configured':
return 'Scan enrichment backend is not configured; AI stages were skipped.';
case 'credential_redacted':
return `${count} ${plural(count, 'credential')} were redacted from scan output.`;
default:
return `${count} ${plural(count, 'warning')} (${code})`;
}
}
function managedDaemonOptionsForScanRun(args: Extract<KtxScanArgs, { command: 'run' }>, io: KtxCliIo) {
if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) {
return undefined;
@ -153,11 +197,26 @@ function writeNeedsAttention(report: KtxScanReport, io: KtxCliIo): void {
}
if (report.warnings.length > 0) {
io.stdout.write(` ${report.warnings.length} ${plural(report.warnings.length, 'warning')}\n`);
for (const warning of report.warnings.slice(0, 5)) {
io.stdout.write(` - ${warningLine(warning)}\n`);
}
if (report.warnings.length > 5) {
io.stdout.write(` - ${report.warnings.length - 5} more warnings in the JSON report\n`);
const groups = groupWarningsByCode(report.warnings);
for (const [code, warnings] of groups) {
io.stdout.write(` - ${describeWarningGroup(code, warnings.length)}\n`);
const first = warnings[0];
if (first) {
io.stdout.write(` ${warningLine(first)}\n`);
}
if (warnings.length > 1) {
const moreTables = warnings
.slice(1)
.map((warning) =>
warning.table ? (warning.column ? `${warning.table}.${warning.column}` : warning.table) : null,
)
.filter((value): value is string => value !== null)
.slice(0, 3);
if (moreTables.length > 0) {
const suffix = warnings.length - 1 > moreTables.length ? `, …` : '';
io.stdout.write(` also: ${moreTables.join(', ')}${suffix}\n`);
}
}
}
}
if (report.capabilityGaps.length > 0) {

204
packages/cli/src/setup-agents.test.ts
View file

@ -37,23 +37,28 @@ describe('setup agents', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('plans project-scoped CLI files for every target', () => {
it('plans project-scoped CLI and research files for every target', () => {
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-code', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
{ kind: 'file', path: join(tempDir, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' },
{ kind: 'file', path: join(tempDir, '.claude/rules/ktx.md'), role: 'rule' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'codex', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md'), role: 'skill' },
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' },
{ kind: 'file', path: join(tempDir, '.codex/instructions/ktx.md'), role: 'rule' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'cursor', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.cursor/rules/ktx.mdc') },
{ kind: 'file', path: join(tempDir, '.cursor/rules/ktx-research.mdc'), role: 'research-skill' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'opencode', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.opencode/commands/ktx.md') },
{ kind: 'file', path: join(tempDir, '.opencode/commands/ktx-research.md'), role: 'research-skill' },
]);
expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'universal', scope: 'project', mode: 'cli' })).toEqual([
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md') },
{ kind: 'file', path: join(tempDir, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' },
]);
});
@ -97,6 +102,31 @@ describe('setup agents', () => {
expect(io.stderr()).toBe('');
});
it('installs the research skill from the runtime asset', async () => {
const io = makeIo();
await expect(
runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'universal',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
),
).resolves.toMatchObject({ status: 'ready' });
const researchSkill = await readFile(join(tempDir, '.agents/skills/ktx-research/SKILL.md'), 'utf-8');
expect(researchSkill).toContain('name: ktx-research');
expect(researchSkill).toContain('Always run `discover_data` before writing SQL.');
expect(researchSkill).toContain('Treat a `dictionary_search` miss as non-authoritative.');
});
it('writes PATH-independent launcher commands for skills', async () => {
const io = makeIo();
@ -123,6 +153,178 @@ describe('setup agents', () => {
expect(skill).not.toContain('sql execute');
});
it('writes Claude Code project MCP config and tracks the json key', async () => {
const io = makeIo();
await expect(
runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'claude-code',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
),
).resolves.toMatchObject({ status: 'ready' });
const mcpJson = JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8')) as {
mcpServers: { ktx: { type: string; url: string; headers?: Record<string, string> } };
};
expect(mcpJson.mcpServers.ktx).toEqual({ type: 'http', url: 'http://localhost:7878/mcp' });
expect(await readKtxAgentInstallManifest(tempDir)).toMatchObject({
entries: expect.arrayContaining([{ kind: 'json-key', path: join(tempDir, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] }]),
});
expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.');
});
it('writes Cursor project MCP config', async () => {
const io = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'cursor',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
);
const cursorJson = JSON.parse(await readFile(join(tempDir, '.cursor/mcp.json'), 'utf-8')) as {
mcpServers: { ktx: { url: string; headers?: Record<string, string> } };
};
expect(cursorJson.mcpServers.ktx).toEqual({ url: 'http://localhost:7878/mcp' });
});
it('prints Codex and opencode snippets without mutating printed-only config files', async () => {
const codexIo = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'codex',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
codexIo.io,
);
expect(codexIo.stdout()).toContain('[mcp_servers.ktx]');
expect(codexIo.stdout()).toContain('url = "http://localhost:7878/mcp"');
const opencodeIo = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'opencode',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
opencodeIo.io,
);
expect(opencodeIo.stdout()).toContain('"mcp"');
expect(opencodeIo.stdout()).toContain('"type": "remote"');
await expect(readFile(join(tempDir, 'opencode.json'), 'utf-8')).rejects.toThrow();
});
it('uses MCP daemon state for port and token metadata without rendering literal tokens', async () => {
await mkdir(join(tempDir, '.ktx'), { recursive: true });
await writeFile(
join(tempDir, '.ktx/mcp.json'),
`${JSON.stringify(
{
schemaVersion: 1,
pid: 999999,
host: '127.0.0.1',
port: 8787,
tokenAuth: true,
projectDir: tempDir,
startedAt: '2026-05-14T00:00:00.000Z',
logPath: join(tempDir, '.ktx/logs/mcp.log'),
},
null,
2,
)}\n`,
'utf-8',
);
const io = makeIo();
const previousToken = process.env.KTX_MCP_TOKEN;
process.env.KTX_MCP_TOKEN = 'secret-token';
try {
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'claude-code',
scope: 'project',
mode: 'cli',
skipAgents: false,
},
io.io,
);
const rendered = JSON.stringify(JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8')));
expect(rendered).toContain('http://127.0.0.1:8787/mcp');
expect(rendered).toContain('Bearer ${KTX_MCP_TOKEN}');
expect(rendered).not.toContain('secret-token');
expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.');
} finally {
if (previousToken === undefined) {
delete process.env.KTX_MCP_TOKEN;
} else {
process.env.KTX_MCP_TOKEN = previousToken;
}
}
});
it('writes Claude Code local MCP config under the project key in ~/.claude.json', async () => {
const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-'));
const previousHome = process.env.HOME;
process.env.HOME = home;
try {
const io = makeIo();
await runKtxSetupAgentsStep(
{
projectDir: tempDir,
inputMode: 'disabled',
yes: true,
agents: true,
target: 'claude-code',
scope: 'local',
mode: 'cli',
skipAgents: false,
},
io.io,
);
const config = JSON.parse(await readFile(join(home, '.claude.json'), 'utf-8')) as {
projects: Record<string, { mcpServers: { ktx: { type: string; url: string } } }>;
};
expect(config.projects[tempDir].mcpServers.ktx).toEqual({ type: 'http', url: 'http://localhost:7878/mcp' });
} finally {
process.env.HOME = previousHome;
await rm(home, { recursive: true, force: true });
}
});
it('removes only manifest-listed files', async () => {
const io = makeIo();
await runKtxSetupAgentsStep(

241
packages/cli/src/setup-agents.ts
View file

@ -1,3 +1,4 @@
import { existsSync } from 'node:fs';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@ -12,9 +13,10 @@ import {
createKtxSetupPromptAdapter,
type KtxSetupPromptOption,
} from './setup-prompts.js';
import { readKtxMcpDaemonStatus } from './managed-mcp-daemon.js';
export type KtxAgentTarget = 'claude-code' | 'codex' | 'cursor' | 'opencode' | 'universal';
export type KtxAgentScope = 'project' | 'global';
export type KtxAgentScope = 'project' | 'global' | 'local';
export type KtxAgentInstallMode = 'cli';
export interface KtxSetupAgentsArgs {
@ -45,18 +47,179 @@ export interface KtxAgentInstallManifest {
installedAt: string;
installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>;
entries: Array<
| { kind: 'file'; path: string; role?: 'skill' | 'rule' }
| { kind: 'file'; path: string; role?: 'skill' | 'rule' | 'research-skill' }
| { kind: 'json-key'; path: string; jsonPath: string[] }
>;
}
type InstallEntry = KtxAgentInstallManifest['entries'][number];
interface KtxMcpEndpointInfo {
url: string;
tokenAuth: boolean;
running: boolean;
}
interface KtxMcpClientInstallResult {
entries: InstallEntry[];
snippets: string[];
notices: string[];
}
interface KtxCliLauncher {
command: string;
args: string[];
}
async function readJsonObject(path: string): Promise<Record<string, unknown>> {
if (!existsSync(path)) return {};
const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error(`Expected JSON object in ${path}`);
}
return parsed as Record<string, unknown>;
}
function objectAtPath(root: Record<string, unknown>, jsonPath: string[]): Record<string, unknown> {
let cursor = root;
for (const segment of jsonPath) {
const current = cursor[segment];
if (!current || typeof current !== 'object' || Array.isArray(current)) {
cursor[segment] = {};
}
cursor = cursor[segment] as Record<string, unknown>;
}
return cursor;
}
async function writeJsonKey(path: string, jsonPath: string[], value: unknown): Promise<void> {
const root = await readJsonObject(path);
const parent = objectAtPath(root, jsonPath.slice(0, -1));
parent[jsonPath.at(-1) as string] = value;
await mkdir(dirname(path), { recursive: true });
await writeFile(path, `${JSON.stringify(root, null, 2)}\n`, 'utf-8');
}
async function resolveMcpEndpoint(projectDir: string): Promise<KtxMcpEndpointInfo> {
const status = await readKtxMcpDaemonStatus({ projectDir }).catch(() => null);
if (status?.kind === 'running') {
return {
url: status.url,
tokenAuth: status.state.tokenAuth,
running: true,
};
}
if (status?.kind === 'stale' && status.state) {
return {
url: `http://${status.state.host}:${status.state.port}/mcp`,
tokenAuth: status.state.tokenAuth || Boolean(process.env.KTX_MCP_TOKEN),
running: false,
};
}
return {
url: 'http://localhost:7878/mcp',
tokenAuth: Boolean(process.env.KTX_MCP_TOKEN),
running: false,
};
}
function tokenHeaders(endpoint: KtxMcpEndpointInfo): Record<string, string> | undefined {
return endpoint.tokenAuth ? { Authorization: 'Bearer ${KTX_MCP_TOKEN}' } : undefined;
}
function claudeMcpEntry(endpoint: KtxMcpEndpointInfo): Record<string, unknown> {
return {
type: 'http',
url: endpoint.url,
...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}),
};
}
function cursorMcpEntry(endpoint: KtxMcpEndpointInfo): Record<string, unknown> {
return {
url: endpoint.url,
...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}),
};
}
function codexSnippet(endpoint: KtxMcpEndpointInfo): string {
if (endpoint.tokenAuth) {
return [
'Codex MCP config does not currently document HTTP headers.',
'Run KTX on loopback without token auth for Codex, or configure headers after Codex documents support.',
].join('\n');
}
return [`[mcp_servers.ktx]`, `url = "${endpoint.url}"`].join('\n');
}
function opencodeSnippet(endpoint: KtxMcpEndpointInfo): string {
return JSON.stringify(
{
mcp: {
ktx: {
type: 'remote',
url: endpoint.url,
enabled: true,
...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}),
},
},
},
null,
2,
);
}
function claudeConfigPath(projectDir: string, scope: KtxAgentScope): { path: string; jsonPath: string[] } {
const home = process.env.HOME ?? '';
if (scope === 'global') {
return { path: join(home, '.claude.json'), jsonPath: ['mcpServers', 'ktx'] };
}
if (scope === 'local') {
return { path: join(home, '.claude.json'), jsonPath: ['projects', resolve(projectDir), 'mcpServers', 'ktx'] };
}
return { path: join(resolve(projectDir), '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] };
}
function cursorConfigPath(projectDir: string, scope: KtxAgentScope): { path: string; jsonPath: string[] } {
const home = process.env.HOME ?? '';
return {
path: scope === 'global' ? join(home, '.cursor/mcp.json') : join(resolve(projectDir), '.cursor/mcp.json'),
jsonPath: ['mcpServers', 'ktx'],
};
}
async function installMcpClientConfig(input: {
projectDir: string;
target: KtxAgentTarget;
scope: KtxAgentScope;
}): Promise<KtxMcpClientInstallResult> {
const endpoint = await resolveMcpEndpoint(input.projectDir);
const entries: InstallEntry[] = [];
const snippets: string[] = [];
const notices: string[] = [];
if (!endpoint.running) {
notices.push('Run `ktx mcp start` to enable the configured KTX MCP server.');
}
if (input.target === 'claude-code') {
const config = claudeConfigPath(input.projectDir, input.scope);
await writeJsonKey(config.path, config.jsonPath, claudeMcpEntry(endpoint));
entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath });
} else if (input.target === 'cursor') {
const config = cursorConfigPath(input.projectDir, input.scope);
await writeJsonKey(config.path, config.jsonPath, cursorMcpEntry(endpoint));
entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath });
} else if (input.target === 'codex') {
snippets.push(`Codex MCP snippet for ~/.codex/config.toml:\n${codexSnippet(endpoint)}`);
} else if (input.target === 'opencode') {
const path = input.scope === 'global' ? '~/.config/opencode/opencode.json' : relative(input.projectDir, join(input.projectDir, 'opencode.json'));
snippets.push(`opencode MCP snippet for ${path}:\n${opencodeSnippet(endpoint)}`);
}
return { entries, snippets, notices };
}
export function agentInstallManifestPath(projectDir: string): string {
return join(resolve(projectDir), '.ktx/agents/install-manifest.json');
}
@ -72,6 +235,7 @@ export function plannedKtxAgentFiles(input: {
const home = process.env.HOME ?? '';
return [
{ kind: 'file', path: join(home, '.claude/skills/ktx/SKILL.md'), role: 'skill' as const },
{ kind: 'file', path: join(home, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' as const },
{ kind: 'file', path: join(home, '.claude/rules/ktx.md'), role: 'rule' as const },
];
}
@ -79,25 +243,44 @@ export function plannedKtxAgentFiles(input: {
const codexHome = process.env.CODEX_HOME ?? join(process.env.HOME ?? '', '.codex');
return [
{ kind: 'file', path: join(codexHome, 'skills/ktx/SKILL.md'), role: 'skill' as const },
{ kind: 'file', path: join(codexHome, 'skills/ktx-research/SKILL.md'), role: 'research-skill' as const },
{ kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const },
];
}
if (input.target === 'cursor' || input.target === 'opencode') {
return [];
}
throw new Error(`Global ${input.target} installation is not supported; omit --global.`);
}
const root = resolve(input.projectDir);
const cliEntries: Partial<Record<KtxAgentTarget, InstallEntry>> = {
'claude-code': { kind: 'file', path: join(root, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
codex: { kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md'), role: 'skill' },
cursor: { kind: 'file', path: join(root, '.cursor/rules/ktx.mdc') },
opencode: { kind: 'file', path: join(root, '.opencode/commands/ktx.md') },
universal: { kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md') },
const cliEntries: Partial<Record<KtxAgentTarget, InstallEntry[]>> = {
'claude-code': [
{ kind: 'file', path: join(root, '.claude/skills/ktx/SKILL.md'), role: 'skill' },
{ kind: 'file', path: join(root, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' },
],
codex: [
{ kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md'), role: 'skill' },
{ kind: 'file', path: join(root, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' },
],
cursor: [
{ kind: 'file', path: join(root, '.cursor/rules/ktx.mdc') },
{ kind: 'file', path: join(root, '.cursor/rules/ktx-research.mdc'), role: 'research-skill' },
],
opencode: [
{ kind: 'file', path: join(root, '.opencode/commands/ktx.md') },
{ kind: 'file', path: join(root, '.opencode/commands/ktx-research.md'), role: 'research-skill' },
],
universal: [
{ kind: 'file', path: join(root, '.agents/skills/ktx/SKILL.md') },
{ kind: 'file', path: join(root, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' },
],
};
const ruleEntries: Partial<Record<KtxAgentTarget, InstallEntry>> = {
'claude-code': { kind: 'file', path: join(root, '.claude/rules/ktx.md'), role: 'rule' },
codex: { kind: 'file', path: join(root, '.codex/instructions/ktx.md'), role: 'rule' },
};
return [cliEntries[input.target], ruleEntries[input.target]].filter(
return [...(cliEntries[input.target] ?? []), ruleEntries[input.target]].filter(
(entry): entry is InstallEntry => entry !== undefined,
);
}
@ -109,6 +292,12 @@ function ktxCliLauncher(): KtxCliLauncher {
};
}
async function readResearchSkillContent(): Promise<string> {
const path = fileURLToPath(new URL('./skills/research/SKILL.md', import.meta.url));
const content = await readFile(path, 'utf-8');
return content.endsWith('\n') ? content : `${content}\n`;
}
function shellQuote(value: string): string {
if (/^[A-Za-z0-9_/:=.,@%+-]+$/.test(value)) {
return value;
@ -283,16 +472,22 @@ export function formatInstallSummary(
projectDir: string,
): string {
const entriesByTarget = new Map<KtxAgentTarget, InstallEntry[]>();
let idx = 0;
for (const install of installs) {
const planned = plannedKtxAgentFiles({ projectDir, ...install });
entriesByTarget.set(install.target, entries.slice(idx, idx + planned.length));
idx += planned.length;
const plannedFilePaths = new Set(
plannedKtxAgentFiles({ projectDir, ...install })
.filter((entry) => entry.kind === 'file')
.map((entry) => entry.path),
);
entriesByTarget.set(
install.target,
entries.filter((entry) => entry.kind === 'file' && plannedFilePaths.has(entry.path)),
);
}
const fileHints: Record<string, string> = {
skill: 'teaches your agent which KTX commands to run',
rule: 'tells your agent when to use KTX',
'research-skill': 'teaches your agent the KTX MCP research workflow',
};
const lines: string[] = [];
@ -304,7 +499,7 @@ export function formatInstallSummary(
install.scope === 'global' ? entry.path : relative(projectDir, entry.path);
if (entry.kind === 'file') {
const isRule = entry.role === 'rule' || fileEntryLabels[install.target] === 'Rule installed';
const label = isRule ? 'Rule installed' : fileEntryLabels[install.target];
const label = entry.role === 'research-skill' ? 'Research skill installed' : isRule ? 'Rule installed' : fileEntryLabels[install.target];
const hint = fileHints[isRule ? 'rule' : (entry.role ?? 'skill')] ?? '';
lines.push(` + ${label} — ${hint}`);
lines.push(` ${displayPath}`);
@ -327,6 +522,8 @@ async function installTarget(input: {
const content =
entry.role === 'rule'
? ruleInstructionContent({ projectDir: input.projectDir })
: entry.role === 'research-skill'
? await readResearchSkillContent()
: cliInstructionContent({ projectDir: input.projectDir, launcher });
await mkdir(dirname(entry.path), { recursive: true });
await writeFile(entry.path, content, 'utf-8');
@ -391,11 +588,25 @@ export async function runKtxSetupAgentsStep(
const installs = targets.map((target) => ({ target, scope: args.scope, mode }));
const entries: InstallEntry[] = [];
const snippets: string[] = [];
const notices = new Set<string>();
try {
for (const install of installs) entries.push(...(await installTarget({ projectDir: args.projectDir, ...install })));
for (const install of installs) {
entries.push(...(await installTarget({ projectDir: args.projectDir, ...install })));
const mcpResult = await installMcpClientConfig({ projectDir: args.projectDir, target: install.target, scope: install.scope });
entries.push(...mcpResult.entries);
for (const snippet of mcpResult.snippets) snippets.push(snippet);
for (const notice of mcpResult.notices) notices.add(notice);
}
await writeManifest(args.projectDir, mergeManifest(args.projectDir, await readKtxAgentInstallManifest(args.projectDir), installs, entries));
await markAgentsComplete(args.projectDir);
io.stdout.write(`\nAgent integration complete\n\n${formatInstallSummary(installs, entries, args.projectDir)}\n`);
for (const snippet of snippets) {
io.stdout.write(`\n${snippet}\n`);
}
for (const notice of notices) {
io.stdout.write(`\n${notice}\n`);
}
return { status: 'ready', projectDir: args.projectDir, installs };
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);

33
packages/cli/src/setup-context.test.ts
View file

@ -198,7 +198,7 @@ describe('setup context build state', () => {
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-abc123',
status: 'running',
status: 'stale',
startedAt: '2026-05-09T10:00:00.000Z',
updatedAt: '2026-05-09T10:00:00.000Z',
primarySourceConnectionIds: ['warehouse'],
@ -207,6 +207,7 @@ describe('setup context build state', () => {
artifactPaths: [],
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-abc123'),
failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.',
sourceProgress: [
{
connectionId: 'warehouse',
@ -623,34 +624,13 @@ describe('setup context build state', () => {
expect(io.stderr()).toContain('No databases or context sources are configured for a KTX context build.');
});
it('normalizes legacy detached and paused setup context states to stale', async () => {
await writeReadyProject(tempDir);
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-old',
status: 'detached' as never,
startedAt: '2026-05-09T09:00:00.000Z',
updatedAt: '2026-05-09T09:00:00.000Z',
primarySourceConnectionIds: ['warehouse'],
contextSourceConnectionIds: [],
reportIds: [],
artifactPaths: [],
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-old'),
});
await expect(readKtxSetupContextState(tempDir)).resolves.toMatchObject({
status: 'stale',
failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.',
});
});
it('starts a fresh foreground build when a stale running state is found', async () => {
it('starts a fresh foreground build when stale state is found', async () => {
await writeReadyProject(tempDir, {
connections: { warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } } },
});
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-running',
status: 'running',
runId: 'setup-context-local-stale',
status: 'stale',
startedAt: '2026-05-09T09:00:00.000Z',
updatedAt: '2026-05-09T09:00:00.000Z',
primarySourceConnectionIds: ['warehouse'],
@ -658,7 +638,8 @@ describe('setup context build state', () => {
reportIds: [],
artifactPaths: [],
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-running'),
commands: contextBuildCommands(tempDir, 'setup-context-local-stale'),
failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.',
});
const io = makeIo();
const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 }));

35
packages/cli/src/setup-context.ts
View file

@ -27,10 +27,8 @@ import {
export type KtxSetupContextBuildStatus =
| 'not_started'
| 'running'
| 'completed'
| 'failed'
| 'interrupted'
| 'stale';
export interface KtxSetupContextCommands {
@ -84,7 +82,6 @@ export interface KtxSetupContextStepArgs {
forcePrompt?: boolean;
allowEmpty?: boolean;
prompt?: boolean;
autoWatch?: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
}
@ -154,14 +151,8 @@ function normalizeState(projectDir: string, value: unknown): KtxSetupContextStat
}
const record = value as Record<string, unknown>;
const rawStatus = typeof record.status === 'string' ? record.status : 'not_started';
const legacyActive = rawStatus === 'detached' || rawStatus === 'paused' || rawStatus === 'running';
const status: KtxSetupContextBuildStatus = legacyActive
? 'stale'
: rawStatus === 'completed' ||
rawStatus === 'failed' ||
rawStatus === 'interrupted' ||
rawStatus === 'not_started' ||
rawStatus === 'stale'
const status: KtxSetupContextBuildStatus =
rawStatus === 'completed' || rawStatus === 'failed' || rawStatus === 'not_started' || rawStatus === 'stale'
? rawStatus
: 'not_started';
const runId = typeof record.runId === 'string' && record.runId.length > 0 ? record.runId : undefined;
@ -187,11 +178,7 @@ function normalizeState(projectDir: string, value: unknown): KtxSetupContextStat
? record.retryableFailedTargets.filter((item): item is string => typeof item === 'string')
: [],
commands: contextBuildCommands(projectDir, runId),
...(typeof record.failureReason === 'string'
? { failureReason: record.failureReason }
: legacyActive
? { failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.' }
: {}),
...(typeof record.failureReason === 'string' ? { failureReason: record.failureReason } : {}),
...(normalizeSourceProgress(record.sourceProgress) ? { sourceProgress: normalizeSourceProgress(record.sourceProgress) } : {}),
};
}
@ -552,9 +539,9 @@ async function runBuild(
const now = deps.now ?? (() => new Date());
const runId = deps.runIdFactory?.() ?? runIdFactory();
const startedAt = now().toISOString();
const runningState: KtxSetupContextState = {
const incompleteState: KtxSetupContextState = {
runId,
status: 'running',
status: 'stale',
startedAt,
updatedAt: startedAt,
primarySourceConnectionIds: targets.primarySourceConnectionIds,
@ -563,8 +550,9 @@ async function runBuild(
artifactPaths: [],
retryableFailedTargets: [],
commands: contextBuildCommands(args.projectDir, runId),
failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.',
};
await writeKtxSetupContextState(args.projectDir, runningState);
await writeKtxSetupContextState(args.projectDir, incompleteState);
let lastSourceProgress: ContextBuildSourceProgressUpdate[] | undefined;
const contextBuild = deps.runContextBuild ?? runContextBuild;
@ -584,7 +572,7 @@ async function runBuild(
const resolvedDir = resolve(args.projectDir);
mkdirSync(join(resolvedDir, '.ktx', 'setup'), { recursive: true });
const progressState = normalizeState(resolvedDir, {
...runningState,
...incompleteState,
sourceProgress: sources,
updatedAt: new Date().toISOString(),
});
@ -600,7 +588,7 @@ async function runBuild(
if (buildResult.exitCode !== 0) {
const updatedAt = now().toISOString();
await writeKtxSetupContextState(args.projectDir, {
...runningState,
...incompleteState,
status: 'failed',
updatedAt,
reportIds: completedReportIds,
@ -616,7 +604,7 @@ async function runBuild(
if (!readiness.ready) {
const updatedAt = now().toISOString();
await writeKtxSetupContextState(args.projectDir, {
...runningState,
...incompleteState,
status: 'failed',
updatedAt,
reportIds: completedReportIds,
@ -635,13 +623,14 @@ async function runBuild(
await markContextComplete(project.projectDir);
const completedAt = now().toISOString();
await writeKtxSetupContextState(args.projectDir, {
...runningState,
...incompleteState,
status: 'completed',
updatedAt: completedAt,
completedAt,
reportIds: completedReportIds,
artifactPaths: completedArtifactPaths,
retryableFailedTargets: [],
failureReason: undefined,
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
});
writeSuccess(project, readiness, targets, io);

4
packages/cli/src/setup-sources.test.ts
View file

@ -1024,6 +1024,8 @@ describe('setup sources step', () => {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
syncMode: 'ALL',
selections: { collections: [], items: [] },
defaultTagNames: [],
},
},
deps: {
@ -1181,6 +1183,8 @@ describe('setup sources step', () => {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
syncMode: 'ALL',
selections: { collections: [], items: [] },
defaultTagNames: [],
},
});
const testPrompts = prompts({

2
packages/cli/src/setup-sources.ts
View file

@ -451,6 +451,8 @@ function buildMetabaseConnection(args: KtxSetupSourcesArgs): KtxProjectConnectio
databaseMappings: { [String(args.metabaseDatabaseId)]: args.sourceWarehouseConnectionId },
syncEnabled: { [String(args.metabaseDatabaseId)]: true },
syncMode: 'ALL',
selections: { collections: [], items: [] },
defaultTagNames: [],
},
};
}

41
packages/cli/src/setup.test.ts
View file

@ -7,7 +7,7 @@ import { writeKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js';
import { contextBuildCommands, readKtxSetupContextState, writeKtxSetupContextState } from './setup-context.js';
import { contextBuildCommands, writeKtxSetupContextState } from './setup-context.js';
import { runDemoTour } from './setup-demo-tour.js';
import { formatKtxSetupStatus, readKtxSetupStatus, runKtxSetup } from './setup.js';
@ -276,7 +276,7 @@ describe('setup status', () => {
});
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-abc123',
status: 'running',
status: 'stale',
startedAt: '2026-05-09T10:00:00.000Z',
updatedAt: '2026-05-09T10:01:00.000Z',
primarySourceConnectionIds: ['warehouse'],
@ -285,6 +285,7 @@ describe('setup status', () => {
artifactPaths: [],
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-abc123'),
failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.',
});
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
@ -311,7 +312,7 @@ describe('setup status', () => {
' url: env:DATABASE_URL',
' metabase:',
' driver: metabase',
' url: env:METABASE_URL',
' api_url: https://metabase.example.test',
' api_key_ref: env:METABASE_API_KEY',
' warehouse_connection_id: warehouse',
'llm:',
@ -1619,40 +1620,6 @@ describe('setup status', () => {
expect(io.stderr()).toContain('KTX context is not ready for agents.');
});
it('does not offer background watch choices from setup status', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'setup:',
' database_connection_ids:',
' - warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-stale',
status: 'running',
startedAt: '2026-05-09T09:00:00.000Z',
updatedAt: '2026-05-09T09:00:00.000Z',
primarySourceConnectionIds: ['warehouse'],
contextSourceConnectionIds: [],
reportIds: [],
artifactPaths: [],
retryableFailedTargets: [],
commands: contextBuildCommands(tempDir, 'setup-context-local-stale'),
});
const status = await readKtxSetupStatus(tempDir);
expect(status.context.status).toBe('stale');
const state = await readKtxSetupContextState(tempDir);
expect(state.status).toBe('stale');
});
it('routes a ready project menu selection to agent setup', async () => {
const calls: string[] = [];
const io = makeIo();

60
packages/cli/src/setup.ts
View file

@ -163,10 +163,7 @@ type KtxSetupFlowStatus =
| 'skipped'
| 'back'
| 'missing-input'
| 'failed'
| 'detached'
| 'paused'
| 'interrupted';
| 'failed';
export interface KtxSetupEntryMenuPromptAdapter {
select(options: { message: string; options: KtxSetupPromptOption[] }): Promise<string>;
@ -408,10 +405,6 @@ function setupContextReady(status: KtxSetupStatus): boolean {
return status.context.ready;
}
function setupContextActive(status: KtxSetupStatus): boolean {
return status.context.status === 'running';
}
function writeContextNotReadyForAgents(projectDir: string, io: KtxCliIo): void {
io.stderr.write('KTX context is not ready for agents.\n\n');
io.stderr.write(`Build context first:\n ktx setup --project-dir ${resolve(projectDir)}\n\n`);
@ -451,27 +444,22 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
args.inputMode !== 'disabled' &&
!args.agents &&
(io.stdout.isTTY === true || deps.entryMenuDeps?.prompts !== undefined);
let autoWatchActiveBuild = false;
setupLoop: while (true) {
entryAction = undefined;
if (canShowEntryMenu) {
const status = await readKtxSetupStatus(args.projectDir);
if (setupContextActive(status)) {
autoWatchActiveBuild = true;
} else {
entryAction = (await runKtxSetupEntryMenu(status, deps.entryMenuDeps)).action;
if (entryAction === 'exit') {
(deps.entryMenuDeps?.prompts ?? createEntryMenuPromptAdapter()).cancel('Setup cancelled.');
return 0;
}
if (entryAction === 'status') {
io.stdout.write(formatKtxSetupStatus(status));
return 0;
}
if (entryAction === 'demo') {
return await runKtxSetupDemoFromEntryMenu(args, io, deps);
}
entryAction = (await runKtxSetupEntryMenu(status, deps.entryMenuDeps)).action;
if (entryAction === 'exit') {
(deps.entryMenuDeps?.prompts ?? createEntryMenuPromptAdapter()).cancel('Setup cancelled.');
return 0;
}
if (entryAction === 'status') {
io.stdout.write(formatKtxSetupStatus(status));
return 0;
}
if (entryAction === 'demo') {
return await runKtxSetupDemoFromEntryMenu(args, io, deps);
}
}
@ -500,30 +488,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
const currentStatus = await readKtxSetupStatus(projectResult.projectDir);
let readyAction: string | undefined;
if (args.inputMode !== 'disabled' && !agentsRequested && setupContextActive(currentStatus)) {
const contextRunner =
deps.context ?? ((contextArgs, contextIo) => runKtxSetupContextStep(contextArgs, contextIo, deps.contextDeps));
const contextResult = await contextRunner(
{
projectDir: projectResult.projectDir,
inputMode: args.inputMode,
allowEmpty: true,
...(autoWatchActiveBuild ? { autoWatch: true } : {}),
},
io,
);
autoWatchActiveBuild = false;
if (contextResult.status === 'back') {
continue;
}
if (contextResult.status === 'failed' || contextResult.status === 'missing-input') {
return 1;
}
if (contextResult.status !== 'ready') {
return 0;
}
}
if (args.inputMode !== 'disabled' && !agentsRequested) {
if (isKtxSetupReady(currentStatus)) {
readyAction = (await runKtxSetupReadyChangeMenu(currentStatus, deps.readyMenuDeps)).action;

49
packages/cli/src/skills/research/SKILL.md Normal file
View file

@ -0,0 +1,49 @@
---
name: ktx-research
description: Use when answering a question that needs data from a KTX-connected database - investigating, analyzing, "how many", "show me", "what's the breakdown of", finding records by value, exploring tables, comparing periods, or any data-investigation request. Triggers even when the user does not say "research"; if the answer requires querying a configured KTX connection, this skill applies.
---
# KTX Research Workflow
You have access to KTX MCP tools for investigating data. Follow this workflow.
<workflow>
1. **Discover** - call `discover_data` first to see what exists across wiki, semantic-layer sources, and raw tables. Returns refs only.
2. **Inspect top hits in parallel** - for each promising ref:
- `kind: 'wiki'` -> `wiki_read`
- `kind: 'sl_source'`, `kind: 'sl_measure'`, or `kind: 'sl_dimension'` -> `sl_read_source`
- `kind: 'table'` or `kind: 'column'` -> `entity_details`
3. **Resolve literals** - if the user named a value such as "Acme Corp" or "status=shipped", call `dictionary_search` to find which column holds it.
4. **Query** -
- Prefer `sl_query` when the semantic layer covers the question.
- Use `sql_execution` only for questions the semantic layer does not cover.
5. **Capture learnings** - at the end of the turn, call `memory_capture` so future turns benefit. Skip when the answer carries no durable knowledge.
</workflow>
<rules>
- Always run `discover_data` before writing SQL. Do not guess table names.
- Prefer the semantic layer over raw SQL when both can answer the question; measures are the source of truth.
- Read entity details before writing SQL against an unfamiliar table. Do not assume column names.
- Treat `sql_execution` as read-only. Writes are rejected by the server.
- Validate value mentions with `dictionary_search` instead of guessing case or spelling. Treat a `dictionary_search` miss as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent.
</rules>
<examples>
**Input:** "How many orders did Acme Corp place last month?"
**Workflow:**
1. `dictionary_search({ values: ["Acme Corp"] })` finds `customers.name`.
2. `discover_data({ query: "orders customer monthly" })` finds an orders semantic-layer source.
3. `sl_read_source({ connectionId: "warehouse", sourceName: "orders_facts" })` confirms the source grain, measures, and dimensions.
4. `sl_query({ connectionId: "warehouse", measures: ["order_count"], filters: ["customer_name = 'Acme Corp'"] })` answers through the semantic layer.
5. `memory_capture({ userMessage, assistantMessage })` captures the durable finding.
---
**Input:** "What columns does the events table have?"
**Workflow:**
1. `discover_data({ query: "events table" })` returns a `table` ref.
2. `entity_details({ connectionId: "warehouse", entities: [{ table: "analytics.events" }] })` returns columns, types, and foreign keys.
3. Answer directly. No query is needed.
</examples>

6
packages/cli/src/sl.ts
View file

@ -213,7 +213,11 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx
if (!source) {
throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`);
}
const result = await validateLocalSlSource(source.yaml, { project, connectionId: args.connectionId });
const result = await validateLocalSlSource(source.yaml, {
project,
connectionId: args.connectionId,
sourceName: args.sourceName,
});
if (!result.valid) {
for (const error of result.errors) {
io.stderr.write(`${error}\n`);

9
packages/cli/src/status-project.ts
View file

@ -16,6 +16,7 @@ import {
red,
yellow,
} from './io/symbols.js';
import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js';
type ProjectStatusLevel = 'ok' | 'warn' | 'fail';
type ProjectVerdict = 'ready' | 'partial' | 'blocked';
@ -76,6 +77,8 @@ interface WarningItem {
fix?: string;
}
const PROJECT_READY_COMMANDS = KTX_NEXT_STEP_DIRECT_COMMANDS.map((step) => step.command);
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
@ -139,7 +142,7 @@ function buildLlmStatus(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): Ll
backend,
model,
status: 'fail',
detail: 'no LLM configured — ktx ask will not work',
detail: 'no LLM configured; research agent will not run',
fix: 'Run: ktx setup (choose an LLM provider)',
};
}
@ -578,7 +581,7 @@ function buildVerdict(
if (llm.status === 'fail') {
return {
verdict: 'blocked',
reason: 'LLM not configured — `ktx ask` will not work.',
reason: 'LLM not configured; research agent will not run.',
nextActions: ['ktx setup'],
};
}
@ -612,7 +615,7 @@ function buildVerdict(
return {
verdict: 'ready',
reason: 'Ready.',
nextActions: ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'],
nextActions: [...PROJECT_READY_COMMANDS],
};
}

3
packages/cli/tsconfig.json
View file

@ -3,7 +3,8 @@
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"jsx": "react-jsx"
"jsx": "react-jsx",
"tsBuildInfoFile": "./node_modules/.cache/tsc.tsbuildinfo"
},
"include": ["src/**/*.ts", "src/**/*.tsx"],
"exclude": ["dist", "node_modules"]

2
packages/connector-bigquery/src/connector.test.ts
View file

@ -100,7 +100,7 @@ const connection = {
dataset_id: 'analytics',
credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }),
location: 'US',
};
} as const;
describe('KtxBigQueryScanConnector', () => {
it('resolves configuration safely', () => {

1
packages/context/package.json
View file

@ -153,6 +153,7 @@
"@types/node": "^25.7.0",
"@types/pg": "^8.20.0",
"@vitest/coverage-v8": "^4.1.6",
"ajv": "8.20.0",
"typescript": "^6.0.3",
"vitest": "^4.1.6"
},

8
packages/context/skills/_shared/identifier-verification.md
View file

@ -8,16 +8,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

16
packages/context/skills/dbt_ingest/SKILL.md
View file

@ -14,14 +14,14 @@ Use this skill for **uploaded** dbt projects (`dbt_project.yml` at stage root, `
|-----|--------|--------|
| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `discover_data` / `entity_details`) | One SL source per physical table; model name may differ from DB name - resolve with `read_raw_file` + warehouse context. |
| `sources:` → `tables:` | Same as models; use `identifier` when present instead of logical `name`. | Schema + name must match how the connection sees tables. |
| Column `description` | `descriptions.user` or merged `descriptions` map on the column | Do not overwrite `dbt` description keys from sync. |
| Column `description` | `column_overrides[].descriptions.user` on the overlay | Do not overwrite `dbt` description keys from sync. |
| `data_tests: not_null` / `unique` | Short hint in column `descriptions` or notes: “dbt: not null”, “dbt: unique” | Full structured metadata lands in manifest via **sync**; the skill keeps bundle-time SL text useful for the agent. |
| `accepted_values` | Add a **brief** line in the column description: allowed values (truncate long lists) | Also mention enum-like use in `discover_data` / filters. |
| `relationships` | Add or confirm `joins:` on the overlay **only** when `to` resolves to a real table via `read_raw_file` + `discover_data` / `entity_details` | If the ref cannot be resolved, capture the intent in a wiki page instead. |
## Physical schema grounding
dbt YAML is documentation and test metadata; it is not permission to invent physical columns. Before writing any table-backed SL source, confirm the real warehouse shape with `discover_data`, `sl_discover`, or `entity_details` and use only confirmed column names in `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr`.
dbt YAML is documentation and test metadata; it is not permission to invent physical columns. Before writing any table-backed SL source, confirm the real warehouse shape with `discover_data`, `sl_discover`, or `entity_details` and use only confirmed column names in `column_overrides:`, computed-only `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr`.
For dbt context-source ingest, the dbt connection is usually not the warehouse connection. Call `sl_discover` without `connectionId` first, then write overlays to the connection that owns the matching manifest-backed source (for example `postgres-warehouse`), not to the dbt connection (for example `dbt-main`). If no matching manifest-backed source is visible on any warehouse connection, do not call `sl_write_source`; record `emit_unmapped_fallback` and keep the fact wiki-only.
@ -41,16 +41,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.
@ -61,7 +61,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
## 1.1 test hints (descriptions / meta)
When YAML shows `accepted_values` or `not_null`, add **short** hints into `columns[].descriptions` (e.g. under `user`) or freeform column notes so chat and validation see intent before the next git sync refreshes `constraints` / `enum_values` in `_schema`. Keep hints under a few words when possible.
When YAML shows `accepted_values` or `not_null`, add **short** hints into `column_overrides[].descriptions` (for example under `user`) or freeform column notes so chat and validation see intent before the next git sync refreshes `constraints` / `enum_values` in `_schema`. Keep hints under a few words when possible.
## Overlap with MetricFlow
@ -71,6 +71,6 @@ If the same bundle also has MetricFlow `semantic_models:` / `metrics:`, the **`m
- Do not run `dbt` CLI or assume `target/` / `manifest.json` exists in the upload.
- Do not invent column names, grain keys, or measure expressions from dbt model names, descriptions, tests, or common naming patterns.
- Do not write `columns:`, `grain:`, or `measures:` for a dbt model unless those exact column names are confirmed by dbt YAML columns or warehouse schema discovery.
- Do not write computed `columns:`, `column_overrides:`, `grain:`, or `measures:` for a dbt model unless those exact column names are confirmed by dbt YAML columns or warehouse schema discovery.
- Do not invent joins from `relationships` tests if the target model/table is not found in SL or the warehouse.
- Do not read `peerFileIndex` paths - use `read_raw_file` only on `rawFiles` and `dependencyPaths` from the WorkUnit.

8
packages/context/skills/historic_sql_patterns/SKILL.md
View file

@ -31,16 +31,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

8
packages/context/skills/historic_sql_table_digest/SKILL.md
View file

@ -27,16 +27,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

8
packages/context/skills/live_database_ingest/SKILL.md
View file

@ -37,16 +37,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

8
packages/context/skills/looker_ingest/SKILL.md
View file

@ -34,16 +34,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

19
packages/context/skills/lookml_ingest/SKILL.md
View file

@ -12,7 +12,7 @@ LookML views map to SL sources, `measure:` to measures, `explore: { join: }` to
| LookML | KTX form | Notes |
|---|---|---|
| `view: X { sql_table_name: …; measure:/dimension:/join: }` | **Overlay** at `<connId>/X.yaml` with `measures`, `columns` (computed), `joins`, `segments` | Manifest-backed; inherit grain/columns |
| `view: X { sql_table_name: …; measure:/dimension:/join: }` | **Overlay** at `<connId>/X.yaml` with `measures`, computed-only `columns`, `column_overrides`, `joins`, `segments` | Manifest-backed; inherit grain/columns |
| `view: X { derived_table: { sql: … } }` | **Standalone** with top-level `sql:`, explicit `grain:` + `columns:` | No manifest entry exists |
| `view: X { sql_always_where: <p> }` | **Standalone** with `sql: SELECT * FROM <base> WHERE <p>` | Enforcement, not opt-in |
| `explore: { join: Y { sql_on: …; relationship: … } }` | `joins:` entry `{ to: Y, on: "<local> = Y.<col>", relationship: … }` | On the overlay or standalone |
@ -64,16 +64,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.
@ -85,11 +85,11 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
**Required flow before writing any overlay or standalone**:
1. Call `sl_discover({ query: "<tableName>" })` for each base table you're about to touch. That returns the real columns.
2. If the table isn't in the manifest, use the warehouse `connectionName`
2. If the table isn't in the manifest, use the warehouse `connectionId`
returned by `discover_data` or the target connection chosen from
`sl_discover`, then call a dialect-appropriate SQL probe with that
connection name, for example:
`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`.
connection id, for example:
`sql_execution({connectionId: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`.
Replace `warehouse`, `analytics`, and `orders` with the verified connection,
schema or dataset, and table from the WorkUnit evidence.
3. Use only those names in `sql:`, `columns:`, and `grain:`. Map each `dimension_group` to ONE `{ name: <physical_col>, type: time, role: time }` entry - never one per timeframe.
@ -136,7 +136,8 @@ KTX overlay at `<connId>/fct_labs.yaml`:
```yaml
name: fct_labs
description: "Lab-order fact table. One row per lab order event."
descriptions:
user: "Lab-order fact table. One row per lab order event."
columns:
- name: is_byol
type: boolean

12
packages/context/skills/metabase_ingest/SKILL.md
View file

@ -57,16 +57,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.
@ -79,7 +79,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
For each card:
1. Analyze `resolvedSql` + `resultMetadata`: identify base tables, aggregations, joins, filters, column types.
2. **REQUIRED before any write**: call `sl_discover` for every candidate target source name. The response tells you whether the name is manifest-backed (`Type: table` or `Type: sql`). For manifest-backed names you MUST use the overlay shape (`name:` + `measures:`/`segments:`/`description:` only - no `sql:`, `table:`, `grain:`, or `columns:`); the tool will reject a standalone write and you'll have wasted the call. If `sl_discover` returns nothing for the name, you can write a standalone source. Also call `sl_read_source` on existing sources you intend to extend so you don't duplicate measures.
2. **REQUIRED before any write**: call `sl_discover` for every candidate target source name. The response tells you whether the name is manifest-backed (`Type: table` or `Type: sql`). For manifest-backed names you MUST use the overlay shape (`name:` plus overlay fields such as `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`, `column_overrides:`, and computed-only `columns:` entries with `expr` + `type`; no `sql:`, `table:`, `grain:`, or base-table `columns:`); the tool will reject a standalone write and you'll have wasted the call. If `sl_discover` returns nothing for the name, you can write a standalone source. Also call `sl_read_source` on existing sources you intend to extend so you don't duplicate measures.
3. Include `rawPaths: ["cards/<id>.json"]` on every `sl_write_source`, `sl_edit_source`, and `wiki_write` call. If one artifact generalizes multiple near-duplicate cards, include each contributing card path and no unrelated cards.
4. Decide:
- Simple aggregation on a table that already has a source → `sl_edit_source` to add a measure.
@ -98,7 +98,7 @@ measures:
expr: "<expression>"
```
Overlay shape: `name:` plus any of `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`. Never include `sql:`, `table:`, `grain:`, or `columns:` on a manifest-backed name - those would shadow the manifest's schema and drop its joins. Overlay `joins:` are merged additively with the manifest's joins (deduped by `to` + `on`); use `disable_joins: ["<on-clause>"]` to suppress a specific manifest join. After the overlay exists, use `sl_edit_source` for further tweaks. See `sl_capture` skill for the canonical overlay rule.
Overlay shape: `name:` plus any of `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`, `exclude_columns:`, `column_overrides:`, or computed-only `columns:` entries with `expr` + `type`. Never include `sql:`, `table:`, `grain:`, or base-table `columns:` on a manifest-backed name — those would shadow the manifest's schema and drop its joins. Use `column_overrides:` for inherited column descriptions. Overlay `joins:` are merged additively with the manifest's joins (deduped by `to` + `on`); use `disable_joins: ["<on-clause>"]` to suppress a specific manifest join. After the overlay exists, use `sl_edit_source` for further tweaks. See `sl_capture` skill for the canonical overlay rule.
**Join discovery:** When your card's SQL references warehouse tables (e.g. in `FROM` or `JOIN` clauses), call `sl_discover({ query: '<table>' })` before writing. The matching manifest entry's `name` is the value you use in `joins: [- to: <name>]` only when the card output exposes a local key that matches the target source grain (for example `account_id = mart_account_segments.account_id`). Do not declare a KTX join just because the card SQL joins that table internally. If the output only exposes display fields such as `account_name`, keep the SQL source self-contained or project the key before adding the join. Use `many_to_one` for FK-to-dimension joins, `one_to_many` for the reverse.

16
packages/context/skills/metricflow_ingest/SKILL.md
View file

@ -12,7 +12,7 @@ A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to
| MetricFlow | KTX form | Notes |
|---|---|---|
| `semantic_model: X { model: ref('t') }` with measures + dimensions | **Overlay** at `<connId>/X.yaml` with `measures`, `columns` (computed), `joins` | The `model:` ref resolves to a manifest table. |
| `semantic_model: X { model: ref('t') }` with measures + dimensions | **Overlay** at `<connId>/X.yaml` with `measures`, computed-only `columns`, `column_overrides`, `joins` | The `model:` ref resolves to a manifest table. |
| `semantic_model: X { model: source('s','t') }` | **Overlay** at `<connId>/X.yaml` over table `t`. | Same shape; `source()` still resolves to a physical table. |
| `semantic_model: X { model: <literal> }` with no manifest entry | **Standalone** with explicit `sql:`, `grain:`, `columns:` | Happens when the dbt manifest isn't available. |
| `semantic_model: Y { extends: X }` | **Merge** Y's measures/dimensions/entities into X's overlay, or write a single overlay named for the most-derived child (Y) containing both X's and Y's primitives | Do not emit a second overlay for X - flatten. |
@ -42,16 +42,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.
@ -82,9 +82,9 @@ The `model:` field on a semantic_model is a string like `ref('table_name')`, `so
If `sl_discover` errors because no such table exists, use `discover_data` and
`entity_details` to find the warehouse target. If a SQL probe is still needed,
call `sql_execution` with the same warehouse connection name, for example:
`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`.
**Never invent column names** - every column in `columns:`, `grain:`, and
call `sql_execution` with the same warehouse connection id, for example:
`sql_execution({connectionId: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`.
**Never invent column names** - every column in computed `columns:`, `column_overrides:`, `grain:`, and
`sql:` must be sourced from raw files, `entity_details`, or a successful SQL
probe.

8
packages/context/skills/notion_synthesize/SKILL.md
View file

@ -79,16 +79,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

7
packages/context/skills/sl/SKILL.md
View file

@ -39,6 +39,10 @@ columns: # computed dimensions only
- name: is_large_order
type: boolean
expr: "amount > 1000"
column_overrides: # metadata patches for inherited columns
- name: status
descriptions:
user: "Order lifecycle status."
segments:
- name: paid_non_refunded
expr: "is_paid = true AND is_refunded = false"
@ -51,6 +55,7 @@ joins:
Rules:
- Do **not** repeat base-table columns, grain, `table`, or `source_type` in an overlay - those are inherited.
- Overlay columns MUST be computed (`expr` + `type`).
- Use `column_overrides` to add descriptions or metadata to inherited manifest columns. Do not put `type` or `expr` in `column_overrides`.
- `exclude_columns` hides specific manifest columns; `disable_joins` suppresses specific auto-detected joins.
### Standalone table sources
@ -110,7 +115,7 @@ An SQL source is a one-shot answer: the aggregation is frozen, callers cannot re
### Columns
Every standalone column requires `name` and `type`. Overlays have computed columns only.
Every standalone column requires `name` and `type`. Overlays have computed columns in `columns:` and manifest column metadata patches in `column_overrides:`.
- `type`: one of `string`, `number`, `boolean`, `time`. Map LookML `date`/`datetime`/`timestamp` → `time`. Map LookML `yesno` → `boolean`.
- `role` (optional): `time` enables time-granularity queries (month, week, day). `default` is the implicit fallback.

40
packages/context/skills/sl_capture/SKILL.md
View file

@ -100,7 +100,33 @@ measures:
**Extract repeated filter bundles into named segments.** If the same predicate appears on multiple measures of the same source, lift it to a `segments[]` entry and have each measure reference it. One edit updates every measure that depends on it.
**Never write a standalone file on a manifest-backed name.** If `sl_discover({ query: "<table-or-source-name>" })` finds an existing schema for that name, you MUST write an overlay (`name:` + `measures:`/`segments:`/`descriptions:` only - no `sql:`, `table:`, `grain:`, `columns:`, `joins:`). A standalone with `sql:` or `table:` on a manifest-backed name clobbers the inherited columns and joins; `sl_write_source` and `sl_validate` both reject this shape with a clear fix hint. Always run `sl_discover` before your first write on any existing name.
**Never write a standalone file on a manifest-backed name.** If `sl_discover({ query: "<table-or-source-name>" })` finds an existing schema for that name, you MUST write an overlay. A standalone with `sql:` or `table:` on a manifest-backed name clobbers the inherited columns and joins; `sl_write_source` and `sl_validate` both reject this shape with a clear fix hint. Always run `sl_discover` before your first write on any existing name.
Overlay before/after examples:
```yaml
# Wrong: patches an inherited manifest column through columns:
name: fct_orders
columns:
- name: status
descriptions:
user: "Order lifecycle status."
```
```yaml
# Right: patch inherited columns with column_overrides:
name: fct_orders
column_overrides:
- name: status
descriptions:
user: "Order lifecycle status."
columns:
- name: is_large_order
type: boolean
expr: "amount > 1000"
```
Overlay YAML may include `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`, `exclude_columns:`, `column_overrides:`, and computed-only `columns:` entries with `expr` and `type`. Do not include `sql:`, `table:`, `grain:`, or base-table `columns:`.
**Prefer overlay decomposition over standalone SQL sources.** Before reaching for `source_type: sql`, check whether the metric decomposes into measures on existing overlays (including cross-source derived measures). Use `source_type: sql` only when:
- The metric requires per-user/per-entity derivation that cannot be expressed as a single `expr` (e.g., `EXISTS` over a time-windowed subset), OR
@ -188,16 +214,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.
@ -213,7 +239,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
3. `sl_read_source({ connectionId, sourceName })` - read the raw YAML before editing.
4. For modifications: `sl_edit_source({ connectionId, sourceName, yaml_edits: [{ oldText, newText, reason }] })` with exact-string replacements. `oldText` must match exactly and be unique in the file.
5. For new sources or full rewrites: `sl_write_source({ connectionId, sourceName, source })` with the full structured source definition.
6. For join discovery: use `sql_execution({connectionName: "warehouse", sql: "SELECT count(*) FROM public.orders o JOIN public.customers c ON c.id = o.customer_id LIMIT 20"})` with the target warehouse connection name and dialect-correct table names to verify the join key exists in both tables and assess cardinality before declaring the join.
6. For join discovery: use `sql_execution({connectionId: "warehouse", sql: "SELECT count(*) FROM public.orders o JOIN public.customers c ON c.id = o.customer_id LIMIT 20"})` with the target warehouse connection id and dialect-correct table names to verify the join key exists in both tables and assess cardinality before declaring the join.
7. Cross-reference knowledge: author the edge once on the **wiki** side via `sl_refs: [source_name]` in the page's front-matter. The reverse edge (wiki pages that cite an SL source) is derived automatically by the reconciler - do not add a `knowledge_refs:` field to SL YAMLs.
8. `sl_validate` - run after writing or editing to surface schema issues, duplicate measure names, and cross-source validation errors. Read-only; the writes are already committed (the squash-at-end flow will collapse them into one commit).
@ -289,7 +315,7 @@ Prior turn: user asked to correlate LTV with protocol count; assistant joined `f
sl_read_source({ connectionId: "warehouse", sourceName: "fct_orders" })
→ no joins section yet
sql_execution({
connectionName: "warehouse",
connectionId: "warehouse",
sql: "SELECT COUNT(*), COUNT(DISTINCT a.admin_user_id) FROM public.fct_orders a JOIN public.fct_mau_multiprotocol b ON a.admin_user_id = b.admin_user_id LIMIT 1"
})
→ confirms cardinality (many orders per MAU row = many_to_one)

8
packages/context/skills/wiki_capture/SKILL.md
View file

@ -60,16 +60,16 @@ Before writing a wiki page or SL source on any topic:
Before emitting any `schema.table` or `schema.table.column` into a wiki body,
SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
2. `entity_details({connectionName, targets: [{display: "<identifier>"}]})` -
2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
confirm the identifier resolves; inspect native types, FK/PK, and
sampleValues.
3. For literal values from the source, such as status codes or plan tiers,
check whether they appear in `entity_details` sampleValues for the relevant
column. If sampleValues is short or the sample may have missed real values,
run a `sql_execution` probe with the same warehouse connection name:
`sql_execution({connectionName, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
run a `sql_execution` probe with the same warehouse connection id:
`sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
4. If the candidate identifier still does not resolve, do one of:
- Use `sql_execution({connectionName, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
- Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
If it errors, the identifier is fictional.
- Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
citing the exact raw path that mentioned it.

12
packages/context/src/agent/agent-runner.service.ts
View file

@ -1,4 +1,4 @@
import { KtxMessageBuilder, type KtxLlmProvider, type KtxModelRole } from '@ktx/llm';
import { KtxMessageBuilder, splitKtxSystemMessages, type KtxLlmProvider, type KtxModelRole } from '@ktx/llm';
import { generateText, stepCountIs, type TelemetrySettings, type Tool } from 'ai';
import { noopLogger, type KtxLogger } from '../core/index.js';
import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from '../llm/index.js';
@ -36,14 +36,6 @@ export interface AgentRunnerServiceDeps {
logger?: KtxLogger;
}
function splitSystemPromptMessages(messages: ReturnType<KtxMessageBuilder['wrapSimple']>['messages']) {
const systemMessages = messages.filter((message) => message.role === 'system');
return {
system: systemMessages.length === 0 ? undefined : systemMessages.length === 1 ? systemMessages[0] : systemMessages,
messages: messages.filter((message) => message.role !== 'system'),
};
}
export class AgentRunnerService {
private readonly logger: KtxLogger;
@ -62,7 +54,7 @@ export class AgentRunnerService {
tools: params.toolSet,
model,
});
const promptMessages = splitSystemPromptMessages(built.messages);
const promptMessages = splitKtxSystemMessages(built.messages);
await this.deps.debugRequestRecorder?.record(
summarizeKtxLlmDebugRequest({

12
packages/context/src/connections/local-warehouse-descriptor.test.ts
View file

@ -36,7 +36,13 @@ describe('localConnectionToWarehouseDescriptor', () => {
});
it('returns null for non-warehouse adapters', () => {
expect(localConnectionToWarehouseDescriptor('looker', { driver: 'looker' })).toBeNull();
expect(
localConnectionToWarehouseDescriptor('looker', {
driver: 'looker',
base_url: 'https://looker.example.com',
client_id: 'client',
}),
).toBeNull();
});
});
@ -48,7 +54,9 @@ describe('local connection info helpers', () => {
});
it('keeps non-warehouse adapter labels for display-only local connection surfaces', () => {
expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase' })).toBe('metabase');
expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase', api_url: 'https://metabase.example.com' })).toBe(
'metabase',
);
expect(localConnectionTypeForConfig('missing-driver', {} as never)).toBe('unknown');
});

17
packages/context/src/connections/notion-config.ts
View file

@ -13,7 +13,20 @@ export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig {
type RawKtxNotionConnectionConfig = Extract<KtxProjectConnectionConfig, { driver: 'notion' }>;
export type KtxNotionConnectionConfig = Omit<
RawKtxNotionConnectionConfig,
| 'auth_token'
| 'auth_token_ref'
| 'crawl_mode'
| 'root_page_ids'
| 'root_database_ids'
| 'root_data_source_ids'
| 'max_pages_per_run'
| 'max_knowledge_creates_per_run'
| 'max_knowledge_updates_per_run'
> & {
driver: 'notion';
auth_token: string | null;
auth_token_ref: string | null;
@ -24,7 +37,7 @@ export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig {
max_pages_per_run: number;
max_knowledge_creates_per_run: number;
max_knowledge_updates_per_run: number;
}
};
export interface RedactedKtxNotionConnectionConfig {
driver: 'notion';

6
packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts
View file

@ -18,6 +18,9 @@ const sqlAnalysis: SqlAnalysisPort = {
async analyzeBatch() {
return new Map();
},
async validateReadOnly() {
return { ok: true };
},
};
const reader: HistoricSqlReader = {
@ -79,6 +82,9 @@ describe('HistoricSqlSourceAdapter', () => {
],
]);
},
async validateReadOnly() {
return { ok: true };
},
};
const adapter = new HistoricSqlSourceAdapter({
sqlAnalysis: batchSqlAnalysis,

1
packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts
View file

@ -159,6 +159,7 @@ function acceptanceSqlAnalysis(): SqlAnalysisPort {
);
},
),
validateReadOnly: vi.fn(async () => ({ ok: true })),
};
}

4
packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts
View file

@ -83,6 +83,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
],
['bad-parse', { tablesTouched: [], columnsByClause: {}, error: 'parse failed' }],
])),
validateReadOnly: vi.fn(async () => ({ ok: true })),
};
await stageHistoricSqlAggregatedSnapshot({
@ -207,6 +208,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
},
],
])),
validateReadOnly: vi.fn(async () => ({ ok: true })),
};
await stageHistoricSqlAggregatedSnapshot({
@ -283,6 +285,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
},
],
])),
validateReadOnly: vi.fn(async () => ({ ok: true })),
};
await stageHistoricSqlAggregatedSnapshot({
@ -403,6 +406,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
},
],
])),
validateReadOnly: vi.fn(async () => ({ ok: true })),
};
await stageHistoricSqlAggregatedSnapshot({

6
packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts
View file

@ -3,6 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { buildDefaultKtxProjectConfig } from '../../../project/index.js';
import { connectionConfigSchema } from '../../../project/driver-schemas.js';
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
describe('Metabase YAML source state and discovery cache', () => {
@ -23,10 +24,11 @@ describe('Metabase YAML source state and discovery cache', () => {
config: {
...buildDefaultKtxProjectConfig(),
connections: {
'prod-metabase': {
'prod-metabase': connectionConfigSchema.parse({
driver: 'metabase',
api_url: 'https://metabase.example.com',
mappings,
},
}),
},
},
};

12
packages/context/src/ingest/adapters/metricflow/import-semantic-models.test.ts
View file

@ -38,7 +38,7 @@ describe('importMetricflowSemanticModels', () => {
const scoped = {
getManifestEntry: vi.fn().mockResolvedValue(null),
isManifestBacked: vi.fn().mockResolvedValue(false),
loadAllSources: vi.fn().mockResolvedValue([]),
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
loadSource: vi.fn().mockResolvedValue(null),
writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
};
@ -104,7 +104,7 @@ describe('importMetricflowSemanticModels', () => {
const scoped = {
getManifestEntry: vi.fn().mockResolvedValue(null),
isManifestBacked: vi.fn().mockResolvedValue(false),
loadAllSources: vi.fn().mockResolvedValue([]),
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
loadSource: vi.fn().mockImplementation((connectionId: string, sourceName: string) =>
Promise.resolve(sourceName === 'orders' ? { name: 'orders' } : null),
),
@ -139,7 +139,7 @@ describe('importMetricflowSemanticModels', () => {
const scoped = {
getManifestEntry: vi.fn().mockResolvedValue(null),
isManifestBacked: vi.fn().mockResolvedValue(false),
loadAllSources: vi.fn().mockResolvedValue([]),
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
loadSource: vi.fn().mockResolvedValue(null),
writeSource: vi.fn().mockRejectedValueOnce(new Error('cannot write orders')).mockResolvedValue({ warnings: [] }),
};
@ -190,7 +190,7 @@ describe('importMetricflowSemanticModels', () => {
isManifestBacked: vi.fn().mockImplementation(async (_connectionId: string, sourceName: string) => {
return sourceName === 'orders';
}),
loadAllSources: vi.fn().mockResolvedValue([]),
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
loadSource: vi.fn().mockResolvedValue(null),
writeSource: vi.fn().mockImplementation(async (_connectionId: string, source: (typeof written)[number]) => {
written.push(source);
@ -268,7 +268,7 @@ describe('importMetricflowSemanticModels', () => {
isManifestBacked: vi.fn().mockImplementation(async (_connectionId: string, sourceName: string) => {
return sourceName === 'orders';
}),
loadAllSources: vi.fn().mockResolvedValue([]),
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
loadSource: vi.fn().mockResolvedValue(null),
writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
};
@ -311,7 +311,7 @@ describe('importMetricflowSemanticModels', () => {
const scoped = {
getManifestEntry: vi.fn().mockResolvedValue(null),
isManifestBacked: vi.fn().mockResolvedValue(false),
loadAllSources: vi.fn().mockResolvedValue([]),
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
loadSource: vi.fn().mockResolvedValue(null),
writeSource: vi
.fn()

2
packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts
View file

@ -71,7 +71,7 @@ export async function importMetricflowSemanticModels(
let crossModelSourcesCreated = 0;
const preexistingSourceNames = new Set(
(await semanticLayerService.loadAllSources(input.connectionId)).map((source) => source.name),
(await semanticLayerService.loadAllSources(input.connectionId)).sources.map((source) => source.name),
);
const modelContexts: MetricflowSemanticModelImportContext[] = [];
const sourceNameByModelRef = new Map<string, string>();

19
packages/context/src/ingest/ingest-bundle.runner.test.ts
View file

@ -187,7 +187,10 @@ const makeDeps = () => {
loadAllSources: vi
.fn()
.mockImplementation((connectionId: string) =>
Promise.resolve(connectionId === 'warehouse-2' ? [{ name: 'looker__orders' }] : []),
Promise.resolve({
sources: connectionId === 'warehouse-2' ? [{ name: 'looker__orders' }] : [],
loadErrors: [],
}),
),
};
const slSearchService = {
@ -1347,7 +1350,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
frontmatter: { sl_refs: ['looker__b2b__sales_pipeline.arr'] },
});
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
Promise.resolve([{ name: `${connectionId}_source` }]),
Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }),
);
deps.agentRunner.runLoop.mockImplementation(async (params: any) => {
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
@ -1447,7 +1450,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
});
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
Promise.resolve([{ name: `${connectionId}_source` }]),
Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }),
);
const postProcessor = {
run: vi.fn().mockResolvedValue({
@ -1631,7 +1634,10 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
const deps = makeDeps();
deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['postgres-warehouse']);
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
Promise.resolve(connectionId === 'postgres-warehouse' ? [{ name: 'stg_accounts' }] : []),
Promise.resolve({
sources: connectionId === 'postgres-warehouse' ? [{ name: 'stg_accounts' }] : [],
loadErrors: [],
}),
);
const runner = buildRunner(deps);
@ -1659,7 +1665,10 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
it('does not resolve qualified fallback table refs by source name alone', async () => {
const deps = makeDeps();
deps.semanticLayerService.loadAllSources.mockResolvedValue([{ name: 'orders', table: 'sales.orders' }]);
deps.semanticLayerService.loadAllSources.mockResolvedValue({
sources: [{ name: 'orders', table: 'sales.orders' }],
loadErrors: [],
});
const runner = buildRunner(deps);
await expect(

6
packages/context/src/ingest/ingest-bundle.runner.ts
View file

@ -300,7 +300,7 @@ export class IngestBundleRunner {
const blocks = await Promise.all(
connectionIds.map(async (connectionId) => {
try {
const sources = await this.deps.semanticLayerService.loadAllSources(connectionId);
const { sources } = await this.deps.semanticLayerService.loadAllSources(connectionId);
const names = sources.map((source) => source.name).sort((left, right) => left.localeCompare(right));
const body = names.length > 0 ? names.join('\n') : '(no sources yet)';
return `## ${connectionId}\n${body}`;
@ -329,7 +329,7 @@ export class IngestBundleRunner {
): Promise<boolean> {
for (const connectionId of connectionIds) {
try {
const sources = await semanticLayerService.loadAllSources(connectionId);
const { sources } = await semanticLayerService.loadAllSources(connectionId);
if (sources.some((source) => semanticSourceMatchesTableRef(source, tableRef))) {
return true;
}
@ -1211,7 +1211,7 @@ export class IngestBundleRunner {
].sort();
for (const connectionId of touchedConnections) {
try {
const allSources = await this.deps.semanticLayerService.loadAllSources(connectionId);
const { sources: allSources } = await this.deps.semanticLayerService.loadAllSources(connectionId);
await this.deps.slSearchService.indexSources(connectionId, allSources);
} catch (err) {
this.logger.warn(

8
packages/context/src/ingest/ingest-runtime-assets.test.ts
View file

@ -94,11 +94,15 @@ describe('ingest runtime assets', () => {
it('packages identifier verification prompt assets', async () => {
const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8');
const legacyConnectionPrefix = ['connection', 'Name'].join('');
expect(shared).toContain('## Identifier Verification Protocol');
expect(shared).toContain('discover_data');
expect(shared).toContain('entity_details');
expect(shared).toContain('sql_execution');
expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT');
expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM');
expect(shared).toContain('sql_execution({connectionId, sql: "SELECT DISTINCT');
expect(shared).toContain('sql_execution({connectionId, sql: "SELECT 1 FROM');
expect(shared).not.toContain(`entity_details({${legacyConnectionPrefix}`);
expect(shared).not.toContain(`sql_execution({${legacyConnectionPrefix}`);
});
});

12
packages/context/src/ingest/local-adapters.test.ts
View file

@ -97,6 +97,9 @@ describe('local ingest adapters', () => {
async analyzeBatch() {
return new Map();
},
async validateReadOnly() {
return { ok: true };
},
};
const adapters = createDefaultLocalIngestAdapters(project, {
historicSql: {
@ -140,6 +143,9 @@ describe('local ingest adapters', () => {
async analyzeBatch() {
return new Map();
},
async validateReadOnly() {
return { ok: true };
},
},
reader,
queryClient,
@ -166,6 +172,9 @@ describe('local ingest adapters', () => {
async analyzeBatch() {
return new Map();
},
async validateReadOnly() {
return { ok: true };
},
},
postgresQueryClient: {
async executeQuery() {
@ -258,6 +267,9 @@ describe('local ingest adapters', () => {
async analyzeBatch() {
return new Map();
},
async validateReadOnly() {
return { ok: true };
},
},
postgresQueryClient: {
async executeQuery() {

5
packages/context/src/ingest/local-mapping-reconcile.test.ts
View file

@ -27,11 +27,12 @@ describe('local mapping yaml reconciliation bridge', () => {
const project = projectWithConnections({
'prod-metabase': {
driver: 'metabase',
api_url: 'https://metabase.example.com',
mappings: {
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
syncMode: 'ONLY',
selections: { collections: [12] },
selections: { collections: [12], items: [] },
defaultTagNames: ['ktx'],
},
},
@ -46,6 +47,8 @@ describe('local mapping yaml reconciliation bridge', () => {
const project = projectWithConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.com',
client_id: 'client',
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
},
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },

5
packages/context/src/ingest/page-triage/page-triage.service.test.ts
View file

@ -227,9 +227,10 @@ describe('PageTriageService', () => {
});
generateTextMock
.mockImplementationOnce((args: any) => {
const systemMessage = args.messages.find((m: { role: string }) => m.role === 'system');
const systemMessage = args.system ?? args.messages.find((m: { role: string }) => m.role === 'system');
const userMessage = args.messages.find((m: { role: string }) => m.role === 'user');
const systemText = systemMessage.content as string;
const systemText =
typeof systemMessage === 'string' ? systemMessage : (systemMessage.content as string);
const userText = userMessage.content as string;
expect(systemText).toContain(
'Reusable templates and scripts are durable knowledge regardless of subject matter.',

6
packages/context/src/ingest/page-triage/page-triage.service.ts
View file

@ -1,7 +1,7 @@
import { createHash } from 'node:crypto';
import { readdir, readFile } from 'node:fs/promises';
import { dirname, join, relative } from 'node:path';
import { KtxMessageBuilder, type KtxLlmProvider } from '@ktx/llm';
import { KtxMessageBuilder, splitKtxSystemMessages, type KtxLlmProvider } from '@ktx/llm';
import { generateText, type ToolSet } from 'ai';
import pLimit from 'p-limit';
import { z } from 'zod';
@ -346,10 +346,12 @@ export class PageTriageService {
tools: {},
model,
});
const split = splitKtxSystemMessages(built.messages);
const result = await this.runGenerateText({
model,
temperature: 0,
messages: built.messages,
...(split.system ? { system: split.system } : {}),
messages: split.messages,
tools: built.tools as ToolSet,
});
return result.text;

44
packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts
View file

@ -1,7 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js';
import type { BaseTool, ToolContext } from '../../../tools/index.js';
import { DiscoverDataTool } from './discover-data.tool.js';
import type { WarehouseCatalogService } from './warehouse-catalog.service.js';
describe('DiscoverDataTool', () => {
const wikiSearchTool = { call: vi.fn() } as unknown as BaseTool & { call: ReturnType<typeof vi.fn> };
@ -36,7 +36,7 @@ describe('DiscoverDataTool', () => {
catalog.searchByName.mockResolvedValue([
{
kind: 'table',
connectionName: 'warehouse',
connectionId: 'warehouse',
ref: { catalog: null, db: 'public', name: 'orders' },
display: 'public.orders',
matchedOn: 'name',
@ -45,28 +45,28 @@ describe('DiscoverDataTool', () => {
});
it('groups wiki, semantic layer, and raw schema hits with routing hints', async () => {
const result = await tool.call({ query: 'orders', connectionName: 'warehouse', limit: 5 }, context);
const result = await tool.call({ query: 'orders', connectionId: 'warehouse', limit: 5 }, context);
expect(result.markdown).toContain('## Wiki Pages');
expect(result.markdown).toContain('use `wiki_read(blockKey)` for full content');
expect(result.markdown).toContain('## Semantic Layer Sources');
expect(result.markdown).toContain('use `sl_read_source(sourceName)` for the YAML');
expect(result.markdown).toContain('## Raw Warehouse Schema');
expect(result.markdown).toContain('use `entity_details({connectionName, targets: [{display}]})`');
expect(result.markdown).toContain('use `entity_details({connectionId, targets: [{display}]})`');
expect(result.structured.raw?.hits).toHaveLength(1);
});
it('includes connectionName on raw schema hits so entity_details can follow up', async () => {
it('includes connectionId on raw schema hits so entity_details can follow up', async () => {
const multiConnectionContext: ToolContext = {
...context,
session: { allowedConnectionNames: new Set(['warehouse', 'analytics']) } as any,
};
catalog.searchByName.mockImplementation(async (connectionName: string, query: string) => [
catalog.searchByName.mockImplementation(async (connectionId: string, query: string) => [
{
kind: 'table',
connectionName,
ref: { catalog: null, db: 'public', name: `${connectionName}_${query}` },
display: `public.${connectionName}_${query}`,
connectionId,
ref: { catalog: null, db: 'public', name: `${connectionId}_${query}` },
display: `public.${connectionId}_${query}`,
matchedOn: 'name',
},
]);
@ -75,16 +75,16 @@ describe('DiscoverDataTool', () => {
expect(catalog.searchByName).toHaveBeenCalledWith('analytics', 'orders', 10);
expect(catalog.searchByName).toHaveBeenCalledWith('warehouse', 'orders', 10);
expect(result.markdown).toContain('connectionName=analytics');
expect(result.markdown).toContain('connectionName=warehouse');
expect(result.markdown).toContain('connectionId=analytics');
expect(result.markdown).toContain('connectionId=warehouse');
expect(result.markdown).toContain(
'entity_details({connectionName: "analytics", targets: [{display: "public.analytics_orders"}]})',
'entity_details({connectionId: "analytics", targets: [{display: "public.analytics_orders"}]})',
);
expect(result.structured.raw?.hits.map((hit) => hit.connectionName)).toEqual(['analytics', 'warehouse']);
expect(result.structured.raw?.hits.map((hit) => hit.connectionId)).toEqual(['analytics', 'warehouse']);
});
it('refuses explicit out-of-scope connection names', async () => {
const result = await tool.call({ query: 'orders', connectionName: 'billing' }, context);
const result = await tool.call({ query: 'orders', connectionId: 'billing' }, context);
expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.');
expect(result.structured).toEqual({ wiki: null, sl: null, raw: null });
@ -99,7 +99,7 @@ describe('DiscoverDataTool', () => {
structured: { sourceName: 'orders' },
});
const result = await tool.call({ sourceName: 'orders', connectionName: 'warehouse' }, context);
const result = await tool.call({ sourceName: 'orders', connectionId: 'warehouse' }, context);
expect(slDiscoverTool.call).toHaveBeenCalledWith({ sourceName: 'orders', connectionId: 'warehouse' }, context);
expect(wikiSearchTool.call).not.toHaveBeenCalled();
@ -112,8 +112,20 @@ describe('DiscoverDataTool', () => {
slDiscoverTool.call.mockResolvedValueOnce({ markdown: '', structured: { totalSources: 0, sources: [] } });
catalog.searchByName.mockResolvedValueOnce([]);
const result = await tool.call({ query: 'customer source', connectionName: 'warehouse' }, context);
const result = await tool.call({ query: 'customer source', connectionId: 'warehouse' }, context);
expect(result.markdown).toContain('No matches for "customer source" across wiki, semantic layer, or raw warehouse schema.');
});
it('uses connectionId as the optional connection filter', () => {
const legacyConnectionField = ['connection', 'Name'].join('');
expect(tool.parseInput({ query: 'orders', connectionId: 'warehouse', limit: 5 })).toEqual({
query: 'orders',
connectionId: 'warehouse',
limit: 5,
});
expect(() => tool.parseInput({ query: 'orders', [legacyConnectionField]: 'warehouse', limit: 5 })).toThrow();
});
});

26
packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts
View file

@ -1,13 +1,13 @@
import { z } from 'zod';
import { WarehouseCatalogService, type RawSchemaHit } from '../../../scan/warehouse-catalog.js';
import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js';
import { WarehouseCatalogService, type RawSchemaHit } from './warehouse-catalog.service.js';
const discoverDataInputSchema = z.object({
query: z.string().optional(),
connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(),
connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(),
limit: z.number().int().positive().max(50).optional().default(10),
sourceName: z.string().optional(),
});
}).strict();
type DiscoverDataInput = z.input<typeof discoverDataInputSchema>;
@ -62,16 +62,16 @@ export class DiscoverDataTool extends BaseTool<typeof discoverDataInputSchema> {
async call(input: DiscoverDataInput, context: ToolContext): Promise<ToolOutput<DiscoverDataStructured>> {
const allowed = allowedConnectionNames(context);
if (input.connectionName && allowed && !allowed.has(input.connectionName)) {
if (input.connectionId && allowed && !allowed.has(input.connectionId)) {
return {
markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`,
markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`,
structured: { wiki: null, sl: null, raw: null },
};
}
if (input.sourceName) {
const sl = await this.deps.slDiscoverTool.call(
{ sourceName: input.sourceName, connectionId: input.connectionName },
{ sourceName: input.sourceName, connectionId: input.connectionId },
context,
);
return { markdown: sl.markdown, structured: { wiki: null, sl: sl.structured, raw: null } };
@ -93,7 +93,7 @@ export class DiscoverDataTool extends BaseTool<typeof discoverDataInputSchema> {
}
const slResult = await this.deps.slDiscoverTool.call(
{ query: query || undefined, connectionId: input.connectionName },
{ query: query || undefined, connectionId: input.connectionId },
context,
);
if (totalSources(slResult.structured) > 0) {
@ -107,23 +107,23 @@ export class DiscoverDataTool extends BaseTool<typeof discoverDataInputSchema> {
}
const catalog = this.deps.catalogFactory(context);
const connections = input.connectionName ? [input.connectionName] : [...(allowed ?? [])].sort();
const connections = input.connectionId ? [input.connectionId] : [...(allowed ?? [])].sort();
const rawHits: RawSchemaHit[] = [];
for (const connectionName of connections) {
rawHits.push(...(await catalog.searchByName(connectionName, query, limit)));
for (const connectionId of connections) {
rawHits.push(...(await catalog.searchByName(connectionId, query, limit)));
}
if (rawHits.length > 0) {
parts.push(
'## Raw Warehouse Schema',
'> use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values',
'> use `entity_details({connectionId, targets: [{display}]})` for full DDL + sample values',
);
parts.push(
rawHits
.slice(0, limit)
.map(
(hit) =>
`- ${hit.kind}: ${hit.display} [connectionName=${hit.connectionName}] (matched on ${hit.matchedOn}) - ` +
`follow up with \`entity_details({connectionName: "${hit.connectionName}", targets: [{display: "${hit.display}"}]})\``,
`- ${hit.kind}: ${hit.display} [connectionId=${hit.connectionId}] (matched on ${hit.matchedOn}) - ` +
`follow up with \`entity_details({connectionId: "${hit.connectionId}", targets: [{display: "${hit.display}"}]})\``,
)
.join('\n'),
);

45
packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts
View file

@ -3,9 +3,9 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { initKtxProject, type KtxLocalProject } from '../../../project/index.js';
import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js';
import type { ToolContext } from '../../../tools/index.js';
import { EntityDetailsTool } from './entity-details.tool.js';
import { WarehouseCatalogService } from './warehouse-catalog.service.js';
describe('EntityDetailsTool', () => {
let tempDir: string;
@ -32,11 +32,11 @@ describe('EntityDetailsTool', () => {
await rm(tempDir, { recursive: true, force: true });
});
async function seedLiveDatabaseScan(connectionName = 'warehouse', syncId = 'sync-1') {
const root = `raw-sources/${connectionName}/live-database/${syncId}`;
async function seedLiveDatabaseScan(connectionId = 'warehouse', syncId = 'sync-1') {
const root = `raw-sources/${connectionId}/live-database/${syncId}`;
await project.fileStore.writeFile(
`${root}/connection.json`,
JSON.stringify({ connectionId: connectionName, driver: 'postgres', extractedAt: '2026-05-12T00:00:00.000Z' }, null, 2),
JSON.stringify({ connectionId, driver: 'postgres', extractedAt: '2026-05-12T00:00:00.000Z' }, null, 2),
'ktx',
'ktx@example.com',
'seed connection',
@ -84,7 +84,7 @@ describe('EntityDetailsTool', () => {
`${root}/enrichment/relationship-profile.json`,
JSON.stringify(
{
connectionId: connectionName,
connectionId,
driver: 'postgres',
tables: [{ table: { catalog: null, db: 'public', name: 'orders' }, rowCount: 12 }],
columns: {
@ -109,7 +109,7 @@ describe('EntityDetailsTool', () => {
}
it('returns scoped table detail for a display target', async () => {
const result = await tool.call({ connectionName: 'warehouse', targets: [{ display: 'public.orders' }] }, context);
const result = await tool.call({ connectionId: 'warehouse', targets: [{ display: 'public.orders' }] }, context);
expect(result.markdown).toContain('### public.orders');
expect(result.markdown).toContain('- status (text, nullable=false)');
@ -120,7 +120,7 @@ describe('EntityDetailsTool', () => {
it('resolves display targets that include a column name', async () => {
const result = await tool.call(
{ connectionName: 'warehouse', targets: [{ display: 'public.orders.status' }] },
{ connectionId: 'warehouse', targets: [{ display: 'public.orders.status' }] },
context,
);
@ -133,7 +133,7 @@ describe('EntityDetailsTool', () => {
it('reports missing explicit columns instead of returning an empty column list', async () => {
const result = await tool.call(
{ connectionName: 'warehouse', targets: [{ display: 'public.orders.plan_tier' }] },
{ connectionId: 'warehouse', targets: [{ display: 'public.orders.plan_tier' }] },
context,
);
@ -146,7 +146,7 @@ describe('EntityDetailsTool', () => {
it('reports missing structured table targets in model-visible markdown', async () => {
const result = await tool.call(
{
connectionName: 'warehouse',
connectionId: 'warehouse',
targets: [{ catalog: null, db: 'public', name: 'orderz' }],
},
context,
@ -161,7 +161,7 @@ describe('EntityDetailsTool', () => {
it('reports missing structured column targets in model-visible markdown', async () => {
const result = await tool.call(
{
connectionName: 'warehouse',
connectionId: 'warehouse',
targets: [{ catalog: null, db: 'public', name: 'orders', column: 'plan_tier' }],
},
context,
@ -175,7 +175,7 @@ describe('EntityDetailsTool', () => {
it('returns a no-scan state distinct from not found', async () => {
const result = await tool.call(
{ connectionName: 'empty', targets: [{ display: 'public.orders' }] },
{ connectionId: 'empty', targets: [{ display: 'public.orders' }] },
{ ...context, session: { ...context.session!, allowedConnectionNames: new Set(['empty']) } },
);
@ -184,9 +184,30 @@ describe('EntityDetailsTool', () => {
});
it('refuses out-of-scope connections', async () => {
const result = await tool.call({ connectionName: 'billing', targets: [{ display: 'public.orders' }] }, context);
const result = await tool.call({ connectionId: 'billing', targets: [{ display: 'public.orders' }] }, context);
expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.');
expect(result.structured.scanAvailable).toBe(false);
});
it('uses connectionId as the public input field', async () => {
const legacyConnectionField = ['connection', 'Name'].join('');
expect(
tool.parseInput({
connectionId: 'warehouse',
targets: [{ display: 'public.orders' }],
}),
).toEqual({
connectionId: 'warehouse',
targets: [{ display: 'public.orders' }],
});
expect(() =>
tool.parseInput({
[legacyConnectionField]: 'warehouse',
targets: [{ display: 'public.orders' }],
}),
).toThrow();
});
});

24
packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts
View file

@ -1,7 +1,7 @@
import { z } from 'zod';
import type { KtxTableRef } from '../../../scan/types.js';
import { WarehouseCatalogService, type TableDetail } from '../../../scan/warehouse-catalog.js';
import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js';
import { WarehouseCatalogService, type TableDetail } from './warehouse-catalog.service.js';
const targetSchema = z.union([
z.object({ display: z.string().min(1) }),
@ -14,9 +14,9 @@ const targetSchema = z.union([
]);
const entityDetailsInputSchema = z.object({
connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
targets: z.array(targetSchema).min(1).max(50),
});
}).strict();
type EntityDetailsInput = z.infer<typeof entityDetailsInputSchema>;
type EntityDetailsTarget = EntityDetailsInput['targets'][number];
@ -47,14 +47,14 @@ function appendMissingTargetMarkdown(parts: string[], target: EntityDetailsTarge
async function resolveTarget(
catalog: WarehouseCatalogService,
connectionName: string,
connectionId: string,
target: EntityDetailsTarget,
): Promise<{ resolved: (KtxTableRef & { column?: string }) | null; candidates: KtxTableRef[] }> {
if ('display' in target) {
return catalog.resolveDisplayTarget(connectionName, target.display);
return catalog.resolveDisplayTarget(connectionId, target.display);
}
const candidateResolution = await catalog.resolveDisplayTarget(connectionName, targetLabel(target));
const candidateResolution = await catalog.resolveDisplayTarget(connectionId, targetLabel(target));
return {
resolved: {
catalog: target.catalog,
@ -107,18 +107,18 @@ export class EntityDetailsTool extends BaseTool<typeof entityDetailsInputSchema>
async call(input: EntityDetailsInput, context: ToolContext): Promise<ToolOutput<EntityDetailsStructured>> {
const allowed = allowedConnectionNames(context);
if (allowed && !allowed.has(input.connectionName)) {
if (allowed && !allowed.has(input.connectionId)) {
return {
markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`,
markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`,
structured: { resolved: [], missing: [], scanAvailable: false },
};
}
const catalog = this.catalogFactory(context);
const scanAvailable = await catalog.hasScan(input.connectionName);
const scanAvailable = await catalog.hasScan(input.connectionId);
if (!scanAvailable) {
return {
markdown: `No live-database scan available for connection "${input.connectionName}"; run \`ktx scan\` first.`,
markdown: `No live-database scan available for connection "${input.connectionId}"; run \`ktx scan\` first.`,
structured: { resolved: [], missing: [], scanAvailable: false },
};
}
@ -128,13 +128,13 @@ export class EntityDetailsTool extends BaseTool<typeof entityDetailsInputSchema>
const missing: EntityDetailsStructured['missing'] = [];
for (const target of input.targets) {
const resolution = await resolveTarget(catalog, input.connectionName, target);
const resolution = await resolveTarget(catalog, input.connectionId, target);
if (!resolution.resolved) {
missing.push({ target, candidates: resolution.candidates });
appendMissingTargetMarkdown(parts, target, resolution.candidates);
continue;
}
const detail = await catalog.getTable({ connectionName: input.connectionName, ...resolution.resolved });
const detail = await catalog.getTable({ connectionId: input.connectionId, ...resolution.resolved });
if (!detail) {
missing.push({ target, candidates: resolution.candidates });
appendMissingTargetMarkdown(parts, target, resolution.candidates);

2
packages/context/src/ingest/tools/warehouse-verification/index.ts
View file

@ -1,10 +1,10 @@
import type { KtxFileStorePort } from '../../../core/index.js';
import type { SlConnectionCatalogPort } from '../../../sl/index.js';
import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js';
import type { BaseTool, ToolContext } from '../../../tools/index.js';
import { DiscoverDataTool } from './discover-data.tool.js';
import { EntityDetailsTool } from './entity-details.tool.js';
import { SqlExecutionTool } from './sql-execution.tool.js';
import { WarehouseCatalogService } from './warehouse-catalog.service.js';
export function createWarehouseVerificationTools(deps: {
connections: SlConnectionCatalogPort;

30
packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts
View file

@ -19,7 +19,7 @@ describe('SqlExecutionTool', () => {
connections.executeQuery.mockResolvedValue({ headers: ['status'], rows: [['paid']], totalRows: 1 });
const result = await tool.call(
{ connectionName: 'warehouse', sql: 'select status from public.orders', rowLimit: 5 },
{ connectionId: 'warehouse', sql: 'select status from public.orders', rowLimit: 5 },
context,
);
@ -34,7 +34,7 @@ describe('SqlExecutionTool', () => {
it.each(['insert into x values (1)', 'drop table x', 'vacuum'])('rejects mutating SQL: %s', async (sql) => {
connections.executeQuery.mockClear();
const result = await tool.call({ connectionName: 'warehouse', sql }, context);
const result = await tool.call({ connectionId: 'warehouse', sql }, context);
expect(result.markdown).toContain('Only read-only SELECT/WITH queries can be executed locally.');
expect(connections.executeQuery).not.toHaveBeenCalled();
@ -44,11 +44,35 @@ describe('SqlExecutionTool', () => {
connections.executeQuery.mockRejectedValue(new Error('relation "orbit_analytics.customer" does not exist'));
const result = await tool.call(
{ connectionName: 'warehouse', sql: 'select 1 from orbit_analytics.customer', rowLimit: 1 },
{ connectionId: 'warehouse', sql: 'select 1 from orbit_analytics.customer', rowLimit: 1 },
context,
);
expect(result.markdown).toContain('relation "orbit_analytics.customer" does not exist');
expect(result.structured.error).toContain('relation "orbit_analytics.customer" does not exist');
});
it('uses connectionId as the public input field', () => {
const legacyConnectionField = ['connection', 'Name'].join('');
expect(
tool.parseInput({
connectionId: 'warehouse',
sql: 'select 1',
rowLimit: 5,
}),
).toEqual({
connectionId: 'warehouse',
sql: 'select 1',
rowLimit: 5,
});
expect(() =>
tool.parseInput({
[legacyConnectionField]: 'warehouse',
sql: 'select 1',
rowLimit: 5,
}),
).toThrow();
});
});

10
packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts
View file

@ -4,10 +4,10 @@ import type { SlConnectionCatalogPort } from '../../../sl/index.js';
import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js';
const sqlExecutionInputSchema = z.object({
connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/),
sql: z.string().min(1),
rowLimit: z.number().int().positive().max(1000).optional().default(100),
});
}).strict();
type SqlExecutionInput = z.input<typeof sqlExecutionInputSchema>;
@ -54,9 +54,9 @@ export class SqlExecutionTool extends BaseTool<typeof sqlExecutionInputSchema> {
async call(input: SqlExecutionInput, context: ToolContext): Promise<ToolOutput<SqlExecutionStructured>> {
const allowed = context.session?.allowedConnectionNames;
if (allowed && !allowed.has(input.connectionName)) {
if (allowed && !allowed.has(input.connectionId)) {
return {
markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`,
markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`,
structured: {
headers: [],
rows: [],
@ -83,7 +83,7 @@ export class SqlExecutionTool extends BaseTool<typeof sqlExecutionInputSchema> {
}
try {
const result = await this.connections.executeQuery(input.connectionName, wrappedSql);
const result = await this.connections.executeQuery(input.connectionId, wrappedSql);
const headers = result.headers ?? [];
const rows = result.rows ?? [];
const rowCount = result.totalRows ?? rows.length;

37
packages/context/src/ingest/wiki-sl-ref-repair.test.ts
View file

@ -44,23 +44,26 @@ describe('repairWikiSlRefs', () => {
})),
};
const semanticLayerService = {
loadAllSources: vi.fn(async () => [
{
name: 'mart_customer_health',
grain: [],
columns: [],
joins: [],
measures: [{ name: 'high_risk_account_count', expr: 'count(*)' }],
segments: [{ name: 'high_risk', expr: "risk_level = 'high'" }],
},
{
name: 'int_procurement_qualifying_actions',
grain: [],
columns: [],
joins: [],
measures: [],
},
]),
loadAllSources: vi.fn(async () => ({
sources: [
{
name: 'mart_customer_health',
grain: [],
columns: [],
joins: [],
measures: [{ name: 'high_risk_account_count', expr: 'count(*)' }],
segments: [{ name: 'high_risk', expr: "risk_level = 'high'" }],
},
{
name: 'int_procurement_qualifying_actions',
grain: [],
columns: [],
joins: [],
measures: [],
},
],
loadErrors: [],
})),
};
const result = await repairWikiSlRefs({

Some files were not shown because too many files have changed in this diff Show more