diff --git a/AGENTS.md b/AGENTS.md index 86dde8e9..15b9c8aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -234,8 +234,8 @@ use `PascalCase` without the suffix. ## Documentation and Specs -- Keep public documentation in `README.md`, package READMEs, and example - READMEs unless the repository intentionally adds a public docs tree. +- Keep public documentation in `README.md`, package READMEs, example READMEs, + and the `docs-site/` Fumadocs tree. - Prefer concrete commands, file paths, and acceptance criteria over broad prose. - When documenting examples, ensure referenced files and commands exist in the @@ -243,6 +243,28 @@ use `PascalCase` without the suffix. - Remove or rewrite stale external app references unless the doc is explicitly historical. +### Updating `docs-site/` After Code Changes + +Before finishing a task, decide whether `docs-site/content/docs/` needs an +update. Update it when your change affects user-visible behavior, including: + +- New, renamed, or removed CLI commands, flags, or subcommands + (`docs-site/content/docs/cli-reference/`) +- Changes to `ktx.yaml`, environment variables, or other configuration users + edit +- New or changed connectors, integrations, or supported drivers + (`docs-site/content/docs/integrations/`) +- Changes to setup, install, or getting-started flows + (`docs-site/content/docs/getting-started/`) +- New concepts, agent capabilities, or workflows users should know about + (`docs-site/content/docs/concepts/`, `docs-site/content/docs/guides/`) + +Skip docs updates for purely internal refactors, test-only changes, or fixes +that do not change user-facing behavior. When you do update docs, follow the +`fumadocs-mdx-structure` skill and keep examples copy-pasteable. If a change +warrants docs but you are out of scope, call it out in your final summary +rather than silently skipping it. + ## LLM and Prompt Development When creating or modifying agent prompts, system prompts, tool descriptions, or diff --git a/README.md b/README.md index 44d7c395..92a2a8dd 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ KTX turns warehouse metadata, semantic definitions, and business knowledge into reviewable project files that agents can use while planning, querying, and updating analytics work. -A KTX project is a directory of plain files — YAML semantic sources, Markdown -wiki pages, and SQLite state — that you commit to git and review in PRs, +A KTX project is a directory of plain files - YAML semantic sources, Markdown +wiki pages, and SQLite state - that you commit to git and review in PRs, just like dbt models. ## Who KTX is for KTX is built for analytics engineers and data teams who want data agents to -work on real analytics systems — not just generate one-off SQL. +work on real analytics systems - not just generate one-off SQL. Use KTX when you want agents to: @@ -89,11 +89,12 @@ ktx connection list --project-dir "$PROJECT_DIR" ktx connection test warehouse --project-dir "$PROJECT_DIR" ``` -The connection test prints the configured driver and discovered table count: +The connection test prints the configured driver and connector-specific status: ```text +Connection test passed: warehouse Driver: sqlite -Tables: 1 +Status: ok ``` ## What's in a project @@ -120,7 +121,7 @@ my-project/ ``` Semantic sources and wiki pages are committed to git. The `.ktx/` directory -holds ephemeral state and is git-ignored — delete it and KTX rebuilds on the +holds ephemeral state and is git-ignored - delete it and KTX rebuilds on the next run. ### Build demo warehouse context @@ -163,7 +164,7 @@ source packages for development, not public release artifacts. KTX integrates with coding agents through CLI skills. The setup wizard configures this automatically. -**CLI skills** — the agent calls `ktx` commands directly through a skill file +**CLI skills** - the agent calls `ktx` commands directly through a skill file installed in your agent's config (e.g., `.claude/skills/ktx/SKILL.md`): ```bash diff --git a/assets/ktx-mascot-dark.svg b/assets/ktx-mascot-dark.svg new file mode 100644 index 00000000..96603a15 --- /dev/null +++ b/assets/ktx-mascot-dark.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/assets/ktx-mascot.svg b/assets/ktx-mascot.svg new file mode 100644 index 00000000..0c20f577 --- /dev/null +++ b/assets/ktx-mascot.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/docs-site/app/global.css b/docs-site/app/global.css index f3425e78..bc4ed8a4 100644 --- a/docs-site/app/global.css +++ b/docs-site/app/global.css @@ -9,7 +9,7 @@ } /* ═══════════════════════════════════════════ - KTX Light Theme — Warm Cream & Taupe + KTX Light Theme - Warm Cream & Taupe ═══════════════════════════════════════════ */ :root { --color-fd-background: #faf9f6; @@ -42,7 +42,7 @@ } /* ═══════════════════════════════════════════ - KTX Dark Theme — Deep Ocean Slate + KTX Dark Theme - Deep Ocean Slate ═══════════════════════════════════════════ */ .dark { --color-fd-background: #0f1719; @@ -79,7 +79,7 @@ body { } /* ═══════════════════════════════════════════ - Typography — Outfit display, Inter body + Typography - Outfit display, Inter body ═══════════════════════════════════════════ */ h1, h2, h3, h4 { font-family: var(--font-display), var(--font-sans), sans-serif; @@ -114,7 +114,7 @@ h2 { border-color: rgba(255, 255, 255, 0.08) !important; } -/* Code blocks — give them a subtle traffic-light feel */ +/* Code blocks - give them a subtle traffic-light feel */ figure[data-rehype-pretty-code-figure], figure:has(> pre) { position: relative; @@ -166,7 +166,7 @@ pre { } /* ═══════════════════════════════════════════ - Code blocks — context-aware modes + Code blocks - context-aware modes ═══════════════════════════════════════════ */ /* Shared wrapper base */ @@ -504,7 +504,7 @@ th { } /* ═══════════════════════════════════════════ - Sidebar — Typographic sections + active rail + Sidebar - Typographic sections + active rail ═══════════════════════════════════════════ */ #nd-sidebar { border-right: 1px solid var(--color-fd-border); @@ -516,7 +516,7 @@ th { backdrop-filter: blur(10px); } -/* Section folder trigger — uppercase tracked label +/* Section folder trigger - uppercase tracked label Fumadocs 15 section wrappers are bare
(no class, no id); content panels and other Radix collapsibles always carry a class attribute, so :not([class]) tightly scopes these rules to section triggers only. */ @@ -601,7 +601,7 @@ th { } /* ═══════════════════════════════════════════ - Cards — refined with multi-layer shadow & lift + Cards - refined with multi-layer shadow & lift ═══════════════════════════════════════════ */ [data-card="true"] { border-radius: 12px !important; @@ -683,7 +683,7 @@ th { } /* ═══════════════════════════════════════════ - Page title area — give docs pages a hero feel + Page title area - give docs pages a hero feel ═══════════════════════════════════════════ */ [data-page-header] h1, article > h1:first-of-type { @@ -724,7 +724,7 @@ article a:not([data-card]):hover { } /* ═══════════════════════════════════════════ - Background atmosphere — gradient blobs (subtle) + Background atmosphere - gradient blobs (subtle) ═══════════════════════════════════════════ */ body::before { content: ""; @@ -973,7 +973,7 @@ body > * { 100% { left: 200%; } } -/* Glow text — use sparingly on hero key phrase */ +/* Glow text - use sparingly on hero key phrase */ .glow-text { position: relative; color: var(--color-fd-primary); diff --git a/docs-site/components/code-block.tsx b/docs-site/components/code-block.tsx index 8362b304..7d6a22af 100644 --- a/docs-site/components/code-block.tsx +++ b/docs-site/components/code-block.tsx @@ -54,7 +54,7 @@ export function CodeBlock(props: Props) { const isOutput = !isTerminal && WIZARD_GLYPHS.test(codeText); const hasTitle = typeof title === "string" && title.length > 0; - // Mode A — Terminal (commands the user types) + // Mode A - Terminal (commands the user types) if (isTerminal) { return (
@@ -77,7 +77,7 @@ export function CodeBlock(props: Props) { ); } - // Mode D — Output preview (wizard prompts, terminal output) + // Mode D - Output preview (wizard prompts, terminal output) if (isOutput) { return (
@@ -90,7 +90,7 @@ export function CodeBlock(props: Props) { ); } - // Mode B — VS Code tab (filename present) + // Mode B - VS Code tab (filename present) if (hasTitle) { return (
@@ -107,7 +107,7 @@ export function CodeBlock(props: Props) { ); } - // Mode C — Minimal default + // Mode C - Minimal default return (
{language && {language}} diff --git a/docs-site/components/copy-button.tsx b/docs-site/components/copy-button.tsx index 01dbd054..876f5de0 100644 --- a/docs-site/components/copy-button.tsx +++ b/docs-site/components/copy-button.tsx @@ -16,7 +16,7 @@ export function CopyButton({ text, className = "" }: Props) { setCopied(true); setTimeout(() => setCopied(false), 1500); } catch { - // Older browsers or denied permission — fail silently + // Older browsers or denied permission - fail silently } }; diff --git a/docs-site/components/logo.tsx b/docs-site/components/logo.tsx index db67a689..4ab8f8ba 100644 --- a/docs-site/components/logo.tsx +++ b/docs-site/components/logo.tsx @@ -1,22 +1,28 @@ export function Logo() { return ( -
+
+
KTX Docs diff --git a/docs-site/content/docs/ai-resources/index.mdx b/docs-site/content/docs/ai-resources/index.mdx index be315453..37bae650 100644 --- a/docs-site/content/docs/ai-resources/index.mdx +++ b/docs-site/content/docs/ai-resources/index.mdx @@ -1,22 +1,37 @@ --- title: AI Resources -description: Machine-readable docs and prompt recipes for coding assistants reading KTX documentation. +description: Machine-readable docs, retrieval paths, and prompt recipes for coding assistants using KTX documentation. --- -Use this section when a coding assistant, IDE agent, or automation system needs to understand the KTX documentation. +Use this section when a coding assistant, IDE agent, or automation system needs +to read, cite, or update KTX documentation. These resources are optimized for +retrieval: agents can fetch small Markdown pages, use the full corpus only when +needed, and copy prompts that point them at current setup and CLI behavior. > **Documentation index** > -> Start with [`/llms.txt`](/llms.txt) to discover the available docs. Use [`/llms-full.txt`](/llms-full.txt) when the assistant needs the complete docs corpus in one Markdown response. +> Start with [`/llms.txt`](/llms.txt) to discover the available docs. Use +> [`/llms-full.txt`](/llms-full.txt) when the assistant needs the complete docs +> corpus in one Markdown response. -## Choose the right path +## What agents can do + +| Need | Recommended path | +|------|------------------| +| Find the right setup or CLI page | Fetch [`/llms.txt`](/llms.txt), then read the smallest matching `.md` page | +| Answer a setup question | Read [Agent Quickstart](/docs/ai-resources/agent-quickstart), then [Quickstart](/docs/getting-started/quickstart) or [ktx setup](/docs/cli-reference/ktx-setup) | +| Quote a command or flag | Read the matching [CLI Reference](/docs/cli-reference) page as Markdown | +| Update docs in this repo | Use [Agent Instructions](/docs/ai-resources/agent-instructions) and verify generated Markdown routes after editing | +| Reuse a prompt | Copy from [Prompt Recipes](/docs/ai-resources/prompt-recipes) | + +## Section map | Goal | Use this page | |------|---------------| -| Tell a coding assistant how to approach KTX docs | [Agent Quickstart](/docs/ai-resources/agent-quickstart) | -| Fetch docs as Markdown instead of HTML | [Markdown Access](/docs/ai-resources/markdown-access) | -| Add lightweight instructions to an assistant prompt | [Agent Instructions](/docs/ai-resources/agent-instructions) | -| Copy prompts for common agent workflows | [Prompt Recipes](/docs/ai-resources/prompt-recipes) | +| Give an assistant a task-first route through the docs | [Agent Quickstart](/docs/ai-resources/agent-quickstart) | +| Fetch docs as Markdown instead of rendered HTML | [Markdown Access](/docs/ai-resources/markdown-access) | +| Add lightweight KTX docs guidance to a system prompt | [Agent Instructions](/docs/ai-resources/agent-instructions) | +| Copy prompts for setup, command lookup, and docs editing | [Prompt Recipes](/docs/ai-resources/prompt-recipes) | ## Available resources @@ -26,13 +41,24 @@ Use this section when a coding assistant, IDE agent, or automation system needs | [`/llms-full.txt`](/llms-full.txt) | Complete docs corpus in one plain-text Markdown response | | `/docs/.md` | Per-page Markdown for any docs page | | Page-level actions | Copy Markdown, view Markdown, or copy MDX from rendered docs pages | -| Prompt recipes | Reusable prompts for docs lookup, setup help, and docs editing | +| Prompt recipes | Reusable prompts for docs lookup, setup help, command discovery, and docs editing | ## Agent usage notes -When an assistant is unsure where to begin, use this order: +When an assistant is unsure where to begin, use this retrieval order: 1. Read [`/llms.txt`](/llms.txt). -2. Fetch the specific Markdown page for the task. -3. Use [Agent Quickstart](/docs/ai-resources/agent-quickstart) to choose the next command or page. -4. Use page-level copy actions when the user wants the exact Markdown or MDX source. +2. Fetch one or two specific Markdown pages for the task. +3. Use [Agent Quickstart](/docs/ai-resources/agent-quickstart) to choose the + next command, guide, or CLI reference page. +4. Use [`/llms-full.txt`](/llms-full.txt) only when the answer requires broad + context across setup, integrations, concepts, and CLI reference. +5. Use page-level copy actions when the user wants exact generated Markdown or + source MDX. + +## Boundaries + +AI Resources explain how agents consume the docs. To install KTX into an +agent client, use [Agent Clients](/docs/integrations/agent-clients). To set up a +project, use [Quickstart](/docs/getting-started/quickstart) or +[`ktx setup`](/docs/cli-reference/ktx-setup). diff --git a/docs-site/content/docs/cli-reference/index.mdx b/docs-site/content/docs/cli-reference/index.mdx new file mode 100644 index 00000000..c4ef07db --- /dev/null +++ b/docs-site/content/docs/cli-reference/index.mdx @@ -0,0 +1,76 @@ +--- +title: "Overview" +description: "Command map and shared options for the KTX CLI." +--- + +The `ktx` CLI sets up local projects, builds agent-ready context, checks +connections, queries semantic-layer sources, searches wiki pages, and manages +the bundled Python runtime. + +## Command Map + +```text +ktx + setup + connection + list + test [connectionId] + ingest [connectionId] + text [files...] + wiki + list + search + sl + list + search + validate + query + status + dev + init [directory] + schema + runtime + install + start + stop + status +``` + +The public context-build entrypoint is `ktx ingest [connectionId]` or +`ktx ingest --all`. + +## Global Options + +| Flag | Description | +|------|-------------| +| `--project-dir ` | KTX project directory. Defaults to `KTX_PROJECT_DIR`, then the nearest `ktx.yaml`, then the current working directory. | +| `--debug` | Print diagnostic dispatch and project-resolution details to stderr. | +| `-v`, `--version` | Show the CLI package name and version. | +| `-h`, `--help` | Show help for the current command. | + +## Project Resolution + +Most commands are project-aware. Pass `--project-dir ` when scripting or +when you are outside the project directory. If you omit it, KTX checks +`KTX_PROJECT_DIR`, then walks upward for the nearest `ktx.yaml`, then falls back +to the current directory. + +## Common Workflows + +```bash +# Start or resume setup +ktx setup + +# Check readiness +ktx status + +# Build one configured connection +ktx ingest warehouse + +# Build every configured connection +ktx ingest --all + +# Search semantic-layer sources and wiki pages +ktx sl search "revenue" +ktx wiki search "revenue recognition" +``` diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 68b7f496..2d61451f 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -4,8 +4,8 @@ description: "List and test configured data sources." --- Inspect configured connections in your KTX project. Connections define how KTX -reaches your data warehouse, BI tools, and context sources. Use `ktx setup` to -add, remove, or reconfigure connections. +reaches databases, warehouses, BI tools, source projects, and knowledge +systems. Use `ktx setup` to add, remove, or reconfigure them. ## Command signature @@ -18,24 +18,21 @@ ktx connection [options] | Subcommand | Description | |-----------|-------------| | `list` | List configured connections | -| `test ` | Test a configured connection | +| `test [connectionId]` | Test one configured connection, or every connection with `--all` | ## Options -The `connection` command has command-level options for listing and testing -existing connections. - -### `connection list` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output | `false` | +`ktx connection` uses the shared global options such as `--project-dir` and +`--debug`. ### `connection test` | Flag | Description | Default | |------|-------------|---------| -| `--json` | Print JSON output | `false` | +| `--all` | Test every configured connection and print a summary list | `false` | + +Project directory resolution defaults to `KTX_PROJECT_DIR`, then the nearest +`ktx.yaml`, then the current working directory. ## Examples @@ -45,6 +42,12 @@ ktx connection list # Test a connection ktx connection test my-warehouse + +# Test every configured connection +ktx connection test --all + +# Test a connection from outside the project +ktx connection test my-warehouse --project-dir ./analytics ``` ## Setup-managed connections @@ -55,24 +58,41 @@ Metabase mapping prompts for BI-to-warehouse mappings. ## Output -Commands with `--json` return machine-readable JSON suitable for scripts and -agents. +`ktx connection list` prints a table of configured ids and drivers. -```json -{ - "connections": [ - { - "id": "my-warehouse", - "driver": "postgres" - } - ] -} +```text +ID DRIVER +my-warehouse postgres +``` + +`ktx connection test ` performs a lightweight connection probe. +Native database connections report `Status: ok` when the connector probe +passes. Source connectors report connector-specific details such as Metabase +database count, Looker user, Notion bot, or Git repo URL. + +```text +Connection test passed: my-warehouse +Driver: postgres +Status: ok +``` + +`ktx connection test --all` prints one row per configured connection and exits +non-zero if any probe fails. + +```text +╭ connection test --all +│ +│ • warehouse postgres ✓ ok Status: ok +│ • metabase metabase ✓ ok Databases: 2 +│ +╰ 2 tested · 2 passed ``` ## Common errors | Error | Cause | Recovery | |-------|-------|----------| +| No connections configured | The project has no entries under `connections` | Run `ktx setup` and add a database or source connection | | Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | | Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Rerun `ktx setup` and update the source mapping selections | | Notion page picker cannot run | The terminal is non-interactive or Notion discovery failed | Rerun interactive `ktx setup`, or use non-interactive setup flags with explicit root page ids | diff --git a/docs-site/content/docs/cli-reference/ktx-dev.mdx b/docs-site/content/docs/cli-reference/ktx-dev.mdx index 16a36393..f124055a 100644 --- a/docs-site/content/docs/cli-reference/ktx-dev.mdx +++ b/docs-site/content/docs/cli-reference/ktx-dev.mdx @@ -3,7 +3,11 @@ title: "ktx dev" description: "Low-level project initialization and runtime management." --- -`ktx dev` contains development-only project initialization and managed runtime commands. Context building lives at the root as [`ktx ingest`](/docs/cli-reference/ktx-ingest). +`ktx dev` contains low-level project initialization and managed Python runtime +commands. Context building lives at the root as +[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with +`ktx setup`; use `ktx dev` when preparing local fixtures, checking the bundled +runtime, or debugging runtime state. ## Command signature @@ -15,43 +19,70 @@ ktx dev [options] | Subcommand | Description | |-----------|-------------| -| `init [directory]` | Initialize a Git-backed KTX project directory | +| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts | +| `schema` | Print a JSON Schema describing `ktx.yaml` | | `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime | ## `dev init` | Flag | Description | Default | |------|-------------|---------| -| `--name ` | Project name written to `ktx.yaml` | — | | `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` | -## `dev runtime` +## `dev schema` -`ktx dev runtime` supports `install`, `start`, `stop`, and `status`. +| Flag | Description | Default | +|------|-------------|---------| +| `--output ` | Write the schema to a file instead of stdout | — | + +## `dev runtime` Subcommands + +| Subcommand | Description | +|-----------|-------------| +| `install` | Install the bundled Python runtime wheel into the managed runtime | +| `start` | Start the KTX-managed Python HTTP daemon | +| `stop` | Stop the KTX-managed Python HTTP daemon | +| `status` | Show managed Python runtime status and readiness checks | + +## `dev runtime` Options | Flag | Description | Default | |------|-------------|---------| | `--feature ` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` | | `--json` | Print JSON output for `status` | `false` | -| `--yes` | Confirm runtime install actions where supported | `false` | -| `--force` | Reinstall or restart where supported | `false` | +| `--yes` | Accepted by `install` for scripted install commands | `false` | +| `--force` | Reinstall for `install`, or restart for `start` | `false` | +| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` | ## Examples ```bash ktx dev init -ktx dev init ./my-project --name "Analytics Context" +ktx dev init ./my-project ktx dev init --force +ktx dev schema +ktx dev schema --output ./ktx.schema.json + ktx dev runtime install --yes +ktx dev runtime install --feature local-embeddings --yes ktx dev runtime status ktx dev runtime start +ktx dev runtime start --feature local-embeddings ktx dev runtime stop +ktx dev runtime stop --all ``` +## Output + +Runtime commands print the runtime root, installed features, daemon URL, daemon +pid, and log paths where relevant. `ktx dev runtime status --json` includes the +runtime status plus readiness checks. + ## Common errors | Error | Cause | Recovery | |-------|-------|----------| | Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx dev runtime status` | | Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx dev runtime install --yes`, then `ktx dev runtime start` | +| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx dev runtime stop --all`, then start the runtime again | diff --git a/docs-site/content/docs/cli-reference/ktx-ingest.mdx b/docs-site/content/docs/cli-reference/ktx-ingest.mdx index e7b8bbe5..a0bca58f 100644 --- a/docs-site/content/docs/cli-reference/ktx-ingest.mdx +++ b/docs-site/content/docs/cli-reference/ktx-ingest.mdx @@ -6,23 +6,25 @@ description: "Build or refresh KTX context from configured connections." `ktx ingest` builds or refreshes KTX context from configured connections. Database connections build schema context. Context-source connections ingest metadata from tools such as dbt, Looker, Metabase, MetricFlow, LookML, and -Notion. +Notion. The current public command is connection-centric: pass one +`connectionId`, or pass `--all`. ## Command signature ```bash ktx ingest [options] [connectionId] +ktx ingest text [options] [files...] ``` Use a connection id to build one configured connection. Use `--all` to build every configured connection. Database connections run before context-source connections when you use `--all`. -## Build options +## `ktx ingest` Options | Flag | Description | Default | |------|-------------|---------| -| `--all` | Build every configured connection | `false` | +| `--all` | Ingest all configured connections | `false` | | `--fast` | Use deterministic database schema ingest | Stored connection default, or `fast` | | `--deep` | Use AI-enriched database ingest | Stored connection default, or `fast` | | `--query-history` | Include database query-history usage patterns | Stored connection default | @@ -30,25 +32,75 @@ connections when you use `--all`. | `--query-history-window-days ` | Query-history lookback window for this run | Stored connection default | | `--plain` | Print plain text output | `true` | | `--json` | Print JSON output | `false` | -| `--no-input` | Disable interactive terminal input | `false` | +| `--no-input` | Disable interactive terminal input | — | `--fast` and `--deep` are mutually exclusive. Depth flags apply only to database connections. Query-history flags apply only to database connections -that support query history. +that support query history. Query-history ingest runs after schema ingest and +requires deep ingest readiness. + +When `--all` selects both databases and context sources, database ingest runs +first, then source ingest and memory updates run for source connections. + +## `ktx ingest text` Options + +Use `ktx ingest text` to capture free-form text artifacts into KTX memory. +Provide files, pass `--text` one or more times, or use `-` as a file argument to +read one item from stdin. + +| Flag | Description | Default | +|------|-------------|---------| +| `--text ` | Text content to ingest; repeat for a batch | `[]` | +| `--connection-id ` | Optional KTX connection id for semantic-layer capture | — | +| `--user-id ` | Memory user id for capture attribution | `local-cli` | +| `--json` | Print JSON output | `false` | +| `--fail-fast` | Stop after the first failed text item | `false` | ## Examples ```bash +# Build one database or source connection ktx ingest warehouse + +# Force deterministic database schema ingest ktx ingest warehouse --fast + +# Force AI-enriched database ingest ktx ingest warehouse --deep + +# Include query-history usage patterns ktx ingest warehouse --deep --query-history ktx ingest warehouse --query-history-window-days 30 + +# Build a source connection ktx ingest notion + +# Build all configured connections ktx ingest --all ktx ingest --all --deep + +# Capture local Markdown notes into memory +ktx ingest text docs/revenue-notes.md --connection-id warehouse + +# Capture one stdin item +printf "Refunds are excluded from net revenue." | ktx ingest text - ``` +## Output + +Plain output summarizes each target and the operations that ran. + +```text +Ingest finished + +Source Database schema Query history Source ingest Memory update +warehouse done done skipped skipped +notion skipped skipped done done +``` + +Use `--json` when a script or agent needs the selected plan and per-target +results. + ## Common errors | Error | Cause | Recovery | @@ -57,3 +109,5 @@ ktx ingest --all --deep | Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` | | Query history is unsupported | The selected database driver does not support query history | Run schema ingest without query-history flags | | No ingest target was selected | No connection id was provided and `--all` was omitted | Run `ktx ingest ` or `ktx ingest --all` | +| Source options were ignored | Depth and query-history flags were supplied for a non-database source | Omit database-only flags when ingesting source connections | +| Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures | diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index d2348231..4de40ecb 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -3,7 +3,14 @@ title: "ktx setup" description: "Set up or resume a local KTX project." --- -Interactive wizard that walks you through configuring LLM credentials, embeddings, database connections, context sources, and agent integrations. When run without flags in a directory that has no `ktx.yaml`, it launches the full guided flow. When run in an existing project, it resumes from the first incomplete step. +`ktx setup` is the guided configuration flow for a local KTX project. It can +create or resume `ktx.yaml`, configure LLM and embedding providers, add +database and context-source connections, build initial context, and install +agent integrations. + +When you run bare `ktx` in an interactive terminal outside any KTX project, the +CLI starts this same setup flow. Inside an existing project, `ktx setup` +resumes from incomplete setup state or opens the setup menu. ## Command signature @@ -11,27 +18,117 @@ Interactive wizard that walks you through configuring LLM credentials, embedding ktx setup [options] ``` -## Options +## Visible Options -### General - -| Flag | Description | Default | -|------|-------------|---------| -| `--project-dir ` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd | -| `--yes` | Accept safe defaults in non-interactive setup | `false` | -| `--no-input` | Disable interactive terminal input | — | - -### Agent Integration +The help output intentionally keeps setup focused on the common interactive +flags. Automation flags are accepted by the same command and are documented +below. | Flag | Description | Default | |------|-------------|---------| | `--agents` | Install agent integration only | `false` | -| `--target ` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — | -| `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` | +| `--target ` | Agent target: `claude-code`, `codex`, `cursor`, `opencode`, or `universal` | - | +| `--global` | Install agent integration into the global target scope for `claude-code` or `codex` | `false` | +| `--yes` | Accept safe defaults in non-interactive setup | `false` | +| `--no-input` | Disable interactive terminal input | - | -The setup wizard is the public configuration interface. It prompts for LLM -credentials, embeddings, database connections, context sources, query history, -and agent integration when those values are needed. +Use the global `--project-dir ` option when setup should target a +specific directory. + +## Automation Options + +These flags are useful for repeatable setup in examples, tests, CI fixtures, and +scripted project creation. They are not shown in `ktx setup --help`. + +### Project Mode + +| Flag | Description | Default | +|------|-------------|---------| +| `--new` | Create a new KTX project before setup | `false` | +| `--existing` | Use an existing KTX project | `false` | + +### LLM Provider + +| Flag | Description | +|------|-------------| +| `--llm-backend ` | LLM backend: `anthropic` or `vertex` | +| `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | +| `--anthropic-api-key-file ` | File containing the Anthropic API key | +| `--anthropic-model ` | Anthropic model ID to validate and save | +| `--vertex-project ` | Vertex AI project ID, `env:NAME`, or `file:/path` reference | +| `--vertex-location ` | Vertex AI location, `env:NAME`, or `file:/path` reference | +| `--skip-llm` | Leave LLM setup incomplete | + +Choose only one Anthropic credential source. Anthropic credential flags are only +valid with the Anthropic backend; Vertex flags are only valid with the Vertex +backend. + +### Embeddings + +| Flag | Description | +|------|-------------| +| `--embedding-backend ` | Embedding backend: `openai` or `sentence-transformers` | +| `--embedding-api-key-env ` | Environment variable containing the embedding provider API key | +| `--embedding-api-key-file ` | File containing the embedding provider API key | +| `--skip-embeddings` | Leave embedding setup incomplete | + +`sentence-transformers` uses the KTX-managed Python runtime. Choose only one +embedding credential source. + +### Databases + +| Flag | Description | +|------|-------------| +| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` | +| `--database-connection-id ` | Existing selected connection id; repeatable | +| `--new-database-connection-id ` | Connection id for one new database connection | +| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite path | +| `--database-schema ` | Database schema or dataset to include; repeatable | +| `--skip-databases` | Leave database setup incomplete | + +KTX needs at least one database connection before it can build database +context. Use `--skip-databases` only when intentionally leaving the project +incomplete. + +### Query History + +| Flag | Description | +|------|-------------| +| `--enable-query-history` | Enable query-history ingest when the selected database supports it | +| `--disable-query-history` | Disable query-history ingest for the selected database | +| `--query-history-window-days ` | Query-history lookback window | +| `--query-history-min-executions ` | Minimum executions for a query-history template | +| `--query-history-service-account-pattern ` | Query-history service-account regex; repeatable | +| `--query-history-redaction-pattern ` | Query-history SQL-literal redaction regex; repeatable | + +Query history setup is supported for Postgres, BigQuery, and Snowflake. Enabling +query history makes deep ingest readiness matter for later `ktx ingest` runs. + +### Context Sources + +| Flag | Description | +|------|-------------| +| `--source ` | Source connector type: `dbt`, `metricflow`, `metabase`, `looker`, `lookml`, or `notion` | +| `--source-connection-id ` | Connection id for source setup | +| `--source-path ` | Local source path for dbt, MetricFlow, or LookML | +| `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML | +| `--source-branch ` | Git branch for source setup | +| `--source-subpath ` | Repo subpath for source setup | +| `--source-auth-token-ref ` | `env:` or `file:` credential reference for source repo auth | +| `--source-url ` | Source service URL for Metabase or Looker | +| `--source-api-key-ref ` | `env:` or `file:` API key reference for Metabase or Notion | +| `--source-client-id ` | Looker client id | +| `--source-client-secret-ref ` | `env:` or `file:` Looker client secret reference | +| `--source-warehouse-connection-id ` | Warehouse connection id used for source mapping | +| `--source-project-name ` | dbt project name override | +| `--source-profiles-path ` | dbt profiles path | +| `--source-target ` | dbt target or source-specific mapping target | +| `--metabase-database-id ` | Metabase database id to map | +| `--notion-crawl-mode ` | Notion crawl mode: `all_accessible` or `selected_roots` | +| `--notion-root-page-id ` | Notion root page id; repeatable | +| `--skip-sources` | Mark optional source setup complete with no sources | + +Choose only one source location: `--source-path` or `--source-git-url`. ## Examples @@ -42,14 +139,37 @@ ktx setup # Run setup for a specific project directory ktx setup --project-dir ./analytics -# Install agent integration for Claude Code only -ktx setup --agents --target claude-code +# Script a Postgres connection that reads its URL from the environment +ktx setup \ + --project-dir ./analytics \ + --no-input \ + --skip-llm \ + --skip-embeddings \ + --database postgres \ + --new-database-connection-id warehouse \ + --database-url env:DATABASE_URL \ + --database-schema public -# Install agent integration globally for Codex -ktx setup --agents --target codex --global +# Enable Postgres query history while setting up a database +ktx setup \ + --project-dir ./analytics \ + --database postgres \ + --new-database-connection-id warehouse \ + --database-url env:DATABASE_URL \ + --enable-query-history \ + --query-history-min-executions 5 -# Check setup readiness -ktx status +# Add a Metabase source mapped to an existing warehouse connection +ktx setup \ + --source metabase \ + --source-connection-id prod_metabase \ + --source-url https://metabase.example.com \ + --source-api-key-ref env:METABASE_API_KEY \ + --source-warehouse-connection-id warehouse \ + --metabase-database-id 1 + +# Install project-scoped agent integration for Codex +ktx setup --agents --target codex ``` ## Output @@ -68,11 +188,16 @@ KTX context built: yes Agent integration ready: yes (codex:project) ``` +Use `ktx status` for repeatable readiness checks after setup exits. + ## Common errors | Error | Cause | Recovery | |-------|-------|----------| | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | -| Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup | -| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests | +| Setup cannot run in CI | Required values are missing and `--no-input` disables prompts | Provide the relevant automation flags or create a fixture `ktx.yaml` | +| Provider health check fails | Provider key, model id, Vertex project, or Vertex location is invalid | Fix the `env:` or `file:` reference and rerun setup | +| `--enable-query-history` is rejected | The selected database driver does not support query history | Use Postgres, BigQuery, or Snowflake, or rerun without query-history flags | +| Source setup rejects location flags | Both `--source-path` and `--source-git-url` were supplied | Choose the local path or the Git URL, not both | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | +| Global agent install is rejected | `--global` was used with a target other than `claude-code` or `codex` | Omit `--global`, or choose `--target claude-code` or `--target codex` | diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index b3e5305f..3d7fd8d9 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -3,7 +3,9 @@ title: "ktx sl" description: "List, search, validate, or query semantic-layer sources." --- -Interact with your project's semantic layer. Semantic sources are YAML definitions that describe your tables, columns, measures, joins, and grain — the vocabulary agents use to generate correct SQL. +Interact with your project's semantic layer. Semantic sources are YAML +definitions that describe tables, columns, measures, joins, segments, and grain: +the vocabulary agents use to generate correct SQL. ## Command signature @@ -26,7 +28,7 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | Filter by KTX connection id | — | +| `--connection-id ` | Filter by KTX connection id | - | | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | @@ -34,8 +36,8 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | Filter by KTX connection id | — | -| `--limit ` | Maximum search results | — | +| `--connection-id ` | Filter by KTX connection id | - | +| `--limit ` | Maximum search results | - | | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | @@ -43,24 +45,29 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | KTX connection id (required) | — | +| `--connection-id ` | KTX connection id (required) | - | ### `sl query` | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | KTX connection id | — | -| `--query-file ` | JSON semantic-layer query file | — | -| `--measure ` | Measure to query; repeatable (at least one required) | — | -| `--dimension ` | Dimension to include; repeatable | — | -| `--filter ` | Filter expression; repeatable | — | -| `--segment ` | Segment to include; repeatable | — | -| `--order-by ` | Order field, optionally suffixed with `:asc` or `:desc`; repeatable | — | -| `--limit ` | Query limit | — | +| `--connection-id ` | KTX connection id | - | +| `--query-file ` | JSON semantic-layer query file | - | +| `--measure ` | Measure to query; repeatable (at least one required) | - | +| `--dimension ` | Dimension to include; repeatable | - | +| `--filter ` | Filter expression; repeatable | - | +| `--segment ` | Segment to include; repeatable | - | +| `--order-by ` | Order field, optionally suffixed with `:asc` or `:desc`; repeatable | - | +| `--limit ` | Query limit | - | | `--include-empty` | Include empty rows | `false` | | `--format ` | Output format: `json` or `sql` | `json` | | `--execute` | Execute the compiled query against the database | `false` | -| `--max-rows ` | Maximum rows to return when executing | — | +| `--yes` | Install the managed Python runtime without prompting when required | `false` | +| `--no-input` | Disable interactive managed runtime installation | - | +| `--max-rows ` | Maximum rows to return when executing | - | + +`sl query` requires at least one `--measure` unless `--query-file` is set. +`--query-file` should point to a JSON semantic-layer query object. ## Examples @@ -113,6 +120,13 @@ ktx sl query \ --execute \ --max-rows 1000 +# Compile or execute without prompting for runtime installation +ktx sl query \ + --connection-id my-warehouse \ + --measure orders.count \ + --execute \ + --yes + # Execute a query from a JSON file ktx sl query \ --connection-id my-warehouse \ @@ -123,7 +137,10 @@ ktx sl query \ ## Output -Semantic-layer commands return human-readable output by default. Use `--json` or `--format json` when an agent needs structured output; use `--format sql` to inspect generated SQL before execution. +Semantic-layer list and search commands return human-readable output by +default. Use `--json` on `list` or `search` when an agent needs structured +output. Use `--format sql` on `query` to inspect generated SQL before +execution, or leave `--format json` for the compiled query and optional rows. ```json { @@ -145,3 +162,4 @@ Semantic-layer commands return human-readable output by default. Use `--json` or | Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` | | Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl search`, inspect the source YAML in your project files, then retry using declared fields | | Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing | +| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx dev runtime install --feature core --yes`, or rerun `ktx sl query --yes` | diff --git a/docs-site/content/docs/cli-reference/ktx-status.mdx b/docs-site/content/docs/cli-reference/ktx-status.mdx index 28f4e981..dae22857 100644 --- a/docs-site/content/docs/cli-reference/ktx-status.mdx +++ b/docs-site/content/docs/cli-reference/ktx-status.mdx @@ -4,8 +4,9 @@ description: "Check KTX setup and project readiness." --- Run the KTX readiness doctor. Inside a KTX project, this checks setup, -project configuration, semantic search, connections, and related diagnostics. -Outside a project, it checks local CLI setup readiness. +project configuration, semantic search, query history, connections, and related +diagnostics. Outside a project, it checks local CLI setup readiness so you know +whether `ktx setup` can run. ## Command signature @@ -18,7 +19,9 @@ ktx status [options] | Flag | Description | Default | |------|-------------|---------| | `--json` | Print JSON output | `false` | -| `--no-input` | Disable interactive terminal input | — | +| `-v`, `--verbose` | Show every check, including passing ones | `false` | +| `--validate` | Only validate the `ktx.yaml` schema; skip readiness checks | `false` | +| `--no-input` | Disable interactive terminal input | - | ## Examples @@ -28,12 +31,21 @@ ktx status # Get status as JSON without interactive input ktx status --json --no-input + +# Show all checks, not only warnings and failures +ktx status --verbose + +# Validate ktx.yaml without running readiness checks +ktx status --validate + +# Check a project from another directory +ktx status --project-dir ./analytics ``` ## Output -`ktx status` prints doctor checks. Agents should use `ktx status --json --no-input` -when they need to branch on readiness state. +`ktx status` prints grouped doctor checks. Agents should use +`ktx status --json --no-input` when they need to branch on readiness state. ```json { @@ -55,4 +67,6 @@ when they need to branch on readiness state. |-------|-------|----------| | No KTX project found | Current directory has no `ktx.yaml` and `KTX_PROJECT_DIR` is unset | `ktx status` runs setup checks; run from a KTX project or set `KTX_PROJECT_DIR` for project checks | | Project config check fails | The project directory is missing or has an invalid `ktx.yaml` | Run `ktx setup` to resume setup | +| Schema validation fails | `ktx.yaml` does not match the current config schema | Run `ktx status --validate --json` for structured issue details, then edit `ktx.yaml` or rerun `ktx setup` | | Semantic search check warns | Embeddings are not configured or the provider probe failed | Run `ktx setup` or inspect the check's `fix` field in JSON output | +| Query history check warns | A database has query history enabled but the warehouse prerequisites are missing | Fix the warehouse extension, grants, or history access, then rerun `ktx status` | diff --git a/docs-site/content/docs/cli-reference/ktx-wiki.mdx b/docs-site/content/docs/cli-reference/ktx-wiki.mdx index a6a0ca01..ad0e53bf 100644 --- a/docs-site/content/docs/cli-reference/ktx-wiki.mdx +++ b/docs-site/content/docs/cli-reference/ktx-wiki.mdx @@ -3,7 +3,9 @@ title: "ktx wiki" description: "List or search wiki pages." --- -Manage wiki pages in your KTX project. Wiki pages are Markdown documents that capture business definitions, rules, and gotchas. Agents search them for context when answering questions about your data. +List and search wiki pages in your KTX project. Wiki pages are Markdown +documents that capture business definitions, rules, and gotchas. Agents search +them for context when answering questions about your data. ## Command signature @@ -18,22 +20,28 @@ ktx wiki [options] | `list` | List local wiki pages | | `search ` | Search local wiki pages | +The current public CLI lists and searches wiki pages. Edit the Markdown files +under `wiki/` directly, or ingest source content with `ktx ingest`, when you +need to add or update wiki knowledge. + ## Options ### `wiki list` | Flag | Description | Default | |------|-------------|---------| -| `--json` | Print JSON output | `false` | | `--user-id ` | Local user id | `local` | +| `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | +| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | ### `wiki search` | Flag | Description | Default | |------|-------------|---------| -| `--json` | Print JSON output | `false` | | `--user-id ` | Local user id | `local` | -| `--limit ` | Maximum search results | — | +| `--limit ` | Maximum search results | - | +| `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | +| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | ## Examples @@ -49,12 +57,17 @@ ktx wiki search "monthly recurring revenue" # Search wiki pages as JSON ktx wiki search "monthly recurring revenue" --json --limit 10 + +# Print search results as TSV +ktx wiki search "monthly recurring revenue" --output plain ``` ## Output -Wiki commands print local wiki page listings and search results. Open the -matching Markdown files directly when you need the full page contents. +Wiki commands print clack-style pretty output in a TTY and TSV-style plain +output when requested. JSON output wraps the items with a command metadata +envelope. Open the matching Markdown files directly when you need the full page +contents. ```json { diff --git a/docs-site/content/docs/cli-reference/meta.json b/docs-site/content/docs/cli-reference/meta.json index 093f75ae..46aafb4c 100644 --- a/docs-site/content/docs/cli-reference/meta.json +++ b/docs-site/content/docs/cli-reference/meta.json @@ -2,6 +2,7 @@ "title": "CLI Reference", "defaultOpen": true, "pages": [ + "index", "ktx-setup", "ktx-connection", "ktx-ingest", diff --git a/docs-site/content/docs/community/contributing.mdx b/docs-site/content/docs/community/contributing.mdx index 1b4e39ce..ac960395 100644 --- a/docs-site/content/docs/community/contributing.mdx +++ b/docs-site/content/docs/community/contributing.mdx @@ -3,7 +3,7 @@ title: Contributing description: How to contribute to KTX. --- -KTX is an open-source project and welcomes contributions — bug fixes, new connectors, documentation improvements, and feature proposals. This page covers how to set up a development environment, navigate the repository, run tests, and submit changes. +KTX is an open-source project and welcomes contributions - bug fixes, new connectors, documentation improvements, and feature proposals. This page covers how to set up a development environment, navigate the repository, run tests, and submit changes. ## Development setup @@ -14,9 +14,9 @@ an analytics project, use the published ### Prerequisites -- **Node.js 22+** and **pnpm** — for the TypeScript workspace -- **Python 3.11+** and **uv** — for the Python semantic layer and daemon -- **Git** — for version control +- **Node.js 22+** and **pnpm** - for the TypeScript workspace +- **Python 3.11+** and **uv** - for the Python semantic layer and daemon +- **Git** - for version control ### Clone and install @@ -72,12 +72,12 @@ packages/ connector-posthog/ # PostHog connector python/ - ktx-sl/ # Semantic layer — grain-aware query planning and SQL generation - ktx-daemon/ # Daemon — portable API server around the semantic layer + ktx-sl/ # Semantic layer - grain-aware query planning and SQL generation + ktx-daemon/ # Daemon - portable API server around the semantic layer examples/ # Example projects and fixtures scripts/ # Workspace scripts (benchmarks, verification, release) -docs/ # Documentation site (Fumadocs) +docs-site/ # Documentation site (Fumadocs) ``` All TypeScript packages are ESM (`"type": "module"`) and use `NodeNext` module resolution. The Python projects use `pyproject.toml` for dependency management. @@ -179,17 +179,17 @@ The `package.json` should follow the pattern of existing connectors: Your connector class must implement `KtxScanConnector`, which requires: -- **`id`** — a string identifier, typically `":"` -- **`driver`** — the `KtxConnectionDriver` value for your database -- **`capabilities`** — a `KtxConnectorCapabilities` object declaring what your connector supports: `tableSampling`, `columnSampling`, `columnStats`, `readOnlySql`, `nestedAnalysis`, `eventStreamDiscovery`, `formalForeignKeys`, `estimatedRowCounts` -- **`introspect()`** — discovers tables, columns, types, and constraints, returning a `KtxSchemaSnapshot` +- **`id`** - a string identifier, typically `":"` +- **`driver`** - the `KtxConnectionDriver` value for your database +- **`capabilities`** - a `KtxConnectorCapabilities` object declaring what your connector supports: `tableSampling`, `columnSampling`, `columnStats`, `readOnlySql`, `nestedAnalysis`, `eventStreamDiscovery`, `formalForeignKeys`, `estimatedRowCounts` +- **`introspect()`** - discovers tables, columns, types, and constraints, returning a `KtxSchemaSnapshot` Optional methods for richer scanning: -- **`sampleColumn()`** — sample values from a specific column -- **`sampleTable()`** — sample rows from a table -- **`columnStats()`** — compute column statistics -- **`executeReadOnly()`** — execute arbitrary read-only SQL +- **`sampleColumn()`** - sample values from a specific column +- **`sampleTable()`** - sample rows from a table +- **`columnStats()`** - compute column statistics +- **`executeReadOnly()`** - execute arbitrary read-only SQL ### Step 3: Add a dialect @@ -212,7 +212,7 @@ Use `packages/connector-sqlite/` as a minimal reference and `packages/connector- ## Code conventions - **TypeScript**: strict types, no `any`, no `as unknown as`. Use `zod` schemas for runtime validation at CLI and config boundaries. Follow the `camelCaseSchema` / `PascalCaseType` naming convention for Zod schemas and inferred types. -- **Python**: type hints on all new code, `pathlib` over `os.path`, explicit exception types over broad `except Exception`, `logger.exception()` for caught exceptions. Use `sqlglot` for SQL parsing — never regex. +- **Python**: type hints on all new code, `pathlib` over `os.path`, explicit exception types over broad `except Exception`, `logger.exception()` for caught exceptions. Use `sqlglot` for SQL parsing - never regex. - **Dependencies**: `pnpm` for Node packages (never `npm` or `bun`), `uv` for Python (never `pip`). - **Dead code**: remove it. Don't leave commented-out code, unused wrappers, or empty directories. @@ -220,11 +220,11 @@ Use `packages/connector-sqlite/` as a minimal reference and `packages/connector- Before submitting a pull request: -1. **Run the relevant checks** — at minimum, `pnpm run type-check` and `pnpm run test` for TypeScript changes, `uv run pytest -q` and `uv run pre-commit run --files [FILES]` for Python changes. -2. **Build if you changed exports** — run `pnpm run build` to verify package exports and `dist/` expectations still align. -3. **Keep changes focused** — one logical change per PR. Don't bundle unrelated refactors. -4. **Follow existing patterns** — match the style and conventions of surrounding code. The codebase favors explicit over clever. -5. **Don't commit artifacts** — `node_modules/`, `.venv/`, `dist/`, coverage output, and local databases should not be committed. +1. **Run the relevant checks** - at minimum, `pnpm run type-check` and `pnpm run test` for TypeScript changes, `uv run pytest -q` and `uv run pre-commit run --files [FILES]` for Python changes. +2. **Build if you changed exports** - run `pnpm run build` to verify package exports and `dist/` expectations still align. +3. **Keep changes focused** - one logical change per PR. Don't bundle unrelated refactors. +4. **Follow existing patterns** - match the style and conventions of surrounding code. The codebase favors explicit over clever. +5. **Don't commit artifacts** - `node_modules/`, `.venv/`, `dist/`, coverage output, and local databases should not be committed. For larger features or architectural changes, open an issue first to discuss the approach. diff --git a/docs-site/content/docs/community/index.mdx b/docs-site/content/docs/community/index.mdx new file mode 100644 index 00000000..25518bc8 --- /dev/null +++ b/docs-site/content/docs/community/index.mdx @@ -0,0 +1,54 @@ +--- +title: Community +description: Contribute to KTX through code, docs, connectors, and examples. +--- + +KTX is an open-source context layer for database agents. The project welcomes +focused contributions that improve setup, integrations, CLI behavior, +documentation, connector coverage, and examples. + +## Where to start + +| Goal | Start here | +|------|------------| +| Prepare a local development checkout | [Contributing](/docs/community/contributing#development-setup) | +| Understand the workspace layout | [Repository structure](/docs/community/contributing#repository-structure) | +| Run verification before a pull request | [Running tests](/docs/community/contributing#running-tests) | +| Add a database connector | [Adding a connector](/docs/community/contributing#adding-a-connector) | +| Update docs for a user-visible CLI or setup change | [PR guidelines](/docs/community/contributing#pr-guidelines) | + +## Contribution areas + +| Area | Good first context | +|------|--------------------| +| CLI and setup | `packages/cli`, especially setup steps, command definitions, status checks, and smoke tests | +| Context engine | `packages/context`, including project config, ingest orchestration, and semantic search | +| Connectors | `packages/connector-*`, plus connector-specific tests and integration docs | +| Python semantic layer | `python/ktx-sl` for planning and SQL generation | +| Python daemon | `python/ktx-daemon` for the portable runtime API | +| Documentation | `docs-site/content/docs` for public docs and `docs-site/tests` for docs behavior | + +## Development loop + +```bash +pnpm install +uv sync --all-groups +pnpm run setup:dev +pnpm run link:dev +ktx-dev status +``` + +Use `ktx-dev` for local CLI testing after linking the development binary. Use +the published `ktx` command when you are testing the released package in a +separate analytics project. + +## Before submitting + +1. Keep the change focused on one behavior, connector, doc area, or workflow. +2. Run the smallest tests that cover the changed surface. +3. Run broader checks when changing shared exports, setup state, or generated files. +4. Update `docs-site/content/docs/` when user-visible setup, CLI, configuration, or integration behavior changes. +5. Do not commit local secrets, generated build output, virtualenvs, dependency directories, or local databases. + +For complete contributor setup and verification commands, read +[Contributing](/docs/community/contributing). diff --git a/docs-site/content/docs/community/meta.json b/docs-site/content/docs/community/meta.json index 0f97320e..a9988c4e 100644 --- a/docs-site/content/docs/community/meta.json +++ b/docs-site/content/docs/community/meta.json @@ -1,5 +1,5 @@ { "title": "Community", "defaultOpen": true, - "pages": ["contributing"] + "pages": ["index", "contributing"] } diff --git a/docs-site/content/docs/concepts/context-as-code.mdx b/docs-site/content/docs/concepts/context-as-code.mdx index 56dca056..e6ebcd7c 100644 --- a/docs-site/content/docs/concepts/context-as-code.mdx +++ b/docs-site/content/docs/concepts/context-as-code.mdx @@ -1,23 +1,23 @@ --- title: Context as Code -description: Treat analytics context like code — version it, review it, merge it. +description: Treat analytics context like code - version it, review it, merge it. --- ## The idea -dbt proved that analytics transformations belong in version control. Before dbt, SQL lived in BI tools, scheduling systems, and spreadsheets — scattered, unreviewed, impossible to audit. "Analytics as code" changed that: put your models in git, review them in PRs, deploy them by merging. +dbt proved that analytics transformations belong in version control. Before dbt, SQL lived in BI tools, scheduling systems, and spreadsheets - scattered, unreviewed, impossible to audit. "Analytics as code" changed that: put your models in git, review them in PRs, deploy them by merging. -KTX applies the same principle to analytics context. Metric definitions, business rules, join relationships, wiki pages — these are artifacts that determine whether an agent produces correct results. They change over time. They need review. They need history. They need to be treated like code. +KTX applies the same principle to analytics context. Metric definitions, business rules, join relationships, wiki pages - these are artifacts that determine whether an agent produces correct results. They change over time. They need review. They need history. They need to be treated like code. A KTX project is a git repository. Semantic sources are YAML files. Wiki pages are Markdown files. Changes are commits. Updates are pull requests. Deployment is a merge. The entire lifecycle of your analytics context follows the same workflow your team already uses for dbt models, application code, and infrastructure. ## Auto-ingestion -Most analytics context already exists — it's in your dbt manifests, LookML models, Metabase questions, and team Notion pages. KTX pulls from these sources automatically through adapters. +Most analytics context already exists - it's in your dbt manifests, LookML models, Metabase questions, and team Notion pages. KTX pulls from these sources automatically through adapters. An ingestion run works like this: -1. **Adapters extract metadata.** Each configured source — dbt, LookML, Metabase, MetricFlow, Notion, or your live database — provides structured metadata about models, metrics, dimensions, questions, and documentation. +1. **Adapters extract metadata.** Each configured source - dbt, LookML, Metabase, MetricFlow, Notion, or your live database - provides structured metadata about models, metrics, dimensions, questions, and documentation. 2. **The LLM agent reconciles.** KTX doesn't blindly overwrite existing context. An LLM agent compares incoming metadata against your current semantic sources and wiki pages. It decides what to create, what to update, and what to leave alone. If your dbt project added a new model, the agent writes a new semantic source. If a Metabase question references a metric you've already defined, the agent skips the duplicate. @@ -66,17 +66,17 @@ metadata, and documentation updates are ready for review each morning. Once merged, agents querying through the KTX CLI see the updated context immediately. No deployment step, no cache invalidation, no restart. The files are the source of truth, and agents read them on every request. -This workflow gives you the same review guarantees you have for dbt models. No semantic source reaches production without a human approving it. But unlike maintaining context manually, the heavy lifting — discovering new tables, drafting source definitions, extracting business rules from documentation — is done by the ingestion agent. You review and approve. You don't write from scratch. +This workflow gives you the same review guarantees you have for dbt models. No semantic source reaches production without a human approving it. But unlike maintaining context manually, the heavy lifting - discovering new tables, drafting source definitions, extracting business rules from documentation - is done by the ingestion agent. You review and approve. You don't write from scratch. ## Feedback loops Context improves over time through two feedback channels. -**Analyst corrections.** When an analytics engineer spots something wrong — a measure formula that doesn't match the business definition, a join that should be `many_to_one` instead of `one_to_many`, a wiki page that's out of date — they edit the YAML or Markdown directly and commit. These corrections become part of the project's git history, and the next ingestion run respects them. If you manually fix a measure definition, KTX won't overwrite it on the next ingest. +**Analyst corrections.** When an analytics engineer spots something wrong - a measure formula that doesn't match the business definition, a join that should be `many_to_one` instead of `one_to_many`, a wiki page that's out of date - they edit the YAML or Markdown directly and commit. These corrections become part of the project's git history, and the next ingestion run respects them. If you manually fix a measure definition, KTX won't overwrite it on the next ingest. -**Agent feedback.** When an agent queries the semantic layer and gets unexpected results — a query that returns no rows because of a bad filter, a join path that produces duplicated results — it can flag the issue. These signals feed back into the context: wiki pages can note known data quality issues, and source definitions can be tightened with better filters, join paths, or grain declarations. +**Agent feedback.** When an agent queries the semantic layer and gets unexpected results - a query that returns no rows because of a bad filter, a join path that produces duplicated results - it can flag the issue. These signals feed back into the context: wiki pages can note known data quality issues, and source definitions can be tightened with better filters, join paths, or grain declarations. -Each of these channels makes the next ingestion cycle better. Analyst corrections teach the system what your team considers authoritative. Agent feedback surfaces gaps in coverage. Context is not a static artifact — it's a living system that converges toward accuracy with every iteration. +Each of these channels makes the next ingestion cycle better. Analyst corrections teach the system what your team considers authoritative. Agent feedback surfaces gaps in coverage. Context is not a static artifact - it's a living system that converges toward accuracy with every iteration. ## Deterministic replay @@ -84,9 +84,9 @@ Every ingestion session in KTX produces a full transcript: every tool call the L This matters for three reasons. -**Debugging.** When a semantic source looks wrong — the grain is off, a join points to the wrong table, a measure formula doesn't match the business definition — you can trace it back to the ingestion session that created it. The transcript shows exactly which adapter provided the input, how the LLM interpreted it, and why it made the decision it did. You don't have to guess. +**Debugging.** When a semantic source looks wrong - the grain is off, a join points to the wrong table, a measure formula doesn't match the business definition - you can trace it back to the ingestion session that created it. The transcript shows exactly which adapter provided the input, how the LLM interpreted it, and why it made the decision it did. You don't have to guess. -**Trust.** Analytics teams need to trust the context that agents consume. Deterministic replay means you can verify any part of the context layer by re-examining the session that produced it. If a stakeholder asks "where did this revenue definition come from?", you have a complete audit trail — from the dbt manifest entry, through the LLM's reconciliation logic, to the YAML file that was written. +**Trust.** Analytics teams need to trust the context that agents consume. Deterministic replay means you can verify any part of the context layer by re-examining the session that produced it. If a stakeholder asks "where did this revenue definition come from?", you have a complete audit trail - from the dbt manifest entry, through the LLM's reconciliation logic, to the YAML file that was written. **Reproducibility.** Because ingestion sessions are recorded as structured transcripts (tool calls and responses, not just logs), they can be replayed for testing and validation. If you change your ingestion configuration or upgrade the LLM, you can replay previous sessions to see how the output would differ. This gives you a safety net for changes that affect how context is generated. diff --git a/docs-site/content/docs/concepts/the-context-layer.mdx b/docs-site/content/docs/concepts/the-context-layer.mdx index a76cbe7d..cb03b7c0 100644 --- a/docs-site/content/docs/concepts/the-context-layer.mdx +++ b/docs-site/content/docs/concepts/the-context-layer.mdx @@ -5,7 +5,7 @@ description: What a context layer is, why agents need one, and how KTX compares ## The problem -Give an agent access to your database and it will generate SQL. It might even produce a decent chart. But ask it a real analytics question — "what's our net revenue trend by segment?" — and things fall apart. +Give an agent access to your database and it will generate SQL. It might even produce a decent chart. But ask it a real analytics question - "what's our net revenue trend by segment?" - and things fall apart. The agent doesn't know that `orders.amount` includes refunds and needs a status filter. It doesn't know that `customers` should join to `orders` on `customer_id`, not `id`. It doesn't know that your team stopped using `legacy_segments` six months ago, or that "enterprise" means contracts over $100k, not just big logos. It sees column names and types. It doesn't see your business. @@ -17,15 +17,170 @@ Analytics engineers already know this pain. It's the same reason you write dbt t The industry has moved through three distinct approaches to getting AI and data to work together. -**Wave one: database access.** Connect an LLM to a database, let it generate SQL. This works for simple lookups — "how many orders last week?" — but breaks on anything that requires business knowledge. The agent guesses at joins, invents metrics, and hallucinates table relationships. Every query is a coin flip. +**Wave one: database access.** Connect an LLM to a database, let it generate SQL. This works for simple lookups - "how many orders last week?" - but breaks on anything that requires business knowledge. The agent guesses at joins, invents metrics, and hallucinates table relationships. Every query is a coin flip. -**Wave two: semantic layers and text-to-SQL.** Add structure. Define metrics in MetricFlow or Cube, expose schemas, build text-to-SQL pipelines. This is better — the agent knows that `revenue` means `sum(amount) where status != 'refunded'` — but building and maintaining that structure by hand is manual, time-consuming, and still limited. Semantic layers define what to calculate, not why, when, or how to interpret the result. The agent can compute net revenue but doesn't know about the February refund anomaly, the segment reclassification, or the fact that `enterprise` changed definition last quarter. +**Wave two: semantic layers and text-to-SQL.** Add structure. Define metrics in MetricFlow or Cube, expose schemas, build text-to-SQL pipelines. This is better - the agent knows that `revenue` means `sum(amount) where status != 'refunded'` - but building and maintaining that structure by hand is manual, time-consuming, and still limited. Semantic layers define what to calculate, not why, when, or how to interpret the result. The agent can compute net revenue but doesn't know about the February refund anomaly, the segment reclassification, or the fact that `enterprise` changed definition last quarter. -**Wave three: agentic context.** AI is no longer just answering questions — it's generating dashboards, writing semantic definitions, proposing dbt models, creating tests and documentation. For that to work, agents need more than metric definitions. They need the full picture: business rules, known data quality issues, relationship maps, historical context, and the institutional knowledge that lives in your team's heads. They need a context layer. +**Wave three: agentic context.** AI is no longer just answering questions - it's generating dashboards, writing semantic definitions, proposing dbt models, creating tests and documentation. For that to work, agents need more than metric definitions. They need the full picture: business rules, known data quality issues, relationship maps, historical context, and the institutional knowledge that lives in your team's heads. They need a context layer. ## What a context layer is -A context layer is the infrastructure that gives agents the business knowledge they need to produce correct analytics artifacts. It includes a semantic layer — that's a critical component — but it's not the whole thing. +A context layer is the infrastructure that gives agents the business knowledge they need to produce correct analytics artifacts. It includes a semantic layer - that's a critical component - but it's not the whole thing. + +
+
+

+ {"How KTX works"} +

+

+ {"KTX pulls structured metadata and human knowledge from your analytics stack, reconciles it into reviewable files, then gives agents a trusted surface for search, SQL generation, validation, and edits."} +

+
+ +
+
+

+ {"Source systems"} +

+
+
+

+ {"Warehouses"} +

+

+ {"schemas, types, row counts, constraints, query history"} +

+
+
+

+ {"Modeling tools"} +

+

+ {"dbt, MetricFlow, LookML"} +

+
+
+

+ {"BI systems"} +

+

+ {"Looker explores, Metabase questions, dashboards"} +

+
+
+

+ {"Notion and team knowledge"} +

+

+ {"runbooks, definitions, policies, analyst notes"} +

+
+
+
+ + + +
+
+

+ {"KTX ingest"} +

+
+
+

{"Extract"}

+

+ {"adapters read metadata, files, APIs, and warehouse evidence"} +

+
+
+

{"Reconcile"}

+

+ {"the ingest agent merges new facts with existing context"} +

+
+
+

{"Verify"}

+

+ {"validation and provenance make each write auditable"} +

+
+
+
+ + + +
+

+ {"KTX project"} +

+
+
+
+ {"semantic-layer/"} +
+
+ {"sources, columns, joins, grain, measures, segments, filters"} +
+
+
+
{"wiki/"}
+
+ {"business definitions, rules, gotchas, semantic references"} +
+
+
+
+ {"raw-sources/"} +
+
+ {"scan artifacts, extracted metadata, relationship evidence"} +
+
+
+
{".ktx/"}
+
+ {"local indexes, embeddings, session state, caches"} +
+
+
+
+ + + +
+

+ {"Agent workflows"} +

+
+
+ {"Search sources and wiki pages"} +
+
+ {"Compile trusted SQL"} +
+
+ {"Explain metrics and provenance"} +
+
+ {"Patch files, validate, open review"} +
+
+
+
+ +
+ {"Reviewed agent and analyst edits flow back into the same YAML and Markdown files, so the next ingest run starts from the team's accepted context."} +
+
KTX organizes context into four pillars: @@ -67,7 +222,7 @@ measures: expr: count(id) ``` -**Wiki pages** are Markdown documents that capture business definitions, rules, and operating context — the kind of context that doesn't fit in a schema definition. Pages have structured frontmatter (summary, tags, semantic layer references) and free-form content. Agents search them when they need to understand why a metric works a certain way, not just how to compute it. +**Wiki pages** are Markdown documents that capture business definitions, rules, and operating context - the kind of context that doesn't fit in a schema definition. Pages have structured frontmatter (summary, tags, semantic layer references) and free-form content. Agents search them when they need to understand why a metric works a certain way, not just how to compute it. ```markdown --- @@ -91,9 +246,9 @@ canonical revenue reporting. **Scan artifacts** are the raw output of KTX's database scanner: table and column metadata, inferred foreign key relationships (even without declared constraints), column statistics, and enrichment reports. They form the foundation that semantic sources are built on. -**Provenance** is the record of how context was created and changed. Every ingestion session records a full transcript — which adapter ran, what the LLM decided, which sources were created or updated, and why. This is what makes the system auditable: you can trace any semantic source back to the ingestion decision that created it. +**Provenance** is the record of how context was created and changed. Every ingestion session records a full transcript - which adapter ran, what the LLM decided, which sources were created or updated, and why. This is what makes the system auditable: you can trace any semantic source back to the ingestion decision that created it. -Together, these four pillars give agents enough context to produce analytics artifacts that match what your team would produce — not just syntactically valid SQL, but the right query for the question. +Together, these four pillars give agents enough context to produce analytics artifacts that match what your team would produce - not just syntactically valid SQL, but the right query for the question. ## How KTX compares @@ -115,7 +270,7 @@ If you do not have a semantic layer, KTX can build an agent-native one from your ## The plain-files philosophy -A KTX project is a directory of plain files. No server to run, no database to manage, no proprietary store to back up. Everything is YAML, Markdown, and SQLite — formats you can read, diff, and version-control with tools you already use. +A KTX project is a directory of plain files. No server to run, no database to manage, no proprietary store to back up. Everything is YAML, Markdown, and SQLite - formats you can read, diff, and version-control with tools you already use. ``` my-project/ @@ -140,7 +295,7 @@ my-project/ └── cache/ # Runtime cache (git-ignored) ``` -Semantic sources and wiki pages are committed to git. The SQLite database holds ephemeral state — schema ingest results, embedding indexes, session logs — and is git-ignored. If you delete it, KTX rebuilds it on the next run. +Semantic sources and wiki pages are committed to git. The SQLite database holds ephemeral state - schema ingest results, embedding indexes, session logs - and is git-ignored. If you delete it, KTX rebuilds it on the next run. This means your analytics context travels with your code. You can fork it, branch it, review it in a PR, and merge it with the same tools you use for dbt models. There's no sync problem between a remote server and your local state. There's no migration to run. The files are the source of truth. diff --git a/docs-site/content/docs/getting-started/introduction.mdx b/docs-site/content/docs/getting-started/introduction.mdx index 70ca9a84..cb8ac0dd 100644 --- a/docs-site/content/docs/getting-started/introduction.mdx +++ b/docs-site/content/docs/getting-started/introduction.mdx @@ -51,7 +51,7 @@ description: How KTX gives analytics agents trusted context for warehouse work. ## Who KTX is for KTX is built for analytics engineers and data teams who want data agents to -work on real analytics systems — not just generate one-off SQL. +work on real analytics systems - not just generate one-off SQL. Use KTX when you want agents to: diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 28728886..335aedfa 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -1,254 +1,286 @@ --- title: Quickstart -description: Set up KTX and build your first context in under 10 minutes. +description: Set up KTX, build local context, and connect your coding agent. --- -This guide walks you through `ktx setup` — an interactive wizard that configures your LLM provider, connects your database, optionally ingests from your existing tools, builds context, and installs agent integration. +This guide gets a local analytics project ready for KTX. You will install the +CLI, run the setup wizard, connect a database, build context, and install agent +rules that teach your coding assistant which KTX commands to run. -If you are a coding assistant trying to decide which KTX docs page to read, start with the [Agent Quickstart](/docs/ai-resources/agent-quickstart). This page is the human setup walkthrough. +If you are a coding assistant choosing a docs route, start with the +[Agent Quickstart](/docs/ai-resources/agent-quickstart). This page is the +human setup walkthrough. -## Workflow summary +## What setup does -Use this sequence when you are setting up KTX in an analytics project: +`ktx setup` is the main project workflow. It can create or resume `ktx.yaml`, +configure model and embedding providers, add database connections, add optional +context sources, build the first context artifacts, and install agent +integration. -1. `npm install -g @kaelio/ktx` — install the published KTX CLI from npm. -2. `ktx setup` — create or resume a KTX project. +When you run bare `ktx` in an interactive terminal outside a KTX project, the +CLI opens the same setup experience. Inside an existing project, `ktx setup` +resumes incomplete work or opens a menu for changing setup, connecting an +agent, checking status, or exploring a demo project. -The setup wizard is stateful. If it exits before completion, rerun `ktx setup` in the same project directory to resume from the first incomplete step. +## Install the CLI -## Install and run setup - -Install the published [`@kaelio/ktx`](https://www.npmjs.com/package/@kaelio/ktx) CLI: +Install the published `@kaelio/ktx` package: ```bash npm install -g @kaelio/ktx ``` -Then run the setup wizard: +Then run setup from the analytics project directory: ```bash ktx setup ``` -The local checkout flow is only for contributors working on KTX itself. See [Contributing](/docs/community/contributing) for that setup. +The local checkout workflow is only for KTX contributors. See +[Contributing](/docs/community/contributing) for that path. -The wizard walks through six steps. You can go back at any point, and if you exit early, rerunning `ktx setup` resumes where you left off. +## Step 1: Choose the project -## Step 1: Configure LLM +In an interactive terminal, setup can create a new KTX project or resume the +nearest existing project. The main project file is `ktx.yaml`. -KTX uses an Anthropic model to enrich schema descriptions, generate semantic sources during ingestion, and reconcile metadata from your tools. +For scripted setup, pass the project directory explicitly: -The wizard asks how to find your API key: - -``` -◆ How should KTX find your Anthropic API key? -│ ○ Use ANTHROPIC_API_KEY from the environment -│ ○ Paste a key and save it as a local secret file +```bash +ktx setup --project-dir ./analytics ``` -If you choose to paste a key, KTX saves it in `.ktx/secrets/anthropic-api-key` with local file permissions. Your `ktx.yaml` stores a `file:` reference, never the raw key. +If setup exits early, rerun `ktx setup` in the same directory. KTX tracks +completed setup steps and resumes from the remaining work. -Next, choose a model: +## Step 2: Configure the LLM -``` -◆ Which Anthropic model should KTX use? -│ ○ Claude Sonnet 4.6 (recommended) -│ ○ Claude Opus 4.6 -│ ○ Claude Haiku 4.5 -│ ○ Enter a model ID manually -``` +KTX uses a Claude model for ingest agents that turn schemas, SQL, BI metadata, +and documents into semantic-layer sources and wiki context. -KTX runs a health check to verify your key and model work before saving. +Setup supports two LLM provider paths: -## Step 2: Configure embeddings +| Provider | Use when | Credential model | +|----------|----------|------------------| +| Anthropic API | You have an Anthropic API key | `ANTHROPIC_API_KEY` or a local `file:` secret | +| Google Vertex AI for Anthropic Claude | Your organization runs Claude through Google Cloud | Application Default Credentials plus Vertex project and location | -KTX uses embeddings for semantic search over sources, wiki content, schema metadata, and relationship evidence. +For Anthropic API, setup can read the key from the environment or save a pasted +key to `.ktx/secrets/anthropic-api-key`. `ktx.yaml` stores an `env:` or `file:` +reference, not the raw key. -``` -◆ Which embedding option should KTX use? -│ ○ Local sentence-transformers embeddings -│ ○ OpenAI embeddings (recommended) -``` +For Vertex AI, setup uses Google Application Default Credentials. It can read +your active `gcloud` project, list visible projects, or accept explicit +`--vertex-project` and `--vertex-location` values. -**OpenAI embeddings** use `text-embedding-3-small` (1536 dimensions) and require an `OPENAI_API_KEY`. +Setup checks the selected model before saving. Anthropic API setup fetches live +Claude model choices when possible and falls back to bundled defaults if model +discovery is unavailable. -**Local embeddings** use `all-MiniLM-L6-v2` (384 dimensions) via the KTX managed Python runtime. No API key is needed. KTX can install and start the runtime during setup; to prepare it ahead of time, run: +## Step 3: Configure embeddings + +KTX uses embeddings for semantic search over semantic-layer sources, wiki +context, schema metadata, and relationship evidence. + +| Backend | Default model | Notes | +|---------|---------------|-------| +| OpenAI | `text-embedding-3-small` | Recommended for hosted embeddings. Requires an OpenAI API key. | +| Local sentence-transformers | `all-MiniLM-L6-v2` | Runs through the KTX-managed Python runtime. No hosted embedding key is required. | + +OpenAI setup reads `OPENAI_API_KEY` or saves a local secret file. Local +sentence-transformers setup can install and start the managed runtime during +setup. To prepare that runtime before setup, run: ```bash ktx dev runtime install --feature local-embeddings --yes ktx dev runtime start --feature local-embeddings ``` -## Step 3: Connect a database +## Step 4: Add a database -Select one or more databases for KTX to connect to. The wizard supports -SQLite, PostgreSQL, MySQL, ClickHouse, SQL Server, BigQuery, and Snowflake. +KTX needs at least one primary database connection before it can build database +context. The wizard supports SQLite, PostgreSQL, MySQL, ClickHouse, SQL Server, +BigQuery, and Snowflake. -For PostgreSQL, you can enter connection details field by field or paste a connection URL: +You can usually enter connection fields interactively or provide a URL. Secret +URLs can be stored as local files under `.ktx/secrets/` or referenced with +`env:NAME` in `ktx.yaml`. -``` -◆ How do you want to connect to PostgreSQL? -│ ○ Enter connection details (host, port, database, user) -│ ○ Paste a connection URL -``` +After saving a connection, setup tests it and builds fast schema context: -If your URL contains credentials, KTX saves it to `.ktx/secrets/` and writes a `file:` reference in `ktx.yaml`. You can also use `env:DATABASE_URL` to reference an environment variable. - -After connecting, KTX automatically runs a connection test and builds fast -schema context: - -``` -Testing postgres-warehouse +```text +Testing warehouse Connection test passed - Driver: PostgreSQL - Tables: 42 -Building schema context for postgres-warehouse +Building schema context for warehouse Running fast database ingest -Schema context complete for postgres-warehouse - Changes: 42 new tables - Database ready - postgres-warehouse - PostgreSQL - schema context complete + warehouse - PostgreSQL - schema context complete ``` -For PostgreSQL, Snowflake, and BigQuery, the wizard can enable query-history -ingest when the warehouse history feature is available. Query history is stored -under `connections..context.queryHistory` in `ktx.yaml`. +PostgreSQL, BigQuery, and Snowflake can also enable query-history ingest. Query +history helps KTX learn common query patterns, joins, service-account filters, +and warehouse-specific usage. -## Step 4: Add context sources +## Step 5: Add context sources -Context sources let KTX ingest metadata from your existing analytics tools. This step is optional — you can skip it and add sources later. +Context sources are optional, but they make the first context layer much richer. +Setup can add: -``` -◆ Which context sources should KTX ingest? -│ ◻ dbt -│ ◻ MetricFlow -│ ◻ Metabase -│ ◻ Looker -│ ◻ LookML -│ ◻ Notion -``` +| Source | Typical input | What KTX learns | +|--------|---------------|-----------------| +| dbt | Local project or Git repo | Models, columns, tests, descriptions, tags | +| MetricFlow | Local project or Git repo | Semantic models, metrics, dimensions, entities | +| LookML | Local files or Git repo | Views, explores, dimensions, measures, joins | +| Looker | API URL and credentials | Explores, looks, dashboards, model metadata | +| Metabase | API URL and key | Questions, dashboards, BI database mappings | +| Notion | Integration token and crawl settings | Business docs and knowledge pages | -For **dbt**, point KTX at a local path or git URL. KTX reads your `dbt_project.yml` and schema files to extract model metadata: +Setup maps BI and source metadata back to your primary warehouse connection so +generated context points at the right tables. -``` -◆ dbt source location -│ ○ Local path -│ ○ Git URL -``` +You can skip this step and add sources later by rerunning `ktx setup`. -For **Metabase** and **Looker**, you provide an API URL and credentials. KTX maps BI databases to your KTX primary source connections so it knows which warehouse tables the BI metadata refers to. +## Step 6: Build context -Context sources are saved to `ktx.yaml` and built during the next step. +The context build turns configured databases and sources into local artifacts +agents can read. It runs database ingest first, then source ingest and memory +updates. -## Step 5: Build context +Fast database ingest records deterministic schema grounding. Deep ingest adds +AI-enriched descriptions, embeddings, relationship evidence, and query-history +context when configured. -This is where KTX builds agent-ready context. It uses the database context -depth saved by setup and ingests metadata from any configured context sources. +When the build finishes, setup verifies that agent-ready context exists: -``` -◆ Build KTX context for agents? -│ ○ Build context now (recommended) -│ ○ Leave context unbuilt and exit setup -``` - -Fast database context builds deterministic schema grounding. Deep database -context also generates AI descriptions, embeddings, and relationship evidence -when those capabilities are configured. - -For a small database (under 50 tables), this can take a few minutes. Larger -warehouses can take longer. Context builds run in the foreground; press -Ctrl+C to stop the current run and rerun `ktx setup` or `ktx ingest` -when you are ready to try again. - -When the build completes, KTX verifies that agent-ready context was produced: - -``` +```text KTX context is ready for agents. Databases: - postgres-warehouse: deep context complete + warehouse: deep context complete Context sources: - dbt-main: memory update complete + dbt_main: memory update complete Verification: Agent context: ready Semantic search: ready ``` -## Step 6: Install agent integration +If a foreground build is interrupted, rerun `ktx setup` or build the same target +with `ktx ingest `. -The final step connects KTX to your coding agent. Choose how agents should access the project: +## Step 7: Install agent integration -``` -◆ How should agents use this KTX project? -│ ○ CLI tools and skills +The final setup step installs project-local rules for your coding assistant. +Supported targets are Claude Code, Codex, Cursor, OpenCode, and universal +`.agents`. + +You can also run this step later: + +```bash +ktx setup --agents --target codex ``` -Then select which agents to install for: +Claude Code and Codex also support global installs: -``` -◆ Which agent targets should KTX install? -│ ◻ Claude Code -│ ◻ Codex -│ ◻ Cursor -│ ◻ OpenCode -│ ◻ Custom agent (.agents) +```bash +ktx setup --agents --target codex --global ``` -**CLI mode** writes a skill file (e.g., `.claude/skills/ktx/SKILL.md`) that teaches the agent to call KTX commands directly. - -**Custom agent** uses the universal `.agents` target for agents that can read project-local skills. +Agent rules are CLI-based. They point agents at the KTX CLI path that created +the file, so agents do not need a separate `ktx` binary in `PATH`. If the CLI +path changes after reinstalling or moving a checkout, rerun `ktx setup --agents`. ## Generated files -KTX writes project state as plain files so agents can inspect and edit changes in git. +KTX writes plain files so people and agents can inspect changes in git. -| Path | Created by | Purpose | -|------|------------|---------| -| `ktx.yaml` | `ktx setup` | Main project configuration: connections, LLM settings, embeddings, and context sources | -| `.ktx/secrets/*` | `ktx setup` when file-backed secrets are selected | Local secret files referenced from `ktx.yaml`; do not commit these | -| `semantic-layer//*.yaml` | context build, ingestion, or direct file edits | Semantic source definitions agents use for SQL generation | -| `wiki/global/*.md` | ingestion, memory capture, or direct file edits | Shared business context and metric definitions | -| `wiki/user//*.md` | memory capture or direct file edits | User-scoped notes for one agent/user context | -| `.claude/skills/ktx/SKILL.md`, `.agents/skills/ktx/SKILL.md` | CLI-mode agent integration setup | Agent instructions for calling public `ktx` commands | +| Path | Purpose | +|------|---------| +| `ktx.yaml` | Project configuration for LLMs, embeddings, connections, context sources, and setup state | +| `.ktx/secrets/*` | Local secret files referenced from `ktx.yaml`; do not commit these | +| `.ktx/setup/*` | Local setup and context-build state | +| `.ktx/agents/install-manifest.json` | Manifest used to manage installed agent files | +| `semantic-layer//*.yaml` | Semantic source definitions used for SQL generation | +| `wiki/global/*.md` | Shared business context and metric definitions | +| `wiki/user//*.md` | User-scoped notes and local context | +| `.claude/skills/ktx/SKILL.md` | Claude Code project skill | +| `.agents/skills/ktx/SKILL.md` | Codex or universal project skill | +| `.cursor/rules/ktx.mdc` | Cursor project rule | +| `.opencode/commands/ktx.md` | OpenCode project command | -## Verify it worked +## Verify setup -Check your project status: +Run: ```bash ktx status ``` -``` +Example output: + +```text KTX project: /home/user/analytics Project ready: yes LLM ready: yes (claude-sonnet-4-6) Embeddings ready: yes (text-embedding-3-small) -Databases configured: yes (postgres-warehouse) -Context sources configured: yes (dbt-main) +Databases configured: yes (warehouse) +Context sources configured: yes (dbt_main) KTX context built: yes -Agent integration ready: yes (claude-code:project) +Agent integration ready: yes (codex:project) ``` +Use JSON when an agent or script needs a structured readiness check: + +```bash +ktx status --json +``` + +## Scripted setup example + +Use non-interactive setup when creating repeatable fixtures or automation: + +```bash +ktx setup \ + --project-dir ./analytics \ + --no-input \ + --skip-llm \ + --skip-embeddings \ + --database postgres \ + --new-database-connection-id warehouse \ + --database-url env:DATABASE_URL \ + --database-schema public +``` + +Then build context: + +```bash +ktx ingest warehouse --fast +``` + +See [ktx setup](/docs/cli-reference/ktx-setup) for the full automation flag +surface. + ## Common errors -| Error or symptom | Likely cause | Recovery | -|------------------|--------------|----------| -| `ktx: command not found` | The KTX package is not installed globally, or the shell cannot find the global binary | Run `npm install -g @kaelio/ktx` and open a new shell | -| LLM health check fails | Missing, invalid, or unauthorized Anthropic API key | Export `ANTHROPIC_API_KEY` or rerun `ktx setup` and choose the file-backed secret option | -| OpenAI embedding check fails | `OPENAI_API_KEY` is missing when OpenAI embeddings are selected | Export `OPENAI_API_KEY`, or rerun setup and choose local sentence-transformers embeddings | -| Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup | -| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | -| `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now | -| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need | +| Symptom | Likely cause | Recovery | +|---------|--------------|----------| +| `ktx: command not found` | The global package is not installed or your shell cannot find it | Reinstall `@kaelio/ktx` and open a new shell | +| Setup resumes the wrong project | `KTX_PROJECT_DIR` or the nearest `ktx.yaml` points somewhere else | Pass `--project-dir ` | +| Anthropic health check fails | API key, model id, or access is invalid | Fix `ANTHROPIC_API_KEY` or rerun setup with a different key or model | +| Vertex AI health check fails | Vertex API, Claude access, project, location, or IAM permissions are missing | Check the project, location, Application Default Credentials, and Vertex AI permissions | +| OpenAI embeddings fail | `OPENAI_API_KEY` is missing or invalid | Export the key or choose local sentence-transformers embeddings | +| Local embeddings fail | Managed Python runtime cannot install or start | Run `ktx dev runtime status`, then install the local embeddings runtime | +| Database test fails | Credentials, network access, database, warehouse, or schema is wrong | Test the same values with the database's native client, then rerun setup | +| Context is not built | Setup saved configuration but skipped or interrupted the build | Run `ktx setup` or `ktx ingest --all` | +| Agent integration is incomplete | Setup skipped the agents step or installed a different target | Run `ktx setup --agents --target ` | ## Next steps -- **Build more context** — learn about [database ingest](/docs/guides/building-context), relationship detection, and source ingestion workflows in the Building Context guide. -- **Refine your semantic layer** — the [Writing Context](/docs/guides/writing-context) guide covers source YAML, measures, joins, and wiki pages. -- **Understand the architecture** — read [The Context Layer](/docs/concepts/the-context-layer) to learn why a context layer is more than a semantic layer. -- **Connect more agents** — see the [Agent Clients](/docs/integrations/agent-clients) integration page for per-tool setup details. +- Build and refresh context with [Building Context](/docs/guides/building-context). +- Edit semantic sources and wiki pages with [Writing Context](/docs/guides/writing-context). +- Connect more tools with [Agent Clients](/docs/integrations/agent-clients). +- Read [The Context Layer](/docs/concepts/the-context-layer) to understand the architecture. diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx index 5dcf2422..c21b7921 100644 --- a/docs-site/content/docs/guides/building-context.mdx +++ b/docs-site/content/docs/guides/building-context.mdx @@ -1,171 +1,195 @@ --- title: Building Context -description: Build database and source context from configured KTX connections. +description: Build and refresh KTX context from databases, source tools, query history, and text. --- -Building context reads your configured connections and writes local context that -agents can use. Database connections produce schema context, and source -connections such as dbt, Looker, Metabase, and Notion produce semantic sources -and wiki pages. +Building context turns configured connections into local semantic-layer sources +and wiki pages. Agents use those files to understand your schema, business +definitions, metric logic, joins, and known caveats before they write SQL. + +Use this guide after `ktx setup` has created `ktx.yaml` and at least one +database or context-source connection. + +## The build loop + +Most projects use this loop: + +1. Check readiness with `ktx status`. +2. Build one connection with `ktx ingest `, or build everything + with `ktx ingest --all`. +3. Search or inspect the generated files under `semantic-layer/` and `wiki/`. +4. Edit source YAML or Markdown when business logic needs refinement. +5. Validate and query representative sources before handing the context to an + agent. + +`ktx ingest --all` runs database connections first, then context-source +connections. That order lets dbt, BI, Notion, and text ingest attach context to +known warehouse tables. ## Database ingest -Database ingest connects to your warehouse and extracts structural metadata. -KTX stores the results locally so agents can understand your schema without -querying the database directly. - -### Running database ingest +Database ingest connects to a configured warehouse and records local schema +context. It gives agents table, column, type, constraint, and row-count +grounding without requiring them to inspect the database directly. ```bash -ktx ingest -``` - -This runs a fast schema ingest by default. You can choose the depth with public -flags: - -| Flag | What it does | -|------|-------------| -| `--fast` | Tables, columns, types, constraints, and row counts | -| `--deep` | Fast ingest plus AI-enriched database context | - -```bash -# Build one connection quickly -ktx ingest my-postgres --fast - -# Build AI-enriched database context -ktx ingest my-postgres --deep +# Build one configured database connection +ktx ingest warehouse # Build all configured connections ktx ingest --all ``` -### Checking results +Depth controls how much context KTX builds: -Every ingest prints a summary and writes local artifacts. Use `ktx status` -after ingest to review project readiness and follow-up setup work: +| Flag | Best for | What it does | +|------|----------|--------------| +| `--fast` | First setup, quick refreshes, CI smoke checks | Deterministic schema ingest with tables, columns, types, constraints, and row counts | +| `--deep` | Agent-ready context for real analysis | Fast ingest plus AI-enriched descriptions, embeddings, relationship evidence, and optional query history | + +Examples: ```bash -ktx status +ktx ingest warehouse --fast +ktx ingest warehouse --deep +ktx ingest --all --deep ``` -### Relationship detection +Deep ingest needs LLM and embedding readiness. If those providers are not +configured, run `ktx setup` or use `--fast`. -Many databases lack declared foreign keys. KTX infers relationships by scoring column pairs across seven signals — name similarity, type compatibility, value overlap, embedding similarity, profile uniqueness, null rate, and structural priors. The weighted score determines each candidate's status: +## Query history -| Score range | Status | Meaning | -|-------------|--------|---------| -| ≥ 0.85 | `accepted` | High confidence — applied automatically | -| 0.55 – 0.84 | `review` | Plausible — needs human review | -| < 0.55 | `rejected` | Low confidence — not applied | +PostgreSQL, BigQuery, and Snowflake can add query-history context. This helps +KTX learn common joins, filters, service-account patterns, redaction rules, and +usage-heavy query templates. -Deep database ingest can include relationship evidence where the connector can -provide it. Relationship review and calibration subcommands are not part of the -current public CLI surface. - -## Ingestion - -Ingestion pulls semantic context from your existing analytics tools — dbt projects, Looker models, Metabase questions, and more — and writes it into your KTX project as semantic sources and wiki pages. - -### How it works - -Each ingest run follows this flow: - -1. An **adapter** extracts metadata from your tool (dbt manifest, LookML files, Metabase API, etc.) -2. An **LLM agent** reconciles the extracted metadata with your existing context — it merges intelligently rather than overwriting -3. **Semantic sources** (YAML) and **wiki pages** (Markdown) are written to your project directory - -### Running an ingest +Enable it during setup, store it under `connections..context.queryHistory`, +or request it for one run: ```bash -ktx ingest my-dbt-source +ktx ingest warehouse --deep --query-history +ktx ingest warehouse --query-history-window-days 30 ``` -Useful output flags: +Use `--no-query-history` when you want to skip a stored query-history setting +for one run. + +## Relationship evidence + +Many databases do not declare all foreign keys. KTX can score relationship +candidates using signals such as name similarity, type compatibility, value +overlap, embedding similarity, uniqueness, null rate, and structural priors. + +The public CLI does not expose separate relationship review subcommands. +Relationship evidence is built as part of deep database ingest when the +connector and readiness checks support it. + +## Context-source ingest + +Context-source connections pull business metadata from tools your team already +uses. The current public `ktx ingest` command is connection-centric: pass one +configured connection id, or pass `--all`. + +```bash +# Build one source connection +ktx ingest dbt_main + +# Build every configured database and source connection +ktx ingest --all +``` + +Supported source types: + +| Driver | Typical source | Output | +|--------|----------------|--------| +| `dbt` | dbt project or Git repo | Semantic sources with model, column, test, tag, and description metadata | +| `metricflow` | MetricFlow project or Git repo | Metrics, dimensions, entities, and semantic joins | +| `lookml` | LookML files or Git repo | Views, explores, dimensions, measures, and joins | +| `looker` | Looker API | Explores, looks, dashboards, and model metadata | +| `metabase` | Metabase API | Questions, dashboards, table metadata, and mappings | +| `notion` | Notion API | Wiki pages and business knowledge | + +Source ingest extracts metadata, reconciles it with existing local context, and +writes semantic-layer YAML plus wiki Markdown. It merges rather than blindly +overwriting local edits. + +## Text ingest + +Use `ktx ingest text` for notes, Markdown files, runbooks, Slack exports, or +other free-form knowledge that should become searchable KTX memory. + +```bash +# Capture a Markdown file +ktx ingest text docs/revenue-notes.md --connection-id warehouse + +# Capture one stdin item +printf "Refunds are excluded from net revenue." | ktx ingest text - + +# Capture direct text +ktx ingest text --text "ARR excludes one-time implementation fees." +``` + +Useful flags: | Flag | Description | |------|-------------| -| `--json` | Output as JSON | -| `--plain` | Plain text output | +| `--connection-id ` | Attach the captured memory to a KTX connection | +| `--user-id ` | Attribute capture to a user scope, default `local-cli` | +| `--json` | Print structured output | +| `--fail-fast` | Stop after the first failed text item | -Foreground context builds do not detach into background control sessions. If a -run is interrupted, rerun `ktx ingest ` or `ktx ingest --all`. +Text ingest is a good fit for small, high-signal documents. For system-specific +connectors such as Notion, dbt, or Metabase, prefer configured source ingest so +KTX can preserve source metadata. -### Supported context sources +## Output and artifacts -| Driver | Source | What gets ingested | -|--------|--------|--------------------| -| `dbt` | dbt project | Model definitions, column descriptions, tests, tags | -| `metricflow` | MetricFlow semantic models | Metrics, dimensions, entities, semantic joins | -| `lookml` | LookML files | Views, explores, dimensions, measures, joins | -| `looker` | Looker API | Explores, looks, dashboard metadata | -| `metabase` | Metabase API | Questions, dashboards, table metadata | -| `notion` | Notion API | Database pages, knowledge articles | +Every ingest run prints a summary. Use `--json` when an agent or script needs a +structured plan and per-target results. -Query history is a database connection facet. Enable it with -`connections..context.queryHistory` or pass `--query-history` for a current -run. See [Context Sources](/docs/integrations/context-sources) for -driver-specific setup and auth configuration. - -### What gets generated - -A typical dbt ingest produces semantic sources and wiki pages in your project: - -**Semantic source** (`semantic-layer/my-postgres/orders.yaml`): - -```yaml title="semantic-layer/my-postgres/orders.yaml" -name: orders -table: public.orders -grain: - - order_id -columns: - - name: order_id - type: string - description: Unique order identifier - - name: customer_id - type: string - description: Foreign key to customers table - - name: order_date - type: time - role: time - description: Date the order was placed - - name: total_amount - type: number - description: Total order value in USD -measures: - - name: total_revenue - expr: SUM(total_amount) - description: Sum of all order values - - name: order_count - expr: COUNT(DISTINCT order_id) - description: Number of distinct orders -joins: - - to: customers - on: orders.customer_id = customers.customer_id - relationship: many_to_one +```bash +ktx ingest --all --json ``` -**Wiki page** (`wiki/global/order-status-definitions.md`): +Typical generated files: -```markdown ---- -summary: Business definitions for order status values -tags: [orders, definitions] -sl_refs: [orders] ---- +| Path | Created by | Purpose | +|------|------------|---------| +| `semantic-layer//*.yaml` | Database and source ingest | Queryable semantic source definitions | +| `wiki/global/*.md` | Source, text, and memory ingest | Shared business definitions and notes | +| `wiki/user//*.md` | Text and memory ingest | User-scoped context | +| `.ktx/setup/context-build.json` | Setup context build | Resume and readiness state for setup | -## Order Statuses +Ingest sessions also record transcripts with tool calls, LLM responses, and +write decisions. Inspect them when you need to debug why a source or wiki page +was written a certain way. -- **pending**: Order placed but not yet processed -- **confirmed**: Payment received, awaiting fulfillment -- **shipped**: Order dispatched to carrier -- **delivered**: Order received by customer -- **cancelled**: Order cancelled before shipment +## Example: first full refresh -Orders in "pending" status for more than 48 hours are flagged for review. +After interactive setup: + +```bash +ktx status +ktx ingest --all --deep +ktx status ``` -### Ingest transcripts +Then inspect what changed: -Every ingest session records a full transcript: tool calls, LLM responses, and -write decisions. Inspect the stored transcript files when you need to debug why -a source was written a certain way. +```bash +git status --short +ktx sl list --json +ktx wiki search "revenue" --json --limit 10 +``` + +## Common errors + +| Symptom | Likely cause | Recovery | +|---------|--------------|----------| +| Connection not configured | The connection id is missing from `ktx.yaml` | Add it with `ktx setup` | +| Deep readiness is missing | LLM or embeddings are not setup-ready | Run `ktx setup`, or rerun with `--fast` | +| Query history is unsupported | The selected database driver does not expose query history | Run schema ingest without query-history flags | +| No target selected | You omitted both a connection id and `--all` | Run `ktx ingest ` or `ktx ingest --all` | +| Source flags have no effect | Depth and query-history flags were supplied for a source connector | Use those flags only for database connections | +| Text ingest stops early | `--fail-fast` stopped on the first failed item | Fix the item or rerun without `--fail-fast` | diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 4a93ae43..192b1c7f 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -1,59 +1,167 @@ --- title: Serving Agents -description: Expose your context to Claude Code, Cursor, Codex, and other coding agents. +description: Expose KTX context to Claude Code, Codex, Cursor, OpenCode, and custom agents. --- -Once you've built and refined your context, expose it to coding agents through -the public KTX CLI. Claude Code, Cursor, Codex, OpenCode, and custom agent -workflows can call the same commands you use at a terminal. +KTX serves agents through the public CLI and project-local instruction files. +Agents do not need a separate server. They read the generated rules, call KTX +commands, inspect local context files, and use JSON output when they need +structured results. -## CLI Commands +## Recommended setup -KTX public commands support JSON output for the context reads that agents use -most often. Use `--project-dir` when the agent is not already running inside the -KTX project directory. - -### Available commands +Run the agent install step from a KTX project: + +```bash +ktx setup --agents +``` + +Or install a specific target: + +```bash +ktx setup --agents --target codex +``` + +Supported targets: + +| Target | Generated project file | +|--------|------------------------| +| Claude Code | `.claude/skills/ktx/SKILL.md` | +| Codex | `.agents/skills/ktx/SKILL.md` | +| Cursor | `.cursor/rules/ktx.mdc` | +| OpenCode | `.opencode/commands/ktx.md` | +| Universal `.agents` | `.agents/skills/ktx/SKILL.md` | + +Claude Code and Codex also support global installs: + +```bash +ktx setup --agents --target claude-code --global +ktx setup --agents --target codex --global +``` + +KTX records installed files in `.ktx/agents/install-manifest.json`. Rerun +`ktx setup --agents` after moving a checkout or reinstalling the CLI so the +generated instructions point at the current CLI path. + +## Agent command set + +All supported agent clients use the same command surface. Use `--project-dir` +when the agent is running outside the KTX project directory. + +### Readiness ```bash -# Check setup and context readiness ktx status --json ``` -**Semantic layer:** +Agents should run this before relying on context. It reports project, LLM, +embedding, database, context-source, context-build, and agent-integration +readiness. + +### Semantic layer discovery ```bash -# List sources ktx sl list --json -ktx sl list --json --connection-id my-postgres -ktx sl search "revenue" --json +ktx sl list --connection-id warehouse --json +ktx sl search "revenue" --json --limit 10 +``` -# Run a query from a JSON file -ktx sl query --json \ - --connection-id my-postgres \ - --query-file query.json \ +Agents use these commands to discover source names, connection ids, measures, +dimensions, and likely files to inspect. + +### Semantic-layer validation and queries + +```bash +ktx sl validate orders --connection-id warehouse +``` + +Compile SQL before executing: + +```bash +ktx sl query \ + --connection-id warehouse \ + --measure orders.total_revenue \ + --dimension orders.created_date \ + --format sql +``` + +Execute only when the task calls for live data: + +```bash +ktx sl query \ + --connection-id warehouse \ + --measure orders.total_revenue \ + --dimension orders.status \ --execute \ --max-rows 100 ``` -**Wiki:** +For complex calls, agents can write a JSON query object and pass it with +`--query-file`. + +### Wiki context ```bash -# Search wiki pages +ktx wiki list --json ktx wiki search "revenue recognition" --json --limit 10 ``` -## Setting Up Your Agent +Agents should search wiki context when a question depends on business +definitions, metric caveats, process rules, or terms that are not obvious from +schema names. -The fastest way to connect an agent is through the setup wizard: +### Context refresh + +Agents can refresh context when the user asks them to: ```bash -ktx setup +ktx ingest warehouse --fast +ktx ingest --all +ktx ingest text docs/revenue-notes.md --connection-id warehouse ``` -The agents step auto-detects installed tools and generates the right -configuration. For manual setup or per-tool details, see the -[Agent Clients](/docs/integrations/agent-clients) integration page. +Use `--deep` only when LLM and embedding setup is ready and the user expects an +AI-enriched refresh. -After configuration, the agent can immediately call KTX commands to list -sources, search wiki pages, and query your semantic layer. +## Good agent behavior + +Agents should: + +- Run `ktx status --json` before using KTX context. +- Use `ktx sl search` and `ktx wiki search` before writing SQL from memory. +- Inspect the relevant YAML or Markdown files after search returns candidates. +- Compile SQL with `ktx sl query --format sql` before executing. +- Use `--max-rows` whenever executing a live query. +- Validate edited semantic sources with `ktx sl validate`. +- Keep generated context changes reviewable in git. + +Agents should not assume a background server, ORPC route, frontend app, or +external migration system exists. KTX is a local context layer with a CLI and +plain project files. + +## Manual setup + +Manual setup is useful for custom agents that can read project-local +instructions but are not yet a named target. + +1. Install the universal target: + + ```bash + ktx setup --agents --target universal + ``` + +2. Configure the agent to read `.agents/skills/ktx/SKILL.md`. +3. Open the agent in the KTX project directory. +4. Ask it to run `ktx status --json` and summarize readiness. + +For per-client notes, see [Agent Clients](/docs/integrations/agent-clients). + +## Troubleshooting + +| Symptom | Likely cause | Recovery | +|---------|--------------|----------| +| Agent says KTX is unavailable | Agent did not load the generated instruction file | Rerun `ktx setup --agents --target ` and restart the agent session | +| Agent command cannot find the project | Agent is running outside the KTX directory | Add `--project-dir ` or open the agent in the project root | +| Generated rules point at a missing CLI path | CLI was moved, rebuilt, or reinstalled | Rerun `ktx setup --agents` | +| Agent cannot find a metric | Context is missing or stale | Run `ktx sl search`, inspect source YAML, then refresh with `ktx ingest` if needed | +| Agent query returns too many rows | The command executed without a result cap | Require `--max-rows` for executed queries | diff --git a/docs-site/content/docs/guides/writing-context.mdx b/docs-site/content/docs/guides/writing-context.mdx index b5a6db5c..fe9d3fdb 100644 --- a/docs-site/content/docs/guides/writing-context.mdx +++ b/docs-site/content/docs/guides/writing-context.mdx @@ -1,295 +1,341 @@ --- title: Writing Context -description: Write and refine semantic sources and wiki pages. +description: Edit semantic sources and wiki pages so agents use your business logic. --- -After building context through scanning and ingestion, you'll want to refine it — edit semantic sources to match your business logic, add wiki pages that capture tribal knowledge, and query your data through the semantic layer to verify everything works. +KTX context is meant to be edited. Ingest gives you a grounded first draft, then +you refine source YAML and wiki Markdown until agents can answer data questions +with the same definitions your team uses. -## Agent workflow summary +Use this guide when you are adding measures, fixing joins, documenting business +rules, or reviewing context changes made by an agent. -Agents should refine context in this order: +## Editing workflow -1. `ktx sl list --json` — discover available sources and connection ids. -2. `ktx sl search --json` — find source candidates for a concept. -3. Edit the source YAML directly in `semantic-layer//`. -4. `ktx sl validate --connection-id ` — verify columns, joins, and table references. -5. `ktx sl query ... --format sql` — compile a representative query without executing it. -6. `ktx wiki search ...` — check business context captured by ingest or memory. +Use this order for most context changes: -## Semantic Sources +1. Discover existing context. -Semantic sources are YAML files that describe your tables, columns, measures, and joins. They're the core of the context layer — the structured definitions that agents use to generate correct SQL. + ```bash + ktx sl list --json + ktx sl search "revenue" --json + ktx wiki search "revenue recognition" --json --limit 10 + ``` -### Listing sources +2. Edit the smallest relevant files under `semantic-layer//` or + `wiki/`. +3. Validate semantic source changes. -```bash -# List all sources across connections -ktx sl list + ```bash + ktx sl validate orders --connection-id warehouse + ``` -# List sources for a specific connection -ktx sl list --connection-id my-postgres +4. Compile a representative query before executing it. -# Output as JSON -ktx sl list --json + ```bash + ktx sl query \ + --connection-id warehouse \ + --measure orders.total_revenue \ + --dimension orders.created_date \ + --format sql + ``` + +5. Search again using likely user wording to confirm the new context is + discoverable. + +## Semantic sources + +Semantic sources are YAML files that describe queryable entities. A source is +usually a table, but it can also point at a custom SQL expression. Sources +define the vocabulary agents use for measures, dimensions, segments, joins, and +grain-aware query planning. + +Source files live at: + +```text +semantic-layer//.yaml ``` -### Searching sources - -```bash -ktx sl search "revenue" --connection-id my-postgres --json -``` - -Search returns ranked source summaries. To inspect or edit a source, open the -YAML file under `semantic-layer//`. - -### The source schema - -A semantic source defines a single queryable entity — usually a table or a SQL expression. Here's a fully annotated example: +### Minimal source ```yaml name: orders -description: Customer orders with line-item totals -table: public.orders # or use `sql:` for a custom SQL expression +description: Customer orders with booked revenue. +table: public.orders grain: - - order_id # columns that uniquely identify a row + - order_id +columns: + - name: order_id + type: string + description: Unique order identifier. + - name: order_date + type: time + role: time + description: Date the order was placed. + - name: total_amount + type: number + description: Booked order value in USD. +measures: + - name: total_revenue + expr: SUM(total_amount) + description: Sum of booked order value before refunds. +``` + +### Full source shape + +```yaml +name: orders +description: Customer orders with line-item totals. +table: public.orders +grain: + - order_id columns: - name: order_id - type: string # string | number | time | boolean - description: Unique order identifier + type: string + description: Unique order identifier. - name: order_date type: time - role: time # marks this as the default time dimension - description: Date the order was placed + role: time + description: Date the order was placed. - name: status type: string - visibility: public # public (default) | internal | hidden - description: Current order status + visibility: public + description: Current order status. - name: _etl_loaded_at type: time - visibility: hidden # hidden columns are excluded from agent queries - description: Internal ETL timestamp + visibility: hidden + description: Internal load timestamp. - name: total_amount type: number - description: Order total in USD + description: Order total in USD. measures: - name: total_revenue expr: SUM(total_amount) - description: Sum of all order values + description: Sum of all order values. - name: order_count expr: COUNT(DISTINCT order_id) - description: Number of distinct orders + description: Number of distinct orders. - name: avg_order_value expr: AVG(total_amount) - description: Average order value + description: Average booked order value. - name: high_value_revenue expr: SUM(total_amount) filter: total_amount > 100 - description: Revenue from orders over $100 + description: Revenue from orders over $100. segments: - - name: us_orders - expr: country = 'US' - description: Orders from US customers + - name: completed_orders + expr: status = 'completed' + description: Orders that completed fulfillment. joins: - to: customers on: orders.customer_id = customers.customer_id - relationship: many_to_one # many_to_one | one_to_many | one_to_one + relationship: many_to_one - to: order_items on: orders.order_id = order_items.order_id relationship: one_to_many - alias: items # optional alias for the joined source + alias: items ``` -Key fields: +### Source fields | Field | Required | Description | |-------|----------|-------------| -| `name` | Yes | Source identifier (lowercase, underscores) | -| `table` or `sql` | Yes | Database table or custom SQL expression (exactly one) | -| `grain` | Yes | Columns that define row uniqueness | -| `columns` | No | Column definitions with type, role, visibility | -| `measures` | No | Aggregation expressions (SUM, COUNT, AVG, etc.) | -| `joins` | No | Relationships to other sources | -| `segments` | No | Named filter conditions | -| `inherits_columns_from` | No | Inherit column metadata from a manifest entry | +| `name` | Yes | Source identifier. Use lowercase words and underscores. | +| `table` or `sql` | Yes | Database table or custom SQL expression. Use exactly one. | +| `grain` | Yes | Columns that uniquely identify a row at the source grain. | +| `columns` | No | Column definitions with type, role, visibility, and descriptions. | +| `measures` | No | Aggregation expressions such as `SUM`, `COUNT`, and `AVG`. | +| `segments` | No | Named predicates agents can reuse. | +| `joins` | No | Relationships to other semantic sources. | +| `inherits_columns_from` | No | Inherit column metadata from a manifest entry. | -Source component fields: +### Component fields | Component | Field | Required | Description | |-----------|-------|----------|-------------| -| Column | `name` | Yes | Column identifier as used in SQL expressions | -| Column | `type` | Yes | Agent-facing type: `string`, `number`, `time`, or `boolean` | -| Column | `role` | No | Special role such as `time` for default time dimensions | -| Column | `visibility` | No | `public`, `internal`, or `hidden` | -| Column | `description` | Strongly recommended | Human-readable business meaning | -| Measure | `name` | Yes | Queryable metric name | -| Measure | `expr` | Yes | SQL aggregation expression at the source grain | -| Measure | `filter` | No | SQL predicate applied only to this measure | -| Measure | `description` | Strongly recommended | Definition agents can cite and compare | -| Segment | `name` | Yes | Reusable filter name | -| Segment | `expr` | Yes | SQL predicate for the segment | -| Join | `to` | Yes | Target semantic source name | -| Join | `on` | Yes | SQL join condition using source names or aliases | -| Join | `relationship` | Yes | `many_to_one`, `one_to_many`, or `one_to_one` | -| Join | `alias` | No | Query alias for repeated or clearer joins | +| Column | `name` | Yes | Column identifier used in SQL expressions. | +| Column | `type` | Yes | Agent-facing type: `string`, `number`, `time`, or `boolean`. | +| Column | `role` | No | Special role such as `time` for default time dimensions. | +| Column | `visibility` | No | `public`, `internal`, or `hidden`. | +| Column | `description` | Strongly recommended | Business meaning and usage notes. | +| Measure | `name` | Yes | Queryable metric name. | +| Measure | `expr` | Yes | SQL aggregation expression at the source grain. | +| Measure | `filter` | No | SQL predicate applied only to this measure. | +| Measure | `description` | Strongly recommended | Definition agents can cite and compare. | +| Segment | `name` | Yes | Reusable filter name. | +| Segment | `expr` | Yes | SQL predicate for the segment. | +| Join | `to` | Yes | Target semantic source name. | +| Join | `on` | Yes | SQL join condition using source names or aliases. | +| Join | `relationship` | Yes | `many_to_one`, `one_to_many`, or `one_to_one`. | +| Join | `alias` | No | Query alias for repeated or clearer joins. | -Column visibility controls what agents see: +### Visibility -| Visibility | Behavior | -|------------|----------| -| `public` | Included in agent queries and listings (default) | -| `internal` | Available for joins and measures but not shown to agents | -| `hidden` | Excluded entirely — useful for ETL columns | +| Visibility | Agent behavior | +|------------|----------------| +| `public` | Included in listings and available for agent queries. | +| `internal` | Available for joins and measures, but not highlighted to agents. | +| `hidden` | Excluded from agent-facing context. Use for ETL fields and sensitive internals. | -### Editing a source +## Measures -Edit source files directly. They live at -`semantic-layer//.yaml` in your project directory. +Good measures have precise names, SQL expressions at the correct grain, and +descriptions that say what is included and excluded. -### Validating sources - -Validation checks a source definition against the actual database schema: - -```bash -ktx sl validate orders --connection-id my-postgres +```yaml +measures: + - name: net_revenue + expr: SUM(total_amount - refunded_amount) + filter: status = 'completed' + description: Completed order revenue after refunds, excluding cancelled orders. ``` -This catches mismatches — columns that don't exist in the table, type mismatches, invalid join targets — before an agent tries to use the source. +Prefer one canonical measure plus wiki synonyms over several nearly identical +measures. If your team uses multiple definitions, document the distinction in a +wiki page and link it with `sl_refs`. -### Querying +## Joins and grain -The semantic layer compiles your measures and dimensions into SQL, optionally executing it against the database: +`grain` and `relationship` prevent agents from producing double-counted SQL. +State the row grain even when it seems obvious. + +```yaml +grain: + - order_id +joins: + - to: customers + on: orders.customer_id = customers.customer_id + relationship: many_to_one +``` + +Use `many_to_one` for dimensions such as customer, account, product, or plan. +Use `one_to_many` only when the target can fan out the source rows, such as +orders to order items. + +## Validate and query + +Validation checks source YAML against the live database schema: + +```bash +ktx sl validate orders --connection-id warehouse +``` + +It catches missing columns, invalid join targets, and table-reference problems +before an agent relies on the source. + +Compile a query to inspect generated SQL: ```bash -# Compile a query to SQL ktx sl query \ - --connection-id my-postgres \ - --measure total_revenue \ - --measure order_count \ - --dimension "order_date" \ - --filter "status = 'completed'" \ - --order-by order_date:desc \ + --connection-id warehouse \ + --measure orders.total_revenue \ + --dimension orders.order_date \ + --filter "orders.status = 'completed'" \ + --order-by orders.order_date:desc \ --limit 10 \ --format sql ``` -This outputs the compiled SQL without executing it. To run the query: +Execute only when you need live rows: ```bash -# Execute and return results ktx sl query \ - --connection-id my-postgres \ - --measure total_revenue \ - --dimension "order_date" \ + --connection-id warehouse \ + --measure orders.total_revenue \ + --dimension orders.status \ --execute \ --max-rows 100 ``` -Query flags: +## Wiki pages -| Flag | Description | -|------|-------------| -| `--measure ` | Measure to query (repeatable, at least one required) | -| `--dimension ` | Dimension to group by (repeatable) | -| `--filter ` | Filter expression (repeatable) | -| `--segment ` | Named segment to apply (repeatable) | -| `--order-by ` | Sort field, optionally with `:asc` or `:desc` (repeatable) | -| `--limit ` | Maximum rows in the compiled query | -| `--format ` | Output format: `json` (default) or `sql` | -| `--execute` | Execute the query against the database | -| `--max-rows ` | Maximum rows to return when executing | -| `--include-empty` | Include empty/null rows in results | +Wiki pages capture business context that does not belong in a single source +file: metric policies, dashboard caveats, company vocabulary, data freshness, +known issues, and source-of-truth notes. -The query planner is grain-aware — it understands the cardinality of joins and avoids chasm traps (double-counting caused by many-to-many fan-outs). When you query measures that span multiple sources, KTX generates sub-queries at the correct grain before joining. +Wiki files live under: -### Workflow: edit and validate a source - -1. Open `semantic-layer/my-postgres/orders.yaml`. -2. Edit the file to add columns, measures, joins, or descriptions. -3. `ktx sl validate orders --connection-id my-postgres` — check the definition against the live schema. -4. `ktx sl query --connection-id my-postgres --measure total_revenue --dimension order_date --format sql` — compile a representative query. - -If validation fails, fix the YAML before asking an agent to use the source. Common validation failures are missing columns, invalid join targets, and measure expressions that reference fields outside the source. - -## Wiki Pages - -Wiki pages are Markdown files that capture business context — definitions, rules, gotchas, and anything an agent needs to understand beyond what the schema tells it. - -### What they are - -When an agent asks "what counts as an active user?" or "why do revenue numbers differ between the dashboard and the SQL query?", the answer isn't in the schema. It's tribal knowledge that lives in Slack threads, Notion pages, or someone's head. Wiki pages make that context searchable and available to agents. - -### Organization - -Wiki pages are organized by scope: - -``` +```text wiki/ -├── global/ # Cross-cutting definitions -│ ├── order-status-definitions.md -│ ├── revenue-recognition-rules.md -│ └── data-freshness-sla.md -└── user/ - └── local/ # User-scoped context - ├── schema-conventions.md - └── known-data-issues.md + global/ + user// ``` -- **Global pages** apply across all connections — business definitions, metric standards, company terminology. -- **User-scoped pages** are private to a user ID — personal notes, local gotchas, or context you do not want shared globally. +Use global pages for shared business rules. Use user-scoped pages for local +notes, personal conventions, or context that should not be shared broadly. -### Editing pages +### Wiki page example -Create and edit wiki pages directly as Markdown files in the `wiki/` -directory. Ingest and memory capture also create these pages automatically. +```markdown +--- +summary: Revenue recognition rules for finance reporting. +tags: [revenue, finance, reporting] +sl_refs: [orders] +external_refs: + - type: notion + id: finance-revenue-policy +--- -Wiki page fields: +## Recognized Revenue + +Recognized revenue includes completed orders after refunds. It excludes +cancelled orders, test orders, implementation fees, and tax. + +Finance reporting uses order completion date, not invoice creation date. +``` + +Useful frontmatter: | Field | Required | Description | |-------|----------|-------------| -| Key | Yes | Stable page identifier used as the Markdown filename | -| Summary | Yes | Short text shown in search results | -| Content | Yes | Full Markdown business context | -| Scope | No | `global` for shared context or `user` for user-scoped notes | -| Tags | No | Search and organization labels | -| External refs | No | Links or identifiers for source-of-truth systems | -| Semantic-layer refs | No | Source names the page explains or constrains | +| `summary` | Yes | Short text shown in search results. | +| `tags` | No | Business terms and synonyms that improve search. | +| `sl_refs` | No | Semantic source names the page explains or constrains. | +| `external_refs` | No | Source-of-truth system links or ids. | -### Listing pages +## Add searchable business context + +1. Search first. + + ```bash + ktx wiki search "active customer definition" --json --limit 10 + ``` + +2. If no page covers the rule, create or edit a Markdown file under + `wiki/global/`. +3. Write a compact `summary` with the wording users are likely to ask. +4. Add tags for synonyms and related business areas. +5. Add `sl_refs` for relevant semantic sources. +6. Search again with a user-like phrase. + +## Review context changes + +Before accepting agent-written context: ```bash -ktx wiki list +git diff -- semantic-layer wiki +ktx sl validate orders --connection-id warehouse +ktx sl search "revenue" --json +ktx wiki search "revenue recognition" --json --limit 10 ``` -### Searching - -```bash -ktx wiki search "revenue recognition" -``` - -Search uses both full-text matching and semantic similarity — it finds relevant pages even when the exact terms don't match. Agents call this automatically when they need business context to answer a question. - -### Workflow: add searchable business context - -1. Search first: `ktx wiki search "order status definitions"`. -2. If no page already covers the rule, create or edit a Markdown file under `wiki/global/`. -3. Include concise frontmatter; agents see the summary before loading full content. -4. Add `tags` values for the business area and `sl_refs` values for related semantic sources. -5. Search again with the user's likely wording to confirm the page is discoverable. +Check that definitions are specific, hidden columns stay hidden, joins have +explicit relationships, and measures compile into the expected SQL. ## Common errors -| Error or symptom | Likely cause | Recovery | -|------------------|--------------|----------| -| `ktx sl validate` reports a missing column | YAML references a column that is absent from the scanned table | Run a fresh scan or update the YAML to match the warehouse schema | -| Query compilation double-counts a measure | Join relationship or grain is missing or wrong | Add `grain` and explicit `relationship` values, then validate and recompile | -| Agent cannot find a metric | Measure name or description does not match business terminology | Add a measure description and a wiki page with common synonyms | -| Wiki search misses a page | Summary and tags do not include likely user wording | Rewrite the summary and add relevant tags, then search again | -| Semantic-layer changes are hard to review | The YAML edit is too large or unfocused | Split the change into smaller source-file edits, then review the git diff | +| Symptom | Likely cause | Recovery | +|---------|--------------|----------| +| `ktx sl validate` reports a missing column | YAML references a column absent from the scanned table | Refresh database context or update the YAML | +| Query compilation double-counts a measure | `grain` or join `relationship` is missing or wrong | Add explicit grain and relationship values, then recompile | +| Agent cannot find a metric | Measure name and description do not match business terminology | Add a clearer measure description and a wiki page with synonyms | +| Wiki search misses a page | Summary, tags, or content do not match user wording | Rewrite the summary and add likely synonyms | +| Context diff is hard to review | One edit changed too many concepts | Split the change into focused source and wiki edits | diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 6f7e7660..de628197 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -7,7 +7,46 @@ KTX integrates with coding agents through CLI skills and command files. These files teach agents to call public `ktx` commands directly from the terminal for semantic-layer context and wiki knowledge. -Run `ktx setup` and select your agent targets, or configure manually using the snippets below. +Run `ktx setup` and select your agent targets, or configure manually using the +snippets below. Setup pins generated skill files to the KTX CLI path that +created them, so agents do not need `ktx` on `PATH`. + +## Install with setup + +```bash +ktx setup --agents +``` + +Use `--target` for one target: + +```bash +ktx setup --agents --target codex +``` + +Use `--global` only with `claude-code` or `codex`: + +```bash +ktx setup --agents --target claude-code --global +ktx setup --agents --target codex --global +``` + +KTX records installed files in `.ktx/agents/install-manifest.json`. That +manifest lets status checks report agent readiness and lets future cleanup +remove only files KTX installed. + +## Generated files + +| Target | Project-scoped files | Global files | +|--------|----------------------|--------------| +| Claude Code | `.claude/skills/ktx/SKILL.md`, `.claude/rules/ktx.md` | `~/.claude/skills/ktx/SKILL.md`, `~/.claude/rules/ktx.md` | +| Codex | `.agents/skills/ktx/SKILL.md`, `.codex/instructions/ktx.md` | `$CODEX_HOME/skills/ktx/SKILL.md`, `$CODEX_HOME/instructions/ktx.md` | +| Cursor | `.cursor/rules/ktx.mdc` | Not supported | +| OpenCode | `.opencode/commands/ktx.md` | Not supported | +| Universal `.agents` | `.agents/skills/ktx/SKILL.md` | Not supported | + +Skill files list pinned `ktx` commands. Rule files tell the agent when KTX is +appropriate, such as data schemas, metrics, dimensions, database structure, and +SQL questions. ## Claude Code @@ -15,11 +54,12 @@ Run `ktx setup` and select your agent targets, or configure manually using the s During setup, select **Claude Code** from the agent targets. KTX writes: -| Mode | File | -|------|------| -| CLI skills | `.claude/skills/ktx/SKILL.md` | +| Scope | Files | +|-------|-------| +| Project | `.claude/skills/ktx/SKILL.md`, `.claude/rules/ktx.md` | +| Global | `~/.claude/skills/ktx/SKILL.md`, `~/.claude/rules/ktx.md` | -Both project-scoped and global installations are supported. Global installs write to `~/.claude/skills/ktx/SKILL.md`. +Both project-scoped and global installations are supported. ### Manual CLI skills configuration @@ -42,6 +82,7 @@ Available commands: ### Workflow tips - Claude Code discovers skills automatically from `.claude/skills/`. +- Claude rules in `.claude/rules/` tell Claude when KTX should be used. - Global installation makes KTX available in all projects without per-project setup. - Keep generated skills committed only when your team wants project-local agent instructions in git. @@ -76,11 +117,13 @@ Create `.cursor/rules/ktx.mdc` with the same content structure as the Claude Cod During setup, select **Codex** from the agent targets. KTX writes: -| Mode | File | -|------|------| -| CLI skills | `.agents/skills/ktx/SKILL.md` | +| Scope | Files | +|-------|-------| +| Project | `.agents/skills/ktx/SKILL.md`, `.codex/instructions/ktx.md` | +| Global | `$CODEX_HOME/skills/ktx/SKILL.md`, `$CODEX_HOME/instructions/ktx.md` | -Both project-scoped and global installations are supported. Global installs write to `$CODEX_HOME/skills/ktx/SKILL.md` (defaults to `~/.codex/skills/ktx/SKILL.md`). +Both project-scoped and global installations are supported. `CODEX_HOME` +defaults to `~/.codex`. ### Manual CLI skills configuration @@ -90,6 +133,7 @@ Create `.agents/skills/ktx/SKILL.md` with the same content structure as Claude C - Set `CODEX_HOME` to customize the global installation directory. - Codex shares the `.agents/` directory structure with the universal format. +- Codex instructions in `.codex/instructions/` tell Codex when KTX should be used. - Global installation makes KTX available across all Codex sessions. --- @@ -143,4 +187,5 @@ All supported agent clients call the same KTX CLI commands: |---|---|---|---|---| | CLI skills | Yes | Yes (.mdc) | Yes | Yes | | Global install | Yes | No | Yes | No | -| Config location | `.claude/skills/ktx/SKILL.md` | `.cursor/rules/ktx.mdc` | `.agents/skills/ktx/SKILL.md` | `.opencode/commands/ktx.md` | +| Rule or instruction file | `.claude/rules/ktx.md` | `.cursor/rules/ktx.mdc` | `.codex/instructions/ktx.md` | `.opencode/commands/ktx.md` | +| Skill file | `.claude/skills/ktx/SKILL.md` | Not separate | `.agents/skills/ktx/SKILL.md` | Not separate | diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx index c2afd8ab..15137056 100644 --- a/docs-site/content/docs/integrations/context-sources.mdx +++ b/docs-site/content/docs/integrations/context-sources.mdx @@ -3,7 +3,7 @@ title: Context Sources description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, and Notion. --- -Context sources feed your existing analytics tooling into KTX. During ingestion, KTX extracts metadata from each source and uses an LLM agent to reconcile it with your existing semantic layer and knowledge base — merging intelligently rather than overwriting. +Context sources feed your existing analytics tooling into KTX. During ingestion, KTX extracts metadata from each source and uses an LLM agent to reconcile it with your existing semantic layer and knowledge base - merging intelligently rather than overwriting. All context sources are configured in `ktx.yaml` under `connections` with their respective `driver` value. @@ -250,7 +250,7 @@ mappings: syncMode: ONLY # ONLY = restrict to mapped DBs ``` -Find Metabase database IDs in **Admin > Databases** — the ID is in the URL when editing a database. +Find Metabase database IDs in **Admin > Databases** - the ID is in the URL when editing a database. --- @@ -353,7 +353,7 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in | Field | Description | Default | |-------|-------------|---------| -| `crawl_mode` | `all_accessible` or `selected_roots` | — | +| `crawl_mode` | `all_accessible` or `selected_roots` | - | | `root_page_ids` | Page IDs to crawl from (for `selected_roots`) | `[]` | | `root_database_ids` | Database IDs to include | `[]` | | `max_pages_per_run` | Pages processed per sync | `1000` | @@ -369,7 +369,7 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in ### Notes -- Notion is knowledge-only — it does not produce semantic layer sources +- Notion is knowledge-only - it does not produce semantic layer sources - Rate limits apply; large workspaces may require multiple ingestion runs - Incremental sync cursors are stored in `.ktx/db.sqlite`; don't add `last_successful_cursor` to `ktx.yaml` diff --git a/docs-site/content/docs/integrations/index.mdx b/docs-site/content/docs/integrations/index.mdx new file mode 100644 index 00000000..8f77a624 --- /dev/null +++ b/docs-site/content/docs/integrations/index.mdx @@ -0,0 +1,70 @@ +--- +title: Integrations +description: Connect KTX to warehouses, analytics tools, and coding agents. +--- + +KTX integrations bring trusted context into an analytics project and make that +context available to coding agents through the CLI. Start with `ktx setup` when +you want the guided flow, then use the integration reference pages for exact +configuration fields, generated files, and manual setup. + +## Integration types + +| Type | What it connects | Start here | +|------|------------------|------------| +| Primary sources | Warehouses and databases that KTX scans for schemas, constraints, row counts, and optional query history | [Primary Sources](/docs/integrations/primary-sources) | +| Context sources | Existing analytics and knowledge tools such as dbt, MetricFlow, LookML, Metabase, Looker, and Notion | [Context Sources](/docs/integrations/context-sources) | +| Agent clients | Claude Code, Codex, Cursor, OpenCode, and universal `.agents` consumers | [Agent Clients](/docs/integrations/agent-clients) | + +## Recommended setup flow + +Use this order for a new project: + +1. Run `ktx setup` from the analytics project directory. +2. Configure an LLM backend and embeddings so KTX can enrich and search context. +3. Add at least one primary source connection. +4. Add optional context sources that describe the same warehouse or business domain. +5. Build context during setup, or run `ktx ingest ` later. +6. Install agent integration with `ktx setup --agents` when the context is ready. + +For repeatable setup, pass `--project-dir`, `--no-input`, and the relevant +automation flags documented in [`ktx setup`](/docs/cli-reference/ktx-setup). + +## What setup writes + +| Path | Purpose | +|------|---------| +| `ktx.yaml` | Main project configuration for providers, embeddings, connections, source mappings, query history, and setup state | +| `.ktx/secrets/*` | Local file-backed secrets when you choose file references during setup | +| `semantic-layer//` | YAML semantic sources generated by database and source ingestion | +| `wiki/` | Markdown business context, definitions, and ingested knowledge | +| `.ktx/agents/install-manifest.json` | Manifest of agent integration files installed by `ktx setup --agents` | +| Agent client files | Skills, rules, or commands that teach agents when and how to call KTX | + +## Common commands + +```bash +# Start or resume the guided flow +ktx setup + +# Add or refresh every configured integration +ktx ingest --all + +# Refresh one configured warehouse, source, or knowledge integration +ktx ingest warehouse + +# Install one project-scoped agent target +ktx setup --agents --target codex + +# Check whether integrations are ready +ktx status +``` + +## Choosing docs + +Read [Primary Sources](/docs/integrations/primary-sources) when you need +database driver fields, authentication formats, query history support, or +warehouse-specific notes. Read [Context Sources](/docs/integrations/context-sources) +when you need source adapter fields, repository authentication, BI tool mapping, +or Notion crawl options. Read [Agent Clients](/docs/integrations/agent-clients) +when you need generated file locations or manual agent configuration. diff --git a/docs-site/content/docs/integrations/meta.json b/docs-site/content/docs/integrations/meta.json index 70fe26ec..20dc642f 100644 --- a/docs-site/content/docs/integrations/meta.json +++ b/docs-site/content/docs/integrations/meta.json @@ -1,5 +1,5 @@ { "title": "Integrations", "defaultOpen": true, - "pages": ["primary-sources", "context-sources", "agent-clients"] + "pages": ["index", "primary-sources", "context-sources", "agent-clients"] } diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index b15d93ab..a3d4db29 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -154,9 +154,9 @@ For multiple schemas: | Primary keys | Yes | Via table constraints | | Foreign keys | No | Not available in Snowflake | | Row count estimates | Yes | From `INFORMATION_SCHEMA.TABLES.ROW_COUNT` | -| Column statistics | No | — | +| Column statistics | No | - | | Query history | Yes | Via `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY` when enabled | -| Table sampling | Yes | — | +| Table sampling | Yes | - | ### Query history @@ -228,12 +228,12 @@ mapping metadata. The BigQuery connector still authenticates with the | Feature | Supported | Notes | |---------|-----------|-------| | Tables & views | Yes | Including materialized views and external tables | -| Primary keys | No | — | +| Primary keys | No | - | | Foreign keys | No | Not available in BigQuery | | Row count estimates | Yes | From table metadata | -| Column statistics | No | — | +| Column statistics | No | - | | Query history | Yes | Via region-scoped `INFORMATION_SCHEMA.JOBS_BY_PROJECT` when enabled | -| Table sampling | Yes | — | +| Table sampling | Yes | - | ### Query history @@ -307,9 +307,9 @@ connections: | Primary keys | Yes | Via `system.columns` | | Foreign keys | No | Not a ClickHouse concept | | Row count estimates | Yes | Via `system.parts` aggregation | -| Column statistics | No | — | -| Query history | No | — | -| Table sampling | Yes | — | +| Column statistics | No | - | +| Query history | No | - | +| Table sampling | Yes | - | ### Dialect notes @@ -364,8 +364,8 @@ connections: | Primary keys | Yes | Via `KEY_COLUMN_USAGE` | | Foreign keys | Yes | Via `REFERENTIAL_CONSTRAINTS` | | Row count estimates | Yes | From `TABLE_ROWS` (InnoDB estimate) | -| Column statistics | No | — | -| Query history | No | — | +| Column statistics | No | - | +| Query history | No | - | | Table sampling | Yes | Uses `RAND()` filter | ### Dialect notes @@ -430,10 +430,10 @@ For multiple schemas: | Primary keys | Yes | Via `TABLE_CONSTRAINTS` and `KEY_COLUMN_USAGE` | | Foreign keys | Yes | Via `REFERENTIAL_CONSTRAINTS` | | Row count estimates | Yes | Via `sys.dm_db_partition_stats` | -| Column statistics | No | — | -| Query history | No | — | -| Table sampling | Yes | — | -| Nested analysis | No | — | +| Column statistics | No | - | +| Query history | No | - | +| Table sampling | Yes | - | +| Nested analysis | No | - | ### Dialect notes @@ -478,7 +478,7 @@ url: sqlite:///path/to/db.sqlite ### Authentication -No authentication required — SQLite is file-based. The file must be readable by the process running KTX. +No authentication required - SQLite is file-based. The file must be readable by the process running KTX. ### Features @@ -488,10 +488,10 @@ No authentication required — SQLite is file-based. The file must be readable b | Primary keys | Yes | Via `PRAGMA table_info()` | | Foreign keys | Yes | Via `PRAGMA foreign_key_list()` (requires `PRAGMA foreign_keys = ON`) | | Row count estimates | Yes | Exact count via `SELECT COUNT(*)` | -| Column statistics | No | — | -| Query history | No | — | -| Table sampling | Yes | — | -| Nested analysis | No | — | +| Column statistics | No | - | +| Query history | No | - | +| Table sampling | Yes | - | +| Nested analysis | No | - | ### Dialect notes diff --git a/docs-site/next-env.d.ts b/docs-site/next-env.d.ts index 9edff1c7..c4b7818f 100644 --- a/docs-site/next-env.d.ts +++ b/docs-site/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/types/routes.d.ts"; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/docs-site/public/brand/ktx-mascot-dark.svg b/docs-site/public/brand/ktx-mascot-dark.svg new file mode 100644 index 00000000..96603a15 --- /dev/null +++ b/docs-site/public/brand/ktx-mascot-dark.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/docs-site/public/brand/ktx-mascot.png b/docs-site/public/brand/ktx-mascot.png deleted file mode 100644 index 9440f1a3..00000000 Binary files a/docs-site/public/brand/ktx-mascot.png and /dev/null differ diff --git a/docs-site/public/brand/ktx-mascot.svg b/docs-site/public/brand/ktx-mascot.svg new file mode 100644 index 00000000..0c20f577 --- /dev/null +++ b/docs-site/public/brand/ktx-mascot.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/docs/superpowers/plans/2026-05-11-demo-guided-tour.md b/docs/superpowers/plans/2026-05-11-demo-guided-tour.md index 3204e111..92b2c6c0 100644 --- a/docs/superpowers/plans/2026-05-11-demo-guided-tour.md +++ b/docs/superpowers/plans/2026-05-11-demo-guided-tour.md @@ -36,7 +36,7 @@ describe('renderDemoBanner', () => { - [ ] **Step 2: Run the test to verify it fails** Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL — module not found +Expected: FAIL - module not found - [ ] **Step 3: Implement `renderDemoBanner` and `waitForDemoNavigation`** @@ -58,7 +58,7 @@ function dim(text: string): string { export function renderDemoBanner(): string { const lines = [ '', - `┌ ${cyan('Demo mode')} — data has been pre-processed and KTX context is already built.`, + `┌ ${cyan('Demo mode')} - data has been pre-processed and KTX context is already built.`, `│ This walkthrough illustrates the setup steps. Selections are pre-filled and read-only.`, '', ]; @@ -145,7 +145,7 @@ describe('renderDemoCardContent', () => { - [ ] **Step 2: Run the test to verify it fails** Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL — `renderDemoCardContent` not exported +Expected: FAIL - `renderDemoCardContent` not exported - [ ] **Step 3: Implement `renderDemoCardContent` and `renderDemoCard`** @@ -243,7 +243,7 @@ describe('DEMO_REPLAY_TARGETS', () => { - [ ] **Step 2: Run the test to verify it fails** Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL — exports not found +Expected: FAIL - exports not found - [ ] **Step 3: Implement replay timeline and target definitions** @@ -388,7 +388,7 @@ function renderDemoContextCompletionSummary(): string { '', `${cyan('★')} KTX finished ingesting demo data`, '', - ' Placeholder — final counts will come from pre-packaged demo results.', + ' Placeholder - final counts will come from pre-packaged demo results.', '', ` ${dim('Press Enter to continue, Escape to go back')}`, '', @@ -459,7 +459,7 @@ describe('renderDemoCompletionSummary', () => { - [ ] **Step 2: Run the test to verify it fails** Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL — exports not found +Expected: FAIL - exports not found - [ ] **Step 3: Implement transition and completion rendering** @@ -469,7 +469,7 @@ Add to `setup-demo-tour.ts`: export function renderDemoAgentTransition(): string { const lines = [ '', - `┌ Demo project is ready — let's connect your agent`, + `┌ Demo project is ready - let's connect your agent`, '│', '│ Your KTX context has been built with demo data.', '│ Select an agent to start using it.', @@ -583,7 +583,7 @@ describe('runDemoTour', () => { - [ ] **Step 2: Run the test to verify it fails** Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL — `runDemoTour` not exported or wrong signature +Expected: FAIL - `runDemoTour` not exported or wrong signature - [ ] **Step 3: Implement `runDemoTour`** @@ -677,7 +677,7 @@ Expected: PASS - [ ] **Step 5: Run type-check** Run: `pnpm --filter @ktx/cli run type-check` -Expected: PASS — all types align with existing interfaces +Expected: PASS - all types align with existing interfaces - [ ] **Step 6: Commit** @@ -736,7 +736,7 @@ async function runKtxSetupDemoFromEntryMenu( } ``` -- [ ] **Step 3: Update imports — remove unused `defaultDemoProjectDir` import if no longer needed elsewhere in setup.ts** +- [ ] **Step 3: Update imports - remove unused `defaultDemoProjectDir` import if no longer needed elsewhere in setup.ts** Check if `defaultDemoProjectDir` is used elsewhere in `setup.ts`. If it's only used in `runKtxSetupDemoFromEntryMenu`, remove the import. If used elsewhere, keep it. @@ -749,7 +749,7 @@ called from the entry menu path. - [ ] **Step 4: Run type-check and tests** Run: `pnpm --filter @ktx/cli run type-check && pnpm --filter @ktx/cli run test` -Expected: PASS — existing tests continue to work, demo tour is now wired in +Expected: PASS - existing tests continue to work, demo tour is now wired in - [ ] **Step 5: Commit** @@ -807,7 +807,7 @@ git commit -m "fix(cli): demo tour adjustments from smoke test" When the user provides the real pre-packaged demo results, update these locations: -1. **`renderDemoContextCompletionSummary()`** in `setup-demo-tour.ts` — replace placeholder text with actual counts (business areas, query definitions, knowledge pages) from the demo data -2. **`buildDemoReplayTimeline()`** in `setup-demo-tour.ts` — adjust timing and progress details to match the real ingestion profile -3. **`demo-assets.ts`** — update `REQUIRED_SEEDED_ASSET_PATHS` and `demoConfig()` if the demo dataset changes from SQLite/Orbit to Postgres/dbt/Metabase/Notion -4. **Pre-packaged asset files** in `packages/cli/assets/demo/` — replace with the new demo dataset +1. **`renderDemoContextCompletionSummary()`** in `setup-demo-tour.ts` - replace placeholder text with actual counts (business areas, query definitions, knowledge pages) from the demo data +2. **`buildDemoReplayTimeline()`** in `setup-demo-tour.ts` - adjust timing and progress details to match the real ingestion profile +3. **`demo-assets.ts`** - update `REQUIRED_SEEDED_ASSET_PATHS` and `demoConfig()` if the demo dataset changes from SQLite/Orbit to Postgres/dbt/Metabase/Notion +4. **Pre-packaged asset files** in `packages/cli/assets/demo/` - replace with the new demo dataset diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md b/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md index 6e6c6aa8..25df4e01 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md @@ -654,11 +654,11 @@ In `docs/content/docs/cli-reference/ktx-setup.mdx`, replace the Historic SQL fla ```markdown | `--enable-historic-sql` | Enable Historic SQL when the selected database supports it | `false` | | `--disable-historic-sql` | Disable Historic SQL for the selected database | `false` | -| `--historic-sql-window-days ` | Historic SQL query-history window in days | — | -| `--historic-sql-min-executions ` | Minimum executions for a Historic SQL template | — | -| `--historic-sql-min-calls ` | Alias for `--historic-sql-min-executions` for one release | — | -| `--historic-sql-service-account-pattern ` | Historic SQL service-account regex; repeatable | — | -| `--historic-sql-redaction-pattern ` | Historic SQL SQL-literal redaction regex; repeatable | — | +| `--historic-sql-window-days ` | Historic SQL query-history window in days | - | +| `--historic-sql-min-executions ` | Minimum executions for a Historic SQL template | - | +| `--historic-sql-min-calls ` | Alias for `--historic-sql-min-executions` for one release | - | +| `--historic-sql-service-account-pattern ` | Historic SQL service-account regex; repeatable | - | +| `--historic-sql-redaction-pattern ` | Historic SQL SQL-literal redaction regex; repeatable | - | ``` - [ ] **Step 4: Update primary source Historic SQL docs** diff --git a/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md b/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md index a9098867..52b55ab1 100644 --- a/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md +++ b/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md @@ -874,7 +874,7 @@ Expected: PASS. The output includes `# fail 0`. - [ ] **Step 2: Verify stale artifact strings are gone from production/docs files** -Run (scans only production and docs files, not test files — test files keep guard assertions that reference the removed strings): +Run (scans only production and docs files, not test files - test files keep guard assertions that reference the removed strings): ```bash rg -n "uv', \\['build', '--package', 'ktx-sl'|uv', \\['build', '--package', 'ktx-daemon'|ktx_sl-0\\.1\\.0|ktx_daemon-0\\.1\\.0|pythonArtifactInstallArgs|pythonVerifySource|verifyPythonArtifacts|standalone Python distributions|installs the Python artifacts directly" scripts/package-artifacts.mjs scripts/release-readiness.mjs README.md examples/package-artifacts/README.md release-policy.json diff --git a/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md b/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md index f48fea36..d8cae96e 100644 --- a/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md +++ b/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md @@ -199,7 +199,7 @@ Modify the raw schema markdown in .slice(0, limit) .map( (hit) => - `- ${hit.kind}: ${hit.display} [connectionName=${hit.connectionName}] (matched on ${hit.matchedOn}) — ` + + `- ${hit.kind}: ${hit.display} [connectionName=${hit.connectionName}] (matched on ${hit.matchedOn}) - ` + `follow up with \`entity_details({connectionName: "${hit.connectionName}", targets: [{display: "${hit.display}"}]})\``, ) .join('\n'), diff --git a/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md b/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md index e336eb31..107379f8 100644 --- a/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md +++ b/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md @@ -2,9 +2,9 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Add a build-time script that prints the full `ktx` CLI command tree (name, aliases, description per node) as an indented text tree, for docs and discovery — without adding a runtime `ktx` subcommand. +**Goal:** Add a build-time script that prints the full `ktx` CLI command tree (name, aliases, description per node) as an indented text tree, for docs and discovery - without adding a runtime `ktx` subcommand. -**Architecture:** Commander.js exposes every registered command as a `Command` instance with `.commands`, `.name()`, `.aliases()`, `.description()` — we walk that tree. The current `runCommanderKtxCli` in `packages/cli/src/cli-program.ts` builds the program inline; we extract that assembly into a pure `buildKtxProgram(...)` helper that any caller can use to materialize the configured root `Command` without parsing argv. A new pure module `command-tree.ts` walks the `Command` into plain data and renders it as indented text. A new TypeScript entrypoint `print-command-tree.ts` compiles alongside `bin.ts` into `dist/print-command-tree.js`, instantiates the program with stub IO/deps, and writes the rendered tree to stdout. A pnpm script under `@ktx/cli` exposes it as `pnpm --filter @ktx/cli run docs:commands`. +**Architecture:** Commander.js exposes every registered command as a `Command` instance with `.commands`, `.name()`, `.aliases()`, `.description()` - we walk that tree. The current `runCommanderKtxCli` in `packages/cli/src/cli-program.ts` builds the program inline; we extract that assembly into a pure `buildKtxProgram(...)` helper that any caller can use to materialize the configured root `Command` without parsing argv. A new pure module `command-tree.ts` walks the `Command` into plain data and renders it as indented text. A new TypeScript entrypoint `print-command-tree.ts` compiles alongside `bin.ts` into `dist/print-command-tree.js`, instantiates the program with stub IO/deps, and writes the rendered tree to stdout. A pnpm script under `@ktx/cli` exposes it as `pnpm --filter @ktx/cli run docs:commands`. **Tech Stack:** TypeScript (NodeNext ESM), Node 22, Commander 14 via `@commander-js/extra-typings`, vitest 4. @@ -12,14 +12,14 @@ ## File Map -- **Modify:** `packages/cli/src/cli-program.ts` — extract `buildKtxProgram` from `runCommanderKtxCli`. -- **Create:** `packages/cli/src/cli-program.test.ts` — vitest tests for the new helper. -- **Create:** `packages/cli/src/command-tree.ts` — pure `walkCommandTree` + `formatCommandTree`. -- **Create:** `packages/cli/src/command-tree.test.ts` — vitest tests against ad-hoc Command trees. -- **Create:** `packages/cli/src/print-command-tree.ts` — script entrypoint; thin glue. -- **Create:** `packages/cli/src/print-command-tree.test.ts` — vitest test that calls the script's exported `main()` with a fake stdout and asserts the rendered tree includes known top-level commands. -- **Modify:** `packages/cli/package.json` — add `docs:commands` script and include the new entry in tsc build output (no change needed if `tsconfig` already globs `src/**/*.ts`, but verify). -- **Modify:** `packages/cli/README.md` (if it exists; otherwise skip) — document `pnpm run docs:commands`. +- **Modify:** `packages/cli/src/cli-program.ts` - extract `buildKtxProgram` from `runCommanderKtxCli`. +- **Create:** `packages/cli/src/cli-program.test.ts` - vitest tests for the new helper. +- **Create:** `packages/cli/src/command-tree.ts` - pure `walkCommandTree` + `formatCommandTree`. +- **Create:** `packages/cli/src/command-tree.test.ts` - vitest tests against ad-hoc Command trees. +- **Create:** `packages/cli/src/print-command-tree.ts` - script entrypoint; thin glue. +- **Create:** `packages/cli/src/print-command-tree.test.ts` - vitest test that calls the script's exported `main()` with a fake stdout and asserts the rendered tree includes known top-level commands. +- **Modify:** `packages/cli/package.json` - add `docs:commands` script and include the new entry in tsc build output (no change needed if `tsconfig` already globs `src/**/*.ts`, but verify). +- **Modify:** `packages/cli/README.md` (if it exists; otherwise skip) - document `pnpm run docs:commands`. Files that change together (cli-program + its test, command-tree + its test, print-command-tree + its test) live next to each other under `packages/cli/src/`, matching the existing convention (e.g. `bin.ts`, `cli-runtime.ts`, `runtime.ts` + `runtime.test.ts`). @@ -27,7 +27,7 @@ Files that change together (cli-program + its test, command-tree + its test, pri ## Task 1: Extract `buildKtxProgram` from `runCommanderKtxCli` -Refactor only — no behavior change. The current code in `cli-program.ts` interleaves program construction with `parseAsync` dispatch. Splitting them lets the new script reuse construction without invoking the CLI. +Refactor only - no behavior change. The current code in `cli-program.ts` interleaves program construction with `parseAsync` dispatch. Splitting them lets the new script reuse construction without invoking the CLI. **Files:** - Modify: `packages/cli/src/cli-program.ts:197-275` (function `runCommanderKtxCli`) @@ -88,7 +88,7 @@ describe('buildKtxProgram', () => { Run: `pnpm --filter @ktx/cli exec vitest run src/cli-program.test.ts` -Expected: FAIL — `buildKtxProgram is not exported from './cli-program.js'` (or similar TS/ESM error). +Expected: FAIL - `buildKtxProgram is not exported from './cli-program.js'` (or similar TS/ESM error). - [ ] **Step 3: Extract `buildKtxProgram` from `runCommanderKtxCli`** @@ -160,19 +160,19 @@ Then rewrite the body of `runCommanderKtxCli` (lines 197-275) to delegate progra }; ``` -Keep the `context` re-declaration only if subsequent code (the `if (argv.length === 0)` branch that calls `runBareInteractiveCommand(program, io, context)`) still needs it. It does — `runBareInteractiveCommand` consumes `context`. Keep `context` exactly as it was after the deletion; do not change `runBareInteractiveCommand`'s signature or behavior. Drop the now-removed individual `register*` calls and their `profileMark` lines from `runCommanderKtxCli`. +Keep the `context` re-declaration only if subsequent code (the `if (argv.length === 0)` branch that calls `runBareInteractiveCommand(program, io, context)`) still needs it. It does - `runBareInteractiveCommand` consumes `context`. Keep `context` exactly as it was after the deletion; do not change `runBareInteractiveCommand`'s signature or behavior. Drop the now-removed individual `register*` calls and their `profileMark` lines from `runCommanderKtxCli`. - [ ] **Step 4: Run the new test to verify it passes** Run: `pnpm --filter @ktx/cli exec vitest run src/cli-program.test.ts` -Expected: PASS — both `it` blocks green. +Expected: PASS - both `it` blocks green. - [ ] **Step 5: Run the full CLI test suite to confirm no regression** Run: `pnpm --filter @ktx/cli run test 2>&1 | tee /tmp/ktx-cli-test-output.log` -Expected: PASS overall. Inspect the log if any previously-passing test now fails — most likely a missing register call (compare to lines 221-249 of the pre-change file). +Expected: PASS overall. Inspect the log if any previously-passing test now fails - most likely a missing register call (compare to lines 221-249 of the pre-change file). - [ ] **Step 6: Type-check** @@ -191,7 +191,7 @@ git commit -m "refactor(cli): extract buildKtxProgram for reuse outside runComma ## Task 2: Pure tree walker `walkCommandTree` -Take a Commander `Command` and produce plain data: `{ name, description, aliases, children }`. No formatting yet. Pure function — depends only on the public `Command` API. +Take a Commander `Command` and produce plain data: `{ name, description, aliases, children }`. No formatting yet. Pure function - depends only on the public `Command` API. **Files:** - Create: `packages/cli/src/command-tree.ts` @@ -254,7 +254,7 @@ describe('walkCommandTree', () => { Run: `pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts` -Expected: FAIL — `walkCommandTree` cannot be resolved. +Expected: FAIL - `walkCommandTree` cannot be resolved. - [ ] **Step 3: Implement `walkCommandTree`** @@ -296,7 +296,7 @@ Expected: no errors. ## Task 3: Indented-text renderer `formatCommandTree` -Render a `CommandTreeNode` as plain text. Each node on its own line: `[ (alias1, alias2)][ — description]`. Indent is two spaces per depth level. Children sorted alphabetically by name to keep output stable across changes that reorder registrar calls. +Render a `CommandTreeNode` as plain text. Each node on its own line: `[ (alias1, alias2)][ - description]`. Indent is two spaces per depth level. Children sorted alphabetically by name to keep output stable across changes that reorder registrar calls. **Files:** - Modify: `packages/cli/src/command-tree.ts` @@ -312,12 +312,12 @@ import { formatCommandTree } from './command-tree.js'; describe('formatCommandTree', () => { it('renders a single node with no children', () => { const node = { name: 'solo', description: 'just me', aliases: [], children: [] }; - expect(formatCommandTree(node)).toBe('solo — just me\n'); + expect(formatCommandTree(node)).toBe('solo - just me\n'); }); it('renders aliases in parentheses before the description', () => { const node = { name: 'cmd', description: 'does things', aliases: ['c', 'co'], children: [] }; - expect(formatCommandTree(node)).toBe('cmd (c, co) — does things\n'); + expect(formatCommandTree(node)).toBe('cmd (c, co) - does things\n'); }); it('omits the dash when description is empty', () => { @@ -338,10 +338,10 @@ describe('formatCommandTree', () => { ], }; expect(formatCommandTree(tree)).toBe( - 'root — top\n' + - ' alpha (al) — a\n' + - ' inner — i\n' + - ' beta — b\n', + 'root - top\n' + + ' alpha (al) - a\n' + + ' inner - i\n' + + ' beta - b\n', ); }); }); @@ -351,7 +351,7 @@ describe('formatCommandTree', () => { Run: `pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts` -Expected: FAIL — `formatCommandTree` is not exported. +Expected: FAIL - `formatCommandTree` is not exported. - [ ] **Step 3: Implement `formatCommandTree`** @@ -367,7 +367,7 @@ export function formatCommandTree(node: CommandTreeNode): string { function appendNode(node: CommandTreeNode, depth: number, lines: string[]): void { const indent = ' '.repeat(depth); const aliasPart = node.aliases.length > 0 ? ` (${node.aliases.join(', ')})` : ''; - const descPart = node.description.length > 0 ? ` — ${node.description}` : ''; + const descPart = node.description.length > 0 ? ` - ${node.description}` : ''; lines.push(`${indent}${node.name}${aliasPart}${descPart}`); const sortedChildren = [...node.children].sort((a, b) => a.name.localeCompare(b.name)); @@ -419,7 +419,7 @@ describe('renderKtxCommandTree', () => { const output = renderKtxCommandTree(); const lines = output.split('\n'); - expect(lines[0]).toMatch(/^ktx( |$|\s—)/); + expect(lines[0]).toMatch(/^ktx( |$|\s-)/); // Top-level commands are indented exactly two spaces. const topLevel = lines @@ -443,7 +443,7 @@ describe('renderKtxCommandTree', () => { Run: `pnpm --filter @ktx/cli exec vitest run src/print-command-tree.test.ts` -Expected: FAIL — module not found. +Expected: FAIL - module not found. - [ ] **Step 3: Implement the script** @@ -495,7 +495,7 @@ if (invokedAsScript) { Run: `pnpm --filter @ktx/cli exec vitest run src/print-command-tree.test.ts` -Expected: PASS — both assertions green. +Expected: PASS - both assertions green. - [ ] **Step 5: Type-check** @@ -572,9 +572,9 @@ git commit -m "chore(cli): add docs:commands pnpm script" After all tasks, confirm: -- [ ] `pnpm --filter @ktx/cli run type-check` — clean -- [ ] `pnpm --filter @ktx/cli run test` — green, including new tests in `cli-program.test.ts`, `command-tree.test.ts`, `print-command-tree.test.ts` -- [ ] `pnpm --filter @ktx/cli run docs:commands` — prints `ktx` followed by indented subcommand tree -- [ ] `git status --short` — only the files listed in the File Map are modified or created; no incidental edits +- [ ] `pnpm --filter @ktx/cli run type-check` - clean +- [ ] `pnpm --filter @ktx/cli run test` - green, including new tests in `cli-program.test.ts`, `command-tree.test.ts`, `print-command-tree.test.ts` +- [ ] `pnpm --filter @ktx/cli run docs:commands` - prints `ktx` followed by indented subcommand tree +- [ ] `git status --short` - only the files listed in the File Map are modified or created; no incidental edits If any check fails, fix in place and re-run before declaring done. diff --git a/docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md b/docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md new file mode 100644 index 00000000..eb8e812d --- /dev/null +++ b/docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md @@ -0,0 +1,808 @@ +# Connection Driver Discriminated Union Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the loose `connectionSchema` in `packages/context/src/project/config.ts` with a Zod 4 discriminated union keyed on `driver`, so that every driver's documented connection fields — including the `mappings` block — appear in the JSON schema emitted by `ktx dev schema`. + +**Architecture:** Add a new module `packages/context/src/project/driver-schemas.ts` that defines one `z.looseObject({ driver: z.literal('x'), ... })` per supported driver and combines them with `z.discriminatedUnion('driver', [...])`. Reuse the existing Metabase/Looker/LookML mapping shapes from `mappings-yaml-schema.ts` by exporting them. Wire the union into `config.ts`. Each per-driver shape stays `looseObject` so today's existing yaml configs with extra fields keep parsing. + +**Tech Stack:** TypeScript (Node 22+, ESM, `NodeNext`), Zod 4 (`^4.4.3`), Vitest, pnpm workspace. + +--- + +## File Structure + +**Create:** +- `packages/context/src/project/driver-schemas.ts` — per-driver Zod schemas + the discriminated union and exported types. +- `packages/context/src/project/driver-schemas.test.ts` — unit tests for each driver schema and the union. + +**Modify:** +- `packages/context/src/project/mappings-yaml-schema.ts` — export the three mapping shapes (`metabaseMappingsSchema`, `lookerMappingsSchema`, `lookmlMappingsSchema`) with `.describe()` annotations and a small description on each field so they surface meaningfully in JSON Schema. +- `packages/context/src/project/config.ts:209-214` — replace `connectionSchema` with the discriminated union imported from `driver-schemas.ts`. Update `KtxProjectConnectionConfig` (line `272`) to be `z.infer` — still works because `connectionSchema` is the union name we keep. +- `packages/context/src/project/index.ts` — re-export `KtxConnectionConfig` per-driver type aliases if useful (optional; only if tests need them). +- `packages/context/src/project/config.test.ts` — add a test that the JSON schema now describes `mappings` for metabase/looker/lookml. + +**No changes needed:** +- `packages/context/src/project/mappings-yaml-schema.ts` parsing helpers (`parseMetabaseMappingBootstrap`, etc.) keep working because `KtxProjectConnectionConfig` still has loose-object semantics per driver. +- Doc files in `docs-site/` already show the `mappings` blocks correctly. + +--- + +## Drivers In Scope + +The discriminated union enumerates the drivers actually used in code, fixtures, and docs (no `fake`/test-only driver — none exist in fixtures, verified via `grep "driver:\s*fake"`). + +Warehouse drivers (read `driver`, `url`; nothing else schema-modeled — kept `looseObject` so warehouse-specific overrides like `historicSql`/`context.queryHistory` pass through): +- `postgres`, `postgresql` (separate literals; KTX normalizes `postgresql` → `postgres` at runtime, but ktx.yaml accepts both) +- `mysql` +- `snowflake` +- `bigquery` +- `sqlite` +- `clickhouse` +- `sqlserver` + +Context-source drivers (model documented fields): +- `metabase` — `api_url`, `api_key`, `api_key_ref`, `network_proxy`/`networkProxy`, `mappings` (metabaseMappingsSchema). +- `looker` — `base_url`, `client_id`, `client_secret`, `client_secret_ref`, `mappings` (lookerMappingsSchema). +- `lookml` — `repoUrl` (camelCase intentional — matches code at `setup-sources.ts:1466`), `branch`, `path`, `auth_token_ref`, `mappings` (lookmlMappingsSchema). +- `notion` — `auth_token`, `auth_token_ref`, `crawl_mode` (`'selected_roots' | 'all_accessible'`), `root_page_ids`, `root_database_ids`, `root_data_source_ids`, `max_pages_per_run`, `max_knowledge_creates_per_run`, `max_knowledge_updates_per_run`. +- `dbt` — `source_dir`, `repo_url`, `branch`, `path`, `auth_token_ref`, `profiles_path`, `target`, `project_name`. +- `metricflow` — `metricflow` (nested object: `repoUrl`, `branch`, `path`, `auth_token_ref`). + +Why not strict-object: existing warehouse connections may carry `historicSql` / `context.queryHistory` blocks and other driver-tunable fields not modeled here. `looseObject` preserves the current pass-through behavior while still surfacing the documented fields in JSON Schema. + +--- + +## Task 1: Export and describe mapping shapes + +Make the three existing mapping schemas reusable and documented. + +**Files:** +- Modify: `packages/context/src/project/mappings-yaml-schema.ts:4-31` +- Test: `packages/context/src/project/mappings-yaml-schema.test.ts` (no behavior change — existing tests must still pass) + +- [ ] **Step 1: Add a failing test that imports the new exports** + +Append to `packages/context/src/project/mappings-yaml-schema.test.ts` (inside the existing `describe` block): + +```typescript +import { + metabaseMappingsSchema, + lookerMappingsSchema, + lookmlMappingsSchema, +} from './mappings-yaml-schema.js'; + +// ...inside describe(...) + +it('exports mapping shapes that parse documented examples', () => { + expect(metabaseMappingsSchema.parse({ databaseMappings: { '1': 'wh' } })).toMatchObject({ + databaseMappings: { '1': 'wh' }, + syncMode: 'ALL', + }); + expect(lookerMappingsSchema.parse({ connectionMappings: { x: 'wh' } })).toEqual({ + connectionMappings: { x: 'wh' }, + }); + expect(lookmlMappingsSchema.parse({ expectedLookerConnectionName: 'x' })).toEqual({ + expectedLookerConnectionName: 'x', + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/mappings-yaml-schema.test.ts` +Expected: FAIL with `metabaseMappingsSchema is not exported` or equivalent module-resolution error. + +- [ ] **Step 3: Add `export` and `.describe()` to the three schemas** + +In `packages/context/src/project/mappings-yaml-schema.ts`, change the three internal `const` declarations: + +```typescript +export const metabaseMappingsSchema = z + .object({ + databaseMappings: z + .record(z.string(), stringTargetSchema) + .default({}) + .describe('Map of Metabase database ID (positive integer string) to KTX connection ID. Use null to explicitly unmap.'), + syncEnabled: z + .record(z.string(), z.boolean()) + .default({}) + .describe('Per-Metabase-database sync toggle, keyed by Metabase database ID string.'), + syncMode: metabaseSyncModeSchema + .default('ALL') + .describe('Sync scope: ALL ingests every mapped DB; ONLY restricts to syncEnabled=true; EXCEPT excludes syncEnabled=true.'), + selections: metabaseSelectionsSchema + .default({ collections: [], items: [] }) + .describe('Optional Metabase collection and item IDs to scope ingest.'), + defaultTagNames: z + .array(z.string().min(1)) + .default([]) + .describe('Default tag names applied to ingested Metabase artifacts.'), + }) + .describe('Metabase database-to-warehouse mapping and sync configuration.'); + +export const lookerMappingsSchema = z + .object({ + connectionMappings: z + .record(z.string().min(1), stringTargetSchema) + .default({}) + .describe('Map of Looker connection name to KTX connection ID. Use null to explicitly unmap.'), + }) + .describe('Looker connection-to-warehouse mapping configuration.'); + +export const lookmlMappingsSchema = z + .object({ + expectedLookerConnectionName: z + .string() + .min(1) + .nullable() + .default(null) + .describe('Looker connection name that LookML models must declare; mismatches block sl_write_source at ingest time.'), + }) + .describe('LookML connection-name expectation for ingest gating.'); +``` + +Leave `metabaseSyncModeSchema`, `metabaseSelectionsSchema`, `stringTargetSchema`, and `positiveIntegerValueSchema` private (no need to export). Leave all parsing helpers (`parseMetabaseMappingBootstrap` etc.) unchanged — they keep working because `.describe()` does not change runtime behavior. + +- [ ] **Step 4: Run test to verify it passes and existing tests still pass** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/mappings-yaml-schema.test.ts` +Expected: PASS for all tests including the new one. + +- [ ] **Step 5: Type-check the package** + +Run: `pnpm --filter @ktx/context run type-check` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/context/src/project/mappings-yaml-schema.ts packages/context/src/project/mappings-yaml-schema.test.ts +git commit -m "refactor(context): export and describe mapping shape schemas" +``` + +--- + +## Task 2: Create the driver-schemas module — warehouse drivers + +Add the new module with the seven warehouse driver schemas first. Smaller surface, easier to validate. + +**Files:** +- Create: `packages/context/src/project/driver-schemas.ts` +- Test: `packages/context/src/project/driver-schemas.test.ts` + +- [ ] **Step 1: Write failing tests for warehouse driver schemas** + +Create `packages/context/src/project/driver-schemas.test.ts`: + +```typescript +import { describe, expect, it } from 'vitest'; +import { connectionConfigSchema } from './driver-schemas.js'; + +describe('connectionConfigSchema (driver discriminated union)', () => { + it.each([ + ['postgres', 'postgres://user:pass@host:5432/db'], // pragma: allowlist secret + ['postgresql', 'postgresql://user:pass@host:5432/db'], // pragma: allowlist secret + ['mysql', 'mysql://user:pass@host:3306/db'], // pragma: allowlist secret + ['snowflake', 'snowflake://account/db'], + ['bigquery', 'bigquery://project/dataset'], + ['sqlite', 'sqlite:///tmp/db.sqlite'], + ['clickhouse', 'clickhouse://host:8123/db'], + ['sqlserver', 'sqlserver://host:1433;database=db'], + ])('parses %s warehouse connection', (driver, url) => { + expect(connectionConfigSchema.parse({ driver, url })).toMatchObject({ driver, url }); + }); + + it('preserves unknown warehouse fields via looseObject passthrough', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'postgres', + url: 'postgres://x', + historicSql: { enabled: true }, + context: { queryHistory: { enabled: false } }, + }); + expect(parsed).toMatchObject({ + driver: 'postgres', + historicSql: { enabled: true }, + context: { queryHistory: { enabled: false } }, + }); + }); + + it('rejects an unknown driver', () => { + expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` +Expected: FAIL — `driver-schemas.js` not found. + +- [ ] **Step 3: Create `driver-schemas.ts` with warehouse drivers only** + +Create `packages/context/src/project/driver-schemas.ts`: + +```typescript +import * as z from 'zod'; + +const warehouseDrivers = [ + 'postgres', + 'postgresql', + 'mysql', + 'snowflake', + 'bigquery', + 'sqlite', + 'clickhouse', + 'sqlserver', +] as const; + +function warehouseConnectionSchema(driver: (typeof warehouseDrivers)[number]) { + return z + .looseObject({ + driver: z.literal(driver), + url: z + .string() + .min(1) + .optional() + .describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'), + }) + .describe(`${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`); +} + +export const connectionConfigSchema = z.discriminatedUnion( + 'driver', + warehouseDrivers.map(warehouseConnectionSchema), +); + +export type KtxConnectionConfig = z.infer; +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` +Expected: PASS for all eight warehouse drivers + passthrough + unknown-driver rejection. + +- [ ] **Step 5: Type-check** + +Run: `pnpm --filter @ktx/context run type-check` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts +git commit -m "feat(context): add driver-schemas module with warehouse drivers" +``` + +--- + +## Task 3: Add Metabase, Looker, LookML driver schemas (the mapping-bearing ones) + +These are the most important drivers — they're why we're doing this refactor. + +**Files:** +- Modify: `packages/context/src/project/driver-schemas.ts` +- Modify: `packages/context/src/project/driver-schemas.test.ts` + +- [ ] **Step 1: Write failing tests** + +Append to `packages/context/src/project/driver-schemas.test.ts`: + +```typescript +describe('connectionConfigSchema — context source drivers with mappings', () => { + it('parses a metabase connection with mappings', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'metabase', + api_url: 'https://metabase.example.com', + api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret + mappings: { + databaseMappings: { '3': 'prod-warehouse' }, + syncEnabled: { '3': true }, + syncMode: 'ONLY', + }, + }); + expect(parsed).toMatchObject({ + driver: 'metabase', + api_url: 'https://metabase.example.com', + mappings: { + databaseMappings: { '3': 'prod-warehouse' }, + syncMode: 'ONLY', + }, + }); + }); + + it('parses a looker connection with connectionMappings', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'looker', + base_url: 'https://looker.example.com', + client_id: 'abc', + client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret + mappings: { connectionMappings: { bigquery_prod: 'wh' } }, + }); + expect(parsed.mappings).toEqual({ connectionMappings: { bigquery_prod: 'wh' } }); + }); + + it('parses a lookml connection with expectedLookerConnectionName', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'lookml', + repoUrl: 'https://github.com/acme/looker.git', + branch: 'main', + mappings: { expectedLookerConnectionName: 'bigquery_prod' }, + }); + expect(parsed.mappings).toEqual({ expectedLookerConnectionName: 'bigquery_prod' }); + }); + + it('rejects metabase mapping with non-integer database key', () => { + expect(() => + connectionConfigSchema.parse({ + driver: 'metabase', + api_url: 'https://x', + mappings: { databaseMappings: { 'abc': 'wh' } }, + }), + ).toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` +Expected: FAIL — `driver: 'metabase'` is not in the discriminated union. + +- [ ] **Step 3: Extend `driver-schemas.ts` with metabase/looker/lookml schemas** + +Edit `packages/context/src/project/driver-schemas.ts` — add imports and the three new schemas, and include them in the union: + +```typescript +import * as z from 'zod'; +import { + lookerMappingsSchema, + lookmlMappingsSchema, + metabaseMappingsSchema, +} from './mappings-yaml-schema.js'; + +// ... (warehouseDrivers + warehouseConnectionSchema stay as-is) ... + +const positiveIntKeyMessage = (field: string) => + `${field} keys must be positive-integer strings (e.g. "1", "42")`; + +const positiveIntKeyRegex = /^[1-9]\d*$/; + +const metabaseMappingsStrictSchema = metabaseMappingsSchema.superRefine((value, ctx) => { + for (const key of Object.keys(value.databaseMappings ?? {})) { + if (!positiveIntKeyRegex.test(key)) { + ctx.addIssue({ code: 'custom', path: ['databaseMappings', key], message: positiveIntKeyMessage('databaseMappings') }); + } + } + for (const key of Object.keys(value.syncEnabled ?? {})) { + if (!positiveIntKeyRegex.test(key)) { + ctx.addIssue({ code: 'custom', path: ['syncEnabled', key], message: positiveIntKeyMessage('syncEnabled') }); + } + } +}); + +const metabaseConnectionSchema = z + .looseObject({ + driver: z.literal('metabase'), + api_url: z.string().url().describe('Metabase instance API URL (e.g. https://metabase.example.com).'), + api_key: z.string().min(1).optional().describe('Literal Metabase API key. Prefer api_key_ref for safety.'), + api_key_ref: z + .string() + .min(1) + .optional() + .describe('Reference to Metabase API key (e.g. env:METABASE_API_KEY or file:/path).'), + network_proxy: z + .looseObject({}) + .optional() + .describe('Optional network proxy configuration (snake_case form).'), + networkProxy: z + .looseObject({}) + .optional() + .describe('Optional network proxy configuration (camelCase form).'), + mappings: metabaseMappingsStrictSchema.optional().describe('Metabase database-to-warehouse mappings and sync configuration.'), + }) + .describe('Metabase context-source connection.'); + +const lookerConnectionSchema = z + .looseObject({ + driver: z.literal('looker'), + base_url: z.string().url().describe('Looker instance base URL (e.g. https://looker.example.com).'), + client_id: z.string().min(1).describe('Looker OAuth client ID.'), + client_secret: z.string().min(1).optional().describe('Literal Looker OAuth client secret. Prefer client_secret_ref.'), + client_secret_ref: z + .string() + .min(1) + .optional() + .describe('Reference to Looker OAuth client secret (e.g. env:LOOKER_CLIENT_SECRET).'), + mappings: lookerMappingsSchema.optional().describe('Looker connection-name to KTX warehouse mappings.'), + }) + .describe('Looker context-source connection.'); + +const lookmlConnectionSchema = z + .looseObject({ + driver: z.literal('lookml'), + repoUrl: z + .string() + .min(1) + .describe('Git URL of the LookML project (https, ssh, or file:). Field is camelCase by convention.'), + branch: z.string().min(1).optional().describe('Git branch (default "main" downstream).'), + path: z.string().optional().describe('Subdirectory within the repo when the LookML project lives in a monorepo.'), + auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos (e.g. env:GITHUB_TOKEN).'), + mappings: lookmlMappingsSchema.optional().describe('LookML expected-connection mapping for ingest gating.'), + }) + .describe('LookML context-source connection.'); + +export const connectionConfigSchema = z.discriminatedUnion( + 'driver', + [ + ...warehouseDrivers.map(warehouseConnectionSchema), + metabaseConnectionSchema, + lookerConnectionSchema, + lookmlConnectionSchema, + ], +); +``` + +Important: the existing `parseMetabaseMappingBootstrap` in `mappings-yaml-schema.ts` already enforces positive-integer keys via `assertPositiveIntegerKeys`. Adding `metabaseMappingsStrictSchema` here gives the same guarantee at the top-level config parse, so a malformed ktx.yaml fails fast at `parseKtxProjectConfig` time rather than at ingest time. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` +Expected: PASS. + +- [ ] **Step 5: Type-check** + +Run: `pnpm --filter @ktx/context run type-check` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts +git commit -m "feat(context): add metabase, looker, lookml driver schemas with mappings" +``` + +--- + +## Task 4: Add Notion, dbt, MetricFlow driver schemas + +The remaining context-source drivers; no `mappings` for these, but plenty of driver-specific fields. + +**Files:** +- Modify: `packages/context/src/project/driver-schemas.ts` +- Modify: `packages/context/src/project/driver-schemas.test.ts` + +- [ ] **Step 1: Write failing tests** + +Append to `packages/context/src/project/driver-schemas.test.ts`: + +```typescript +describe('connectionConfigSchema — notion / dbt / metricflow', () => { + it('parses a notion connection with selected_roots crawl', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['abc', 'def'], + max_pages_per_run: 500, + }); + expect(parsed).toMatchObject({ + driver: 'notion', + crawl_mode: 'selected_roots', + root_page_ids: ['abc', 'def'], + max_pages_per_run: 500, + }); + }); + + it('rejects notion with unknown crawl_mode', () => { + expect(() => + connectionConfigSchema.parse({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'everything', + }), + ).toThrow(); + }); + + it('parses a dbt connection from a local source_dir', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'dbt', + source_dir: '/tmp/dbt-project', + target: 'dev', + }); + expect(parsed).toMatchObject({ driver: 'dbt', source_dir: '/tmp/dbt-project', target: 'dev' }); + }); + + it('parses a metricflow connection with nested config', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/sl.git', + branch: 'main', + }, + }); + expect(parsed).toMatchObject({ + driver: 'metricflow', + metricflow: { repoUrl: 'https://github.com/acme/sl.git' }, + }); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` +Expected: FAIL — `driver: 'notion'` etc. not in union. + +- [ ] **Step 3: Extend `driver-schemas.ts`** + +Add to `packages/context/src/project/driver-schemas.ts` before the final `connectionConfigSchema` export: + +```typescript +const notionConnectionSchema = z + .looseObject({ + driver: z.literal('notion'), + auth_token: z.string().min(1).optional().describe('Literal Notion integration token. Prefer auth_token_ref.'), + auth_token_ref: z + .string() + .min(1) + .optional() + .describe('Reference to Notion integration token (e.g. env:NOTION_TOKEN).'), + crawl_mode: z + .enum(['selected_roots', 'all_accessible']) + .optional() + .describe('Crawl scope. "selected_roots" requires at least one of root_page_ids, root_database_ids, root_data_source_ids.'), + root_page_ids: z.array(z.string().min(1)).optional().describe('Notion page IDs to crawl when crawl_mode is selected_roots.'), + root_database_ids: z.array(z.string().min(1)).optional().describe('Notion database IDs to crawl when crawl_mode is selected_roots.'), + root_data_source_ids: z + .array(z.string().min(1)) + .optional() + .describe('Notion data source IDs to crawl when crawl_mode is selected_roots.'), + max_pages_per_run: z + .number() + .int() + .min(1) + .max(10000) + .optional() + .describe('Maximum Notion pages fetched in a single ingest run.'), + max_knowledge_creates_per_run: z + .number() + .int() + .min(0) + .max(25) + .optional() + .describe('Maximum new wiki pages created per run.'), + max_knowledge_updates_per_run: z + .number() + .int() + .min(0) + .max(100) + .optional() + .describe('Maximum existing wiki pages updated per run.'), + }) + .describe('Notion context-source connection.'); + +const dbtConnectionSchema = z + .looseObject({ + driver: z.literal('dbt'), + source_dir: z.string().min(1).optional().describe('Absolute or project-relative path to a local dbt project.'), + repo_url: z.string().min(1).optional().describe('Git URL of the dbt project (https, ssh, or file:).'), + branch: z.string().min(1).optional().describe('Git branch when using repo_url.'), + path: z.string().optional().describe('Subdirectory within the repo when the dbt project lives in a monorepo.'), + auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'), + profiles_path: z.string().optional().describe('Override path to dbt profiles.yml.'), + target: z.string().min(1).optional().describe('dbt target name (e.g. dev, prod).'), + project_name: z.string().min(1).optional().describe('Override auto-detected dbt project name.'), + }) + .describe('dbt context-source connection.'); + +const metricflowConnectionSchema = z + .looseObject({ + driver: z.literal('metricflow'), + metricflow: z + .looseObject({ + repoUrl: z.string().min(1).describe('Git URL of the MetricFlow / SL project.'), + branch: z.string().min(1).optional().describe('Git branch (default "main").'), + path: z.string().optional().describe('Subdirectory within the repo when the SL config lives in a monorepo.'), + auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'), + }) + .describe('Nested MetricFlow configuration block.'), + }) + .describe('MetricFlow / SL context-source connection.'); +``` + +Then update the final union: + +```typescript +export const connectionConfigSchema = z.discriminatedUnion( + 'driver', + [ + ...warehouseDrivers.map(warehouseConnectionSchema), + metabaseConnectionSchema, + lookerConnectionSchema, + lookmlConnectionSchema, + notionConnectionSchema, + dbtConnectionSchema, + metricflowConnectionSchema, + ], +); +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` +Expected: PASS. + +- [ ] **Step 5: Type-check** + +Run: `pnpm --filter @ktx/context run type-check` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts +git commit -m "feat(context): add notion, dbt, metricflow driver schemas" +``` + +--- + +## Task 5: Wire the discriminated union into `config.ts` + +Now switch the top-level `connectionSchema` to the new union. This is the change that flips JSON-schema output. + +**Files:** +- Modify: `packages/context/src/project/config.ts:209-214, 272` +- Test: `packages/context/src/project/config.test.ts` — add a JSON-schema assertion. + +- [ ] **Step 1: Write a failing test for the JSON schema output** + +Append to `packages/context/src/project/config.test.ts`: + +```typescript +import { generateKtxProjectConfigJsonSchema } from './config.js'; + +describe('generateKtxProjectConfigJsonSchema', () => { + it('emits the metabase mappings shape under connections', () => { + const schema = generateKtxProjectConfigJsonSchema(); + const serialized = JSON.stringify(schema); + expect(serialized).toContain('databaseMappings'); + expect(serialized).toContain('connectionMappings'); + expect(serialized).toContain('expectedLookerConnectionName'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/config.test.ts` +Expected: FAIL — the strings are not in the emitted schema yet because `connectionSchema` is still loose. + +- [ ] **Step 3: Replace `connectionSchema` in `config.ts`** + +In `packages/context/src/project/config.ts`, delete lines `209-214`: + +```typescript +const connectionSchema = z + .looseObject({ + driver: z.string().min(1).optional().describe('Connector driver identifier (e.g. "postgres", "bigquery", "snowflake").'), + url: z.string().optional().describe('Connection URL or DSN. Format depends on the driver; may contain environment-variable references.'), + }) + .describe('A single database/connector connection entry. Additional driver-specific fields are accepted and passed through.'); +``` + +Replace with an import + re-bind at the top of the file (after the existing imports): + +```typescript +import { connectionConfigSchema } from './driver-schemas.js'; + +const connectionSchema = connectionConfigSchema; +``` + +(Re-binding to the local name `connectionSchema` keeps the rest of the file unchanged, including the export of `KtxProjectConnectionConfig` at line `272`.) + +- [ ] **Step 4: Run the new test plus existing config tests** + +Run: `pnpm --filter @ktx/context exec vitest run src/project/` +Expected: PASS for all tests. + +If any existing test fails (e.g. a fixture used an undocumented driver string), update the fixture or expand the union — do not loosen the union. + +- [ ] **Step 5: Run the full context test suite to catch downstream regressions** + +Run: `pnpm --filter @ktx/context run test` +Expected: PASS. + +- [ ] **Step 6: Type-check the workspace** + +Run: `pnpm run type-check` +Expected: PASS. `KtxProjectConnectionConfig` is now a union; any consumer that destructured fields not present on every driver branch will surface here. + +If type-check fails in a consumer, the fix is usually `if (connection.driver === 'metabase')` style narrowing — or, for code that already does this dynamically (e.g. `String(connection.driver).toLowerCase() === 'metabase'`), an explicit cast at the call site is acceptable. Do not add `as any`; prefer narrowing. + +- [ ] **Step 7: Commit** + +```bash +git add packages/context/src/project/config.ts packages/context/src/project/config.test.ts +git commit -m "refactor(context): make connectionSchema a driver-discriminated union" +``` + +--- + +## Task 6: Verify the user-visible result and CLI smoke + +Confirm the original bug is fixed and the CLI behavior is unchanged. + +**Files:** none modified in this task. + +- [ ] **Step 1: Build the CLI** + +Run: `pnpm run build` +Expected: PASS. + +- [ ] **Step 2: Confirm `ktx dev schema | rg -i mapping` now returns hits** + +Run: `node scripts/run-ktx.mjs -- dev schema | rg -i mapping` +Expected: multiple lines, including the `databaseMappings`, `connectionMappings`, `expectedLookerConnectionName` keys and their descriptions. + +- [ ] **Step 3: Run the CLI smoke** + +Run: `pnpm --filter @ktx/cli run smoke` +Expected: PASS. + +- [ ] **Step 4: Run the broader workspace test suite** + +Run: `pnpm run test 2>&1 | tee /tmp/ktx-test-output.log` +Expected: PASS. Inspect `/tmp/ktx-test-output.log` if anything fails. + +- [ ] **Step 5: Run pre-commit on changed files** + +Run: `pnpm run check` +Expected: PASS. + +- [ ] **Step 6: Knip dead-code sweep (in case we introduced unused exports)** + +Run: `pnpm run dead-code` +Expected: PASS — or, if Knip flags `KtxConnectionConfig` as unused, decide whether to export it from `packages/context/src/project/index.ts` (preferred — it documents intent) or drop the export. + +If exporting: add to `packages/context/src/project/index.ts`: + +```typescript +export type { KtxConnectionConfig } from './driver-schemas.js'; +``` + +- [ ] **Step 7: Final commit if any docs / index changes were made** + +```bash +git status --short +# If only docs/index were touched in step 6: +git add packages/context/src/project/index.ts +git commit -m "chore(context): re-export KtxConnectionConfig from project package" +``` + +--- + +## Self-Review + +**1. Spec coverage:** Original request was "I need to be able to see full schema" with chosen approach option 1 (discriminated union). Task 5 step 2 verifies that `ktx dev schema | rg -i mapping` now returns hits. Task 6 step 2 is the explicit end-to-end check. All catalogued drivers (warehouse + metabase + looker + lookml + notion + dbt + metricflow) have a schema and a test. ✅ + +**2. Placeholder scan:** No "TBD", "add validation", "similar to Task N", or skipped code. Every step has the actual code or command. ✅ + +**3. Type consistency:** +- `connectionConfigSchema` is defined in Task 2 and extended (not renamed) in Tasks 3–4. ✅ +- `KtxConnectionConfig` (new type) appears only in `driver-schemas.ts` and the optional re-export in Task 6. `KtxProjectConnectionConfig` (existing type at `config.ts:272`) keeps its name. ✅ +- `metabaseMappingsSchema`, `lookerMappingsSchema`, `lookmlMappingsSchema` — Task 1 exports them; Task 3 imports them by the same names. ✅ +- `metabaseMappingsStrictSchema` is defined and used in Task 3 only. ✅ +- The `warehouseDrivers` array and `warehouseConnectionSchema` helper are introduced in Task 2 and reused unchanged in Task 4's union extension. ✅ + +--- + +## Execution Handoff + +Plan complete and saved to `docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md`. Two execution options: + +**1. Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration. + +**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints. + +Which approach? diff --git a/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md b/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md index 34585743..a2f12a75 100644 --- a/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md +++ b/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md @@ -1,4 +1,4 @@ -# Demo Guided Tour — Design Spec +# Demo Guided Tour - Design Spec ## Problem @@ -40,7 +40,7 @@ Copy pre-packaged assets (demo DB, replay, context artifacts) ┌────────────────────────────────────────────────────────────────┐ │ Demo banner (persistent, shown on every step) │ │ │ -│ Demo mode — data has been pre-processed and KTX context is │ +│ Demo mode - data has been pre-processed and KTX context is │ │ already built. This walkthrough illustrates the setup steps. │ │ Selections are pre-filled and read-only. │ └────────────────────────────────────────────────────────────────┘ @@ -67,7 +67,7 @@ Context build replay │ ▼ Transition message: - "Demo project is ready — let's connect your agent" + "Demo project is ready - let's connect your agent" │ ▼ Interactive agents step (real runKtxSetupAgentsStep()) @@ -89,7 +89,7 @@ Final summary: Shown at the top of every read-only step. Uses clack box-drawing style: ``` -┌ Demo mode — data has been pre-processed and KTX context is already built. +┌ Demo mode - data has been pre-processed and KTX context is already built. │ This walkthrough illustrates the setup steps. Selections are pre-filled and read-only. ``` @@ -170,7 +170,7 @@ Completion summary uses the existing format: ★ KTX finished ingesting your data ✓ Analyzed X business areas - ✓ Reconciled — 0 conflicts + ✓ Reconciled - 0 conflicts KTX created: 📊 X query definitions @@ -187,7 +187,7 @@ The exact counts and artifact names come from the pre-packaged demo results A brief message bridges from the read-only tour to the interactive step: ``` -┌ Demo project is ready — let's connect your agent +┌ Demo project is ready - let's connect your agent │ │ Your KTX context has been built with demo data. │ Select an agent to start using it. @@ -240,13 +240,13 @@ the pre-packaged replay file at an accelerated playback rate. | `packages/cli/src/setup.ts` | Add `demoMode` flag to setup loop; skip models/embeddings; dispatch to demo cards for databases/sources; show demo banner; demo completion summary | | `packages/cli/src/setup-demo-cards.ts` | New file: `renderDemoCard()` helper, demo banner renderer, demo step definitions | | `packages/cli/src/setup-context.ts` | Support replay mode for demo: feed pre-packaged events at accelerated pace through existing progress view | -| `packages/cli/src/demo.ts` | Remove or simplify `runKtxSetupDemoFromEntryMenu()` — now dispatches to the main setup loop with `demoMode: true` | +| `packages/cli/src/demo.ts` | Remove or simplify `runKtxSetupDemoFromEntryMenu()` - now dispatches to the main setup loop with `demoMode: true` | | `packages/cli/src/demo-assets.ts` | Update asset list if new demo data is provided; ensure demo project setup writes valid `ktx.yaml` for agent use | ## Open Items - **Demo data**: User will provide improved pre-packaged results (Postgres, dbt, Metabase, Notion). Current demo assets may need updating. -- **Replay speed**: Exact acceleration factor TBD — should feel brisk but +- **Replay speed**: Exact acceleration factor TBD - should feel brisk but give users time to read source names and status transitions. Start with ~2x real-time and adjust. diff --git a/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md b/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md index c9c0aa10..d27b6ea8 100644 --- a/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md +++ b/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md @@ -1,4 +1,4 @@ -# Historic SQL Ingestion — Redesign +# Historic SQL Ingestion - Redesign **Status:** draft **Date:** 2026-05-11 @@ -16,12 +16,12 @@ Concrete pain points observed: - The output is **rigid and shallow**: deterministic slot classification (constant / categorical / runtime) and triage-signal buckets do not produce narrative an agent can use. The current downstream skills (`historic_sql_ingest`, `historic_sql_curator`) try to recover narrative from these templates but at high cost. - Lots of moving parts (baseline files, reset detection, atomic per-connection commit, slot heuristics, ranking formula) for what is fundamentally "find interesting queries and tell agents about them." -The end goal — per the user — is for ingested content to be **searchable by `ktx wiki search` and `ktx sl search` to help consumer research agents do data analysis and agentic BI**. +The end goal - per the user - is for ingested content to be **searchable by `ktx wiki search` and `ktx sl search` to help consumer research agents do data analysis and agentic BI**. ## 2. Design principles 1. **LLMs are the right tool for narrative and clustering.** Deterministic heuristics (slot classification, ranking formulas, categorical expansion) get replaced by LLM judgement applied to aggregated, bucketed inputs. -2. **The adapter stays LLM-free.** The existing convention — adapters are deterministic, skills do LLM work — is preserved. +2. **The adapter stays LLM-free.** The existing convention - adapters are deterministic, skills do LLM work - is preserved. 3. **One pipeline across dialects.** A single reader interface, a single staging shape, a single set of skills. Dialect-specific behavior lives only in the snapshot query. 4. **No work where no signal changed.** Daily reruns should LLM only the things that actually changed. 5. **Lean context for caller agents.** Each retrieval tier (search hit → source read → pattern read) carries only what the agent needs to make the next decision. The principle lives in prompt instructions, not in defensive schema constraints. @@ -53,9 +53,9 @@ Reader (unified) ─▶ Aggregated snapshot ─▶ Batch SQL parse ─▶ Bu └──────────────────────────┬───────────────────────────────────────────────────┘ ▼ onPullSucceeded() projection (no LLM): - Pass A — merge `usage` into _schema/{shard}.yaml (per-shard atomic, scan-managed keys) - Pass B — write/update pattern wiki pages (slug stability + stale handling) - Pass C — trigger SL search re-index for changed sources + Pass A - merge `usage` into _schema/{shard}.yaml (per-shard atomic, scan-managed keys) + Pass B - write/update pattern wiki pages (slug stability + stale handling) + Pass C - trigger SL search re-index for changed sources ``` ## 4. Hot path (LLM-free) @@ -78,7 +78,7 @@ interface HistoricSqlReader { ### 4.2 Snapshot queries (one per dialect) -**Postgres** — `pg_stat_statements` collapsed to `queryid`: +**Postgres** - `pg_stat_statements` collapsed to `queryid`: ```sql SELECT queryid::text AS template_id, @@ -95,7 +95,7 @@ HAVING SUM(calls) >= @min_executions `firstSeen` derives from `pg_stat_statements_info.stats_reset`; `lastSeen` is `now()`. `p50RuntimeMs` / `p95RuntimeMs` collapse to `mean_ms`. `errorRate = 0` (PG doesn't track failures in PGSS). -**BigQuery** — warehouse-side aggregation over `INFORMATION_SCHEMA.JOBS_BY_PROJECT`: +**BigQuery** - warehouse-side aggregation over `INFORMATION_SCHEMA.JOBS_BY_PROJECT`: ```sql SELECT query_hash AS template_id, @@ -115,7 +115,7 @@ GROUP BY query_hash HAVING COUNT(*) >= @min_executions ``` -**Snowflake** — analogous, over `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY`: +**Snowflake** - analogous, over `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY`: ```sql SELECT query_hash AS template_id, @@ -154,22 +154,22 @@ Per-row parse failures are non-fatal: the template loses table grounding (exclud ### 4.4 Filtering (three layers) -**Layer A — Warehouse-side (in the SQL above):** +**Layer A - Warehouse-side (in the SQL above):** - Noise prefixes (`SHOW`, `DESCRIBE`, `EXPLAIN`, `USE`, `SET`). - System catalogs (`INFORMATION_SCHEMA`, `SNOWFLAKE.ACCOUNT_USAGE`, `pg_*`, `system.*`). - DDL / non-analytical statement types via `statement_type` / `query_type` columns (PG falls back to prefix regex). -- Trivial probes (`SELECT 1`, `SELECT NOW()`, `SELECT VERSION()`) — configurable. +- Trivial probes (`SELECT 1`, `SELECT NOW()`, `SELECT VERSION()`) - configurable. - Minimum executions threshold (`@min_executions`, default 5). -- Trailing window (`@window_days`, default 90) — BQ/SF only. +- Trailing window (`@window_days`, default 90) - BQ/SF only. -**Layer B — Post-fetch, in-memory:** +**Layer B - Post-fetch, in-memory:** - Service-account exclusion/inclusion via configurable regex patterns; three modes (`exclude` default, `include`, `mark-only`). -- Orchestrator boilerplate (dbt/Looker/Metabase markers) — default `mark-only` (do not drop; dbt-generated queries are often the actual business logic). +- Orchestrator boilerplate (dbt/Looker/Metabase markers) - default `mark-only` (do not drop; dbt-generated queries are often the actual business logic). - Failed-query filter (BQ/SF only): templates with `errorRate > 0.9 AND executions < 10`. -**Layer C — Post-parse:** +**Layer C - Post-parse:** - Zero-table templates (parsed cleanly but touch no real tables) are dropped from per-table bucketization and from patterns. @@ -187,7 +187,7 @@ In-memory pass: a single template touching N tables ends up in N table buckets. patterns-input.json ``` -`manifest.json` is small (summary, window, counts, warnings — schema in §9). +`manifest.json` is small (summary, window, counts, warnings - schema in §9). `tables/{schema}.{name}.json` contains **bucketed** content so that DiffSet content hashes are stable when nothing material changed: @@ -223,7 +223,7 @@ Bucket bands are defined deterministically in code (e.g. `executionsBucket`: `<1 ### 4.7 `chunk()` (trivial, convention-following) -One `WorkUnit` per `tables/*.json` file (handled by `historic_sql_table_digest`) + one `WorkUnit` referencing `patterns-input.json` (handled by `historic_sql_patterns`). No custom diff logic — the framework's `DiffSetComputerPort` already filters to changed files. +One `WorkUnit` per `tables/*.json` file (handled by `historic_sql_table_digest`) + one `WorkUnit` referencing `patterns-input.json` (handled by `historic_sql_patterns`). No custom diff logic - the framework's `DiffSetComputerPort` already filters to changed files. ## 5. Cold path (LLM, via skills) @@ -262,7 +262,7 @@ No hard length/cap constraints in the schema. Concision is a behavioral instruct ### 5.2 `historic_sql_patterns` -One invocation per run (or a small handful if `patterns-input.json` exceeds a context budget — split deterministically by `tablesTouched` cardinality stratification). +One invocation per run (or a small handful if `patterns-input.json` exceeds a context budget - split deterministically by `tablesTouched` cardinality stratification). **Prompt:** identifies recurring analytical intents that span ≥2 tables with ≥mid executionsBucket and ≥2-5 distinct users. Output is a list of `PatternOutput`. @@ -288,18 +288,18 @@ export const patternOutputSchema = z.object({ After all skills complete and evidence is committed, run two passes. Both are pure data transformations, no LLM calls. -**Pass A — `_schema` shard reconciliation:** +**Pass A - `_schema` shard reconciliation:** 1. Collect all `historic_sql_table_usage` evidence written this run. 2. Group by `shardKey` (`catalog.schema`). 3. For each shard: - Load existing `_schema/{shardKey}.yaml`. - - For each table entry: if new evidence exists, merge under `usage` via `mergeUsagePreservingExternal()` (only `historicSql`-managed keys touched; user-added keys preserved — same pattern as `mergeDescriptionsPreservingExternal` at `local-enrichment-artifacts.ts:237-242`). + - For each table entry: if new evidence exists, merge under `usage` via `mergeUsagePreservingExternal()` (only `historicSql`-managed keys touched; user-added keys preserved - same pattern as `mergeDescriptionsPreservingExternal` at `local-enrichment-artifacts.ts:237-242`). - For tables previously present with `historicSql`-managed `usage` but absent from this run's snapshot: set `usage.staleSince = lastSnapshotSeenAt`, clear other historicSql-managed fields. - Atomic write to `_schema/{shardKey}.yaml`. 4. Trigger SL search re-index for changed sources via the existing flow (`sl-search.service.ts:91-99` detects search-text drift). -**Pass B — wiki pattern pages:** +**Pass B - wiki pattern pages:** 1. Collect all `historic_sql_pattern` evidence written this run. 2. Load existing wiki pages with tags `['historic-sql', 'pattern']` for this connection. @@ -312,21 +312,21 @@ After all skills complete and evidence is committed, run two passes. Both are pu ## 6. Search-surface plumbing -### 6.1 `ktx wiki search` — no plumbing required +### 6.1 `ktx wiki search` - no plumbing required Pattern pages are written to `knowledge/global/historic-sql/{slug}.md` and are discovered by the existing `searchLocalKnowledgePages()` walk. Tags `['historic-sql', 'pattern']` enable faceted search. -### 6.2 `ktx sl search` — small extension +### 6.2 `ktx sl search` - small extension -**6.2.1 — `SemanticLayerSource.usage` field** +**6.2.1 - `SemanticLayerSource.usage` field** Add an optional `usage` field to `SemanticLayerSource` in `packages/context/src/sl/schemas.ts`, reusing the same `tableUsageOutputSchema` from `skill-schemas.ts`. Single source of truth end-to-end. -**6.2.2 — `_schema` → `SemanticLayerSource` projection carries `usage`** +**6.2.2 - `_schema` → `SemanticLayerSource` projection carries `usage`** The existing projection step in `local-sl.ts` (or wherever the manifest reader builds `SemanticLayerSource` objects) needs one new field copy: `entry.usage → source.usage`. -**6.2.3 — `buildSemanticLayerSourceSearchText()` extension** +**6.2.3 - `buildSemanticLayerSourceSearchText()` extension** Extend the function at `sl-search.service.ts:8-74` to include usage content in the FTS5/embedding text: @@ -344,11 +344,11 @@ if (source.usage) { } ``` -**6.2.4 — Re-index trigger** +**6.2.4 - Re-index trigger** Already wired. Per-source content-hash detection at `sl-search.service.ts:91-99` ensures only sources whose `usage` changed re-embed. -**6.2.5 — Query-mode result enrichment** +**6.2.5 - Query-mode result enrichment** Extend the search result shape returned by `agent sl list --query` to include `score` and an FTS5 `snippet()` per hit. Implementation: small SQL change in `sqlite-sl-sources-index.ts` to select `snippet(local_sl_sources_fts, ...)` alongside the source row. @@ -510,7 +510,7 @@ export const stagedManifestSchema = z.object({ }); ``` -In `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` — the **single source of truth for LLM I/O shapes**, imported by the prompt builder, the evidence parser, the projection step, the `SemanticLayerSource` type, and the `_schema` manifest entry type: +In `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` - the **single source of truth for LLM I/O shapes**, imported by the prompt builder, the evidence parser, the projection step, the `SemanticLayerSource` type, and the `_schema` manifest entry type: ```typescript export const tableUsageOutputSchema = z.object({ @@ -541,10 +541,10 @@ export type PatternOutput = z.infer; **Extensions to existing types:** -- `packages/context/src/sl/schemas.ts` — `SemanticLayerSource.usage: tableUsageOutputSchema.optional()`. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` — `LiveDatabaseManifestTableEntry.usage?: TableUsageOutput`. +- `packages/context/src/sl/schemas.ts` - `SemanticLayerSource.usage: tableUsageOutputSchema.optional()`. +- `packages/context/src/ingest/adapters/live-database/manifest.ts` - `LiveDatabaseManifestTableEntry.usage?: TableUsageOutput`. -The `_schema/{shard}.yaml` manifest version need not bump — `usage` is an additive, optional field. Validators must allow unknown future keys (audit during step 1 of §10). +The `_schema/{shard}.yaml` manifest version need not bump - `usage` is an additive, optional field. Validators must allow unknown future keys (audit during step 1 of §10). ## 10. Cutover plan @@ -554,22 +554,22 @@ Hard cutover. No parallel codepaths. Single coordinated PR (or PR train). Within `packages/context/src/ingest/adapters/historic-sql/`: -- `stage.ts` — rewritten -- `stage-pgss.ts` — **deleted** (no baseline tracking) -- `stage-pgss.test.ts`, `stage-pgss-golden.test.ts` — **deleted** -- `historic-sql.adapter.ts` — rewritten -- `historic-sql.adapter.test.ts` — rewritten -- `chunk.ts` / `chunk.test.ts` — rewritten (becomes trivial) -- `detect.ts` / `detect.test.ts` — trivial update -- `postgres-pgss-query-history-reader.ts` — rewritten as `postgres-pgss-reader.ts`; baseline-tracking code removed -- `bigquery-query-history-reader.ts` / `snowflake-query-history-reader.ts` — rewritten; cursor logic removed; warehouse-side GROUP BY -- `types.ts` — rewritten +- `stage.ts` - rewritten +- `stage-pgss.ts` - **deleted** (no baseline tracking) +- `stage-pgss.test.ts`, `stage-pgss-golden.test.ts` - **deleted** +- `historic-sql.adapter.ts` - rewritten +- `historic-sql.adapter.test.ts` - rewritten +- `chunk.ts` / `chunk.test.ts` - rewritten (becomes trivial) +- `detect.ts` / `detect.test.ts` - trivial update +- `postgres-pgss-query-history-reader.ts` - rewritten as `postgres-pgss-reader.ts`; baseline-tracking code removed +- `bigquery-query-history-reader.ts` / `snowflake-query-history-reader.ts` - rewritten; cursor logic removed; warehouse-side GROUP BY +- `types.ts` - rewritten - **new** `skill-schemas.ts` -- `errors.ts` — keep (probe errors); prune unused +- `errors.ts` - keep (probe errors); prune unused -Old skills `historic_sql_ingest` and `historic_sql_curator` — audit; if only consumed by historic-sql, delete. +Old skills `historic_sql_ingest` and `historic_sql_curator` - audit; if only consumed by historic-sql, delete. -`expandCategoricalTemplates`, `classifySlot`, `rankTemplate`, slot-related types — gone. +`expandCategoricalTemplates`, `classifySlot`, `rankTemplate`, slot-related types - gone. ### 10.2 Existing artifacts @@ -596,7 +596,7 @@ Old skills `historic_sql_ingest` and `historic_sql_curator` — audit; if only c - `historic_sql_table_digest` + `historic_sql_patterns`. - `onPullSucceeded` projection passes. - One-time legacy cleanup. -5. **Delete the old codepath** — same PR as step 3, ideally. +5. **Delete the old codepath** - same PR as step 3, ideally. 6. **Docs + setup wizard** updates. ### 10.4 Verification before merging @@ -612,11 +612,11 @@ Old skills `historic_sql_ingest` and `historic_sql_curator` — audit; if only c ### 10.5 Out of scope - Embedding-based pattern clustering (rejected in favor of LLM-driven intent detection). -- Wiki shard pages (rejected — patterns are sparse; per-page is correct). +- Wiki shard pages (rejected - patterns are sparse; per-page is correct). - Incremental dialect-by-dialect rollout behind a flag. -- A `ktx historic-sql migrate` command — cleanup runs automatically once. +- A `ktx historic-sql migrate` command - cleanup runs automatically once. - Framework-level `raw-sources/` retention policy (separate concern; not introduced here). -- Per-table wiki pages (the very problem `_schema` shards exist to avoid — see §11). +- Per-table wiki pages (the very problem `_schema` shards exist to avoid - see §11). ### 10.6 Risks @@ -632,23 +632,23 @@ Old skills `historic_sql_ingest` and `historic_sql_curator` — audit; if only c Documented so future readers don't relitigate. -**Per-table wiki pages** — one `.md` per table under `knowledge/global/historic-sql/`. Rejected: reintroduces the per-table-file proliferation problem (`writeLocalKnowledgePage` writes one file per page) that `_schema` shards exist to avoid. ~800 markdown files for a 1000-table warehouse, ~100 churning daily. +**Per-table wiki pages** - one `.md` per table under `knowledge/global/historic-sql/`. Rejected: reintroduces the per-table-file proliferation problem (`writeLocalKnowledgePage` writes one file per page) that `_schema` shards exist to avoid. ~800 markdown files for a 1000-table warehouse, ~100 churning daily. -**Single-file all-usage page** — one giant page containing every table. Rejected: ~700 KB blob; FTS5 snippets all come from the same source; `wiki read` returns an unusable mass. +**Single-file all-usage page** - one giant page containing every table. Rejected: ~700 KB blob; FTS5 snippets all come from the same source; `wiki read` returns an unusable mass. -**One file per table in a new `_usage/` directory** — same file-count problem as per-table wiki, plus needs new search plumbing. +**One file per table in a new `_usage/` directory** - same file-count problem as per-table wiki, plus needs new search plumbing. -**New parallel `_usage/{shard}.yaml` shards** — same sharding benefit as merging into `_schema` but without riding SL search. Plumbing required without offsetting win. +**New parallel `_usage/{shard}.yaml` shards** - same sharding benefit as merging into `_schema` but without riding SL search. Plumbing required without offsetting win. -**One wiki page per `catalog.schema`** — workable, but pages get large (200 tables per page) and only rides wiki search, not SL search. The chosen design rides both. +**One wiki page per `catalog.schema`** - workable, but pages get large (200 tables per page) and only rides wiki search, not SL search. The chosen design rides both. -**Single staged `snapshot.json`** — to reduce `raw-sources/` accumulation. Rejected: required custom diff logic in `chunk()`, broke framework convention, saved bounded disk for a framework-level concern (sync retention). Per-table staged files with bucketed content is cleaner. +**Single staged `snapshot.json`** - to reduce `raw-sources/` accumulation. Rejected: required custom diff logic in `chunk()`, broke framework convention, saved bounded disk for a framework-level concern (sync retention). Per-table staged files with bucketed content is cleaner. -**Embedding-based pattern clustering** — using sentence-transformer embeddings to cluster templates into themes before naming via LLM. Rejected: reintroduces clustering hyperparameters and determinism the redesign aims to avoid. The LLM does the grouping in one call from the full template list, no embedding step. +**Embedding-based pattern clustering** - using sentence-transformer embeddings to cluster templates into themes before naming via LLM. Rejected: reintroduces clustering hyperparameters and determinism the redesign aims to avoid. The LLM does the grouping in one call from the full template list, no embedding step. -**Skip pattern pages entirely** — ship only `_schema` enrichment for a leaner v1. Rejected: leaves `ktx wiki search` empty of historic-sql content (loses one of two stated consumption surfaces) and forces agents to synthesize cross-cutting intents from fragmented per-table mentions. +**Skip pattern pages entirely** - ship only `_schema` enrichment for a leaner v1. Rejected: leaves `ktx wiki search` empty of historic-sql content (loses one of two stated consumption surfaces) and forces agents to synthesize cross-cutting intents from fragmented per-table mentions. -**TypeScript-native SQL parser** instead of sqlglot via daemon — `node-sql-parser`, `pgsql-parser` (WASM), etc. Rejected: materially worse dialect coverage on Snowflake/BigQuery edge cases; duplicates parser logic when KTX already uses sqlglot elsewhere (`python/ktx-daemon/src/ktx_daemon/lookml.py`); AGENTS.md explicitly mandates sqlglot. Batch endpoint on the existing daemon achieves the perf win. +**TypeScript-native SQL parser** instead of sqlglot via daemon - `node-sql-parser`, `pgsql-parser` (WASM), etc. Rejected: materially worse dialect coverage on Snowflake/BigQuery edge cases; duplicates parser logic when KTX already uses sqlglot elsewhere (`python/ktx-daemon/src/ktx_daemon/lookml.py`); AGENTS.md explicitly mandates sqlglot. Batch endpoint on the existing daemon achieves the perf win. **Hard length/count caps in zod output schemas** (e.g. `narrative.max(250)`, `commonFilters.max(5)`). Rejected: arbitrary thresholds, brittle retry-on-violation paths, defensive coding for a soft concern. Concision belongs in prompt instructions; the schema validates shape. @@ -671,7 +671,7 @@ For a large warehouse (~800 touched tables): first-run ~$20–30, daily ~$0.20 ## 13. Open questions -- Exact bucket thresholds for `executionsBucket`, `distinctUsersBucket`, etc. — to be chosen during implementation based on what produces stable hashes in practice. +- Exact bucket thresholds for `executionsBucket`, `distinctUsersBucket`, etc. - to be chosen during implementation based on what produces stable hashes in practice. - Final naming of the daemon endpoint (`/sql/analyze-batch` vs alternatives). -- Whether `historic_sql_ingest` / `historic_sql_curator` skills are consumed elsewhere — audit during step 1. -- Whether to delete legacy wiki pages automatically or behind a confirmation flag — design assumes automatic. +- Whether `historic_sql_ingest` / `historic_sql_curator` skills are consumed elsewhere - audit during step 1. +- Whether to delete legacy wiki pages automatically or behind a confirmation flag - design assumes automatic. diff --git a/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md b/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md index 074f00e5..84764501 100644 --- a/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md +++ b/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md @@ -2,7 +2,7 @@ **Date:** 2026-05-12 **Author:** Andrey Avtomonov -**Status:** Design — pending implementation plan +**Status:** Design - pending implementation plan ## Background and motivation @@ -16,7 +16,7 @@ A real-world inspection (project `/tmp/ktx-proj-1`) surfaced two failure modes t Root cause analysis (`packages/context/skills/notion_synthesize/SKILL.md`, `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`, `packages/context/src/wiki/tools/wiki-write.tool.ts`) showed three contributing factors: - The synthesis LLM has no verification primitive that distinguishes a real warehouse identifier from a fabricated one. `sl_discover` only finds objects already promoted into the semantic layer; raw warehouse scans (which already exist on disk under `raw-sources//live-database//`) are not surfaced to the LLM at all. -- `wiki_write` performs no body-text validation — anything the LLM emits is written. +- `wiki_write` performs no body-text validation - anything the LLM emits is written. - The skill prompt itself uses `orbit_analytics.customer` as a canonical example string (`SKILL.md:70`), reinforcing the same fictional name the LLM ends up emitting. Kaelio's server-side ingest WU agent (`/Users/andrey/conductor/workspaces/kaelio-main2/douala/server/src/tools/toolset-factory.service.ts`) had four verification tools that KTX dropped during the open-source extraction: `discover_data`, `entity_details`, `dictionary_search`, and `sql_execution`. The underlying connector infrastructure (`KtxScanConnector`, dialect classes, `assertReadOnlySql`, `SemanticLayerService.executeQuery`) is present in KTX, so the gap is at the tool layer, not the platform layer. @@ -115,7 +115,7 @@ export type SupportedDriver = 'postgres'|'postgresql'|'mysql'|'sqlserver'|'snowf export function getDialectForDriver(driver: SupportedDriver): KtxDialect; ``` -Sync dispatch. The connectors' existing dialect classes already expose the same shape — `formatTableName(KtxTableRef)`, `quoteIdentifier(string)`, `mapToDimensionType(nativeType)`. The implementation plan introduces a minimal `KtxDialect` interface that these classes already satisfy structurally; no connector-internal changes required. Used by tools only for display-string parsing and error-message formatting; tools never construct executable SQL. +Sync dispatch. The connectors' existing dialect classes already expose the same shape - `formatTableName(KtxTableRef)`, `quoteIdentifier(string)`, `mapToDimensionType(nativeType)`. The implementation plan introduces a minimal `KtxDialect` interface that these classes already satisfy structurally; no connector-internal changes required. Used by tools only for display-string parsing and error-message formatting; tools never construct executable SQL. ## Tool contracts @@ -139,7 +139,7 @@ Type: table | Native columns: 11 | PK: account_id | FKs: parent_account_id → o Description: One row per customer account… Columns: -- account_id (text, nullable=false, PK) — sample: ["acct_001","acct_002",…] +- account_id (text, nullable=false, PK) - sample: ["acct_001","acct_002",…] - parent_account_id (text, nullable=true, FK → orbit_raw.accounts.account_id) - account_name (text, nullable=false) - … @@ -147,7 +147,7 @@ Columns: Profile: rowCount=4321 distinctCount(account_id)=4321 nullRate(parent_account_id)=0.62 ``` -When `column` is provided in a target, output is scoped to that one column. When a target doesn't resolve, output is `Not found in scan. Closest matches: …` with up to 5 candidates from `searchByName`. When the connection has no `live-database` scan, output is `No live-database scan available for connection ""; run \`ktx scan\` first.` — distinct from the "not found" state. +When `column` is provided in a target, output is scoped to that one column. When a target doesn't resolve, output is `Not found in scan. Closest matches: …` with up to 5 candidates from `searchByName`. When the connection has no `live-database` scan, output is `No live-database scan available for connection ""; run \`ktx scan\` first.` - distinct from the "not found" state. Structured output: `{ resolved: TableDetail[], missing: Array<{target, candidates}>, scanAvailable: boolean }`. @@ -165,14 +165,14 @@ input = { Pipeline: -1. `assertReadOnlySql(sql)` — regex rejects anything starting with `insert|update|delete|merge|alter|drop|create|truncate|grant|revoke|copy|call|do|vacuum|analyze|refresh`. -2. `limitSqlForExecution(sql, rowLimit)` — wraps as `select * from () as ktx_query_result limit N`. +1. `assertReadOnlySql(sql)` - regex rejects anything starting with `insert|update|delete|merge|alter|drop|create|truncate|grant|revoke|copy|call|do|vacuum|analyze|refresh`. +2. `limitSqlForExecution(sql, rowLimit)` - wraps as `select * from () as ktx_query_result limit N`. 3. `SemanticLayerService.executeQuery(connectionName, wrappedSql)`. 4. Format as markdown table; first ~20 rows inline; if truncated, append `… +N more rows`. Structured output: `{ headers, rows, rowCount, truncated, sql, wrappedSql }`. -Connector errors surface verbatim (e.g., Postgres `relation "orbit_analytics.customer" does not exist`). That error message is the most valuable verification signal — it tells the LLM the identifier is fictional. +Connector errors surface verbatim (e.g., Postgres `relation "orbit_analytics.customer" does not exist`). That error message is the most valuable verification signal - it tells the LLM the identifier is fictional. Refuses `connectionName` not in `allowedConnectionNames`. Each connector's driver-level read-only enforcement (Postgres read-only transaction, BigQuery query-only jobs) is a second defence under the regex gate. @@ -189,9 +189,9 @@ input = { Composes three searches and groups output into three sections, omitting empty sections: -1. **Wiki Pages** — `wiki_search({query, limit})`. Routing hint: *use `wiki_read(blockKey)` for full content*. -2. **Semantic Layer Sources** — `sl_discover({query, connectionName})`. Routing hint: *use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details*. -3. **Raw Warehouse Schema** — `WarehouseCatalogService.searchByName(connectionName, query, limit)`. Routing hint: *use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values*. +1. **Wiki Pages** - `wiki_search({query, limit})`. Routing hint: *use `wiki_read(blockKey)` for full content*. +2. **Semantic Layer Sources** - `sl_discover({query, connectionName})`. Routing hint: *use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details*. +3. **Raw Warehouse Schema** - `WarehouseCatalogService.searchByName(connectionName, query, limit)`. Routing hint: *use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values*. When `sourceName` is set, delegates entirely to `sl_discover` inspect mode and skips other sections. When all three sections are empty, output is `No matches for "" across wiki, semantic layer, or raw warehouse schema. Try broader terms; this concept may not exist yet.` @@ -215,7 +215,7 @@ const warehouseTools = createWarehouseVerificationTools({ // alongside emit_unmapped_fallback. ``` -`createWarehouseVerificationTools` returns `Record` with three keys. The set is wired into every adapter's synthesis stage — no per-adapter opt-in. +`createWarehouseVerificationTools` returns `Record` with three keys. The set is wired into every adapter's synthesis stage - no per-adapter opt-in. ## Skill-prompt updates @@ -227,12 +227,12 @@ const warehouseTools = createWarehouseVerificationTools({ ## Identifier Verification Protocol Before writing a wiki page or SL source on any topic: -1. `discover_data({query: ""})` — see what wikis, SL sources, and raw tables +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw tables already exist. Prefer updating existing pages over creating new ones. Before emitting any `schema.table` or `schema.table.column` into a wiki body, SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: -2. `entity_details({connectionName, targets: [{display: ""}]})` — +2. `entity_details({connectionName, targets: [{display: ""}]})` - confirm the identifier resolves; inspect native types, FK/PK, and sampleValues. 3. For literal values from the source (status codes, plan tiers): check whether they appear in `entity_details`' `sampleValues` for the relevant column. @@ -241,7 +241,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: 4. If the candidate identifier still doesn't resolve, do one of: (a) Use `sql_execution` with `SELECT 1 FROM LIMIT 0`. If it errors, the identifier is fictional. - (b) Wrap the identifier in `[unverified — from ]` in the wiki body, + (b) Wrap the identifier in `[unverified - from ]` in the wiki body, citing the exact raw path that mentioned it. (c) When recording `emit_unmapped_fallback` with `no_physical_table`, include the failing probe error in `clarification`. @@ -271,10 +271,10 @@ Two skills are deliberately excluded from updates: `ingest_triage` (read-only tr ### Cleanups beyond the four-tool addition -- `notion_synthesize/SKILL.md:70` — remove `orbit_analytics.customer` (placeholder). -- `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts:67` — same example string in the Zod `.describe()` — replace with `.`. -- `dbt_ingest/SKILL.md:24` — fix `wiki_sl_search` and `sl_describe_table` (neither tool exists in KTX). -- `packages/context/src/sl/tools/sl-warehouse-validation.ts:93` — inline error message references the non-existent `sl_describe_table`. Replace with `sl_read_source`. +- `notion_synthesize/SKILL.md:70` - remove `orbit_analytics.customer` (placeholder). +- `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts:67` - same example string in the Zod `.describe()` - replace with `.
`. +- `dbt_ingest/SKILL.md:24` - fix `wiki_sl_search` and `sl_describe_table` (neither tool exists in KTX). +- `packages/context/src/sl/tools/sl-warehouse-validation.ts:93` - inline error message references the non-existent `sl_describe_table`. Replace with `sl_read_source`. ## Testing strategy @@ -294,7 +294,7 @@ Two skills are deliberately excluded from updates: `ingest_triage` (read-only tr - Extend `packages/context/src/ingest/ingest-bundle.runner.test.ts` to verify the three new tools are present in both WU-stage and reconcile-stage tool maps and refuse out-of-scope `connectionName` values. - New fixture-based test: stage a small `raw-sources//live-database//` directory with 2 tables + 1 enrichment profile, then call each tool through the runner's tool map and assert the markdown contains the expected fields. Uses the same fake-LLM harness as `notion.adapter.test.ts`. -- One end-to-end regression test reproducing the `orbit_analytics.customer` hallucination: a fake Notion page mentioning the fictional table is fed to the synthesis stage; the run produces a wiki page where the fictional name is wrapped in `[unverified — …]` or omitted, not promoted to `tables:` frontmatter. +- One end-to-end regression test reproducing the `orbit_analytics.customer` hallucination: a fake Notion page mentioning the fictional table is fed to the synthesis stage; the run produces a wiki page where the fictional name is wrapped in `[unverified - …]` or omitted, not promoted to `tables:` frontmatter. ### Prompt-bundling tests @@ -306,7 +306,7 @@ Extend `packages/context/src/memory/memory-runtime-assets.test.ts`: ### Performance guards -`WarehouseCatalogService` caches the per-connection table list per stage (one WorkUnit's lifetime). Tests assert second call is a cache hit. No DB index for `searchByName` in this iteration — linear scan over scan artefacts is acceptable up to ~50K columns. If volume warrants it later, a follow-up PR adds a SQLite FTS index. +`WarehouseCatalogService` caches the per-connection table list per stage (one WorkUnit's lifetime). Tests assert second call is a cache hit. No DB index for `searchByName` in this iteration - linear scan over scan artefacts is acceptable up to ~50K columns. If volume warrants it later, a follow-up PR adds a SQLite FTS index. ## Rollout @@ -323,7 +323,7 @@ Skill prompts land last so they can reference the tools that already exist. ## Out of scope -- **Hard write-time validation in `wiki_write` / `emit_unmapped_fallback`.** A complementary spec covers regex-based identifier validation at the write boundary. Defence-in-depth — separate concern. +- **Hard write-time validation in `wiki_write` / `emit_unmapped_fallback`.** A complementary spec covers regex-based identifier validation at the write boundary. Defence-in-depth - separate concern. - **SQLite FTS index for `searchByName`.** Deferred until the linear scan benchmark fails. - **`raw_schema_search` as a standalone tool.** `discover_data`'s raw section covers the concept-search case. - **`semantic_query` in the synthesis toolset.** `semantic_query` will exist in KTX for the research/chat-time agent; it is deliberately excluded from synthesis because synthesis creates SL sources rather than queries them. diff --git a/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md b/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md index 3697f47b..f6e4835e 100644 --- a/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md +++ b/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md @@ -2,7 +2,7 @@ **Date:** 2026-05-13 **Author:** Andrey Avtomonov -**Status:** Design — pending implementation plan +**Status:** Design - pending implementation plan ## Background diff --git a/examples/local-warehouse/ktx.yaml b/examples/local-warehouse/ktx.yaml index a967e31c..fb8ae027 100644 --- a/examples/local-warehouse/ktx.yaml +++ b/examples/local-warehouse/ktx.yaml @@ -1,4 +1,3 @@ -project: local-warehouse connections: warehouse: driver: postgres diff --git a/examples/orbit-relationship-verification/ktx.yaml b/examples/orbit-relationship-verification/ktx.yaml index 1251d5a1..b1d30961 100644 --- a/examples/orbit-relationship-verification/ktx.yaml +++ b/examples/orbit-relationship-verification/ktx.yaml @@ -1,4 +1,3 @@ -project: orbit-relationship-verification connections: orbit: driver: sqlite diff --git a/packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md b/packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md index 162ade23..77eff917 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md +++ b/packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md @@ -12,7 +12,7 @@ refs: ## Customer Update Communication Standard -**Source:** Notion — People & Operating Norms, last edited 2026-05-07 +**Source:** Notion - People & Operating Norms, last edited 2026-05-07 --- diff --git a/packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md b/packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md index 2059329d..7dece131 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md +++ b/packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md @@ -12,23 +12,23 @@ refs: ## New Hire Week-One Onboarding Policy -**Source:** Notion — People & Operating Norms, last edited 2026-05-07 +**Source:** Notion - People & Operating Norms, last edited 2026-05-07 **Owner:** Manager (not People Ops) --- ## Policy -Every new hire must understand **four things by end of week one**. The manager — not People Ops — is responsible for supplying this context. +Every new hire must understand **four things by end of week one**. The manager - not People Ops - is responsible for supplying this context. ## Required Week-One Knowledge | # | What the new hire must understand | |---|---| -| 1 | **What Orbit sells** — the core procurement workflow product and value proposition | -| 2 | **Why procurement workflow gets messy inside a customer** — the pain points that make Orbit necessary | -| 3 | **Which team handles which part of the customer lifecycle** — team lanes and ownership boundaries | -| 4 | **What their first useful project is** — a concrete, scoped piece of work they can contribute to immediately | +| 1 | **What Orbit sells** - the core procurement workflow product and value proposition | +| 2 | **Why procurement workflow gets messy inside a customer** - the pain points that make Orbit necessary | +| 3 | **Which team handles which part of the customer lifecycle** - team lanes and ownership boundaries | +| 4 | **What their first useful project is** - a concrete, scoped piece of work they can contribute to immediately | ## Ownership diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md index dc97bdc2..940f8189 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md @@ -21,7 +21,7 @@ tables: # Activation KPI Glossary **Owner team:** Growth -**Source:** Notion — Orbit Demo Home / Data Team - Onboarding / Activation KPI Glossary, last edited 2026-05-07 +**Source:** Notion - Orbit Demo Home / Data Team - Onboarding / Activation KPI Glossary, last edited 2026-05-07 Use this when a question is about signup-to-habit behavior. Orbit uses activation language across Growth, Product, and CS conversations. @@ -41,7 +41,7 @@ A customer is **activated** when **all three** of the following happen **within | 2. Email Verified | `customer.email_verified_at` is not null | `orbit_analytics.customer` | | 3. First Project | At least one row in `orbit_analytics.project` for the customer | `orbit_analytics.project` | | 4. Team Invite | At least one row in `orbit_analytics.invite` for the customer | `orbit_analytics.invite` | -| 5. Activated | All of (2), (3), and (4) within 14 days of (1) | — | +| 5. Activated | All of (2), (3), and (4) within 14 days of (1) | - | ## Conversion-Rate KPIs @@ -51,7 +51,7 @@ A customer is **activated** when **all three** of the following happen **within | **D14 Activation Rate** | `activated_customers_within_14_days / signups_in_cohort` | | **Time-to-Activate** | `median(activated_at - created_at)` in hours | -Growth conversations typically use D7 and D14 Activation Rate. Product and CS may ask about individual funnel steps — confirm whether they mean the full activation definition or only one stage. +Growth conversations typically use D7 and D14 Activation Rate. Product and CS may ask about individual funnel steps - confirm whether they mean the full activation definition or only one stage. ## Source Notes diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md index 3216d94d..c273b015 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md @@ -12,7 +12,7 @@ sl_refs: - mart_account_activity --- -# Activation Policy Change — January 2026 +# Activation Policy Change - January 2026 **Governed metric key:** `activated_accounts` **Owner team:** growth @@ -23,8 +23,8 @@ sl_refs: The activation workflow changed on **2026-01-15**. All activation events are tagged with `policy_version`: -- `pre_2026_01_15` — events before the workflow update -- `post_2026_01_15` — events after the workflow update +- `pre_2026_01_15` - events before the workflow update +- `post_2026_01_15` - events after the workflow update ## Activation Event Types diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md index 4cb34b76..6e2d20f6 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md @@ -13,7 +13,7 @@ sl_refs: - mart_account_segments --- -# ARR — Contract-First Definition +# ARR - Contract-First Definition **Governed metric key:** `arr` **Owner team:** finance @@ -30,10 +30,10 @@ The dbt test on `mart_arr_daily.arr_cents` asserts the value equals **1,874,200, ## Intermediate model -`int_active_contract_arr` — active contract ARR as of 2026-03-31 (grain: `contract_id`). +`int_active_contract_arr` - active contract ARR as of 2026-03-31 (grain: `contract_id`). ## Related -- `stg_contracts` — contract records (status: draft, active, cancelled, expired) -- `stg_subscriptions` — fallback ARR source (status: active, cancelled, past_due, trialing) -- `mart_arr_daily` — board-prep daily ARR mart +- `stg_contracts` - contract records (status: draft, active, cancelled, expired) +- `stg_subscriptions` - fallback ARR source (status: active, cancelled, past_due, trialing) +- `mart_arr_daily` - board-prep daily ARR mart diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md index 83645aeb..88e3abe5 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md @@ -15,14 +15,14 @@ refs: # Orbit Company Overview -**Source:** Notion — Orbit Demo Home / Company Overview + Orbit Demo Home (root), last edited 2026-05-07 +**Source:** Notion - Orbit Demo Home / Company Overview + Orbit Demo Home (root), last edited 2026-05-07 ## What Orbit Sells Orbit sells procurement workflow and spend-control software. The core value proposition: route purchase requests, collect approvals, onboard suppliers, and issue purchase orders without turning every exception into a status hunt. **Primary buyers:** Finance, Procurement, Business Operations. -**Daily users:** department admins, office managers, IT leads, legal ops partners — anyone who has to get a vendor through the building. +**Daily users:** department admins, office managers, IT leads, legal ops partners - anyone who has to get a vendor through the building. ## Product Workflow diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md index 56deb3f3..fb53e1cd 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md @@ -21,18 +21,18 @@ sl_refs: ## Risk Levels -`low`, `medium`, `high` — derived from two signal types: +`low`, `medium`, `high` - derived from two signal types: 1. **Support ticket signals** (`stg_support_tickets`): open or pending tickets with severity `high` or `critical` increase risk. 2. **Procurement activity signals** (`stg_purchase_requests`, `stg_purchase_orders`): recent qualifying procurement actions reduce risk. ## Intermediate Model -`int_customer_health_signals` — combines open critical ticket count and recent procurement action count per account. +`int_customer_health_signals` - combines open critical ticket count and recent procurement action count per account. ## Mart -`mart_customer_health` — account-grain risk mart as of **2026-03-31**. +`mart_customer_health` - account-grain risk mart as of **2026-03-31**. - `account_id`: dbt not_null, unique - `risk_level`: dbt accepted_values [low, medium, high] diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md index f7e57215..d5f9faf2 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md @@ -13,7 +13,7 @@ refs: ## Customer Stakeholder Needs by Role -**Source:** Notion — Product & Customers, last edited 2026-05-07 +**Source:** Notion - Product & Customers, last edited 2026-05-07 --- @@ -26,7 +26,7 @@ These are recurring, role-specific customer needs observed across accounts. Use | Role | Primary Need | Implication | |---|---|---| | **Finance** | Committed spend visibility earlier in the procurement cycle | Surface budget commitments at request approval, not at PO creation | -| **Department leaders** | Request speed — faster time from request to approval | Reduce approval routing friction; minimize back-and-forth | +| **Department leaders** | Request speed - faster time from request to approval | Reduce approval routing friction; minimize back-and-forth | | **Procurement** | Supplier file complete before the first invoice | Supplier onboarding must be finished before PO is issued, not after | | **Legal** | Fewer emergency reviews | Route contracts with legal implications earlier; avoid last-minute escalations | | **Customer Success (internal)** | Renewal risk visible before the account is already annoyed | CS needs leading indicators of dissatisfaction, not lagging ones | diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md index 2c9f2c65..a893eb70 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md @@ -20,7 +20,7 @@ tables: **Table:** `orbit_analytics.customer` **Grain:** one row per signed-up customer -**Source:** Notion — Orbit Demo Home / Data Team - Onboarding / Orbit Customers Source, last edited 2026-05-07 +**Source:** Notion - Orbit Demo Home / Data Team - Onboarding / Orbit Customers Source, last edited 2026-05-07 Use this when a question needs customer identity, plan tier, signup timing, recent activity, or the standard customer joins. @@ -29,7 +29,7 @@ Use this when a question needs customer identity, plan tier, signup timing, rece | Column | Type | Notes | |---|---|---| | `id` | number | Primary key, surrogate key | -| `email` | string | Login email, unique — **do not use as join key** | +| `email` | string | Login email, unique - **do not use as join key** | | `name` | string | Display name | | `country` | string | ISO 3166-1 alpha-2 code | | `plan_tier` | string | One of `free`, `pro`, `enterprise` | diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md index 3640a693..1fd84b9e 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md @@ -12,7 +12,7 @@ refs: ## How We Work -**Source:** Notion — Orbit Demo Home / How We Work, last edited 2026-05-07 +**Source:** Notion - Orbit Demo Home / How We Work, last edited 2026-05-07 --- @@ -30,7 +30,7 @@ refs: |---|---| | **Monday** | Commitments and dependency checks | | **Tuesday – Thursday** | Customer calls, product work, implementation, and building | -| **Friday** | Closing loops — review what shipped, what slipped, and write down any decisions | +| **Friday** | Closing loops - review what shipped, what slipped, and write down any decisions | Use this rhythm when scheduling work, meetings, or reviews. Do not schedule decision-making meetings on Fridays; use Friday to record decisions already made. @@ -64,11 +64,11 @@ These are explicitly codified rules Orbit has identified as recurring failure mo - **Escalations are coordination tools, not indicators of individual failure.** Escalating is the correct behavior when a problem exceeds the current team's ability to resolve it alone. - When escalating, the person escalating must: 1. Bring in the right people (those with authority or context to unblock). - 2. Summarize current state clearly — what has been tried, what is blocked, and why. + 2. Summarize current state clearly - what has been tried, what is blocked, and why. 3. Name the customer impact explicitly. 4. Keep updates moving until the risk is resolved or a workaround is established. - Escalations that stall because no one owns the next update are a process failure, not a customer failure. -- An escalation is closed when the risk is resolved or a documented workaround is in place — not when the immediate noise stops. +- An escalation is closed when the risk is resolved or a documented workaround is in place - not when the immediate noise stops. --- diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md index ea0c4a9e..9f813b4a 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md @@ -14,7 +14,7 @@ refs: ## Known Product Gaps and Friction Points -**Source:** Notion — Product & Customers (Notes from Recent Customer Calls), last edited 2026-05-07 +**Source:** Notion - Product & Customers (Notes from Recent Customer Calls), last edited 2026-05-07 --- diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md index 74a936f6..88d3f35a 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md @@ -33,8 +33,8 @@ tables: ## Key measures (SL: `mart_account_activity`) -- `avg_pre_policy_activation_rate` — `avg(pre_policy_30_day_activation_rate)` -- `avg_post_policy_activation_rate` — `avg(post_policy_30_day_activation_rate)` +- `avg_pre_policy_activation_rate` - `avg(pre_policy_30_day_activation_rate)` +- `avg_post_policy_activation_rate` - `avg(post_policy_30_day_activation_rate)` ## Common query patterns diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md index 04085359..1471cb18 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md @@ -37,10 +37,10 @@ tables: ## Key measures (SL: `mart_account_segments`) -- `account_count` — `count(*)` -- `total_contract_arr_cents` — `sum(contract_arr_cents)` -- `active_contract_arr_cents` — `sum(contract_arr_cents)` where `contract_status = 'active'` -- `active_contract_arr_millions` — active ARR in $M +- `account_count` - `count(*)` +- `total_contract_arr_cents` - `sum(contract_arr_cents)` +- `active_contract_arr_cents` - `sum(contract_arr_cents)` where `contract_status = 'active'` +- `active_contract_arr_millions` - active ARR in $M ## Common query patterns diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md index 5b3db7dd..08338f13 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md @@ -31,8 +31,8 @@ tables: ## Key measures (SL: `mart_arr_daily`) -- `total_arr_cents` — `sum(arr_cents)` -- `arr_millions` — `round(sum(arr_cents) / 100000000.0, 3)` — ARR in $M +- `total_arr_cents` - `sum(arr_cents)` +- `arr_millions` - `round(sum(arr_cents) / 100000000.0, 3)` - ARR in $M ## Common query patterns diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md index 288c8201..2a42b739 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md @@ -38,8 +38,8 @@ tables: ## Key measures (SL: `mart_nrr_quarterly`) -- `avg_nrr` — `avg(net_revenue_retention)` across all rows -- `avg_nrr_enterprise` — `avg(net_revenue_retention)` filtered to `segment = 'enterprise'` +- `avg_nrr` - `avg(net_revenue_retention)` across all rows +- `avg_nrr_enterprise` - `avg(net_revenue_retention)` filtered to `segment = 'enterprise'` - `total_expansion_arr_cents`, `total_contraction_arr_cents`, `total_churned_arr_cents` ## Common query patterns diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md index ab3de364..a50894e7 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md @@ -32,8 +32,8 @@ tables: ## Key measures (SL: `mart_procurement_activity`) -- `total_active_requesters` — `sum(active_requesters)` -- `active_requesters_200k_threshold` — `sum(active_requesters)` where `contract_arr_threshold_cents = 20000000` +- `total_active_requesters` - `sum(active_requesters)` +- `active_requesters_200k_threshold` - `sum(active_requesters)` where `contract_arr_threshold_cents = 20000000` ## Common query patterns @@ -44,4 +44,4 @@ tables: - `active_requesters` counts non-internal, non-test requesters on large active contracts. See [orbit-procurement-qualifying-actions](orbit-procurement-qualifying-actions). - The standard threshold is `contract_arr_threshold_cents = 20000000` ($200k ARR). -- Always filter by `contract_arr_threshold_cents` — the table contains rows for multiple threshold values. +- Always filter by `contract_arr_threshold_cents` - the table contains rows for multiple threshold values. diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md index 8deb5ffe..c5d8a07c 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md @@ -36,12 +36,12 @@ tables: ## Key measures (SL: `mart_revenue_daily`) -- `total_gross_revenue_cents` — `sum(gross_revenue_cents)` -- `total_credits_cents` — `sum(credits_cents)` -- `total_refunds_cents` — `sum(refunds_cents)` -- `total_net_revenue_cents` — `sum(net_revenue_cents)` -- `net_revenue_millions` — `round(sum(net_revenue_cents) / 100000000.0, 3)` -- `gross_revenue_millions` — `round(sum(gross_revenue_cents) / 100000000.0, 3)` +- `total_gross_revenue_cents` - `sum(gross_revenue_cents)` +- `total_credits_cents` - `sum(credits_cents)` +- `total_refunds_cents` - `sum(refunds_cents)` +- `total_net_revenue_cents` - `sum(net_revenue_cents)` +- `net_revenue_millions` - `round(sum(net_revenue_cents) / 100000000.0, 3)` +- `gross_revenue_millions` - `round(sum(gross_revenue_cents) / 100000000.0, 3)` ## Common query patterns diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md index 28055b9b..8e8b8da4 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md @@ -15,7 +15,7 @@ sl_refs: - mart_nrr_quarterly --- -# Orbit Metabase SQL Library — Patterns & Conventions +# Orbit Metabase SQL Library - Patterns & Conventions Collection **7 "SQL Library"** (parent: Orbit Showcase, collection 5) contains reference queries that demonstrate how to write Metabase native SQL against the Orbit analytics marts. Cards here are intentionally illustrative; several have `dashboardCount: 0` and are not embedded in live dashboards. diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md index 0b966d8c..64155513 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md @@ -13,7 +13,7 @@ sl_refs: - mart_nrr_quarterly --- -# NRR — Discount Expiration Treatment +# NRR - Discount Expiration Treatment **Governed metric key:** `net_revenue_retention` **Owner team:** analytics diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md index 08126dd5..e41f8b9a 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md @@ -12,7 +12,7 @@ sl_refs: - mart_procurement_activity --- -# Procurement — Qualifying Actions & Weekly Active Requesters +# Procurement - Qualifying Actions & Weekly Active Requesters **Governed metric key:** `weekly_active_requesters` **Owner team:** product diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md index f2b72b43..8c1a9ddf 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md @@ -13,7 +13,7 @@ refs: ## Orbit Product Design Principles -**Source:** Notion — Product & Customers, last edited 2026-05-07 +**Source:** Notion - Product & Customers, last edited 2026-05-07 --- diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md index abf8e747..7dc4e361 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md @@ -13,7 +13,7 @@ refs: ## Product Review Checklist -**Source:** Notion — Product & Customers, last edited 2026-05-07 +**Source:** Notion - Product & Customers, last edited 2026-05-07 --- diff --git a/packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md index 65004331..b5276380 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md +++ b/packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md @@ -12,7 +12,7 @@ sl_refs: - mart_revenue_daily --- -# Revenue — Gross-to-Net Reconciliation +# Revenue - Gross-to-Net Reconciliation **Governed metric key:** `net_revenue` **Owner team:** finance @@ -25,7 +25,7 @@ sl_refs: net_revenue = gross_revenue - credits - refunds ``` -All amounts are in **cents** (USD only — `stg_invoices.currency` is asserted to be `USD`). +All amounts are in **cents** (USD only - `stg_invoices.currency` is asserted to be `USD`). ## Components @@ -38,12 +38,12 @@ All amounts are in **cents** (USD only — `stg_invoices.currency` is asserted t ## Intermediate model -`int_revenue_components` — daily gross, credit, refund, and net revenue components. +`int_revenue_components` - daily gross, credit, refund, and net revenue components. ## Quality Gates - `reconciliation_check` must be `true` on every row of `mart_revenue_daily`. -- `assert_february_2026_net_revenue` — a dbt singular test covering February 2026 net revenue total. +- `assert_february_2026_net_revenue` - a dbt singular test covering February 2026 net revenue total. ## Line Item Types (`stg_invoice_line_items`) diff --git a/packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md b/packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md index 65693ee6..a7a0bcd1 100644 --- a/packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md +++ b/packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md @@ -14,7 +14,7 @@ refs: ## Sales Ops → Customer Success Implementation Handoff -**Source:** Notion — People & Operating Norms, last edited 2026-05-07 +**Source:** Notion - People & Operating Norms, last edited 2026-05-07 **Owner:** Sales Ops (sender), Customer Success (receiver) --- @@ -27,7 +27,7 @@ Sales Ops must complete the handoff **before the first implementation call**. Cu | Field | Notes | |---|---| -| Current plan | Starter / Growth / Enterprise — use canonical plan name | +| Current plan | Starter / Growth / Enterprise - use canonical plan name | | Account segment | self_serve / commercial / enterprise (see `orbit-plan-segment-normalization`) | | Contract shape | Term, ARR, any discounts or custom terms | | Renewal contact | Named person on the customer side responsible for renewal | @@ -38,7 +38,7 @@ Sales Ops must complete the handoff **before the first implementation call**. Cu - **Sales Ops** is responsible for populating and delivering the handoff before the first implementation call. - **Customer Success** is responsible for flagging missing fields to Sales Ops before the call, not during or after. -- If a field is unknown at handoff time, Sales Ops must note it explicitly as "unknown — to be resolved by [date]" rather than leaving it blank. +- If a field is unknown at handoff time, Sales Ops must note it explicitly as "unknown - to be resolved by [date]" rather than leaving it blank. ## Common Failure Mode @@ -51,7 +51,7 @@ Handoffs that omit contract shape or renewal contact force CS to re-engage Sales - Enterprise accounts with parent/child account structures require extra care during handoff. - Small assumptions made during handoff in these accounts tend to produce large downstream problems (billing mismatches, approval routing failures, supplier onboarding gaps). - When the account has parent/child complexity, Sales Ops must explicitly flag it in the handoff and document the account hierarchy before the first implementation call. -- CS should treat any undocumented parent/child relationship as a blocker — do not proceed with implementation setup until the structure is confirmed. +- CS should treat any undocumented parent/child relationship as a blocker - do not proceed with implementation setup until the structure is confirmed. --- diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index cfbc86b0..e8bdf445 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -20,7 +20,7 @@ export interface KtxCliCommandContext { deps: KtxCliDeps; packageInfo: KtxCliPackageInfo; setExitCode: (code: number) => void; - runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KtxCliIo) => Promise; + runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise; writeDebug?: (command: string, commandContext: CommandWithGlobalOptions) => void; } @@ -32,14 +32,14 @@ export interface OutputModeOptions { } interface KtxCommanderProgramOptions { - runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KtxCliIo) => Promise; + runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise; } export interface BuildKtxProgramOptions { io: KtxCliIo; deps: KtxCliDeps; packageInfo: KtxCliPackageInfo; - runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KtxCliIo) => Promise; + runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise; setExitCode?: (code: number) => void; } @@ -58,6 +58,8 @@ type CommandPathNode = CommandWithGlobalOptions & { const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status']); const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx dev init']); const COMMANDS_WITH_OWN_MISSING_PROJECT_HANDLING = new Set(['status']); +const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']); +const GLOBAL_OPTIONS_WITHOUT_VALUE = new Set(['--debug', '--help', '-h', '--version', '-v']); class KtxProjectMissingAbortError extends Error { readonly isKtxProjectMissingAbort = true; @@ -72,24 +74,6 @@ function isKtxProjectMissingAbortError(error: unknown): error is KtxProjectMissi (typeof error === 'object' && error !== null && (error as { isKtxProjectMissingAbort?: unknown }).isKtxProjectMissingAbort === true) ); } -const REMOVED_COMMAND_PATHS = new Set([ - 'scan', - 'wiki read', - 'wiki write', -]); -const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']); -const OPTIONS_WITH_VALUE = new Set([ - '--project-dir', - '--query-history-window-days', - '--user-id', - '--limit', - '--format', - '--connection-id', - '--source-name', - '--query-file', - '--max-rows', -]); - export interface CommandWithGlobalOptions { opts: () => object; optsWithGlobals?: () => object; @@ -336,43 +320,32 @@ function formatCliError(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function commandPathFromArgv(argv: string[]): string[] { - const path: string[] = []; - for (let index = 0; index < argv.length && path.length < 2; index += 1) { +function firstTopLevelCommandToken(argv: string[]): string | null { + for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === undefined) { continue; } if (arg === '--') { - break; + return null; } - if ((path.length === 0 ? GLOBAL_OPTIONS_WITH_VALUE : OPTIONS_WITH_VALUE).has(arg)) { + if (GLOBAL_OPTIONS_WITH_VALUE.has(arg)) { index += 1; continue; } - const optionsWithValue = path.length === 0 ? GLOBAL_OPTIONS_WITH_VALUE : OPTIONS_WITH_VALUE; - if ([...optionsWithValue].some((option) => arg.startsWith(`${option}=`))) { + if ([...GLOBAL_OPTIONS_WITH_VALUE].some((option) => arg.startsWith(`${option}=`))) { continue; } - if (path.length === 0 && arg === '--debug') { + if (GLOBAL_OPTIONS_WITHOUT_VALUE.has(arg) || arg.startsWith('-')) { continue; } - if (arg.startsWith('-')) { - continue; - } - path.push(arg); + return arg; } - return path; + return null; } -function removedCommandName(argv: string[]): string | null { - const path = commandPathFromArgv(argv); - if (path.length === 0) { - return null; - } - - const pathKey = path.join(' '); - return REMOVED_COMMAND_PATHS.has(pathKey) ? path.at(-1) ?? null : null; +function isKnownTopLevelCommand(program: Command, commandName: string): boolean { + return program.commands.some((command) => command.name() === commandName || command.aliases().includes(commandName)); } async function runBareInteractiveCommand( @@ -489,9 +462,9 @@ export async function runCommanderKtxCli( return 0; } - const removedCommand = removedCommandName(argv); - if (removedCommand) { - io.stderr.write(`error: unknown command '${removedCommand}'\n`); + const topLevelCommand = firstTopLevelCommandToken(argv); + if (topLevelCommand && !isKnownTopLevelCommand(program, topLevelCommand)) { + io.stderr.write(`error: unknown command '${topLevelCommand}'\n`); return 1; } diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index a2147904..cc1a2b98 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -59,14 +59,10 @@ export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo { }; } -async function runInit( - args: { projectDir: string; projectName?: string; force: boolean }, - io: KtxCliIo, -): Promise { +async function runInit(args: { projectDir: string; force: boolean }, io: KtxCliIo): Promise { const { initKtxProject } = await import('@ktx/context/project'); const result = await initKtxProject({ projectDir: args.projectDir, - projectName: args.projectName, force: args.force, }); @@ -77,7 +73,7 @@ async function runInit( } export async function runInitForCommander( - args: { projectDir: string; projectName?: string; force: boolean }, + args: { projectDir: string; force: boolean }, io: KtxCliIo, ): Promise { return await runInit(args, io); diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 920512f5..88f4b921 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -82,7 +82,7 @@ describe('runKtxConnection', () => { it('lists configured connections without resolving secrets', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' }, @@ -100,7 +100,7 @@ describe('runKtxConnection', () => { it('prints an empty-state message that points at setup instead of removed connection add', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const io = makeIo(); await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0); @@ -111,7 +111,7 @@ describe('runKtxConnection', () => { it('tests a native connection by calling connector.testConnection (not introspect)', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { warehouse: { driver: 'sqlite' }, }); @@ -136,7 +136,7 @@ describe('runKtxConnection', () => { it('reports the connector error and still cleans up when native testConnection fails', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { warehouse: { driver: 'sqlite' }, }); @@ -155,7 +155,7 @@ describe('runKtxConnection', () => { it('tests a configured Metabase connection through the Metabase runtime client', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { prod_metabase: { driver: 'metabase', @@ -201,7 +201,7 @@ describe('runKtxConnection', () => { it('tests a Looker connection through the Looker client', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { bi_looker: { driver: 'looker', @@ -230,7 +230,7 @@ describe('runKtxConnection', () => { it('falls back to userId when Looker metadata has no display name', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { bi_looker: { driver: 'looker', @@ -255,7 +255,7 @@ describe('runKtxConnection', () => { it('reports the Looker error when testConnection fails', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { bi_looker: { driver: 'looker', @@ -277,7 +277,7 @@ describe('runKtxConnection', () => { it('tests a Notion connection by retrieving the bot user', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { docs: { driver: 'notion', @@ -302,7 +302,7 @@ describe('runKtxConnection', () => { it('falls back to bot id when Notion bot has no name', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { docs: { driver: 'notion', @@ -323,7 +323,7 @@ describe('runKtxConnection', () => { it('tests a dbt connection via testRepoConnection (success)', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); process.env.DBT_TOKEN = 'gh_token_abc'; // pragma: allowlist secret await writeConnections(projectDir, { 'dbt-main': { @@ -354,7 +354,7 @@ describe('runKtxConnection', () => { it('reports the git error when testRepoConnection fails for dbt', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { 'dbt-main': { driver: 'dbt', @@ -377,7 +377,7 @@ describe('runKtxConnection', () => { it('tests a LookML connection via testRepoConnection with camelCase repoUrl', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { lookml_main: { driver: 'lookml', @@ -400,7 +400,7 @@ describe('runKtxConnection', () => { it('tests a MetricFlow connection via the nested metricflow block', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { mf_main: { driver: 'metricflow', @@ -422,7 +422,7 @@ describe('runKtxConnection', () => { it('--all: prints a single coherent list with one row per connection', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { warehouse: { driver: 'sqlite' }, docs: { driver: 'notion', auth_token: 'secret_token', crawl_mode: 'all_accessible' }, // pragma: allowlist secret @@ -450,7 +450,7 @@ describe('runKtxConnection', () => { it('--all: marks failing connections, keeps passing ones, and returns non-zero', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeConnections(projectDir, { warehouse: { driver: 'sqlite' }, broken: { driver: 'sqlite' }, @@ -476,7 +476,7 @@ describe('runKtxConnection', () => { it('--all: shows an empty-state message when no connections are configured', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const io = makeIo(); await expect(runKtxConnection({ command: 'test-all', projectDir }, io.io)).resolves.toBe(0); @@ -488,16 +488,18 @@ describe('runKtxConnection', () => { it('rejects unknown drivers with a helpful error', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeConnections(projectDir, { - mystery: { driver: 'duckdb' }, - }); + await initKtxProject({ projectDir }); + await writeFile( + join(projectDir, 'ktx.yaml'), + 'connections:\n mystery:\n driver: duckdb\n', + 'utf-8', + ); const io = makeIo(); await expect( runKtxConnection({ command: 'test', projectDir, connectionId: 'mystery' }, io.io), ).resolves.toBe(1); - expect(io.stderr()).toContain('uses driver "duckdb"'); - expect(io.stderr()).toContain('Supported:'); + expect(io.stderr()).toContain('connections.mystery.driver'); + expect(io.stderr()).toContain('postgres'); }); }); diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index 487e5468..8d6b48ad 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -40,7 +40,7 @@ function projectWithConnections(connections: KtxProjectConfig['connections']): K return { projectDir: '/tmp/project', config: { - ...buildDefaultKtxProjectConfig('warehouse'), + ...buildDefaultKtxProjectConfig(), connections, }, }; diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index aae9f1a2..674af8cf 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -52,7 +52,6 @@ export function defaultDemoProjectDir(): string { function demoConfig(databasePath: string): string { return [ - 'project: ktx-demo-orbit', 'connections:', ` ${DEMO_CONNECTION_ID}:`, ' driver: sqlite', diff --git a/packages/cli/src/dev.test.ts b/packages/cli/src/dev.test.ts index e2c72012..bda96692 100644 --- a/packages/cli/src/dev.test.ts +++ b/packages/cli/src/dev.test.ts @@ -72,10 +72,10 @@ describe('dev Commander tree', () => { const testIo = makeIo(); try { - await expect(runKtxCli(['dev', 'init', projectDir, '--name', 'warehouse'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['dev', 'init', projectDir], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`); - await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('project: warehouse'); + await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.not.toContain('project:'); expect(testIo.stderr()).toBe(''); } finally { await rm(tempDir, { recursive: true, force: true }); @@ -92,7 +92,7 @@ describe('dev Commander tree', () => { try { await expect( - runKtxCli(['--project-dir', projectDir, 'dev', 'init', '--name', 'global-init'], testIo.io), + runKtxCli(['--project-dir', projectDir, 'dev', 'init'], testIo.io), ).resolves.toBe(0); expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`); diff --git a/packages/cli/src/dev.ts b/packages/cli/src/dev.ts index 80a86ec4..12ad6f46 100644 --- a/packages/cli/src/dev.ts +++ b/packages/cli/src/dev.ts @@ -25,19 +25,17 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont .command('init') .description('Initialize a Git-backed KTX project directory for maintenance scripts') .argument('[directory]', 'Project directory') - .option('--name ', 'Project name written to ktx.yaml') .option('--force', 'Rewrite ktx.yaml and scaffold files in an existing project', false) .action( async ( projectDir: string | undefined, - commandOptions: { name?: string; force?: boolean }, + commandOptions: { force?: boolean }, command: CommandWithGlobalOptions, ) => { context.setExitCode( await context.runInit( { projectDir: projectDir ? resolve(projectDir) : resolveCommandProjectDir(command), - ...(commandOptions.name ? { projectName: commandOptions.name } : {}), force: commandOptions.force === true, }, context.io, diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index 5a9a3fdd..daeb5b96 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -1,6 +1,6 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { formatDoctorReport, @@ -64,6 +64,11 @@ describe('formatDoctorReport', () => { expect(output).toContain('Node 22+ · pnpm 10.20+'); expect(output).not.toContain('v22.16.0'); expect(output).toContain('Everything ready.'); + expect(output).toContain('ktx status --json'); + expect(output).toContain('ktx sl list'); + expect(output).toContain('ktx wiki list'); + expect(output).not.toContain('ktx scan'); + expect(output).not.toContain('ktx sl ask'); }); it('shows the underlying detail for a single-check group on the group line', () => { @@ -328,7 +333,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'storrage:', ' state: sqlite', 'ingest:', @@ -359,7 +363,7 @@ describe('runKtxDoctor', () => { it('emits structured JSON when ktx.yaml fails Zod validation', async () => { await writeFile( join(tempDir, 'ktx.yaml'), - ['project: warehouse', 'storrage: {}', ''].join('\n'), + ['storrage: {}', ''].join('\n'), 'utf-8', ); const testIo = makeIo(); @@ -387,7 +391,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -418,7 +421,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -452,7 +454,7 @@ describe('runKtxDoctor', () => { const out = testIo.stdout(); expect(out).toContain('KTX status'); - expect(out).toContain('· warehouse'); + expect(out).toContain(`· ${basename(tempDir)}`); expect(out).toContain('Connections (1)'); expect(out).toContain('LLM'); expect(out).toContain('anthropic'); @@ -465,10 +467,10 @@ describe('runKtxDoctor', () => { it('includes Postgres query-history readiness in project doctor output', async () => { process.env.ANTHROPIC_API_KEY = 'test-key'; // pragma: allowlist secret process.env.OPENAI_API_KEY = 'test-key'; // pragma: allowlist secret + process.env.WAREHOUSE_DATABASE_URL = 'postgresql://reader@example.test/warehouse'; await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -520,15 +522,20 @@ describe('runKtxDoctor', () => { expect(out).toContain('pg_stat_statements ready (PostgreSQL 16.4)'); expect(out).toContain('info: pg_stat_statements.max is 1000'); expect(out).not.toContain('Update the Postgres parameter group or config'); + expect(out).toContain('ktx status --json'); + expect(out).toContain('ktx sl list'); + expect(out).toContain('ktx wiki list'); + expect(out).not.toContain('ktx scan'); + expect(out).not.toContain('ktx sl ask'); delete process.env.ANTHROPIC_API_KEY; delete process.env.OPENAI_API_KEY; + delete process.env.WAREHOUSE_DATABASE_URL; }); it('returns blocked verdict when LLM is not configured', async () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -548,6 +555,7 @@ describe('runKtxDoctor', () => { ).resolves.toBe(1); expect(testIo.stdout()).toContain('no LLM configured'); + expect(testIo.stdout()).not.toContain('ktx ask'); expect(testIo.stdout()).toContain('ktx setup'); }); @@ -558,7 +566,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -620,7 +627,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -660,7 +666,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -695,7 +700,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -724,7 +728,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'storrage:', ' state: sqlite', 'ingest:', @@ -752,7 +755,7 @@ describe('runKtxDoctor', () => { it('emits structured JSON issues when validation fails', async () => { await writeFile( join(tempDir, 'ktx.yaml'), - ['project: warehouse', 'storrage: {}', ''].join('\n'), + ['storrage: {}', ''].join('\n'), 'utf-8', ); const testIo = makeIo(); @@ -788,7 +791,6 @@ describe('runKtxDoctor', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index b1845ae3..efb87e2b 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -5,6 +5,7 @@ import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import type { KtxConfigIssue } from '@ktx/context/project'; +import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js'; import type { BuildProjectStatusOptions } from './status-project.js'; const execFileAsync = promisify(execFile); @@ -287,7 +288,7 @@ interface RenderOptions { command?: 'setup' | 'project'; } -const NEXT_STEPS_PROJECT = ['ktx scan', 'ktx wiki', 'ktx sl ask "…"']; +const NEXT_STEPS_PROJECT = KTX_NEXT_STEP_DIRECT_COMMANDS.map((step) => step.command); export function formatDoctorReport(report: DoctorReport, options: Partial = {}): string { const opts: RenderOptions = { diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 1e2da422..86a7468c 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -102,7 +102,7 @@ describe('runKtxCli', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-')); - await writeFile(join(tempDir, 'ktx.yaml'), 'project: cli-dispatch-fixture\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), '{}\n', 'utf-8'); }); afterEach(async () => { @@ -502,7 +502,7 @@ describe('runKtxCli', () => { it('keeps representative JSON command stdout parseable', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const commands = [ ['--project-dir', projectDir, 'status', '--json'], ['--project-dir', projectDir, 'sl', 'list', '--json'], @@ -580,7 +580,7 @@ describe('runKtxCli', () => { try { delete process.env.KTX_PROJECT_DIR; - await writeFile(join(tempDir, 'ktx.yaml'), 'project: revenue\nconnections: {}\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); process.chdir(tempDir); await expect(runKtxCli([], testIo.io, { setup })).resolves.toBe(0); @@ -1482,7 +1482,7 @@ describe('runKtxCli', () => { it('dispatches public connection subcommands through the existing connection implementation', async () => { const tempDir = await mkdtemp(join(tmpdir(), 'ktx-connection-dispatch-')); - await writeFile(join(tempDir, 'ktx.yaml'), 'project: connection-dispatch\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), '{}\n', 'utf-8'); const connection = vi.fn(async () => 0); await expect( diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index 7b65e33a..41affbb9 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -106,10 +106,10 @@ export async function writeWarehouseConfig(projectDir: string): Promise { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' prod-metabase:', ' driver: metabase', + ' api_url: https://metabase.example.test', ' warehouse_a:', ' driver: postgres', 'ingest:', @@ -126,7 +126,6 @@ export async function writeMetabaseConfig(projectDir: string): Promise { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -488,7 +487,6 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync await writeFile( join(projectDir, 'ktx.yaml'), [ - `project: metabase-sync-mode-${input.name}`, 'connections:', ' prod-metabase:', ' driver: metabase', diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index 7626918c..52dcbc38 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -633,7 +633,6 @@ describe('runKtxIngest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: metabase-cli', 'connections:', ' prod-metabase:', ' driver: metabase', @@ -1099,7 +1098,7 @@ describe('runKtxIngest', () => { it('passes managed daemon options to adapters and pull-config options when no explicit daemon URL is set', async () => { const projectDir = join(tempDir, 'managed-daemon-ingest-project'); - await initKtxProject({ projectDir, projectName: 'managed-daemon-ingest-project' }); + await initKtxProject({ projectDir }); await writeWarehouseConfig(projectDir); const createdAdapters: SourceAdapter[] = [ { source: 'fake', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, @@ -1159,7 +1158,6 @@ describe('runKtxIngest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: historic-sql-project', 'connections:', ' warehouse:', ' driver: postgres', @@ -1224,7 +1222,6 @@ describe('runKtxIngest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: historic-sql-progress-project', 'connections:', ' warehouse:', ' driver: postgres', @@ -1353,7 +1350,6 @@ describe('runKtxIngest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: historic-sql-step-progress-project', 'connections:', ' warehouse:', ' driver: postgres', @@ -1446,7 +1442,6 @@ describe('runKtxIngest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: historic-sql-concurrent-progress-project', 'connections:', ' warehouse:', ' driver: postgres', @@ -1596,7 +1591,6 @@ describe('runKtxIngest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: looker-cli', 'connections:', ' prod-looker:', ' driver: looker', diff --git a/packages/cli/src/io/print-list.ts b/packages/cli/src/io/print-list.ts index bd7ab20a..b05e12f2 100644 --- a/packages/cli/src/io/print-list.ts +++ b/packages/cli/src/io/print-list.ts @@ -43,13 +43,13 @@ export interface PrintListArgs { io: KtxCliIo; } -export interface KtxJsonResultEnvelope { +interface KtxJsonResultEnvelope { kind: string; data: T; meta?: Record; } -export function writeJsonResult(io: KtxCliIo, envelope: KtxJsonResultEnvelope): void { +function writeJsonResult(io: KtxCliIo, envelope: KtxJsonResultEnvelope): void { io.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`); } diff --git a/packages/cli/src/knowledge.test.ts b/packages/cli/src/knowledge.test.ts index 2486d621..523d8a1b 100644 --- a/packages/cli/src/knowledge.test.ts +++ b/packages/cli/src/knowledge.test.ts @@ -1,8 +1,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from '@ktx/context/project'; +import { initKtxProject, loadKtxProject } from '@ktx/context/project'; import type { KtxEmbeddingPort } from '@ktx/context'; +import { writeLocalKnowledgePage } from '@ktx/context/wiki'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { runKtxKnowledge } from './knowledge.js'; @@ -40,6 +41,28 @@ class FakeEmbeddingPort implements KtxEmbeddingPort { } } +interface WikiPageFixture { + key?: string; + summary?: string; + content?: string; + tags?: string[]; + slRefs?: string[]; +} + +async function seedWikiPage(projectDir: string, fixture: WikiPageFixture = {}): Promise { + const project = await loadKtxProject({ projectDir }); + await writeLocalKnowledgePage(project, { + key: fixture.key ?? 'metrics-revenue', + scope: 'GLOBAL', + userId: 'local', + summary: fixture.summary ?? 'Revenue', + content: fixture.content ?? 'Revenue is paid order value.', + tags: fixture.tags ?? ['finance'], + refs: [], + slRefs: fixture.slRefs ?? ['orders'], + }); +} + describe('runKtxKnowledge', () => { let tempDir: string; @@ -51,36 +74,10 @@ describe('runKtxKnowledge', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('writes, reads, lists, and searches wiki pages', async () => { + it('lists and searches wiki pages', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - const writeIo = makeIo(); - await expect( - runKtxKnowledge( - { - command: 'write', - projectDir, - key: 'metrics-revenue', - scope: 'GLOBAL', - userId: 'local', - summary: 'Revenue', - content: 'Revenue is paid order value.', - tags: ['finance'], - refs: [], - slRefs: ['orders'], - }, - writeIo.io, - ), - ).resolves.toBe(0); - expect(writeIo.stdout()).toContain('Wrote wiki/global/metrics-revenue.md'); - - const readIo = makeIo(); - await expect( - runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local' }, readIo.io), - ).resolves.toBe(0); - expect(readIo.stdout()).toContain('# metrics-revenue'); - expect(readIo.stdout()).toContain('Revenue is paid order value.'); + await initKtxProject({ projectDir }); + await seedWikiPage(projectDir); const listIo = makeIo(); await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local' }, listIo.io)).resolves.toBe(0); @@ -93,27 +90,10 @@ describe('runKtxKnowledge', () => { expect(searchIo.stdout()).toContain('metrics-revenue'); }); - it('prints wiki list, search, and read as public JSON envelopes', async () => { + it('prints wiki list and search as public JSON envelopes', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - await expect( - runKtxKnowledge( - { - command: 'write', - projectDir, - key: 'metrics-revenue', - scope: 'GLOBAL', - userId: 'local', - summary: 'Revenue', - content: 'Revenue is paid order value.', - tags: ['finance'], - refs: [], - slRefs: ['orders'], - }, - makeIo().io, - ), - ).resolves.toBe(0); + await initKtxProject({ projectDir }); + await seedWikiPage(projectDir); const listIo = makeIo(); await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local', json: true }, listIo.io)).resolves.toBe( @@ -137,53 +117,11 @@ describe('runKtxKnowledge', () => { data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] }, meta: { command: 'wiki search' }, }); - - const readIo = makeIo(); - await expect( - runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local', json: true }, readIo.io), - ).resolves.toBe(0); - expect(JSON.parse(readIo.stdout())).toMatchObject({ - kind: 'wiki.page', - data: { - key: 'metrics-revenue', - summary: 'Revenue', - content: 'Revenue is paid order value.', - }, - }); - }); - - it('rejects slash-delimited write keys with a flat-key suggestion', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - const writeIo = makeIo(); - await expect( - runKtxKnowledge( - { - command: 'write', - projectDir, - key: 'orbit/company-overview', - scope: 'GLOBAL', - userId: 'local', - summary: 'Orbit', - content: 'Orbit overview.', - tags: [], - refs: [], - slRefs: [], - }, - writeIo.io, - ), - ).resolves.toBe(1); - - expect(writeIo.stderr()).toContain( - 'Invalid wiki key "orbit/company-overview". Wiki keys must be flat; use "orbit-company-overview".', - ); - expect(writeIo.stdout()).toBe(''); }); it('explains empty search results for a project without wiki pages', async () => { const projectDir = join(tempDir, 'empty-project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const searchIo = makeIo(); await expect( @@ -197,25 +135,14 @@ describe('runKtxKnowledge', () => { it('uses configured embeddings for semantic wiki search', async () => { const projectDir = join(tempDir, 'semantic-project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - await expect( - runKtxKnowledge( - { - command: 'write', - projectDir, - key: 'active-contract-arr-open-tickets', - scope: 'GLOBAL', - userId: 'local', - summary: 'Active Contract ARR Ranked by Open Support Ticket Count', - content: 'Accounts ranked by annual recurring contract value and support ticket load.', - tags: ['historic-sql'], - refs: [], - slRefs: [], - }, - makeIo().io, - ), - ).resolves.toBe(0); + await initKtxProject({ projectDir }); + await seedWikiPage(projectDir, { + key: 'active-contract-arr-open-tickets', + summary: 'Active Contract ARR Ranked by Open Support Ticket Count', + content: 'Accounts ranked by annual recurring contract value and support ticket load.', + tags: ['historic-sql'], + slRefs: [], + }); const searchIo = makeIo(); await expect( diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index 66109301..d98df9e8 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -5,20 +5,16 @@ import { } from '@ktx/context'; import { loadKtxProject } from '@ktx/context/project'; import { - type LocalKnowledgeScope, type LocalKnowledgeSearchResult, type LocalKnowledgeSummary, listLocalKnowledgePages, - readLocalKnowledgePage, searchLocalKnowledgePages, - writeLocalKnowledgePage, } from '@ktx/context/wiki'; import { resolveOutputMode } from './io/mode.js'; -import { printList, type PrintListColumn, writeJsonResult } from './io/print-list.js'; +import { printList, type PrintListColumn } from './io/print-list.js'; export type KtxKnowledgeArgs = | { command: 'list'; projectDir: string; userId: string; output?: string; json?: boolean } - | { command: 'read'; projectDir: string; key: string; userId: string; json?: boolean } | { command: 'search'; projectDir: string; @@ -27,18 +23,6 @@ export type KtxKnowledgeArgs = output?: string; json?: boolean; limit?: number; - } - | { - command: 'write'; - projectDir: string; - key: string; - scope: LocalKnowledgeScope; - userId: string; - summary: string; - content: string; - tags: string[]; - refs: string[]; - slRefs: string[]; }; type KtxKnowledgeIo = import('./cli-runtime.js').KtxCliIo; @@ -104,25 +88,6 @@ export async function runKtxKnowledge( }); return 0; } - if (args.command === 'read') { - const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId }); - if (!page) { - throw new Error(`Wiki page "${args.key}" was not found`); - } - if (args.json) { - writeJsonResult(io, { - kind: 'wiki.page', - data: page, - meta: { command: 'wiki read' }, - }); - return 0; - } - io.stdout.write(`# ${page.key}\n\n`); - io.stdout.write(`Scope: ${page.scope}\n`); - io.stdout.write(`Summary: ${page.summary}\n\n`); - io.stdout.write(`${page.content}\n`); - return 0; - } if (args.command === 'search') { const results = await searchLocalKnowledgePages(project, { query: args.query, @@ -153,18 +118,6 @@ export async function runKtxKnowledge( }); return 0; } - - const write = await writeLocalKnowledgePage(project, { - key: args.key, - scope: args.scope, - userId: args.userId, - summary: args.summary, - content: args.content, - tags: args.tags, - refs: args.refs, - slRefs: args.slRefs, - }); - io.stdout.write(`Wrote ${write.path}\n`); return 0; } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/src/local-adapters.test.ts index 12f8d652..dbb03f2b 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/src/local-adapters.test.ts @@ -40,7 +40,6 @@ describe('CLI local ingest adapters', () => { await writeProject( tempDir, [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -71,7 +70,6 @@ describe('CLI local ingest adapters', () => { await writeProject( tempDir, [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -103,7 +101,6 @@ describe('CLI local ingest adapters', () => { await writeProject( tempDir, [ - 'project: warehouse', 'connections:', ' bq:', ' driver: bigquery', @@ -136,7 +133,6 @@ describe('CLI local ingest adapters', () => { await writeProject( tempDir, [ - 'project: warehouse', 'connections:', ' sf:', ' driver: snowflake', @@ -172,7 +168,6 @@ describe('CLI local ingest adapters', () => { await writeProject( tempDir, [ - 'project: warehouse', 'connections:', ' bq:', ' driver: bigquery', diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index d287b563..b9672bfa 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -39,11 +39,10 @@ describe('createKtxCliScanConnector', () => { }); it('creates a native sqlite connector from standalone config', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -61,11 +60,10 @@ describe('createKtxCliScanConnector', () => { }); it('passes canonical BigQuery YAML scan limits through to the connector', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: bigquery', @@ -94,12 +92,11 @@ describe('createKtxCliScanConnector', () => { expect(bigQueryMock.constructorInputs[0]).not.toHaveProperty('maxBytesBilled'); }); - it('throws for structural daemon-only fallback configs', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + it('rejects daemon-only fallback driver configs at config parse time', async () => { + await initKtxProject({ projectDir: tempDir }); await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: duckdb', @@ -108,19 +105,17 @@ describe('createKtxCliScanConnector', () => { ].join('\n'), 'utf-8', ); - const project = await loadKtxProject({ projectDir: tempDir }); - await expect(createKtxCliScanConnector(project, 'warehouse')).rejects.toThrow( - 'Connection "warehouse" uses driver "duckdb", which has no native standalone KTX scan connector', + await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow( + /connections\.warehouse\.driver:.*Invalid discriminator value/, ); }); - it('throws a clear error when the connection block has no driver field', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + it('rejects connection blocks with no driver field at config parse time', async () => { + await initKtxProject({ projectDir: tempDir }); await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' type: postgres', @@ -129,10 +124,9 @@ describe('createKtxCliScanConnector', () => { ].join('\n'), 'utf-8', ); - const project = await loadKtxProject({ projectDir: tempDir }); - await expect(createKtxCliScanConnector(project, 'warehouse')).rejects.toThrow( - 'Connection "warehouse" has no `driver` field in ktx.yaml', + await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow( + /connections\.warehouse\.driver:.*Invalid discriminator value/, ); }); }); diff --git a/packages/cli/src/project-dir.test.ts b/packages/cli/src/project-dir.test.ts index b5006bcc..25a9b585 100644 --- a/packages/cli/src/project-dir.test.ts +++ b/packages/cli/src/project-dir.test.ts @@ -6,7 +6,7 @@ import { runKtxCli, type KtxCliDeps } from './index.js'; async function makeFixtureProject(prefix: string): Promise { const dir = await mkdtemp(join(tmpdir(), prefix)); - await writeFile(join(dir, 'ktx.yaml'), 'project: project-dir-fixture\n', 'utf-8'); + await writeFile(join(dir, 'ktx.yaml'), '{}\n', 'utf-8'); return dir; } @@ -138,7 +138,7 @@ describe('project directory defaults', () => { const projectDir = join(root, 'warehouse'); const nestedDir = join(projectDir, 'nested', 'deeper'); await mkdir(nestedDir, { recursive: true }); - await writeFile(join(projectDir, 'ktx.yaml'), 'project: warehouse\n', 'utf-8'); + await writeFile(join(projectDir, 'ktx.yaml'), '{}\n', 'utf-8'); const expectedProjectDir = await realpath(projectDir); const publicIngest = vi.fn(async () => 0); diff --git a/packages/cli/src/project-resolver.test.ts b/packages/cli/src/project-resolver.test.ts index 74ef02bc..39dab27b 100644 --- a/packages/cli/src/project-resolver.test.ts +++ b/packages/cli/src/project-resolver.test.ts @@ -48,7 +48,7 @@ describe('resolveKtxProjectDir', () => { const project = join(tempDir, 'warehouse'); const nested = join(project, 'nested', 'deeper'); await mkdir(nested, { recursive: true }); - await writeFile(join(project, 'ktx.yaml'), 'project: warehouse\n', 'utf-8'); + await writeFile(join(project, 'ktx.yaml'), '{}\n', 'utf-8'); expect(resolveKtxProjectDir({ env: {}, cwd: nested })).toBe(resolve(project)); expect(findNearestKtxProjectDir(nested)).toBe(resolve(project)); diff --git a/packages/cli/src/public-ingest.test.ts b/packages/cli/src/public-ingest.test.ts index e1c0e612..d34a5785 100644 --- a/packages/cli/src/public-ingest.test.ts +++ b/packages/cli/src/public-ingest.test.ts @@ -41,7 +41,7 @@ function projectWithConnections(connections: KtxProjectConfig['connections']): K return { projectDir: '/tmp/project', config: { - ...buildDefaultKtxProjectConfig('warehouse'), + ...buildDefaultKtxProjectConfig(), connections, }, }; @@ -51,7 +51,7 @@ function deepReadyProject( connections: KtxProjectConfig['connections'], relationshipsEnabled = true, ): KtxPublicIngestProject { - const config = buildDefaultKtxProjectConfig('warehouse'); + const config = buildDefaultKtxProjectConfig(); return { projectDir: '/tmp/project', config: { @@ -85,7 +85,7 @@ describe('buildPublicIngestPlan', () => { it('plans warehouse connections as scan targets and source connections as source ingest targets', () => { const project = projectWithConnections({ warehouse: { driver: 'postgres' }, - prod_metabase: { driver: 'metabase' }, + prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' }, docs: { driver: 'notion' }, }); @@ -745,7 +745,7 @@ describe('runKtxPublicIngest', () => { const io = makeIo(); const project = projectWithConnections({ warehouse: { driver: 'postgres' }, - prod_metabase: { driver: 'metabase' }, + prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' }, }); const runScan = vi.fn(async () => 1); const runIngest = vi.fn(async () => 0); diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 487bd935..5fe4a342 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -316,7 +316,7 @@ describe('runKtxScan', () => { }); it('runs structural scans and prints a dev-friendly plain summary', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn( async (_input: RunLocalScanOptions): Promise => ({ runId: 'scan-run-1', @@ -377,7 +377,7 @@ describe('runKtxScan', () => { }); it('passes managed daemon options to local ingest adapters when no explicit daemon URL is set', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const createLocalIngestAdapters = vi.fn(() => []); const runLocalScan = vi.fn( async (_input: RunLocalScanOptions): Promise => ({ @@ -421,7 +421,7 @@ describe('runKtxScan', () => { }); it('explains warnings, capability gaps, and relationships in human scan summaries', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn( async (_input: RunLocalScanOptions): Promise => ({ runId: 'scan-run-1', @@ -472,7 +472,7 @@ describe('runKtxScan', () => { }); it('prints review-only relationship summaries and validation capability warnings', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const reviewOnlyReport: KtxScanReport = { ...reportWithAttention, capabilityGaps: [], @@ -525,7 +525,7 @@ describe('runKtxScan', () => { }); it('passes a scan progress port and prints TTY progress messages', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn(async (input: RunLocalScanOptions): Promise => { await input.progress?.update(0.15, 'Inspecting database schema'); await input.progress?.update(0.55, 'Semantic layer comparison found 5 changes across 18 tables'); @@ -572,7 +572,7 @@ describe('runKtxScan', () => { }); it('uses injected structured progress without requiring TTY progress output', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const progressEvents: Array<{ progress: number; message?: string; transient?: boolean }> = []; const structuredProgress = { async update(progress: number, message?: string, options?: { transient?: boolean }) { @@ -674,7 +674,7 @@ describe('runKtxScan', () => { }); it('flushes transient TTY progress messages before printing scan failures', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn(async (input: RunLocalScanOptions): Promise => { await input.progress?.update(0.42, 'Generating descriptions 3/35 tables', { transient: true }); throw new Error('scan failed'); @@ -711,7 +711,7 @@ describe('runKtxScan', () => { }); it('does not print live progress messages for non-TTY output', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn(async (input: RunLocalScanOptions): Promise => { await input.progress?.update(0.15, 'Inspecting database schema'); return { @@ -747,7 +747,7 @@ describe('runKtxScan', () => { }); it('uses terminal-aware visual styling only for TTY output', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn( async (_input: RunLocalScanOptions): Promise => ({ runId: 'scan-run-1', @@ -807,7 +807,7 @@ describe('runKtxScan', () => { }); it('honors NO_COLOR for TTY scan summaries', async () => { - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn( async (_input: RunLocalScanOptions): Promise => ({ runId: 'scan-run-1', @@ -853,11 +853,10 @@ describe('runKtxScan', () => { it('passes native CLI adapters into local scan runs for mysql configs', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-native-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: mysql', @@ -901,11 +900,10 @@ describe('runKtxScan', () => { it('creates a native connector for standalone relationship scans', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-relationships-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -955,11 +953,10 @@ describe('runKtxScan', () => { it('routes standalone postgres scans through the native connector before daemon fallback', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-native-postgres-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1021,11 +1018,10 @@ describe('runKtxScan', () => { it('passes native CLI adapters into local scan runs for clickhouse configs', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-native-clickhouse-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: clickhouse', @@ -1072,11 +1068,10 @@ describe('runKtxScan', () => { it('passes native CLI adapters into local scan runs for sqlserver configs', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-native-sqlserver-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlserver', @@ -1138,11 +1133,10 @@ describe('runKtxScan', () => { it('passes native CLI adapters into local scan runs for bigquery configs', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-native-bigquery-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: bigquery', @@ -1203,11 +1197,10 @@ describe('runKtxScan', () => { it('passes native CLI adapters into local scan runs for snowflake configs', async () => { const tempProject = await mkdtemp(join(tmpdir(), 'ktx-scan-cli-native-snowflake-')); - await initKtxProject({ projectDir: tempProject, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempProject }); await writeFile( join(tempProject, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: snowflake', diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index 19647a3f..ee5c7718 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -30,7 +30,7 @@ describe('setup agents', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-')); await mkdir(join(tempDir, '.ktx', 'agents'), { recursive: true }); - await writeFile(join(tempDir, 'ktx.yaml'), 'project: revenue\nconnections: {}\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); }); afterEach(async () => { diff --git a/packages/cli/src/setup-context.test.ts b/packages/cli/src/setup-context.test.ts index 66c8f6fe..61a9019a 100644 --- a/packages/cli/src/setup-context.test.ts +++ b/packages/cli/src/setup-context.test.ts @@ -50,7 +50,7 @@ type ReadyProjectOverrides = Omit, 'ingest' | 'llm' | }; async function writeReadyProject(projectDir: string, overrides: ReadyProjectOverrides = {}) { - const defaults = buildDefaultKtxProjectConfig('revenue'); + const defaults = buildDefaultKtxProjectConfig(); const readyConfig: KtxProjectConfig = { ...defaults, setup: { database_connection_ids: ['warehouse'] }, @@ -595,7 +595,6 @@ describe('setup context build state', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'connections: {}', 'llm:', ' provider:', diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index cc69e6cc..86a73fa3 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -119,7 +119,7 @@ describe('setup databases step', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-databases-')); - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); }); afterEach(async () => { @@ -242,7 +242,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -575,7 +574,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -622,7 +620,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -770,7 +767,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -815,7 +811,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -864,7 +859,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -936,7 +930,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1010,7 +1003,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1079,7 +1071,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1146,7 +1137,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1646,7 +1636,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1939,7 +1928,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -2019,7 +2007,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' analytics:', ' driver: bigquery', @@ -2074,7 +2061,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -2123,7 +2109,6 @@ describe('setup databases step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index c251b45c..aadb8a9f 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -60,7 +60,7 @@ describe('setup embeddings step', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-embeddings-')); - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); }); afterEach(async () => { @@ -446,11 +446,10 @@ describe('setup embeddings step', () => { it('preserves already completed embeddings setup when no embedding args request changes', async () => { await mkdir(join(tempDir, '.ktx'), { recursive: true }); - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse', force: true }); + await initKtxProject({ projectDir: tempDir, force: true }); await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'setup:', ' database_connection_ids: []', 'connections: {}', diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index d9fef97d..fc41cf1d 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -92,7 +92,7 @@ describe('setup Anthropic model step', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-models-')); - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir: tempDir }); }); afterEach(async () => { @@ -1049,11 +1049,10 @@ describe('setup Anthropic model step', () => { it('preserves already completed llm setup when no model args request changes', async () => { await mkdir(join(tempDir, '.ktx'), { recursive: true }); - await initKtxProject({ projectDir: tempDir, projectName: 'warehouse', force: true }); + await initKtxProject({ projectDir: tempDir, force: true }); await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'setup:', ' database_connection_ids: []', 'connections: {}', @@ -1099,7 +1098,6 @@ describe('setup Anthropic model step', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: warehouse', 'setup:', ' database_connection_ids: []', 'connections: {}', diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index e20b9544..578beb8b 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -76,11 +76,10 @@ describe('setup project step', () => { it('loads an existing project with --existing and drops config setup progress', async () => { const projectDir = join(tempDir, 'warehouse'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -196,7 +195,7 @@ describe('setup project step', () => { expect.objectContaining({ message: `Create KTX project at ${projectDir}?` }), ); expect(prompts.text).not.toHaveBeenCalled(); - expect(result.status === 'ready' ? result.project.config.project : '').toBe('ktx-project'); + expect(result.status === 'ready' ? result.project.configPath : '').toBe(join(projectDir, 'ktx.yaml')); expect(testIo.stdout()).toContain(`│ KTX will create:\n│ ${projectDir}`); await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined(); }); diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index fa2dd3ed..23884b0c 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -1,7 +1,7 @@ import { existsSync } from 'node:fs'; import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { basename, join, resolve } from 'node:path'; +import { join, resolve } from 'node:path'; import { initKtxProject, type KtxLocalProject, @@ -156,7 +156,7 @@ async function persistProjectStep(project: KtxLocalProject): Promise { const initProject = deps.initProject ?? initKtxProject; - const initialized = await initProject({ projectDir, projectName: basename(projectDir) || 'ktx-project' }); + const initialized = await initProject({ projectDir }); return await persistProjectStep(initialized); } diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index f39dde62..bfcb54bc 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -79,7 +79,7 @@ describe('setup sources step', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-sources-')); projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'sources' }); + await initKtxProject({ projectDir }); }); afterEach(async () => { @@ -1024,6 +1024,8 @@ describe('setup sources step', () => { databaseMappings: { '1': 'warehouse' }, syncEnabled: { '1': true }, syncMode: 'ALL', + selections: { collections: [], items: [] }, + defaultTagNames: [], }, }, deps: { @@ -1181,6 +1183,8 @@ describe('setup sources step', () => { databaseMappings: { '1': 'warehouse' }, syncEnabled: { '1': true }, syncMode: 'ALL', + selections: { collections: [], items: [] }, + defaultTagNames: [], }, }); const testPrompts = prompts({ diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 3b141f58..7b4425f0 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -451,6 +451,8 @@ function buildMetabaseConnection(args: KtxSetupSourcesArgs): KtxProjectConnectio databaseMappings: { [String(args.metabaseDatabaseId)]: args.sourceWarehouseConnectionId }, syncEnabled: { [String(args.metabaseDatabaseId)]: true }, syncMode: 'ALL', + selections: { collections: [], items: [] }, + defaultTagNames: [], }, }; } diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index 7e479c0e..ff1261c3 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -68,7 +68,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'llm:', ' provider:', ' backend: anthropic', @@ -109,7 +108,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'llm:', ' provider:', ...fixture.providerLines, @@ -129,7 +127,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -162,7 +159,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -183,7 +179,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -206,7 +201,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids: []', 'connections:', @@ -230,7 +224,7 @@ describe('setup status', () => { it('reports agent status from the install manifest', async () => { await mkdir(join(tempDir, '.ktx', 'agents'), { recursive: true }); - await writeFile(join(tempDir, 'ktx.yaml'), 'project: revenue\nconnections: {}\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); await writeFile( join(tempDir, '.ktx/agents/install-manifest.json'), JSON.stringify( @@ -256,7 +250,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -309,7 +302,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -319,7 +311,7 @@ describe('setup status', () => { ' url: env:DATABASE_URL', ' metabase:', ' driver: metabase', - ' url: env:METABASE_URL', + ' api_url: https://metabase.example.test', ' api_key_ref: env:METABASE_API_KEY', ' warehouse_connection_id: warehouse', 'llm:', @@ -370,7 +362,7 @@ describe('setup status', () => { }); it('prints the readiness checklist for an existing project', async () => { - await writeFile(join(tempDir, 'ktx.yaml'), 'project: revenue\nconnections: {}\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); const rendered = formatKtxSetupStatus(await readKtxSetupStatus(tempDir)); @@ -503,7 +495,7 @@ describe('setup status', () => { ), ).resolves.toBe(0); - await writeFile(join(tempDir, 'ktx.yaml'), 'project: revenue\nconnections: {}\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); await expect( runKtxSetup( @@ -589,7 +581,7 @@ describe('setup status', () => { }); it('lets Back from new project creation return to the first setup intent menu', async () => { - const existingConfig = 'project: revenue\nconnections: {}\n'; + const existingConfig = 'connections: {}\n'; await writeFile(join(tempDir, 'ktx.yaml'), existingConfig, 'utf-8'); const entryChoices = ['new-project', 'exit']; @@ -645,7 +637,7 @@ describe('setup status', () => { const existingProjectDir = join(tempDir, 'existing'); const newProjectDir = join(tempDir, 'fresh'); await mkdir(existingProjectDir, { recursive: true }); - const existingConfig = 'project: revenue\nconnections: {}\n'; + const existingConfig = 'connections: {}\n'; await writeFile(join(existingProjectDir, 'ktx.yaml'), existingConfig, 'utf-8'); const projectChoices = ['custom', 'create']; @@ -722,7 +714,7 @@ describe('setup status', () => { const existingProjectDir = join(tempDir, 'existing'); const newProjectDir = join(tempDir, 'fresh'); await mkdir(existingProjectDir, { recursive: true }); - await writeFile(join(existingProjectDir, 'ktx.yaml'), 'project: revenue\nconnections: {}\n', 'utf-8'); + await writeFile(join(existingProjectDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); const projectChoices = ['custom', 'create']; const projectPrompts = { @@ -1147,7 +1139,7 @@ describe('setup status', () => { }); it('lets Back from the first setup step return to the entry menu instead of exiting', async () => { - await writeFile(join(tempDir, 'ktx.yaml'), 'project: test\nconnections: {}\n', 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); const testIo = makeIo(); const entryChoices = ['setup', 'exit']; @@ -1254,7 +1246,7 @@ describe('setup status', () => { it('runs sources after database setup', async () => { const calls: string[] = []; const io = makeIo(); - await writeFile(join(tempDir, 'ktx.yaml'), ['project: revenue', 'connections: {}', ''].join('\n'), 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); await expect( runKtxSetup( @@ -1315,7 +1307,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -1374,7 +1365,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'connections:', ' warehouse:', ' driver: postgres', @@ -1430,7 +1420,7 @@ describe('setup status', () => { it('runs context after sources and before agents in full setup', async () => { const calls: string[] = []; const io = makeIo(); - await writeFile(join(tempDir, 'ktx.yaml'), ['project: revenue', 'connections: {}', ''].join('\n'), 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); await expect( runKtxSetup( @@ -1543,7 +1533,7 @@ describe('setup status', () => { it('runs agent setup after context succeeds in --agents mode', async () => { const calls: string[] = []; const io = makeIo(); - await writeFile(join(tempDir, 'ktx.yaml'), ['project: revenue', 'connections: {}', ''].join('\n'), 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); await expect( runKtxSetup( @@ -1596,7 +1586,7 @@ describe('setup status', () => { projectDir: tempDir, installs: [{ target: 'codex' as const, scope: 'project' as const, mode: 'cli' as const }], })); - await writeFile(join(tempDir, 'ktx.yaml'), ['project: revenue', 'connections: {}', ''].join('\n'), 'utf-8'); + await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); await expect( runKtxSetup( @@ -1633,7 +1623,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -1671,7 +1660,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids: []', 'connections: {}', @@ -1778,7 +1766,6 @@ describe('setup status', () => { await writeFile( join(tempDir, 'ktx.yaml'), [ - 'project: revenue', 'setup:', ' database_connection_ids: []', 'connections: {}', diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index d89a4eec..cf458f1d 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -1,5 +1,5 @@ import { existsSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { basename, join, resolve } from 'node:path'; import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest'; import { ktxLocalStateDbPath, @@ -317,7 +317,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise ({ diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index 14f18337..16041a31 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -44,7 +44,7 @@ async function seedSlSource(input: { sourceName?: string; yaml?: string; }): Promise { - const project = await initKtxProject({ projectDir: input.projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: input.projectDir }); await project.fileStore.writeFile( `semantic-layer/${input.connectionId ?? 'warehouse'}/${input.sourceName ?? 'orders'}.yaml`, input.yaml ?? ORDERS_YAML, @@ -139,7 +139,7 @@ describe('runKtxSl', () => { it('fails validation when a table-backed source declares columns absent from a matching warehouse manifest', async () => { const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir }); await project.fileStore.writeFile( 'semantic-layer/postgres-warehouse/_schema/orbit_analytics.yaml', `tables: @@ -189,7 +189,7 @@ joins: [] it('runs sl query and prints SQL output', async () => { const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir }); project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', @@ -246,7 +246,7 @@ joins: [] it('runs sl query from a JSON query file', async () => { const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir }); project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', @@ -313,7 +313,7 @@ joins: [] it('creates default sl query compute through the managed runtime helper', async () => { const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir }); project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', @@ -374,7 +374,7 @@ joins: [] it('executes sl query through the injected query executor', async () => { const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', @@ -459,7 +459,7 @@ joins: [] it('executes sl query against a local SQLite connection through the default executor', async () => { const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir }); const dbPath = join(projectDir, 'warehouse.db'); const db = new Database(dbPath); db.exec(` @@ -475,7 +475,6 @@ joins: [] await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 3b657780..d63be434 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -80,7 +80,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -218,7 +217,6 @@ describe('standalone built ktx CLI smoke', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: gateway-smoke', 'llm:', ' provider:', ' backend: gateway', diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index 7a272183..2aab1e5c 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -1,3 +1,4 @@ +import { basename } from 'node:path'; import type { KtxConfigIssue, KtxLocalProject, @@ -8,6 +9,7 @@ import type { } from '@ktx/context/project'; import type { PostgresPgssProbeResult } from '@ktx/context/ingest'; import type { DoctorCheck } from './doctor.js'; +import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js'; type ProjectStatusLevel = 'ok' | 'warn' | 'fail'; type ProjectVerdict = 'ready' | 'partial' | 'blocked'; @@ -68,6 +70,8 @@ interface WarningItem { fix?: string; } +const PROJECT_READY_COMMANDS = KTX_NEXT_STEP_DIRECT_COMMANDS.map((step) => step.command); + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -131,7 +135,7 @@ function buildLlmStatus(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): Ll backend, model, status: 'fail', - detail: 'no LLM configured — ktx ask will not work', + detail: 'no LLM configured; research agent will not run', fix: 'Run: ktx setup (choose an LLM provider)', }; } @@ -570,7 +574,7 @@ function buildVerdict( if (llm.status === 'fail') { return { verdict: 'blocked', - reason: 'LLM not configured — `ktx ask` will not work.', + reason: 'LLM not configured; research agent will not run.', nextActions: ['ktx setup'], }; } @@ -604,7 +608,7 @@ function buildVerdict( return { verdict: 'ready', reason: 'Ready.', - nextActions: ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'], + nextActions: [...PROJECT_READY_COMMANDS], }; } @@ -650,7 +654,7 @@ export async function buildProjectStatus(project: KtxLocalProject, options: Buil const { verdict, reason, nextActions } = buildVerdict(llm, embeddings, connections, queryHistory, warnings); return { - projectName: config.project, + projectName: basename(project.projectDir) || project.projectDir, projectDir: project.projectDir, config: configStatus, llm, diff --git a/packages/connector-bigquery/src/connector.test.ts b/packages/connector-bigquery/src/connector.test.ts index c78770e6..0de940df 100644 --- a/packages/connector-bigquery/src/connector.test.ts +++ b/packages/connector-bigquery/src/connector.test.ts @@ -100,7 +100,7 @@ const connection = { dataset_id: 'analytics', credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }), location: 'US', -}; +} as const; describe('KtxBigQueryScanConnector', () => { it('resolves configuration safely', () => { diff --git a/packages/context/prompts/memory_agent_backfill.md b/packages/context/prompts/memory_agent_backfill.md index fdf7211d..167b8f4d 100644 --- a/packages/context/prompts/memory_agent_backfill.md +++ b/packages/context/prompts/memory_agent_backfill.md @@ -3,7 +3,7 @@ You are backfilling knowledge from a historical chat transcript or archived SQL -Moderately conservative. Historical content is not directly steering current work, so spurious captures will surface in future chats and annoy users. But genuine patterns are worth saving — these backfills exist because the content is known to contain value. +Moderately conservative. Historical content is not directly steering current work, so spurious captures will surface in future chats and annoy users. But genuine patterns are worth saving - these backfills exist because the content is known to contain value. Capture only when the signal is unambiguous: a metric definition stated plainly, a reusable SQL pattern, a documented correction, a durable business rule. Skip casual chatter and ambiguous interpretations. @@ -12,10 +12,10 @@ Capture only when the signal is unambiguous: a metric definition stated plainly, 1. Read the wiki and SL indexes to avoid creating duplicates. 2. If the content has wiki-style signal, load the `wiki_capture` skill and follow its workflow. 3. If the content has SL-style signal, load the `sl` skill and follow its Part 3 workflow. -4. Prefer updating existing entries over creating new ones — backfills often duplicate existing knowledge. +4. Prefer updating existing entries over creating new ones - backfills often duplicate existing knowledge. 5. When done, exit the loop. -Wiki writes follow the session's scope selection (USER for user-scoped enabled, GLOBAL otherwise). The `wiki_write` tool picks automatically — focus on capture judgment. +Wiki writes follow the session's scope selection (USER for user-scoped enabled, GLOBAL otherwise). The `wiki_write` tool picks automatically - focus on capture judgment. diff --git a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md index 3813afcb..6282c37d 100644 --- a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md +++ b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md @@ -3,12 +3,12 @@ You are the reconciliation agent for a multi-file ingest bundle. Stage 3 WorkUni -Parsimonious. Stage 3 WUs already loaded `ingest_triage` and handled conflicts they saw. Your sweep is the safety net for contradictions that are only visible when you can see the whole job at once — e.g. two WUs that each looked clean in isolation but collectively form a near-duplicate cluster. Do not redo work Stage 3 already did. +Parsimonious. Stage 3 WUs already loaded `ingest_triage` and handled conflicts they saw. Your sweep is the safety net for contradictions that are only visible when you can see the whole job at once - e.g. two WUs that each looked clean in isolation but collectively form a near-duplicate cluster. Do not redo work Stage 3 already did. 1. Load `ingest_triage`, then `sl_capture` + `wiki_capture`. -2. Call `stage_list()` for the full index of this job's writes. If it is empty AND you have no evictions, exit — the runner short-circuits this case but the skill still teaches you to bail fast. +2. Call `stage_list()` for the full index of this job's writes. If it is empty AND you have no evictions, exit - the runner short-circuits this case but the skill still teaches you to bail fast. 3. If the system prompt includes ``, apply those pins before flagging a same-name or near-duplicate conflict. A pinned `canonicalArtifactKey` keeps the contested name when it is present in the Stage Index; competing variants keep or receive disambiguated names. 4. Sweep both exact-key conflicts and near-duplicate writes. Compare WUs that wrote overlapping SL source names, overlapping wiki keys, the same `tables:` or `sl_refs:` action details, or obviously equivalent topic titles under different wiki keys. Call `stage_diff` to see the actual difference, and use `wiki_read`/`sl_read_source` when two different keys appear to describe the same table, metric, or source-of-truth mapping. If they're the same content, leave one canonical artifact and record the duplicate as subsumed. If they differ per `ingest_triage` rules, apply the correct resolution (rename + capture; election of canonical; silent replace for expression-only re-ingest change; or pinned canonical), then call `emit_conflict_resolution` with the artifact key and decision. 5. For any `wiki_write`, `wiki_remove`, `sl_write_source`, or `sl_edit_source` call you make during reconciliation, include `rawPaths` with only the raw paths that directly caused that reconciliation action. @@ -24,6 +24,6 @@ Wiki keys must be flat slugs, not directory paths. If a Stage 3 page used a path -- Do not overwrite a Stage 3 WU's resolution that already matches `ingest_triage` output — that's churn. -- Do not treat two SL sources with the same logical meaning but legitimately different domains (e.g. `finance.revenue` and `marketing.revenue`) as a conflict — that's by design. +- Do not overwrite a Stage 3 WU's resolution that already matches `ingest_triage` output - that's churn. +- Do not treat two SL sources with the same logical meaning but legitimately different domains (e.g. `finance.revenue` and `marketing.revenue`) as a conflict - that's by design. diff --git a/packages/context/prompts/memory_agent_external_ingest.md b/packages/context/prompts/memory_agent_external_ingest.md index dd84651a..6f81ae0a 100644 --- a/packages/context/prompts/memory_agent_external_ingest.md +++ b/packages/context/prompts/memory_agent_external_ingest.md @@ -10,19 +10,19 @@ A single artifact typically produces multiple actions: one SL source per table/v 1. Review the wiki and SL indexes in the prompt. Prefer updating existing entries over creating duplicates. -2. Load the `sl` skill for SL-writes and `wiki_capture` for wiki-writes. Both skills describe schema, decision rules, and editing patterns — follow them. +2. Load the `sl` skill for SL-writes and `wiki_capture` for wiki-writes. Both skills describe schema, decision rules, and editing patterns - follow them. 3. For each distinct element in the artifact (table/view, measure, dimension group, derived column, computed filter, business rule, alias): decide whether it belongs in the SL, in the wiki, or both. 4. Write SL sources first (so they have stable names), then wiki pages that reference them via `sl_refs`. -5. When the artifact mixes data definitions with business rules, capture BOTH — one in each store, linked. +5. When the artifact mixes data definitions with business rules, capture BOTH - one in each store, linked. 6. When you're done, exit the loop without calling any more tools. -All wiki writes go to the GLOBAL scope — they will be visible to every user of this KTX project. Phrase wiki pages as objective business knowledge, not personal preference. The `wiki_write` tool handles scope selection automatically for external ingest. +All wiki writes go to the GLOBAL scope - they will be visible to every user of this KTX project. Phrase wiki pages as objective business knowledge, not personal preference. The `wiki_write` tool handles scope selection automatically for external ingest. - Do not fabricate measures, joins, or rules that aren't in the artifact. - Do not invent column names. If a type is unclear, omit it rather than guess. -- Do not mirror presentation hints (LookML `link:`, `map_layer_name:`, HTML formatting) into SL — those belong in wiki if anywhere. +- Do not mirror presentation hints (LookML `link:`, `map_layer_name:`, HTML formatting) into SL - those belong in wiki if anywhere. diff --git a/packages/context/prompts/memory_agent_research.md b/packages/context/prompts/memory_agent_research.md index 6090e5bb..c771255d 100644 --- a/packages/context/prompts/memory_agent_research.md +++ b/packages/context/prompts/memory_agent_research.md @@ -1,5 +1,5 @@ -You capture durable knowledge from an analytics assistant's chat turn. The user just asked a question, the assistant answered, and you are running after the turn to decide what — if anything — is worth saving for future chats. +You capture durable knowledge from an analytics assistant's chat turn. The user just asked a question, the assistant answered, and you are running after the turn to decide what - if anything - is worth saving for future chats. @@ -21,7 +21,7 @@ Skip: 2. Identify durable knowledge OR reusable data patterns in the turn. 3. If the turn has wiki-style signal (preferences, definitions, conventions), load the `wiki_capture` skill and follow its workflow. 4. If the turn has SL-style signal (reusable metric aggregations, new joins, derived dimensions), load the `sl` skill and follow its Part 3 (capture) workflow. -5. A single turn can produce BOTH a wiki page and an SL source — load both skills and author the edge once on the wiki via `sl_refs: [source_name]`. The reverse edge (wiki pages that cite the SL source) is derived by the reconciler; do not set `knowledge_refs:` on the SL side. +5. A single turn can produce BOTH a wiki page and an SL source - load both skills and author the edge once on the wiki via `sl_refs: [source_name]`. The reverse edge (wiki pages that cite the SL source) is derived by the reconciler; do not set `knowledge_refs:` on the SL side. 6. When you're done, exit the loop without calling any more tools. Do NOT emit a final text summary. diff --git a/packages/context/prompts/skills/page_triage_classifier.md b/packages/context/prompts/skills/page_triage_classifier.md index 5a6d7e23..561ed035 100644 --- a/packages/context/prompts/skills/page_triage_classifier.md +++ b/packages/context/prompts/skills/page_triage_classifier.md @@ -12,11 +12,11 @@ Reusable templates and scripts are durable knowledge regardless of subject matte Analytics evidence (BI tools like Looker, Metabase, Tableau) is durable knowledge of *how the organization defines its metrics and segments*. The `signals.objectType` tells you what you are looking at: -- `looker_explore` (or any explore-like analytics surface) -> `full` by default. Explores enumerate dimensions, measures, and joins — these are the canonical schema-of-the-business and warrant the full WorkUnit agent so each measure can become a candidate. Skip only if the excerpt is empty or contains zero measures and zero descriptive text. +- `looker_explore` (or any explore-like analytics surface) -> `full` by default. Explores enumerate dimensions, measures, and joins - these are the canonical schema-of-the-business and warrant the full WorkUnit agent so each measure can become a candidate. Skip only if the excerpt is empty or contains zero measures and zero descriptive text. - `looker_dashboard` (or any named dashboard with tile queries, filters, calculated fields) -> `full` when it has multiple tiles or named metrics, `light` when one or two tiles with trivial fields, `skip` only when usage hints make it clear it is unused (e.g. `queryCount30d` and `uniqueUsers30d` are both zero) AND there are no calculated fields, filters, or named tiles worth extracting. - `looker_look` (or any saved query) -> `light` when the query is a simple field listing, `full` when it has custom calculations, non-trivial filters, or aggregation expressions, `skip` only when usage is zero AND the query is a default field listing. -Treat dashboard/Look filter values, saved aggregations, calculated fields, and named tiles as candidate metric/segment definitions — they are durable. Do **not** mark BI evidence as `skip` solely because it is "configuration" or "tied to a data model"; that is exactly the durable knowledge we want to capture. +Treat dashboard/Look filter values, saved aggregations, calculated fields, and named tiles as candidate metric/segment definitions - they are durable. Do **not** mark BI evidence as `skip` solely because it is "configuration" or "tied to a data model"; that is exactly the durable knowledge we want to capture. Examples: diff --git a/packages/context/skills/dbt_ingest/SKILL.md b/packages/context/skills/dbt_ingest/SKILL.md index 75ab8b78..a3ce0151 100644 --- a/packages/context/skills/dbt_ingest/SKILL.md +++ b/packages/context/skills/dbt_ingest/SKILL.md @@ -6,13 +6,13 @@ callers: [memory_agent] # dbt → KTX (bundle ingest) -Use this skill for **uploaded** dbt projects (`dbt_project.yml` at stage root, `models/**`, `sources/**`, `schema.yml`). There is **no** `fetch()` in v1 — scheduled `dbt parse` / `manifest.json` pulls are out of scope; host-provided dbt sync may still backfill structured test metadata into `_schema` on the next sync. +Use this skill for **uploaded** dbt projects (`dbt_project.yml` at stage root, `models/**`, `sources/**`, `schema.yml`). There is **no** `fetch()` in v1 - scheduled `dbt parse` / `manifest.json` pulls are out of scope; host-provided dbt sync may still backfill structured test metadata into `_schema` on the next sync. ## Mapping (models / sources → SL) | dbt | KTX | Notes | |-----|--------|--------| -| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `discover_data` / `entity_details`) | One SL source per physical table; model name may differ from DB name — resolve with `read_raw_file` + warehouse context. | +| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `discover_data` / `entity_details`) | One SL source per physical table; model name may differ from DB name - resolve with `read_raw_file` + warehouse context. | | `sources:` → `tables:` | Same as models; use `identifier` when present instead of logical `name`. | Schema + name must match how the connection sees tables. | | Column `description` | `column_overrides[].descriptions.user` on the overlay | Do not overwrite `dbt` description keys from sync. | | `data_tests: not_null` / `unique` | Short hint in column `descriptions` or notes: “dbt: not null”, “dbt: unique” | Full structured metadata lands in manifest via **sync**; the skill keeps bundle-time SL text useful for the agent. | @@ -73,4 +73,4 @@ If the same bundle also has MetricFlow `semantic_models:` / `metrics:`, the **`m - Do not invent column names, grain keys, or measure expressions from dbt model names, descriptions, tests, or common naming patterns. - Do not write computed `columns:`, `column_overrides:`, `grain:`, or `measures:` for a dbt model unless those exact column names are confirmed by dbt YAML columns or warehouse schema discovery. - Do not invent joins from `relationships` tests if the target model/table is not found in SL or the warehouse. -- Do not read `peerFileIndex` paths — use `read_raw_file` only on `rawFiles` and `dependencyPaths` from the WorkUnit. +- Do not read `peerFileIndex` paths - use `read_raw_file` only on `rawFiles` and `dependencyPaths` from the WorkUnit. diff --git a/packages/context/skills/ingest_triage/SKILL.md b/packages/context/skills/ingest_triage/SKILL.md index c7cee225..77872e75 100644 --- a/packages/context/skills/ingest_triage/SKILL.md +++ b/packages/context/skills/ingest_triage/SKILL.md @@ -4,7 +4,7 @@ description: Classify and resolve conflicts detected during bundle ingest (struc callers: [memory_agent] --- -# Ingest Triage — conflict classification and resolution +# Ingest Triage - conflict classification and resolution This skill is loaded in two contexts: - By a Stage 3 WorkUnit agent when `sl_discover` reveals that a prior WU (or a prior sync) already wrote something that overlaps with what the current WU is about to write. @@ -15,12 +15,12 @@ Apply the rules below before every write that could collide with an existing art ## Decision tree 1. **Is this the same artifact I'm producing now, or a different one with the same name?** - Read both. If names match and content matches (modulo whitespace): no conflict — skip the write, the prior one stands. + Read both. If names match and content matches (modulo whitespace): no conflict - skip the write, the prior one stands. 2. **If content differs, is it an expression-only change (e.g. a different `sql:` body for the same measure name, same grain, same columns)?** Re-ingest change (expression-only): silently replace via `sl_edit_source`. No flag. -3. **If the difference is structural — grain, columns, filter, join shape — is the current bundle the re-ingest of a previously-ingested bundle (i.e. `priorProvenance` has a row for this raw file and artifact)?** +3. **If the difference is structural - grain, columns, filter, join shape - is the current bundle the re-ingest of a previously-ingested bundle (i.e. `priorProvenance` has a row for this raw file and artifact)?** Re-ingest change (semantic break): replace + flag. Record in the IngestReport's `conflicts_resolved` list with `flagged_for_human: true`. 4. **If there's no prior-sync row (both are from THIS job), check for same-ingest contradictions:** @@ -37,11 +37,11 @@ Apply the rules below before every write that could collide with an existing art ## Why same-ingest vs re-ingest differs -Within ONE bundle there's no user signal telling us which duplicate wins — we capture all variants and flag. Across bundles, re-uploading IS the signal that the new state is intended — we replace silently for expression changes and flag for semantic breaks. +Within ONE bundle there's no user signal telling us which duplicate wins - we capture all variants and flag. Across bundles, re-uploading IS the signal that the new state is intended - we replace silently for expression changes and flag for semantic breaks. ## Naming disambiguation hints -When you rename to disambiguate, prefer domain suffixes that match the containing view/table/collection name: `customers.churn_risk_score` → `customers.churn_risk_engagement_based` (if the `customer_churn` view computes it from engagement); `billing.churn_risk_score` → `billing.churn_risk_billing_based`. Avoid numeric suffixes (`churn_risk_1`, `churn_risk_2`) — they disclose nothing. +When you rename to disambiguate, prefer domain suffixes that match the containing view/table/collection name: `customers.churn_risk_score` → `customers.churn_risk_engagement_based` (if the `customer_churn` view computes it from engagement); `billing.churn_risk_score` → `billing.churn_risk_billing_based`. Avoid numeric suffixes (`churn_risk_1`, `churn_risk_2`) - they disclose nothing. ## Applying canonical pins @@ -62,7 +62,7 @@ When you perform rename + capture, also write one page named ` }` | **Standalone** with `sql: SELECT * FROM WHERE

` | Enforcement, not opt-in | | `explore: { join: Y { sql_on: …; relationship: … } }` | `joins:` entry `{ to: Y, on: " = Y.

", relationship: … }` | On the overlay or standalone | | `conditionally_filter` / `always_filter` | `segments: [{ name, expr }]` | Callers reference by name | -| Manifest entry | `_schema/*.yaml` | **Never edit** — auto-imported | +| Manifest entry | `_schema/*.yaml` | **Never edit** - auto-imported | Type map: `date`/`datetime`/`timestamp` → `time`; `yesno` → `boolean`; `number` → `number`; `string` → `string`. Ignore `drill_fields:` (UI only). @@ -92,14 +92,14 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: `sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. Replace `warehouse`, `analytics`, and `orders` with the verified connection, schema or dataset, and table from the WorkUnit evidence. -3. Use only those names in `sql:`, `columns:`, and `grain:`. Map each `dimension_group` to ONE `{ name: , type: time, role: time }` entry — never one per timeframe. +3. Use only those names in `sql:`, `columns:`, and `grain:`. Map each `dimension_group` to ONE `{ name: , type: time, role: time }` entry - never one per timeframe. | LookML input | KTX `columns:` entry | |---|---| | `dimension_group: month { type: time; timeframes: [month]; sql: ${TABLE}.month_date ;; }` | `{ name: month_date, type: time, role: time }` | -| `dimension_group: date { type: time; timeframes: [raw, date, week, month]; sql: ${TABLE}.date ;; }` | `{ name: date, type: time, role: time }` — single entry, NOT `date_raw`/`date_date`/`date_week` | +| `dimension_group: date { type: time; timeframes: [raw, date, week, month]; sql: ${TABLE}.date ;; }` | `{ name: date, type: time, role: time }` - single entry, NOT `date_raw`/`date_date`/`date_week` | -**After every `sl_write_source`**: call `sl_validate`. It runs `SELECT * FROM () LIMIT 0` against the connection. If a column name was invented, the warehouse's `Unrecognized name: …` error comes back verbatim. Treat that as a hard failure — re-read the real columns with `sl_discover` and rewrite. +**After every `sl_write_source`**: call `sl_validate`. It runs `SELECT * FROM () LIMIT 0` against the connection. If a column name was invented, the warehouse's `Unrecognized name: …` error comes back verbatim. Treat that as a hard failure - re-read the real columns with `sl_discover` and rewrite. ## Provenance markers @@ -110,13 +110,13 @@ When a wiki mixes LookML source prose with `sl_discover` output, tag sections: Customers fan out many-to-one into `accounts` via `account_id`. -`customers.admin_user_id` is nullable — orphan rows exist. +`customers.admin_user_id` is nullable - orphan rows exist. ``` Invisible in most renderers; lets a future pass audit provenance. -## Example 1 — overlay (thin wrapper) +## Example 1 - overlay (thin wrapper) LookML (excerpt): @@ -155,7 +155,7 @@ joins: relationship: many_to_one ``` -## Example 2 — standalone from `derived_table` +## Example 2 - standalone from `derived_table` ```lookml view: lab_results { @@ -189,7 +189,7 @@ measures: - { name: avg_delta, expr: "avg(delta)" } ``` -## Example 3 — standalone with `sql_always_where` +## Example 3 - standalone with `sql_always_where` ```lookml view: rpt_daily_braze_email { diff --git a/packages/context/skills/metabase_ingest/SKILL.md b/packages/context/skills/metabase_ingest/SKILL.md index 31bd979f..aefd067f 100644 --- a/packages/context/skills/metabase_ingest/SKILL.md +++ b/packages/context/skills/metabase_ingest/SKILL.md @@ -84,7 +84,7 @@ For each card: 4. Decide: - Simple aggregation on a table that already has a source → `sl_edit_source` to add a measure. - Join between tables that should be linked in the SL graph → `sl_edit_source` to add a join. - - Complex derived SQL (CTEs, multi-layer aggregation, scoring models) → `sl_write_source` with `source_type: sql`. When the SQL projects/filters from a single manifest-backed base table, set `inherits_columns_from: ` so columns inherit type and description from the manifest — see `sl_capture` skill for the slim form. Use `sl_discover` to discover the manifest key from the table reference in the SQL (it accepts `MARTS.CONSIGNMENTS`, `ANALYTICS.MARTS.CONSIGNMENTS`, or `CONSIGNMENTS`). + - Complex derived SQL (CTEs, multi-layer aggregation, scoring models) → `sl_write_source` with `source_type: sql`. When the SQL projects/filters from a single manifest-backed base table, set `inherits_columns_from: ` so columns inherit type and description from the manifest - see `sl_capture` skill for the slim form. Use `sl_discover` to discover the manifest key from the table reference in the SQL (it accepts `MARTS.CONSIGNMENTS`, `ANALYTICS.MARTS.CONSIGNMENTS`, or `CONSIGNMENTS`). - New base table not yet in the semantic layer → `sl_write_source` with `source_type: table`. - Trivial query (`SELECT *`, simple `COUNT(*)` with no business logic) → do nothing; the runner will record this card as `action_type='skipped'`. - Duplicate of an existing measure → same as trivial; do nothing for this card. @@ -102,7 +102,7 @@ Overlay shape: `name:` plus any of `measures:`, `segments:`, `descriptions:`, `j **Join discovery:** When your card's SQL references warehouse tables (e.g. in `FROM` or `JOIN` clauses), call `sl_discover({ query: '
' })` before writing. The matching manifest entry's `name` is the value you use in `joins: [- to: ]` only when the card output exposes a local key that matches the target source grain (for example `account_id = mart_account_segments.account_id`). Do not declare a KTX join just because the card SQL joins that table internally. If the output only exposes display fields such as `account_name`, keep the SQL source self-contained or project the key before adding the join. Use `many_to_one` for FK-to-dimension joins, `one_to_many` for the reverse. -**Hard rule on join columns (prevents broken joins):** For every join you declare, the local column on the left of `on:` MUST be (a) present in your source's projected output and (b) a key/ID column, never a display value. If the natural FK isn't in your SELECT, add it to SELECT before declaring the join. Joining `account_name = mart_account_segments.account_id` is always wrong — names are not identifiers and the equality produces zero matches. The validator rejects this with a "display value to identifier" error; the tool will refuse to save it. Add `account_id` to your SELECT and join on `account_id = mart_account_segments.account_id`, or omit the join entirely. +**Hard rule on join columns (prevents broken joins):** For every join you declare, the local column on the left of `on:` MUST be (a) present in your source's projected output and (b) a key/ID column, never a display value. If the natural FK isn't in your SELECT, add it to SELECT before declaring the join. Joining `account_name = mart_account_segments.account_id` is always wrong - names are not identifiers and the equality produces zero matches. The validator rejects this with a "display value to identifier" error; the tool will refuse to save it. Add `account_id` to your SELECT and join on `account_id = mart_account_segments.account_id`, or omit the join entirely. ## priorProvenance @@ -114,7 +114,7 @@ If the WU prompt includes a `priorProvenance` section for a card, it tells you w ## Deduplication -Before writing, scan all cards in this WU for near-duplicate groups — cards whose `resolvedSql` shares the same CTEs, base tables, joins, and aggregation structure but differs only in: +Before writing, scan all cards in this WU for near-duplicate groups - cards whose `resolvedSql` shares the same CTEs, base tables, joins, and aggregation structure but differs only in: - Trailing filters (e.g. `date_trunc(week, date)` vs `date_trunc(month, date)`). - Minor `WHERE` clause variations. - Column aliases or output column subsets. @@ -124,7 +124,7 @@ When you find a group of near-duplicates: 1. Create ONE generalized source from the most comprehensive card in the group. 2. Strip card-specific trailing filters from the SQL so the source covers all variants (e.g. keep daily grain instead of filtering to week/month). 3. If each card had a distinct measure or filter, add them as separate measures on the single source. -4. For all cards except the canonical one, do nothing — they'll be recorded as `action_type='skipped'` automatically by the runner. +4. For all cards except the canonical one, do nothing - they'll be recorded as `action_type='skipped'` automatically by the runner. Do NOT merge cards with fundamentally different business logic, even if they share CTEs. @@ -132,7 +132,7 @@ Do NOT merge cards with fundamentally different business logic, even if they sha When a card's `resolvedSql` contains `GROUP BY` with aggregation functions (`SUM`, `COUNT`, `AVG`, …): -1. **Detect**: simple aggregation on base tables/joins — `SELECT` with `GROUP BY`, no complex CTEs or window functions. +1. **Detect**: simple aggregation on base tables/joins - `SELECT` with `GROUP BY`, no complex CTEs or window functions. 2. **Decompose**: strip the `GROUP BY` and aggregation functions. Keep `FROM`, `JOIN`, and `WHERE` intact. 3. **Expose row-level columns**: include the grouped-by columns AND the raw columns being aggregated (e.g. `money_out` instead of `SUM(money_out) AS total_money_out`). 4. **Define aggregations as measures**: convert each aggregation into a KSL measure (e.g. `sum(money_out)`). @@ -144,17 +144,17 @@ Exception: keep the pre-aggregated SQL when the query involves multi-CTE pipelin Every card carries a `resolvedSql` field. Check the staged card's `resolutionStatus` first: -- `resolutionStatus: "resolved"` — `{{#N}}` references are inlined and `[[ ... ]]` optional clauses have been dropped locally. If the resolved SQL contains no other parameters the SQL is executable as-is. If the card had **required** (non-bracketed) `{{ var }}` placeholders, the SQL is prefixed with a placeholder-warning comment block listing every dummy substitution Metabase made — see "Step A" below. -- `resolutionStatus: "fallback"` — Metabase failed to resolve. The SQL still contains `{{#N}}`, `{{#N-name}} alias`, `{{ var }}`, and `[[ ... ]]` syntax. Do the translation steps below before writing a source. +- `resolutionStatus: "resolved"` - `{{#N}}` references are inlined and `[[ ... ]]` optional clauses have been dropped locally. If the resolved SQL contains no other parameters the SQL is executable as-is. If the card had **required** (non-bracketed) `{{ var }}` placeholders, the SQL is prefixed with a placeholder-warning comment block listing every dummy substitution Metabase made - see "Step A" below. +- `resolutionStatus: "fallback"` - Metabase failed to resolve. The SQL still contains `{{#N}}`, `{{#N-name}} alias`, `{{ var }}`, and `[[ ... ]]` syntax. Do the translation steps below before writing a source. -### Step A — Handle dummy-substituted placeholders (resolved cards only) +### Step A - Handle dummy-substituted placeholders (resolved cards only) When a card has a required `{{ var }}` outside any `[[ ]]` block, the resolver substitutes a **dummy value** purely so Metabase's parser will accept the query. The resulting SQL is prefixed with a comment like: ```sql -- PLACEHOLDER_WARNING: this SQL was extracted from a Metabase card with -- unbound template parameters. The placeholders below were substituted with DUMMY --- values to satisfy Metabase's parser — they DO NOT represent intended filters. +-- values to satisfy Metabase's parser - they DO NOT represent intended filters. -- Drop the corresponding clauses (or expose them as runtime SL filters) before -- persisting this SQL as a semantic-layer source. -- {{ auction_end }} (type=dimension, widget=date/all-options) → '2020-01-01~2020-12-31' @@ -165,7 +165,7 @@ WHERE start_date >= '2020-01-01' AND start_date < '2021-01-01' AND status = 'pla For each listed placeholder: locate the WHERE clause(s) in the SQL that reference the dummy literal and **drop them**, then strip the warning comment. SL chat-time filters compose narrowing predicates dynamically, so the source should represent the unfiltered dataset. -For `fallback` cards, dropping is simpler — the SQL still has the `[[ ... ]]` brackets and `{{ var }}` placeholders intact: +For `fallback` cards, dropping is simpler - the SQL still has the `[[ ... ]]` brackets and `{{ var }}` placeholders intact: ```sql -- before: @@ -177,18 +177,18 @@ WHERE 1=1 WHERE 1=1 ``` -### Step B — Inline `{{#N}}` references (fallback cards only) +### Step B - Inline `{{#N}}` references (fallback cards only) Resolved cards already have `{{#N}}` inlined for you. For `fallback` cards, each `{{#N}}` (or `{{#N-some-slug}}`) in the SQL refers to another card's `resolvedSql`. The referenced card is in the WU's `rawFiles` or `dependencyPaths`. Read it with `read_raw_file`, then inline its SQL. If the reference has an alias (`from {{#5996-listing-interactions}} tb`), the **outer** SQL probably uses that alias (`select tb.* ...`, `tb.column_name`, etc.). When you inline, you must EITHER: -1. **Pick a single base table inside the inlined SQL and rename its alias to the outer alias.** Useful when the inlined card is `SELECT * FROM listings JOIN ...` — set the LISTINGS alias to `tb` and `tb.*` keeps working in the outer query. +1. **Pick a single base table inside the inlined SQL and rename its alias to the outer alias.** Useful when the inlined card is `SELECT * FROM listings JOIN ...` - set the LISTINGS alias to `tb` and `tb.*` keeps working in the outer query. 2. **Replace the outer alias references with explicit columns from the inlined SQL.** Useful when the inlined card has multiple JOINs and `tb.*` is ambiguous. Never leave the outer alias dangling: after inlining, **grep your SQL for the outer alias name and rewrite or remove every reference**. A leftover `tb.*` with no `tb` table is the most common failure mode here. -### Step C — Inlining cleanup checklist +### Step C - Inlining cleanup checklist After Steps A and B, your SQL must: @@ -209,11 +209,11 @@ For `source_type: sql`: - If `sl_discover` resolves the table, it is not outside the manifest. Do not write an `unmapped-table-*` fallback for resolved `orbit_raw`, `mart`, or other manifest-backed sources just because they appear inside card SQL. - If `sl_discover` cannot resolve a referenced table at all, write a single-line `wiki_write` with key `unmapped-table-` and `rawPaths: ["cards/.json"]` so the gap is documented, then call `emit_unmapped_fallback` with the staged card path as `rawPath`, `reason: "missing_target_table"`, `tableRef: ""`, and `fallback: "wiki_only"`. Do not use this fallback if `sl_discover` resolved the table/source. -Joins on manifest-backed names compose: the manifest's joins are inherited automatically, and any overlay `joins:` are merged on top (deduped by `to` + `on`). Use `disable_joins: [""]` in the overlay to suppress a specific manifest join. If `sl_discover` shows a manifest-backed source with `Joins: 0` and the warehouse FK metadata is genuinely absent, declaring application-level joins via the overlay is fair game — bootstrap with `sl_write_source` (overlay shape above), then refine via `sl_edit_source`. +Joins on manifest-backed names compose: the manifest's joins are inherited automatically, and any overlay `joins:` are merged on top (deduped by `to` + `on`). Use `disable_joins: [""]` in the overlay to suppress a specific manifest join. If `sl_discover` shows a manifest-backed source with `Joins: 0` and the warehouse FK metadata is genuinely absent, declaring application-level joins via the overlay is fair game - bootstrap with `sl_write_source` (overlay shape above), then refine via `sl_edit_source`. ## Cross-card references (`{{#N}}`) -Resolved cards (`resolutionStatus: "resolved"`) have these inlined for you. Unresolved cards (`resolutionStatus: "fallback"`) need manual handling — see "SQL translation from raw native to KSL" above. +Resolved cards (`resolutionStatus: "resolved"`) have these inlined for you. Unresolved cards (`resolutionStatus: "fallback"`) need manual handling - see "SQL translation from raw native to KSL" above. ## Provenance markers @@ -237,7 +237,7 @@ Source definitions must follow ktx-sl YAML conventions: - `columns`: all columns with correct types (`string`, `number`, `time`, `boolean`). - Time columns: mark with `role: time`. - `joins`: use correct `relationship` types (`many_to_one` for FK→PK, `one_to_many` for reverse). -- `joins.on`: `local_column = TARGET_SOURCE.target_column` — the right side MUST include the target source name. +- `joins.on`: `local_column = TARGET_SOURCE.target_column` - the right side MUST include the target source name. - `measures.expr`: aggregation expression (e.g. `"sum(amount)"`); optional `filter` for business rules; required `description`. Measure naming: descriptive `snake_case` (e.g. `total_revenue`, `avg_order_value`). @@ -250,4 +250,4 @@ Measure naming: descriptive `snake_case` (e.g. `total_revenue`, `avg_order_value - If two measures differ only by a filter (e.g. `revenue` vs `paid_revenue`), they are distinct. - Use the card's `name` + `description` to write meaningful measure descriptions. - When multiple cards in a WU are near-duplicates, create ONE generalized source; the runner will skip the rest automatically. -- Process every card in the WU — don't stop early. +- Process every card in the WU - don't stop early. diff --git a/packages/context/skills/metricflow_ingest/SKILL.md b/packages/context/skills/metricflow_ingest/SKILL.md index 54d45b6e..42caf604 100644 --- a/packages/context/skills/metricflow_ingest/SKILL.md +++ b/packages/context/skills/metricflow_ingest/SKILL.md @@ -15,7 +15,7 @@ A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to | `semantic_model: X { model: ref('t') }` with measures + dimensions | **Overlay** at `/X.yaml` with `measures`, computed-only `columns`, `column_overrides`, `joins` | The `model:` ref resolves to a manifest table. | | `semantic_model: X { model: source('s','t') }` | **Overlay** at `/X.yaml` over table `t`. | Same shape; `source()` still resolves to a physical table. | | `semantic_model: X { model: }` with no manifest entry | **Standalone** with explicit `sql:`, `grain:`, `columns:` | Happens when the dbt manifest isn't available. | -| `semantic_model: Y { extends: X }` | **Merge** Y's measures/dimensions/entities into X's overlay, or write a single overlay named for the most-derived child (Y) containing both X's and Y's primitives | Do not emit a second overlay for X — flatten. | +| `semantic_model: Y { extends: X }` | **Merge** Y's measures/dimensions/entities into X's overlay, or write a single overlay named for the most-derived child (Y) containing both X's and Y's primitives | Do not emit a second overlay for X - flatten. | | `measures: [{ name, agg, expr }]` | `measures: [{ name, expr: "()" }]` | Aggregation inlined. `agg: count_distinct` → `count(distinct ...)`. | | `entities: [{ name, type: primary }]` | `grain: []` on the overlay/standalone | Primary/unique entities drive grain. | | `entities: [{ name, type: foreign }]` | `joins:` entry joining to the primary-entity's semantic_model | Only when a matching primary is discoverable. | @@ -24,10 +24,10 @@ A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to | `metrics: [{ type: derived, type_params: { expr, metrics } }]` | **Derived measure** on whichever source owns the referenced measures, with `expr:` referencing measure names | If the metric spans models, still write it once on the source owning the "primary" measure (the one the agent judges most central). Mention the cross-model chain in the description. | | `metrics: [{ type: ratio, type_params: { numerator, denominator } }]` | Same as derived; `expr: "numerator / NULLIF(denominator, 0)"` if no explicit expr | Safe-division by default. | | `metrics: [{ type: cumulative, type_params: { window, grain_to_date } }]` | **Standalone** source with a window-function SQL; reference the resulting column as a normal measure | KTX SL has no first-class cumulative primitive (spec Non-goals). | -| `metrics: [{ type: conversion }]` | **Flag for human** — do NOT write. Emit a wiki note describing the intended semantics. | No KTX equivalent in v1. | +| `metrics: [{ type: conversion }]` | **Flag for human** - do NOT write. Emit a wiki note describing the intended semantics. | No KTX equivalent in v1. | | Metric not mappable | Wiki page `-definition.md` with the full YAML body quoted | Capture the intent even if we can't emit SL. | -Type map: MetricFlow `time` to KTX `time`; `categorical` to `string`; `number` to `number`; `boolean` to `boolean`. Follow `expr` over `name` when both differ — `expr` is the physical column. +Type map: MetricFlow `time` to KTX `time`; `categorical` to `string`; `number` to `number`; `boolean` to `boolean`. Follow `expr` over `name` when both differ - `expr` is the physical column. Verify each MetricFlow model source table with entity_details before producing the corresponding sl_write_source. @@ -67,7 +67,7 @@ Within one WorkUnit, multiple semantic_models linked by `extends:` are guarantee 1. Start with the most-derived child (the one that no other semantic_model extends). 2. Walk the `extends:` chain upward, accumulating measures, dimensions, entities. 3. Write ONE overlay/standalone, named for the most-derived child's SL-appropriate name (not the base). -4. Parents that lack their own distinctive content should NOT get a separate overlay. If a parent has unique measures a child doesn't inherit, consider whether the base is used elsewhere — if yes, write both; if no, still one overlay. +4. Parents that lack their own distinctive content should NOT get a separate overlay. If a parent has unique measures a child doesn't inherit, consider whether the base is used elsewhere - if yes, write both; if no, still one overlay. 5. Measure/dimension name collisions: child wins, but note the overridden parent in the overlay's description or in a sibling wiki page. The spec's worked example has `orders`, `orders_ext` (extends orders), and `metrics/orders_final.yml` (defines `revenue` referencing both). The right output is ONE overlay named `orders_ext` (or `orders` if the team's naming favors the base) containing `order_count`, `gross_amount`, `refund_amount`, and a derived `revenue` measure. Provenance tags point to all three source files. @@ -88,9 +88,9 @@ call `sql_execution` with the same warehouse connection name, for example: `sql:` must be sourced from raw files, `entity_details`, or a successful SQL probe. -After every `sl_write_source`, call `sl_validate`. The warehouse will reject invented columns with `Unrecognized name: ` — treat as a hard failure and re-read the schema. +After every `sl_write_source`, call `sl_validate`. The warehouse will reject invented columns with `Unrecognized name: ` - treat as a hard failure and re-read the schema. -## Cumulative metrics — sql-standalone fallback +## Cumulative metrics - sql-standalone fallback KTX SL has no first-class `window:` or `grain_to_date:` primitive in v1 (spec Non-goals). Translate a MetricFlow cumulative metric to a standalone SL source with a window-function SQL: @@ -125,7 +125,7 @@ measures: Pick the time column based on the semantic_model's `defaults.agg_time_dimension` (e.g. `ordered_at`). If the MetricFlow config omits it, probe the base table for time-typed columns and choose the most obvious. After writing the standalone SQL source, call `emit_unmapped_fallback` with `rawPath` set to the MetricFlow file path, `reason: "cumulative_metric_unsupported"`, and `fallback: "sql_standalone"`. -## Conversion metrics — flag for human +## Conversion metrics - flag for human ```yaml metrics: @@ -159,7 +159,7 @@ name: orders_ext Line ranges (`#L-`) point to the exact YAML span within the file (the `semantic_models:` entry for its own `name`). Use `read_raw_span` to identify those ranges before writing. -## Example 1 — single semantic_model to overlay +## Example 1 - single semantic_model to overlay ```yaml # MetricFlow: @@ -185,7 +185,7 @@ measures: grain: [order_id] ``` -## Example 2 — extends chain → one flattened overlay +## Example 2 - extends chain → one flattened overlay ```yaml # MetricFlow: @@ -232,7 +232,7 @@ measures: grain: [order_id] ``` -## Example 3 — derived metric spanning two semantic_models +## Example 3 - derived metric spanning two semantic_models ```yaml # models/sales.yml @@ -256,7 +256,7 @@ metrics: metrics: [{name: revenue}, {name: cost}] ``` -Because the WorkUnit bundles all three files (cross-component union via the metric), write the derived measure on ONE of the two sources — pick the source whose domain "owns" the metric (here, `sales` — margin is inherently a sales metric). Cross-source references aren't native in KTX SL; treat the metric's operands as already-resolvable in the target source's query context OR emit a standalone SQL that joins the two tables: +Because the WorkUnit bundles all three files (cross-component union via the metric), write the derived measure on ONE of the two sources - pick the source whose domain "owns" the metric (here, `sales` - margin is inherently a sales metric). Cross-source references aren't native in KTX SL; treat the metric's operands as already-resolvable in the target source's query context OR emit a standalone SQL that joins the two tables: ```yaml # /sales.yaml @@ -269,7 +269,7 @@ measures: ``` ```yaml -# /margin.yaml — standalone because it spans two tables +# /margin.yaml - standalone because it spans two tables # # # @@ -292,7 +292,7 @@ measures: Also write a wiki page at `wiki/global/margin-metric.md` explaining the cross-source origin. -## Example 4 — filtered metric creates a new measure +## Example 4 - filtered metric creates a new measure ```yaml metrics: diff --git a/packages/context/skills/notion_synthesize/SKILL.md b/packages/context/skills/notion_synthesize/SKILL.md index 1b5417e3..e799ce7c 100644 --- a/packages/context/skills/notion_synthesize/SKILL.md +++ b/packages/context/skills/notion_synthesize/SKILL.md @@ -67,7 +67,7 @@ Search existing wiki pages for the same `tables:` or `sl_refs:` frontmatter and - Do not create SL sources under the Notion connection just because a page mentions a warehouse, dbt, Looker, or Metabase object. Use the mapped warehouse/source connection after discovery, or emit an unmapped fallback and write wiki-only. - Distinguish fallback reasons precisely: if a non-Notion warehouse/dbt connection exists but `sl_discover` cannot find the named table/source, use `no_physical_table`; reserve `no_connection_mapping` for cases where there is no plausible non-Notion target connection at all. - If `sl_discover` resolves the table/source, do not call `emit_unmapped_fallback` for that table. Use the resolved source for `sl_refs`, overlay edits, or wiki-only documentation. -- When calling `emit_unmapped_fallback`, pass the table or source identifier as `tableRef` (e.g. `tableRef: ".
"`) — the tool generates the canonical detail string from the reason code and `tableRef`. Use the optional `clarification` field only to add context that does not contradict the reason. Do not restate the reason in `clarification`. +- When calling `emit_unmapped_fallback`, pass the table or source identifier as `tableRef` (e.g. `tableRef: ".
"`) - the tool generates the canonical detail string from the reason code and `tableRef`. Use the optional `clarification` field only to add context that does not contradict the reason. Do not restate the reason in `clarification`. ## Identifier Verification Protocol diff --git a/packages/context/skills/sl/SKILL.md b/packages/context/skills/sl/SKILL.md index a71a992e..d5f334fe 100644 --- a/packages/context/skills/sl/SKILL.md +++ b/packages/context/skills/sl/SKILL.md @@ -1,6 +1,6 @@ --- name: sl -description: KTX's semantic layer — a structured catalog of sources (tables/views), measures, joins, and segments expressed as YAML. Covers the schema and how to query it via `sl_query`. Use when the task involves querying pre-defined metrics (ARR, churn, retention, LTV, MAU) or reading SL source YAML to understand the catalog. Capture is handled by the `sl_capture` skill (memory-agent only). +description: KTX's semantic layer - a structured catalog of sources (tables/views), measures, joins, and segments expressed as YAML. Covers the schema and how to query it via `sl_query`. Use when the task involves querying pre-defined metrics (ARR, churn, retention, LTV, MAU) or reading SL source YAML to understand the catalog. Capture is handled by the `sl_capture` skill (memory-agent only). --- # Semantic Layer @@ -8,10 +8,10 @@ description: KTX's semantic layer — a structured catalog of sources (tables/vi KTX's semantic layer (SL) is a structured catalog. Each **source** represents a table, a SQL view, or an overlay that enriches a manifest-backed table with measures, computed columns, joins, and named segments. The catalog is the single source of truth for reusable business metrics. This skill covers two parts: -- **Part 1** — Schema reference (what an SL source looks like). -- **Part 2** — Querying via `sl_query`. +- **Part 1** - Schema reference (what an SL source looks like). +- **Part 2** - Querying via `sl_query`. -Capture (when and how to add new patterns to the SL) is a separate concern handled by the memory-agent — see the `sl_capture` skill if you are running in capture mode. The research agent **reads** and **queries** the SL via the tools described here; it does not write to it. +Capture (when and how to add new patterns to the SL) is a separate concern handled by the memory-agent - see the `sl_capture` skill if you are running in capture mode. The research agent **reads** and **queries** the SL via the tools described here; it does not write to it. For capture-time identifier verification, load `sl_capture`. Synthesis writer skills must verify warehouse identifiers with `discover_data`, @@ -19,7 +19,7 @@ skills must verify warehouse identifiers with `discover_data`, --- -## Part 1 — Schema reference +## Part 1 - Schema reference An SL source is a YAML file at `semantic-layer//.yaml`. There are three flavors: @@ -34,7 +34,7 @@ descriptions: measures: - name: total_revenue expr: sum(amount) - description: Total order revenue — filter by status or region at query time + description: Total order revenue - filter by status or region at query time columns: # computed dimensions only - name: is_large_order type: boolean @@ -53,7 +53,7 @@ joins: ``` Rules: -- Do **not** repeat base-table columns, grain, `table`, or `source_type` in an overlay — those are inherited. +- Do **not** repeat base-table columns, grain, `table`, or `source_type` in an overlay - those are inherited. - Overlay columns MUST be computed (`expr` + `type`). - Use `column_overrides` to add descriptions or metadata to inherited manifest columns. Do not put `type` or `expr` in `column_overrides`. - `exclude_columns` hides specific manifest columns; `disable_joins` suppresses specific auto-detected joins. @@ -111,7 +111,7 @@ measures: expr: count(*) ``` -An SQL source is a one-shot answer: the aggregation is frozen, callers cannot re-group or re-filter by columns the SQL has collapsed, and the source is disconnected from the join graph. Prefer overlays + measures over SQL sources when possible — the `sl_capture` skill covers when SQL is justified. +An SQL source is a one-shot answer: the aggregation is frozen, callers cannot re-group or re-filter by columns the SQL has collapsed, and the source is disconnected from the join graph. Prefer overlays + measures over SQL sources when possible - the `sl_capture` skill covers when SQL is justified. ### Columns @@ -124,7 +124,7 @@ Every standalone column requires `name` and `type`. Overlays have computed colum ### Grain -`grain: [col_a, col_b]` — the set of columns that uniquely identify one row. The query engine uses grain to prevent fan-out in joins. Overlays inherit grain from the manifest unless they override. +`grain: [col_a, col_b]` - the set of columns that uniquely identify one row. The query engine uses grain to prevent fan-out in joins. Overlays inherit grain from the manifest unless they override. ### Joins @@ -133,7 +133,7 @@ joins: - to: customers # target source name on: "customer_id = customers.id" # local_col = TARGET.target_col relationship: many_to_one # or one_to_many, one_to_one - alias: primary_customer # optional — lets you join the same target twice + alias: primary_customer # optional - lets you join the same target twice ``` - `on` format: `local_col = TARGET.target_col`. Always qualify the right side with the target source name. @@ -145,13 +145,13 @@ joins: measures: - name: total_arr expr: sum(arr_amount) - description: Sum of ARR — filter by plan_name at query time + description: Sum of ARR - filter by plan_name at query time filter: "is_active = true" segments: [paid_non_refunded] ``` - `name` (required, snake_case). -- `expr` (required): any valid SQL aggregate — `sum(x)`, `count(*)`, `count(distinct user_id)`, `avg(score)`. +- `expr` (required): any valid SQL aggregate - `sum(x)`, `count(*)`, `count(distinct user_id)`, `avg(score)`. - `description` (required on capture): what the measure computes and how to use it. - `filter` (optional): SQL predicate applied as a WHERE clause specific to this measure. - `segments` (optional): names of segments defined on the same source. The engine AND-composes each segment's `expr` into this measure's effective filter. @@ -167,23 +167,23 @@ segments: description: Orders that were paid and not refunded ``` -Named, reusable boolean predicates scoped to one source. Reference by bare name in a measure's `segments: []`, or by dotted form `source.segment_name` in an `sl_query`. Segments are predicates only — they are NOT selectable as dimensions. If you need to group by the predicate, add a `columns[]` entry instead. +Named, reusable boolean predicates scoped to one source. Reference by bare name in a measure's `segments: []`, or by dotted form `source.segment_name` in an `sl_query`. Segments are predicates only - they are NOT selectable as dimensions. If you need to group by the predicate, add a `columns[]` entry instead. ### Cross-references with the wiki -The reverse edge (wiki pages that cite this source) is derived automatically from each wiki's `sl_refs:` — you don't emit anything on the SL side. Author the edge once on the wiki via `sl_refs:`; the post-write reconciler populates the knowledge↔SL index. +The reverse edge (wiki pages that cite this source) is derived automatically from each wiki's `sl_refs:` - you don't emit anything on the SL side. Author the edge once on the wiki via `sl_refs:`; the post-write reconciler populates the knowledge↔SL index. --- -## Part 2 — Querying via `sl_query` +## Part 2 - Querying via `sl_query` The `sl_query` tool generates correct SQL from a structured query. It handles joins, fan-out prevention, aggregation correctness, and filter classification automatically. Prefer it over writing raw SQL whenever the SL has the relevant sources. ### When to prefer sl_query over raw SQL - A pre-defined measure already exists (`source.measure_name` appears in the catalog). -- The question combines fields from multiple sources — the engine resolves the join path automatically. -- The question asks for a standard metric (revenue, ARR, churn, retention, LTV, conversion, MAU, etc.) — even if no pre-defined measure exists, a runtime aggregation over a catalog column is usually correct. +- The question combines fields from multiple sources - the engine resolves the join path automatically. +- The question asks for a standard metric (revenue, ARR, churn, retention, LTV, conversion, MAU, etc.) - even if no pre-defined measure exists, a runtime aggregation over a catalog column is usually correct. Use raw SQL (`sql_execution`) only when: - The computation requires multi-step CTEs whose intermediate grain is not a column in any source. @@ -206,17 +206,17 @@ Use raw SQL (`sql_execution`) only when: - **`measures`**: mix pre-defined refs (`source.measure`) and runtime aggregations (`sum(source.column)`). - **`dimensions`**: column refs or `{ field, granularity }` objects for time grains (`day`, `week`, `month`, `quarter`, `year`). - **`filters`**: free-form SQL predicates. The engine auto-classifies each as WHERE or HAVING based on whether it references an aggregated measure. -- **`segments`**: dotted `source.segment_name`. Each segment is AND-ed into the effective filter of every measure whose base source matches. Segments never become a global WHERE — use `filters` for cross-source predicates. +- **`segments`**: dotted `source.segment_name`. Each segment is AND-ed into the effective filter of every measure whose base source matches. Segments never become a global WHERE - use `filters` for cross-source predicates. - **`order_by`**: string or `{ field, direction }`. Direction defaults to `asc`. - **`limit`**: integer row cap. ### Join resolution -You don't specify a base table. The engine infers the set of sources needed from the fields you reference and resolves the shortest join path through the catalog's declared joins. If no path exists between two sources, the query fails with a path-not-found error — check `discover_data` or `sl_discover` to see which sources are connected. +You don't specify a base table. The engine infers the set of sources needed from the fields you reference and resolves the shortest join path through the catalog's declared joins. If no path exists between two sources, the query fails with a path-not-found error - check `discover_data` or `sl_discover` to see which sources are connected. ### Worked examples -Cross-source query — engine resolves `account_health_scores → accounts ← opportunities` automatically: +Cross-source query - engine resolves `account_health_scores → accounts ← opportunities` automatically: ```json { diff --git a/packages/context/skills/sl_capture/SKILL.md b/packages/context/skills/sl_capture/SKILL.md index 9f57a858..22e55859 100644 --- a/packages/context/skills/sl_capture/SKILL.md +++ b/packages/context/skills/sl_capture/SKILL.md @@ -1,10 +1,10 @@ --- name: sl_capture -description: How to capture new reusable patterns into KTX's semantic layer — when a measure, segment, or join belongs in the catalog and how to write it generically so it stays small and useful over time. Loaded by the post-turn memory-agent only. The research agent does not write to the SL. +description: How to capture new reusable patterns into KTX's semantic layer - when a measure, segment, or join belongs in the catalog and how to write it generically so it stays small and useful over time. Loaded by the post-turn memory-agent only. The research agent does not write to the SL. callers: [memory_agent] --- -# Semantic Layer — Capture +# Semantic Layer - Capture This skill covers **when** and **how** to capture new patterns into the semantic layer. For schema reference and query grammar, load the `sl` skill first. @@ -13,8 +13,8 @@ When the current turn produces a reusable pattern (business metric, derived view ## SQL dialect The user-facing prompt includes a `Warehouse:` line under the SL Sources index -(e.g. `Warehouse: BIGQUERY`). All `expr` strings — measure expressions, segment -predicates, computed-column SQL — execute on that warehouse and must use its +(e.g. `Warehouse: BIGQUERY`). All `expr` strings - measure expressions, segment +predicates, computed-column SQL - execute on that warehouse and must use its syntax. Date arithmetic in particular varies by dialect: - **BigQuery**: `transaction_date >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)` (when the column is `TIMESTAMP`); `event_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 90 DAY)` (when `DATE`). @@ -22,7 +22,7 @@ syntax. Date arithmetic in particular varies by dialect: - **Snowflake**: `transaction_date >= dateadd(day, -90, current_timestamp())`. Match the column's manifest type (`type: time` → TIMESTAMP/DATETIME on the -warehouse) — comparing TIMESTAMP to a DATE-arithmetic result fails on +warehouse) - comparing TIMESTAMP to a DATE-arithmetic result fails on BigQuery. After every `sl_edit_source`/`sl_write_source`, the inline validator runs a `LIMIT 1` warehouse probe per measure and surfaces dialect mismatches; if you see an error trailer, fix the expression and retry rather than leaving @@ -68,12 +68,12 @@ Callers filter `region = 'US'` at query time. **Bake constants in only when the filter has named business meaning that won't change** (`enterprise_arr` for a contractually defined tier), cannot be expressed via the source's dimensions, or comes from a regulated/fixed list. **Time anchors and value lists belong in callers' filters, not in measure expressions or source SQL.** -- Anti-pattern (date anchor inlined): `expr: count(distinct case when transaction_date >= '2026-04-12' then customer_id end)` — the date will need editing every time the question shifts, and every reader has to discover it. -- Anti-pattern (value list inlined in source SQL): `WHERE product_category_1 IN ('Testosterone', 'Weight Loss', …)` — locks the source to today's catalog and blocks callers from broadening or narrowing. +- Anti-pattern (date anchor inlined): `expr: count(distinct case when transaction_date >= '2026-04-12' then customer_id end)` - the date will need editing every time the question shifts, and every reader has to discover it. +- Anti-pattern (value list inlined in source SQL): `WHERE product_category_1 IN ('Testosterone', 'Weight Loss', …)` - locks the source to today's catalog and blocks callers from broadening or narrowing. - Preferred: a generic measure (`count(distinct customer_id)`) plus either a named segment that captures the *meaning* of the anchor (`gh_new_products_since_launch`) or a query-time filter. Callers compose; the source stays small. - A date is durable to bake in only when it represents a regulatory cutover, a contractually fixed boundary, or a one-time event that reshapes how the source itself is read. -**If you create a segment whose expr matches a measure's filter, the measure MUST reference the segment via `segments: [segment_name]` rather than re-inlining the predicate.** This is the canonical pattern even with a single measure — duplicating the predicate inline defeats the purpose of naming it. +**If you create a segment whose expr matches a measure's filter, the measure MUST reference the segment via `segments: [segment_name]` rather than re-inlining the predicate.** This is the canonical pattern even with a single measure - duplicating the predicate inline defeats the purpose of naming it. Anti-pattern: ```yaml @@ -132,18 +132,18 @@ Overlay YAML may include `measures:`, `segments:`, `descriptions:`, `joins:`, `d - The metric requires per-user/per-entity derivation that cannot be expressed as a single `expr` (e.g., `EXISTS` over a time-windowed subset), OR - The metric requires multi-step CTEs whose intermediate grain is not a column in any existing source. -When an `sql` source is unavoidable, note in its `descriptions` map which SL gap forced the choice so it can be retired once the primitive ships. It must target a name NOT in the manifest — pick a distinct one (e.g. `mrr_waterfall_rollup`, not `fct_orders`). +When an `sql` source is unavoidable, note in its `descriptions` map which SL gap forced the choice so it can be retired once the primitive ships. It must target a name NOT in the manifest - pick a distinct one (e.g. `mrr_waterfall_rollup`, not `fct_orders`). ## Slim standalone sources via `inherits_columns_from` When a standalone SQL source filters or projects from a single manifest-backed base table (the common pattern for derived views like `aav_consignments` over `MARTS.CONSIGNMENTS`), set `inherits_columns_from:` to the base table's manifest key and list only column **names** in `columns:`. Compose-time enrichment fills `type`, `descriptions`, and `role` from the matching manifest column. -Discover the manifest key with `sl_discover` — pass the bare name (`CONSIGNMENTS`), the fully-qualified path (`ANALYTICS.MARTS.CONSIGNMENTS`), or any suffix; the tool resolves all forms and prints the canonical key in its output. +Discover the manifest key with `sl_discover` - pass the bare name (`CONSIGNMENTS`), the fully-qualified path (`ANALYTICS.MARTS.CONSIGNMENTS`), or any suffix; the tool resolves all forms and prints the canonical key in its output. ```yaml name: aav_consignments descriptions: - user: AAV consignments — filtered view of MARTS.CONSIGNMENTS for the auto-auction-vaulting channel. + user: AAV consignments - filtered view of MARTS.CONSIGNMENTS for the auto-auction-vaulting channel. source_type: sql sql: | SELECT CONSIGNED_ITEM_ID, CASH_ADV_AMOUNT, ALT_VALUE_COMBINED, my_derived_flag @@ -157,7 +157,7 @@ columns: - { name: CONSIGNED_ITEM_ID } # type/descriptions inherited from manifest - { name: CASH_ADV_AMOUNT } - { name: ALT_VALUE_COMBINED } - - { name: my_derived_flag, type: boolean, expr: "CASH_ADV_AMOUNT > 0", descriptions: { user: "Computed locally — has any cash advance." } } + - { name: my_derived_flag, type: boolean, expr: "CASH_ADV_AMOUNT > 0", descriptions: { user: "Computed locally - has any cash advance." } } measures: - name: total_cash_advance expr: sum(CASH_ADV_AMOUNT) @@ -165,12 +165,12 @@ measures: Rules: -- Inheritance fills only **blank** fields. If you set a `description` locally, it wins — useful when the base description is misleading in the filtered view. +- Inheritance fills only **blank** fields. If you set a `description` locally, it wins - useful when the base description is misleading in the filtered view. - A column not in the manifest (a derived/aliased column, or one from a different table in a `JOIN`) needs its own `type` and `description` declared. -- If `inherits_columns_from` doesn't resolve, the source still loads, but every column without a type triggers a validator error on the warehouse probe — `sl_discover` first to confirm the key. -- Don't use `inherits_columns_from` for sources backed by `table:` (those should be overlays — see the rule against shadowing the manifest above). +- If `inherits_columns_from` doesn't resolve, the source still loads, but every column without a type triggers a validator error on the warehouse probe - `sl_discover` first to confirm the key. +- Don't use `inherits_columns_from` for sources backed by `table:` (those should be overlays - see the rule against shadowing the manifest above). -## Refinement — replace, don't append +## Refinement - replace, don't append When the user corrects a prior answer, the existing measure is wrong by the user's own standard. Replace it, don't add a parallel measure. @@ -234,14 +234,14 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: ## Tool sequence -1. `sl_discover` — see what source files exist. -2. `sl_discover({ query: "" })` — **REQUIRED before the first write on any name**. Shows columns/joins/grain from the manifest. If the call returns a schema, you MUST write an overlay, not a standalone. Skipping this is the #1 cause of accidentally shadowing the manifest. -3. `sl_read_source({ connectionId, sourceName })` — read the raw YAML before editing. +1. `sl_discover` - see what source files exist. +2. `sl_discover({ query: "" })` - **REQUIRED before the first write on any name**. Shows columns/joins/grain from the manifest. If the call returns a schema, you MUST write an overlay, not a standalone. Skipping this is the #1 cause of accidentally shadowing the manifest. +3. `sl_read_source({ connectionId, sourceName })` - read the raw YAML before editing. 4. For modifications: `sl_edit_source({ connectionId, sourceName, yaml_edits: [{ oldText, newText, reason }] })` with exact-string replacements. `oldText` must match exactly and be unique in the file. 5. For new sources or full rewrites: `sl_write_source({ connectionId, sourceName, source })` with the full structured source definition. 6. For join discovery: use `sql_execution({connectionName: "warehouse", sql: "SELECT count(*) FROM public.orders o JOIN public.customers c ON c.id = o.customer_id LIMIT 20"})` with the target warehouse connection name and dialect-correct table names to verify the join key exists in both tables and assess cardinality before declaring the join. -7. Cross-reference knowledge: author the edge once on the **wiki** side via `sl_refs: [source_name]` in the page's front-matter. The reverse edge (wiki pages that cite an SL source) is derived automatically by the reconciler — do not add a `knowledge_refs:` field to SL YAMLs. -8. `sl_validate` — run after writing or editing to surface schema issues, duplicate measure names, and cross-source validation errors. Read-only; the writes are already committed (the squash-at-end flow will collapse them into one commit). +7. Cross-reference knowledge: author the edge once on the **wiki** side via `sl_refs: [source_name]` in the page's front-matter. The reverse edge (wiki pages that cite an SL source) is derived automatically by the reconciler - do not add a `knowledge_refs:` field to SL YAMLs. +8. `sl_validate` - run after writing or editing to surface schema issues, duplicate measure names, and cross-source validation errors. Read-only; the writes are already committed (the squash-at-end flow will collapse them into one commit). ## Editing patterns @@ -250,13 +250,13 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: - Do NOT modify existing measures or their descriptions unless the current turn explicitly corrects them. - During bundle/external ingest, include `rawPaths` on every `sl_write_source`/`sl_edit_source` call with only the raw files that directly support the SL action. -## Worked example — additive overlay +## Worked example - additive overlay Conversation: - User: "What was the average order value last quarter?" - Assistant fell back to SQL: `SELECT AVG(amount) FROM orders WHERE order_date >= ...` -Existing index: `orders [measures=0, joins=0] — candidate for enrichment`. +Existing index: `orders [measures=0, joins=0] - candidate for enrichment`. ``` sl_discover() @@ -279,9 +279,9 @@ sl_validate({ connectionId: "warehouse" }) → clean ``` -The overlay only contains `name` and `measures` — no columns, grain, or table. Those are inherited from the manifest. +The overlay only contains `name` and `measures` - no columns, grain, or table. Those are inherited from the manifest. -## Worked example — refinement (replace) +## Worked example - refinement (replace) Prior turn: - [user] "How many active users do we have per region?" @@ -307,7 +307,7 @@ sl_validate({ connectionId: "warehouse" }) If you only added a new measure, the old incorrect `active_count` would stay and future queries would keep answering the wrong question. -## Worked example — new join +## Worked example - new join Prior turn: user asked to correlate LTV with protocol count; assistant joined `fct_orders` with `fct_mau_multiprotocol` on `admin_user_id` in raw SQL. @@ -341,6 +341,6 @@ Always verify joins with `sql_execution` before adding them. - A measure whose filter matches a segment MUST reference the segment via `segments: [name]`. - Extract repeated predicates into named segments. - Use computed dimensions for derived categories. -- When the user corrects a prior answer, replace — don't append. +- When the user corrects a prior answer, replace - don't append. - Always run `sl_validate` after writing to surface issues. - If nothing is worth capturing, respond without calling any SL write tool. diff --git a/packages/context/skills/wiki_capture/SKILL.md b/packages/context/skills/wiki_capture/SKILL.md index d57a39ad..55601f99 100644 --- a/packages/context/skills/wiki_capture/SKILL.md +++ b/packages/context/skills/wiki_capture/SKILL.md @@ -1,6 +1,6 @@ --- name: wiki_capture -description: KTX's knowledge base — wiki pages for durable, reusable business knowledge. Covers capture workflow for user preferences, metric definitions, organizational conventions, and cross-references between wiki pages and semantic-layer sources. Loaded by the post-turn memory-agent only. The research agent reads wiki via `wiki_read`/`wiki_search` but does not write it. +description: KTX's knowledge base - wiki pages for durable, reusable business knowledge. Covers capture workflow for user preferences, metric definitions, organizational conventions, and cross-references between wiki pages and semantic-layer sources. Loaded by the post-turn memory-agent only. The research agent reads wiki via `wiki_read`/`wiki_search` but does not write it. callers: [memory_agent] --- @@ -8,14 +8,14 @@ callers: [memory_agent] ## Role -The knowledge base stores durable, reusable business knowledge for an analytics assistant. Each page is a self-contained rule, definition, or convention that answers "how should this concept be handled in this organization?" — written once and reused across chats. +The knowledge base stores durable, reusable business knowledge for an analytics assistant. Each page is a self-contained rule, definition, or convention that answers "how should this concept be handled in this organization?" - written once and reused across chats. Scope selection is handled by the runtime: - When user-scoped knowledge is enabled AND the caller is a chat turn, writes go to the user's **personal** scope. - When the caller is an admin-driven ingest (`sourceType: 'external_ingest'`), writes go to the **global** scope. - When user-scoped knowledge is disabled, all writes go to the global scope. -The `wiki_write` tool picks the right scope based on the session. Capture logic does not need to choose — focus on whether the content is worth capturing at all. +The `wiki_write` tool picks the right scope based on the session. Capture logic does not need to choose - focus on whether the content is worth capturing at all. ## What to capture @@ -30,8 +30,8 @@ Do NOT capture: - One-off requests ("answer under 100 words"). - Temporary instructions scoped to the current chat. - Ad-hoc formatting preferences. -- Information already present in the semantic layer (column names, join paths, measure formulas — those belong in SL). -- **Query results, snapshots, or time-bounded benchmark tables.** Numbers go stale; pasting "Oct 2025: 25%, Nov 2025: 19.9%, …" creates misinformation as soon as new data lands. Reference the SL source by name (`sl_refs`) and let future query tools pull live data — the wiki captures the *rule* (definition, exclusion, segmentation), the SL source captures the *measure*, and query execution captures the *current values*. +- Information already present in the semantic layer (column names, join paths, measure formulas - those belong in SL). +- **Query results, snapshots, or time-bounded benchmark tables.** Numbers go stale; pasting "Oct 2025: 25%, Nov 2025: 19.9%, …" creates misinformation as soon as new data lands. Reference the SL source by name (`sl_refs`) and let future query tools pull live data - the wiki captures the *rule* (definition, exclusion, segmentation), the SL source captures the *measure*, and query execution captures the *current values*. - **Interpretive narrative tied to a specific snapshot** ("M1 retention degraded sharply from Dec 2025"). The observation is anchored to data that will move; the actionable convention (e.g., "always exclude in-progress cohorts") may be worth capturing on its own, but the snapshot-specific commentary is not. If nothing is worth capturing, respond without calling any tool. @@ -40,13 +40,13 @@ If nothing is worth capturing, respond without calling any tool. 1. Read the wiki index (provided in the prompt) and decide whether the turn introduces durable knowledge. 2. **Before writing**, search for related content so cross-references are accurate: - - `discover_data` first when a page relates to data or SL concepts — find + - `discover_data` first when a page relates to data or SL concepts - find existing wiki pages, SL sources, and raw warehouse schema together. - - `wiki_search` with the topic — find related wiki pages to populate `refs`. - - `sl_discover` with the concept — if the page defines a metric (revenue, churn, retention, LTV, ARR, MRR, CAC, attribution, etc.), find matching SL sources or measures to populate `sl_refs`. If no matches, pass `sl_refs: []` so future readers know you checked. + - `wiki_search` with the topic - find related wiki pages to populate `refs`. + - `sl_discover` with the concept - if the page defines a metric (revenue, churn, retention, LTV, ARR, MRR, CAC, attribution, etc.), find matching SL sources or measures to populate `sl_refs`. If no matches, pass `sl_refs: []` so future readers know you checked. 3. If updating an existing page, `wiki_read` it first. Use the returned `structured.content` or markdown body as the exact stored text for targeted replacements; current tags, refs, and sl_refs are returned in structured metadata. 4. `wiki_write` to create or update. Prefer merging into an existing page over creating a new one. -5. `wiki_remove` only when a page is truly obsolete — not to replace stale content (update it instead). +5. `wiki_remove` only when a page is truly obsolete - not to replace stale content (update it instead). For bundle/external ingest, include `rawPaths` on every `wiki_write`/`wiki_remove` call with only the raw files that directly support that wiki action. This keeps ingest provenance tied to the actual source file, not every file in the WorkUnit. @@ -82,7 +82,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: - **Keys** are short kebab-case topic identifiers: `leads-source-filter`, `revenue-definition`, `churn-calculation`. No namespacing, no prefixes. - **Summary** is a one-line hook (≤200 chars) shown in the index. -- **Content** is concise markdown — actionable rules, not prose. +- **Content** is concise markdown - actionable rules, not prose. ``` ## [Topic Title] @@ -116,8 +116,8 @@ All three fields use REPLACE semantics on update: Two modes: -- **Full content** — pass `content` to rewrite the whole page. Use when the page structure needs to change. -- **Targeted edits** — pass `replacements: [{ oldText, newText }]` to apply exact-string replacements. Use for small updates; preserves the rest of the page. +- **Full content** - pass `content` to rewrite the whole page. Use when the page structure needs to change. +- **Targeted edits** - pass `replacements: [{ oldText, newText }]` to apply exact-string replacements. Use for small updates; preserves the rest of the page. When editing, read the page first so the edit matches exact whitespace and indentation. @@ -125,7 +125,7 @@ When editing, read the page first so the edit matches exact whitespace and inden Organization (GLOBAL) pages are read-only from a user's personal-scope session. To override a global rule for a single user, write a personal page with the **same key**. At read time the USER page wins. -## Worked example — capturing a metric with cross-references +## Worked example - capturing a metric with cross-references User says: "Going forward, the official refund rate is total refunded amount divided by total gross transaction amount." @@ -133,7 +133,7 @@ User says: "Going forward, the official refund rate is total refunded amount div wiki_list_tags() → existing tags include "finance" wiki_search({ query: "refund revenue paid orders" }) - → returns `revenue-definition` (related — defines paid-orders filter) + → returns `revenue-definition` (related - defines paid-orders filter) sl_discover({ query: "refund rate" }) → returns fct_orders (score 0.08), fct_gaap_revenue (0.06) sl_read_source({ connectionId: "warehouse", sourceName: "fct_orders" }) @@ -155,6 +155,6 @@ Search-then-write order matters. Cross-references are part of the page's identit - Read existing pages before updating them. - Prefer merging into an existing page over creating a new one. - Prefer fewer, richer pages over many thin ones. -- Write content as clear, actionable rules — not narrative prose. +- Write content as clear, actionable rules - not narrative prose. - Discover cross-references via search before writing, not after. - If nothing is worth capturing, respond without calling any tool. diff --git a/packages/context/src/connections/local-warehouse-descriptor.test.ts b/packages/context/src/connections/local-warehouse-descriptor.test.ts index 5864c833..0eee9f34 100644 --- a/packages/context/src/connections/local-warehouse-descriptor.test.ts +++ b/packages/context/src/connections/local-warehouse-descriptor.test.ts @@ -36,7 +36,13 @@ describe('localConnectionToWarehouseDescriptor', () => { }); it('returns null for non-warehouse adapters', () => { - expect(localConnectionToWarehouseDescriptor('looker', { driver: 'looker' })).toBeNull(); + expect( + localConnectionToWarehouseDescriptor('looker', { + driver: 'looker', + base_url: 'https://looker.example.com', + client_id: 'client', + }), + ).toBeNull(); }); }); @@ -48,7 +54,9 @@ describe('local connection info helpers', () => { }); it('keeps non-warehouse adapter labels for display-only local connection surfaces', () => { - expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase' })).toBe('metabase'); + expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase', api_url: 'https://metabase.example.com' })).toBe( + 'metabase', + ); expect(localConnectionTypeForConfig('missing-driver', {} as never)).toBe('unknown'); }); diff --git a/packages/context/src/connections/notion-config.ts b/packages/context/src/connections/notion-config.ts index c7068512..24dd5d4b 100644 --- a/packages/context/src/connections/notion-config.ts +++ b/packages/context/src/connections/notion-config.ts @@ -13,7 +13,20 @@ export const KTX_NOTION_ORG_KNOWLEDGE_WARNING = type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots'; -export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig { +type RawKtxNotionConnectionConfig = Extract; + +export type KtxNotionConnectionConfig = Omit< + RawKtxNotionConnectionConfig, + | 'auth_token' + | 'auth_token_ref' + | 'crawl_mode' + | 'root_page_ids' + | 'root_database_ids' + | 'root_data_source_ids' + | 'max_pages_per_run' + | 'max_knowledge_creates_per_run' + | 'max_knowledge_updates_per_run' +> & { driver: 'notion'; auth_token: string | null; auth_token_ref: string | null; @@ -24,7 +37,7 @@ export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig { max_pages_per_run: number; max_knowledge_creates_per_run: number; max_knowledge_updates_per_run: number; -} +}; export interface RedactedKtxNotionConnectionConfig { driver: 'notion'; diff --git a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index 8f583d9c..5540c991 100644 --- a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -166,7 +166,6 @@ async function writeHistoricSqlProject(project: KtxLocalProject): Promise { }); it('projects table and pattern evidence into semantic-layer and wiki retrieval surfaces', async () => { - const initialized = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + const initialized = await initKtxProject({ projectDir: join(tempDir, 'project') }); const project = await writeHistoricSqlProject(initialized); const sqlAnalysis = acceptanceSqlAnalysis(); const agentRunner = new HistoricSqlAcceptanceAgentRunner(); diff --git a/packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts b/packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts index 1139ea4d..4fdafba5 100644 --- a/packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { buildDefaultKtxProjectConfig } from '../../../project/index.js'; +import { connectionConfigSchema } from '../../../project/driver-schemas.js'; import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js'; describe('Metabase YAML source state and discovery cache', () => { @@ -21,12 +22,13 @@ describe('Metabase YAML source state and discovery cache', () => { function projectWithMetabaseMappings(mappings: Record) { return { config: { - ...buildDefaultKtxProjectConfig('metabase-cache-test'), + ...buildDefaultKtxProjectConfig(), connections: { - 'prod-metabase': { + 'prod-metabase': connectionConfigSchema.parse({ driver: 'metabase', + api_url: 'https://metabase.example.com', mappings, - }, + }), }, }, }; diff --git a/packages/context/src/ingest/local-adapters.test.ts b/packages/context/src/ingest/local-adapters.test.ts index e8cbf5a5..17269698 100644 --- a/packages/context/src/ingest/local-adapters.test.ts +++ b/packages/context/src/ingest/local-adapters.test.ts @@ -16,7 +16,7 @@ describe('local ingest adapters', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-adapters-')); const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); project = await loadKtxProject({ projectDir }); }); diff --git a/packages/context/src/ingest/local-bundle-ingest.test.ts b/packages/context/src/ingest/local-bundle-ingest.test.ts index fe781b33..79071134 100644 --- a/packages/context/src/ingest/local-bundle-ingest.test.ts +++ b/packages/context/src/ingest/local-bundle-ingest.test.ts @@ -314,11 +314,10 @@ describe('canonical local ingest', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-full-ingest-')); const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -443,7 +442,6 @@ describe('canonical local ingest', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -521,11 +519,10 @@ describe('canonical local ingest', () => { it('runs historic-SQL evidence projection through the local bundle post-processor', async () => { const projectDir = join(tempDir, 'historic-sql-project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -605,11 +602,10 @@ describe('canonical local ingest', () => { it('rejects direct Metabase scheduled pulls before requiring a local ingest LLM provider', async () => { const projectDir = join(tempDir, 'metabase-project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -637,7 +633,7 @@ describe('canonical local ingest', () => { it('runs full MetricFlow local ingest from a dbt repo fixture through the canonical runner', async () => { const projectDir = join(tempDir, 'metricflow-run-project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const fixtureDir = join(tempDir, 'metricflow-fixture'); await mkdir(join(fixtureDir, 'models'), { recursive: true }); @@ -685,7 +681,6 @@ describe('canonical local ingest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -767,7 +762,6 @@ describe('canonical local ingest', () => { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: local-mf', 'connections:', ' warehouse:', ' driver: postgres', @@ -801,11 +795,10 @@ describe('canonical local ingest', () => { it('runs scheduled Looker ingest through the canonical local runner and records SL target evidence', async () => { const projectDir = join(tempDir, 'looker-project'); - await initKtxProject({ projectDir, projectName: 'looker-runtime' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: looker-runtime', 'connections:', ' prod-looker:', ' driver: looker', diff --git a/packages/context/src/ingest/local-bundle-runtime.test.ts b/packages/context/src/ingest/local-bundle-runtime.test.ts index c6bd0539..bee28653 100644 --- a/packages/context/src/ingest/local-bundle-runtime.test.ts +++ b/packages/context/src/ingest/local-bundle-runtime.test.ts @@ -24,11 +24,10 @@ describe('createLocalBundleIngestRuntime', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-bundle-runtime-')); const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -149,7 +148,6 @@ describe('createLocalBundleIngestRuntime', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', diff --git a/packages/context/src/ingest/local-mapping-reconcile.test.ts b/packages/context/src/ingest/local-mapping-reconcile.test.ts index 4a5d2740..903ef43f 100644 --- a/packages/context/src/ingest/local-mapping-reconcile.test.ts +++ b/packages/context/src/ingest/local-mapping-reconcile.test.ts @@ -27,11 +27,12 @@ describe('local mapping yaml reconciliation bridge', () => { const project = projectWithConnections({ 'prod-metabase': { driver: 'metabase', + api_url: 'https://metabase.example.com', mappings: { databaseMappings: { '1': 'prod-warehouse' }, syncEnabled: { '1': true }, syncMode: 'ONLY', - selections: { collections: [12] }, + selections: { collections: [12], items: [] }, defaultTagNames: ['ktx'], }, }, @@ -46,6 +47,8 @@ describe('local mapping yaml reconciliation bridge', () => { const project = projectWithConnections({ 'prod-looker': { driver: 'looker', + base_url: 'https://looker.example.com', + client_id: 'client', mappings: { connectionMappings: { analytics: 'prod-warehouse' } }, }, 'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' }, diff --git a/packages/context/src/ingest/local-stage-ingest.test.ts b/packages/context/src/ingest/local-stage-ingest.test.ts index 3a0beaa5..d12ce167 100644 --- a/packages/context/src/ingest/local-stage-ingest.test.ts +++ b/packages/context/src/ingest/local-stage-ingest.test.ts @@ -17,7 +17,6 @@ async function writeWarehouseConfig(projectDir: string): Promise { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -34,7 +33,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -88,7 +86,7 @@ describe('local ingest', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-ingest-')); const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeWarehouseConfig(projectDir); project = await loadKtxProject({ projectDir }); }); @@ -574,7 +572,6 @@ describe('local ingest', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' notion-main:', ' driver: notion', diff --git a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts b/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts index 9188bc68..24a14863 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +++ b/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts @@ -15,7 +15,7 @@ describe('EntityDetailsTool', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-entity-details-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); await seedLiveDatabaseScan(); tool = new EntityDetailsTool(() => new WarehouseCatalogService({ fileStore: project.fileStore })); context = { diff --git a/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts index c2ab1f36..03340ace 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts +++ b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts @@ -11,7 +11,7 @@ describe('WarehouseCatalogService', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-warehouse-catalog-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); }); afterEach(async () => { diff --git a/packages/context/src/llm/local-config.test.ts b/packages/context/src/llm/local-config.test.ts index 5d114e12..59ad34b7 100644 --- a/packages/context/src/llm/local-config.test.ts +++ b/packages/context/src/llm/local-config.test.ts @@ -194,7 +194,7 @@ describe('local KTX embedding config', () => { it('constructs deterministic embeddings from the default project config', () => { const createKtxEmbeddingProvider = vi.fn(() => ({}) as never); const provider = createLocalKtxEmbeddingProviderFromConfig( - buildDefaultKtxProjectConfig('warehouse').ingest.embeddings, + buildDefaultKtxProjectConfig().ingest.embeddings, { createKtxEmbeddingProvider }, ); diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index fab2f076..4d01b846 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -71,7 +71,7 @@ describe('createLocalProjectMcpContextPorts', () => { } it('lists local project connections from ktx.yaml', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', @@ -84,7 +84,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('tests a local project connection through the native scan connector factory', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', @@ -120,7 +120,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('triggers canonical bundle ingest and reads status, report, and replay through MCP ports', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', }; @@ -216,7 +216,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('returns child run metadata for local Metabase fan-out triggers', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections = { 'prod-metabase': { driver: 'metabase', @@ -339,7 +339,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('writes, reads, and searches global wiki pages', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); const ports = createLocalProjectMcpContextPorts(project); await expect( @@ -383,7 +383,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('writes, lists, reads, and validates semantic-layer sources', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); const ports = createLocalProjectMcpContextPorts(project); await expect( @@ -449,7 +449,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('returns semantic-layer hybrid search metadata through local project ports', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); await writeLocalSlSource(project, { connectionId: 'warehouse', sourceName: 'orders', @@ -518,7 +518,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('returns historic SQL usage frequency and snippet through semantic-layer list search', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); await project.fileStore.writeFile( 'semantic-layer/warehouse/_schema/public.yaml', `tables: @@ -566,7 +566,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('uses configured local embeddings for semantic-layer search when available', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.ingest.embeddings = { backend: 'none', dimensions: 2 }; await writeLocalSlSource(project, { connectionId: 'warehouse', @@ -607,7 +607,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('rejects path traversal keys before touching the project directory', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); const ports = createLocalProjectMcpContextPorts(project); await expect( @@ -626,7 +626,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('uses semantic compute for validation and compile-only sl_query when supplied', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', @@ -712,7 +712,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('executes local MCP sl_query when a query executor is configured', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', @@ -770,7 +770,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('exposes detailed local ingest trigger and status ports when local ingest is enabled', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres' }; project.config.ingest.adapters = ['fake']; project.config.ingest.embeddings = { @@ -890,7 +890,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('passes local ingest pull-config options into runLocalIngest', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres' }; project.config.ingest.adapters = ['looker']; const runLocalIngest = vi.fn(async () => ({ @@ -949,7 +949,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('triggers fetch-capable local ingest without sourceDir config', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', @@ -1024,7 +1024,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); it('lists and reads only artifacts that belong to a local scan report', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', @@ -1140,6 +1140,6 @@ describe('createLocalProjectMcpContextPorts', () => { }), ).resolves.toBeNull(); await expect(ports.scan?.listArtifacts?.({ runId: 'missing' })).resolves.toBeNull(); - await expect(readFile(join(project.projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('project: warehouse'); + await expect(readFile(join(project.projectDir, 'ktx.yaml'), 'utf-8')).resolves.not.toContain('project:'); }); }); diff --git a/packages/context/src/mcp/server.test.ts b/packages/context/src/mcp/server.test.ts index 193d8f67..e02f2574 100644 --- a/packages/context/src/mcp/server.test.ts +++ b/packages/context/src/mcp/server.test.ts @@ -168,7 +168,7 @@ describe('createKtxMcpServer', () => { it('runs MCP memory_capture against a local project memory port', async () => { const tempDir = await mkdtemp(join(tmpdir(), 'ktx-mcp-local-memory-')); try { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); const agentRunner = { runLoop: async ({ toolSet, diff --git a/packages/context/src/memory/local-memory.test.ts b/packages/context/src/memory/local-memory.test.ts index 83b22146..f0b870eb 100644 --- a/packages/context/src/memory/local-memory.test.ts +++ b/packages/context/src/memory/local-memory.test.ts @@ -89,7 +89,7 @@ describe('createLocalProjectMemoryCapture', () => { }); it('captures a wiki page through the local memory agent and persists pollable status', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); const agentRunner = { runLoop: async ({ toolSet, @@ -144,7 +144,7 @@ describe('createLocalProjectMemoryCapture', () => { }); it('captures a semantic-layer source for a named local connection id', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); + const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { driver: 'postgres' }; const agentRunner = { runLoop: async ({ diff --git a/packages/context/src/project/config.test.ts b/packages/context/src/project/config.test.ts index be6d5219..164282eb 100644 --- a/packages/context/src/project/config.test.ts +++ b/packages/context/src/project/config.test.ts @@ -11,7 +11,6 @@ describe('KTX project config', () => { it.each(['status', 'replay', 'run', 'watch'])('accepts former ingest subcommand name "%s" as a connection id', (connectionId) => { expect( parseKtxProjectConfig(` -project: reserved-test connections: ${connectionId}: driver: postgres @@ -24,8 +23,7 @@ connections: }); it('builds the default standalone project config', () => { - expect(buildDefaultKtxProjectConfig('warehouse')).toEqual({ - project: 'warehouse', + expect(buildDefaultKtxProjectConfig()).toEqual({ connections: {}, storage: { state: 'sqlite', @@ -84,15 +82,14 @@ connections: }); it('round-trips through YAML with stable defaults', () => { - const serialized = serializeKtxProjectConfig(buildDefaultKtxProjectConfig('warehouse')); + const serialized = serializeKtxProjectConfig(buildDefaultKtxProjectConfig()); const parsed = parseKtxProjectConfig(serialized); - expect(serialized).toContain('project: warehouse'); + expect(serialized).not.toContain('project:'); expect(serialized).not.toContain('live-database'); expect(serialized).toContain( ' embeddings:\n backend: deterministic\n model: deterministic\n dimensions: 8', ); - expect(parsed.project).toBe('warehouse'); expect(parsed.ingest.adapters).toEqual([]); expect(parsed.ingest.embeddings).toEqual({ backend: 'deterministic', @@ -103,7 +100,6 @@ connections: it('parses and serializes setup warehouse metadata without setup progress', () => { const config = parseKtxProjectConfig(` -project: revenue setup: database_connection_ids: - warehouse @@ -126,7 +122,6 @@ connections: it('parses global direct Anthropic LLM config', () => { const config = parseKtxProjectConfig(` -project: demo llm: provider: backend: anthropic @@ -166,7 +161,6 @@ ingest: it('parses global Vertex LLM config', () => { const config = parseKtxProjectConfig(` -project: demo llm: provider: backend: vertex @@ -188,7 +182,6 @@ llm: it('parses gateway LLM, OpenAI scan embeddings, and sentence-transformers ingest embeddings', () => { const config = parseKtxProjectConfig(` -project: demo llm: provider: backend: gateway @@ -232,7 +225,6 @@ scan: it('parses scan relationship settings', () => { const config = parseKtxProjectConfig(` -project: demo scan: relationships: enabled: false @@ -273,7 +265,6 @@ scan: it('parses the scan relationship validation budget sentinel', () => { const config = parseKtxProjectConfig(` -project: demo scan: relationships: validationBudget: all @@ -285,7 +276,6 @@ scan: it('rejects out-of-range scan relationship numeric settings', () => { const yaml = ` -project: demo scan: relationships: acceptThreshold: 2 @@ -316,7 +306,6 @@ scan: it('rejects invalid scan relationship validation budget strings', () => { const yaml = ` -project: demo scan: relationships: validationBudget: infinite @@ -327,7 +316,6 @@ scan: it('rejects unsupported local LLM and embedding fields', () => { expect(() => parseKtxProjectConfig(` -project: demo ingest: llm: backend: anthropic @@ -336,7 +324,6 @@ ingest: expect(() => parseKtxProjectConfig(` -project: demo scan: enrichment: backend: gateway @@ -345,7 +332,6 @@ scan: expect(() => parseKtxProjectConfig(` -project: demo scan: enrichment: mode: llm @@ -356,7 +342,6 @@ scan: expect(() => parseKtxProjectConfig(` -project: demo ingest: embeddings: provider: gateway @@ -368,7 +353,6 @@ ingest: it('rejects gateway embedding configs', () => { expect(() => parseKtxProjectConfig(` -project: demo ingest: embeddings: backend: gateway @@ -379,7 +363,6 @@ ingest: expect(() => parseKtxProjectConfig(` -project: demo scan: enrichment: mode: llm @@ -392,9 +375,9 @@ scan: }); it('fills optional sections when a minimal config is loaded', () => { - const config = parseKtxProjectConfig('project: local\n'); + const config = parseKtxProjectConfig('{}\n'); - expect(config).toEqual(buildDefaultKtxProjectConfig('local')); + expect(config).toEqual(buildDefaultKtxProjectConfig()); expect(config.ingest.embeddings).toEqual({ backend: 'deterministic', model: 'deterministic', @@ -406,14 +389,15 @@ scan: expect(() => parseKtxProjectConfig('- nope\n')).toThrow('ktx.yaml must contain a YAML object'); }); - it('rejects configs with a missing project name', () => { - expect(() => parseKtxProjectConfig('connections: {}\n')).toThrow('ktx.yaml field "project" is required'); + it('accepts configs without a project name', () => { + expect(parseKtxProjectConfig('connections: {}\n')).toMatchObject({ + connections: {}, + }); }); it('rejects unknown top-level fields under strict mode', () => { expect(() => parseKtxProjectConfig(` -project: demo storrage: state: sqlite `), @@ -423,13 +407,12 @@ storrage: describe('validateKtxProjectConfig', () => { it('returns ok: true with no issues for a valid config', () => { - const result = validateKtxProjectConfig('project: warehouse\n'); + const result = validateKtxProjectConfig('connections: {}\n'); expect(result).toEqual({ ok: true, issues: [] }); }); it('collects every schema issue without throwing', () => { const result = validateKtxProjectConfig(` -project: "" storage: search: not-a-real-backend scan: @@ -441,7 +424,6 @@ scan: const paths = result.issues.map((issue) => issue.path); expect(paths).toEqual( expect.arrayContaining([ - 'project', 'storage.search', 'scan.relationships.acceptThreshold', ]), @@ -450,7 +432,6 @@ scan: it('attaches migration hints for known deprecated keys', () => { const result = validateKtxProjectConfig(` -project: demo ingest: llm: backend: anthropic @@ -499,18 +480,15 @@ describe('generateKtxProjectConfigJsonSchema', () => { it('exposes every top-level ktx.yaml section under properties', () => { const properties = schema.properties as Record; - expect(Object.keys(properties).sort()).toEqual( - ['agent', 'connections', 'ingest', 'llm', 'memory', 'project', 'scan', 'setup', 'storage'].sort(), - ); + expect(Object.keys(properties).sort()).toEqual(['agent', 'connections', 'ingest', 'llm', 'memory', 'scan', 'setup', 'storage'].sort()); }); - it('marks "project" as required', () => { - expect(schema.required).toEqual(expect.arrayContaining(['project'])); + it('does not require any top-level fields', () => { + expect(schema.required).toBeUndefined(); }); it('carries .describe() text on top-level fields', () => { const properties = schema.properties as Record; - expect(properties.project?.description).toMatch(/Project identifier/); expect(properties.llm?.description).toMatch(/LLM/); expect(properties.scan?.description).toMatch(/Schema-scan/); }); @@ -531,4 +509,11 @@ describe('generateKtxProjectConfigJsonSchema', () => { const relationships = scan?.properties?.relationships as { properties?: Record }; expect(relationships?.properties?.acceptThreshold?.description).toMatch(/auto-accepted/); }); + + it('emits the mappings shapes under connections', () => { + const serialized = JSON.stringify(schema); + expect(serialized).toContain('databaseMappings'); + expect(serialized).toContain('connectionMappings'); + expect(serialized).toContain('expectedLookerConnectionName'); + }); }); diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index 55da5413..178721c4 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -1,6 +1,7 @@ import { KTX_MODEL_ROLES } from '@ktx/llm'; import YAML from 'yaml'; import * as z from 'zod'; +import { connectionConfigSchema } from './driver-schemas.js'; const KTX_LLM_BACKENDS = ['none', 'anthropic', 'vertex', 'gateway'] as const; const KTX_EMBEDDING_BACKENDS = ['none', 'deterministic', 'openai', 'sentence-transformers'] as const; @@ -206,12 +207,7 @@ const storageSchema = z }) .describe('Storage backends and commit policy for KTX state and search indexes.'); -const connectionSchema = z - .looseObject({ - driver: z.string().min(1).optional().describe('Connector driver identifier (e.g. "postgres", "bigquery", "snowflake").'), - url: z.string().optional().describe('Connection URL or DSN. Format depends on the driver; may contain environment-variable references.'), - }) - .describe('A single database/connector connection entry. Additional driver-specific fields are accepted and passed through.'); +const connectionSchema = connectionConfigSchema; const agentSchema = z .strictObject({ @@ -242,11 +238,6 @@ const memorySchema = z const ktxProjectConfigSchema = z .strictObject({ - project: z - .string({ error: 'ktx.yaml field "project" is required' }) - .trim() - .min(1, 'ktx.yaml field "project" is required') - .describe('Project identifier; used in logs, ktx state files, and as the default workspace name.'), setup: setupSchema.optional().describe('Setup-wizard state. Written by `ktx setup`; may be omitted.'), connections: z .record(z.string(), connectionSchema) @@ -336,8 +327,8 @@ function formatZodError(error: z.ZodError, input: unknown): string { .join('\n'); } -export function buildDefaultKtxProjectConfig(projectName = 'ktx-project'): KtxProjectConfig { - return ktxProjectConfigSchema.parse({ project: projectName }); +export function buildDefaultKtxProjectConfig(): KtxProjectConfig { + return ktxProjectConfigSchema.parse({}); } export function parseKtxProjectConfig(raw: string): KtxProjectConfig { diff --git a/packages/context/src/project/driver-schemas.test.ts b/packages/context/src/project/driver-schemas.test.ts new file mode 100644 index 00000000..89862546 --- /dev/null +++ b/packages/context/src/project/driver-schemas.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from 'vitest'; +import { connectionConfigSchema } from './driver-schemas.js'; + +describe('connectionConfigSchema (driver discriminated union)', () => { + it.each([ + ['postgres', 'postgres://user:pass@host:5432/db'], // pragma: allowlist secret + ['postgresql', 'postgresql://user:pass@host:5432/db'], // pragma: allowlist secret + ['mysql', 'mysql://user:pass@host:3306/db'], // pragma: allowlist secret + ['snowflake', 'snowflake://account/db'], + ['bigquery', 'bigquery://project/dataset'], + ['sqlite', 'sqlite:///tmp/db.sqlite'], + ['clickhouse', 'clickhouse://host:8123/db'], + ['sqlserver', 'sqlserver://host:1433;database=db'], + ])('parses %s warehouse connection', (driver, url) => { + expect(connectionConfigSchema.parse({ driver, url })).toMatchObject({ driver, url }); + }); + + it('preserves unknown warehouse fields via looseObject passthrough', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'postgres', + url: 'postgres://x', + historicSql: { enabled: true }, + context: { queryHistory: { enabled: false } }, + }); + expect(parsed).toMatchObject({ + driver: 'postgres', + historicSql: { enabled: true }, + context: { queryHistory: { enabled: false } }, + }); + }); + + it('rejects an unknown driver', () => { + expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow(); + }); +}); + +describe('connectionConfigSchema - context source drivers with mappings', () => { + it('parses a metabase connection with mappings', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'metabase', + api_url: 'https://metabase.example.com', + api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret + mappings: { + databaseMappings: { '3': 'prod-warehouse' }, + syncEnabled: { '3': true }, + syncMode: 'ONLY', + }, + }); + expect(parsed).toMatchObject({ + driver: 'metabase', + api_url: 'https://metabase.example.com', + mappings: { + databaseMappings: { '3': 'prod-warehouse' }, + syncMode: 'ONLY', + }, + }); + }); + + it('parses a looker connection with connectionMappings', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'looker', + base_url: 'https://looker.example.com', + client_id: 'abc', + client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret + mappings: { connectionMappings: { bigquery_prod: 'wh' } }, + }); + expect(parsed.mappings).toEqual({ connectionMappings: { bigquery_prod: 'wh' } }); + }); + + it('parses a lookml connection with expectedLookerConnectionName', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'lookml', + repoUrl: 'https://github.com/acme/looker.git', + branch: 'main', + mappings: { expectedLookerConnectionName: 'bigquery_prod' }, + }); + expect(parsed.mappings).toEqual({ expectedLookerConnectionName: 'bigquery_prod' }); + }); + + it('rejects metabase mapping with non-integer database key', () => { + expect(() => + connectionConfigSchema.parse({ + driver: 'metabase', + api_url: 'https://x', + mappings: { databaseMappings: { abc: 'wh' } }, + }), + ).toThrow(); + }); +}); + +describe('connectionConfigSchema - notion / dbt / metricflow', () => { + it('parses a notion connection with selected_roots crawl', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['abc', 'def'], + max_pages_per_run: 500, + }); + expect(parsed).toMatchObject({ + driver: 'notion', + crawl_mode: 'selected_roots', + root_page_ids: ['abc', 'def'], + max_pages_per_run: 500, + }); + }); + + it('rejects notion with unknown crawl_mode', () => { + expect(() => + connectionConfigSchema.parse({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'everything', + }), + ).toThrow(); + }); + + it('parses a dbt connection from a local source_dir', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'dbt', + source_dir: '/tmp/dbt-project', + target: 'dev', + }); + expect(parsed).toMatchObject({ driver: 'dbt', source_dir: '/tmp/dbt-project', target: 'dev' }); + }); + + it('parses a metricflow connection with nested config', () => { + const parsed = connectionConfigSchema.parse({ + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/sl.git', + branch: 'main', + }, + }); + expect(parsed).toMatchObject({ + driver: 'metricflow', + metricflow: { repoUrl: 'https://github.com/acme/sl.git' }, + }); + }); +}); diff --git a/packages/context/src/project/driver-schemas.ts b/packages/context/src/project/driver-schemas.ts new file mode 100644 index 00000000..1815975d --- /dev/null +++ b/packages/context/src/project/driver-schemas.ts @@ -0,0 +1,205 @@ +import * as z from 'zod'; +import { + lookerMappingsSchema, + lookmlMappingsSchema, + metabaseMappingsSchema, +} from './mappings-yaml-schema.js'; + +const warehouseDrivers = [ + 'postgres', + 'postgresql', + 'mysql', + 'snowflake', + 'bigquery', + 'sqlite', + 'clickhouse', + 'sqlserver', +] as const; + +type WarehouseDriver = (typeof warehouseDrivers)[number]; + +function warehouseConnectionSchema(driver: Driver) { + return z + .looseObject({ + driver: z.literal(driver), + url: z + .string() + .min(1) + .optional() + .describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'), + }) + .describe( + `${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`, + ); +} + +const warehouseConnectionSchemas = [ + warehouseConnectionSchema('postgres'), + warehouseConnectionSchema('postgresql'), + warehouseConnectionSchema('mysql'), + warehouseConnectionSchema('snowflake'), + warehouseConnectionSchema('bigquery'), + warehouseConnectionSchema('sqlite'), + warehouseConnectionSchema('clickhouse'), + warehouseConnectionSchema('sqlserver'), +] as const; + +const positiveIntKeyMessage = (field: string) => `${field} keys must be positive-integer strings (e.g. "1", "42")`; + +const positiveIntKeyRegex = /^[1-9]\d*$/; + +const metabaseMappingsStrictSchema = metabaseMappingsSchema.superRefine((value, ctx) => { + for (const key of Object.keys(value.databaseMappings ?? {})) { + if (!positiveIntKeyRegex.test(key)) { + ctx.addIssue({ + code: 'custom', + path: ['databaseMappings', key], + message: positiveIntKeyMessage('databaseMappings'), + }); + } + } + for (const key of Object.keys(value.syncEnabled ?? {})) { + if (!positiveIntKeyRegex.test(key)) { + ctx.addIssue({ + code: 'custom', + path: ['syncEnabled', key], + message: positiveIntKeyMessage('syncEnabled'), + }); + } + } +}); + +const metabaseConnectionSchema = z + .looseObject({ + driver: z.literal('metabase'), + api_url: z.string().url().describe('Metabase instance API URL (e.g. https://metabase.example.com).'), + api_key: z.string().min(1).optional().describe('Literal Metabase API key. Prefer api_key_ref for safety.'), + api_key_ref: z + .string() + .min(1) + .optional() + .describe('Reference to Metabase API key (e.g. env:METABASE_API_KEY or file:/path).'), + network_proxy: z.looseObject({}).optional().describe('Optional network proxy configuration (snake_case form).'), + networkProxy: z.looseObject({}).optional().describe('Optional network proxy configuration (camelCase form).'), + mappings: metabaseMappingsStrictSchema + .optional() + .describe('Metabase database-to-warehouse mappings and sync configuration.'), + }) + .describe('Metabase context-source connection.'); + +const lookerConnectionSchema = z + .looseObject({ + driver: z.literal('looker'), + base_url: z.string().url().describe('Looker instance base URL (e.g. https://looker.example.com).'), + client_id: z.string().min(1).describe('Looker OAuth client ID.'), + client_secret: z.string().min(1).optional().describe('Literal Looker OAuth client secret. Prefer client_secret_ref.'), + client_secret_ref: z + .string() + .min(1) + .optional() + .describe('Reference to Looker OAuth client secret (e.g. env:LOOKER_CLIENT_SECRET).'), + mappings: lookerMappingsSchema.optional().describe('Looker connection-name to KTX warehouse mappings.'), + }) + .describe('Looker context-source connection.'); + +const lookmlConnectionSchema = z + .looseObject({ + driver: z.literal('lookml'), + repoUrl: z + .string() + .min(1) + .describe('Git URL of the LookML project (https, ssh, or file:). Field is camelCase by convention.'), + branch: z.string().min(1).optional().describe('Git branch (default "main" downstream).'), + path: z.string().optional().describe('Subdirectory within the repo when the LookML project lives in a monorepo.'), + auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos (e.g. env:GITHUB_TOKEN).'), + mappings: lookmlMappingsSchema.optional().describe('LookML expected-connection mapping for ingest gating.'), + }) + .describe('LookML context-source connection.'); + +const notionConnectionSchema = z + .looseObject({ + driver: z.literal('notion'), + auth_token: z.string().min(1).optional().describe('Literal Notion integration token. Prefer auth_token_ref.'), + auth_token_ref: z + .string() + .min(1) + .optional() + .describe('Reference to Notion integration token (e.g. env:NOTION_TOKEN).'), + crawl_mode: z + .enum(['selected_roots', 'all_accessible']) + .optional() + .describe( + 'Crawl scope. "selected_roots" requires at least one of root_page_ids, root_database_ids, root_data_source_ids.', + ), + root_page_ids: z.array(z.string().min(1)).optional().describe('Notion page IDs to crawl when crawl_mode is selected_roots.'), + root_database_ids: z + .array(z.string().min(1)) + .optional() + .describe('Notion database IDs to crawl when crawl_mode is selected_roots.'), + root_data_source_ids: z + .array(z.string().min(1)) + .optional() + .describe('Notion data source IDs to crawl when crawl_mode is selected_roots.'), + max_pages_per_run: z + .number() + .int() + .min(1) + .max(10000) + .optional() + .describe('Maximum Notion pages fetched in a single ingest run.'), + max_knowledge_creates_per_run: z + .number() + .int() + .min(0) + .max(25) + .optional() + .describe('Maximum new wiki pages created per run.'), + max_knowledge_updates_per_run: z + .number() + .int() + .min(0) + .max(100) + .optional() + .describe('Maximum existing wiki pages updated per run.'), + }) + .describe('Notion context-source connection.'); + +const dbtConnectionSchema = z + .looseObject({ + driver: z.literal('dbt'), + source_dir: z.string().min(1).optional().describe('Absolute or project-relative path to a local dbt project.'), + repo_url: z.string().min(1).optional().describe('Git URL of the dbt project (https, ssh, or file:).'), + branch: z.string().min(1).optional().describe('Git branch when using repo_url.'), + path: z.string().optional().describe('Subdirectory within the repo when the dbt project lives in a monorepo.'), + auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'), + profiles_path: z.string().optional().describe('Override path to dbt profiles.yml.'), + target: z.string().min(1).optional().describe('dbt target name (e.g. dev, prod).'), + project_name: z.string().min(1).optional().describe('Override auto-detected dbt project name.'), + }) + .describe('dbt context-source connection.'); + +const metricflowConnectionSchema = z + .looseObject({ + driver: z.literal('metricflow'), + metricflow: z + .looseObject({ + repoUrl: z.string().min(1).describe('Git URL of the MetricFlow / SL project.'), + branch: z.string().min(1).optional().describe('Git branch (default "main").'), + path: z.string().optional().describe('Subdirectory within the repo when the SL config lives in a monorepo.'), + auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'), + }) + .describe('Nested MetricFlow configuration block.'), + }) + .describe('MetricFlow / SL context-source connection.'); + +export const connectionConfigSchema = z.discriminatedUnion('driver', [ + ...warehouseConnectionSchemas, + metabaseConnectionSchema, + lookerConnectionSchema, + lookmlConnectionSchema, + notionConnectionSchema, + dbtConnectionSchema, + metricflowConnectionSchema, +]); + +export type KtxConnectionConfig = z.infer; diff --git a/packages/context/src/project/index.ts b/packages/context/src/project/index.ts index 96e4d366..a0c08767 100644 --- a/packages/context/src/project/index.ts +++ b/packages/context/src/project/index.ts @@ -15,6 +15,7 @@ export { serializeKtxProjectConfig, validateKtxProjectConfig, } from './config.js'; +export type { KtxConnectionConfig } from './driver-schemas.js'; export type { LocalGitFileStoreDeps } from './local-git-file-store.js'; export { LocalGitFileStore } from './local-git-file-store.js'; export { ktxLocalStateDbPath } from './local-state-db.js'; diff --git a/packages/context/src/project/mappings-yaml-schema.test.ts b/packages/context/src/project/mappings-yaml-schema.test.ts index 5497bba9..f7001a70 100644 --- a/packages/context/src/project/mappings-yaml-schema.test.ts +++ b/packages/context/src/project/mappings-yaml-schema.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; import { + lookerMappingsSchema, + lookmlMappingsSchema, + metabaseMappingsSchema, parseConnectionMappingBootstrap, parseLookmlMappingBootstrap, parseLookerMappingBootstrap, @@ -82,4 +85,17 @@ describe('ktx.yaml mapping bootstrap schema', () => { }), ).toMatchObject({ adapter: 'looker', connectionId: 'prod-looker' }); }); + + it('exports mapping shapes that parse documented examples', () => { + expect(metabaseMappingsSchema.parse({ databaseMappings: { '1': 'wh' } })).toMatchObject({ + databaseMappings: { '1': 'wh' }, + syncMode: 'ALL', + }); + expect(lookerMappingsSchema.parse({ connectionMappings: { x: 'wh' } })).toEqual({ + connectionMappings: { x: 'wh' }, + }); + expect(lookmlMappingsSchema.parse({ expectedLookerConnectionName: 'x' })).toEqual({ + expectedLookerConnectionName: 'x', + }); + }); }); diff --git a/packages/context/src/project/mappings-yaml-schema.ts b/packages/context/src/project/mappings-yaml-schema.ts index 67f14627..43ad6515 100644 --- a/packages/context/src/project/mappings-yaml-schema.ts +++ b/packages/context/src/project/mappings-yaml-schema.ts @@ -1,5 +1,4 @@ import * as z from 'zod'; -import type { KtxProjectConnectionConfig } from './config.js'; const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']); const positiveIntegerValueSchema = z.number().int().positive(); @@ -11,24 +10,48 @@ const metabaseSelectionsSchema = z items: z.array(positiveIntegerValueSchema).default([]), }); -const metabaseMappingsSchema = z +export const metabaseMappingsSchema = z .object({ - databaseMappings: z.record(z.string(), stringTargetSchema).default({}), - syncEnabled: z.record(z.string(), z.boolean()).default({}), - syncMode: metabaseSyncModeSchema.default('ALL'), - selections: metabaseSelectionsSchema.default({ collections: [], items: [] }), - defaultTagNames: z.array(z.string().min(1)).default([]), - }); + databaseMappings: z + .record(z.string(), stringTargetSchema) + .default({}) + .describe('Map of Metabase database ID (positive integer string) to KTX connection ID. Use null to explicitly unmap.'), + syncEnabled: z + .record(z.string(), z.boolean()) + .default({}) + .describe('Per-Metabase-database sync toggle, keyed by Metabase database ID string.'), + syncMode: metabaseSyncModeSchema + .default('ALL') + .describe('Sync scope: ALL ingests every mapped DB; ONLY restricts to syncEnabled=true; EXCEPT excludes syncEnabled=true.'), + selections: metabaseSelectionsSchema + .default({ collections: [], items: [] }) + .describe('Optional Metabase collection and item IDs to scope ingest.'), + defaultTagNames: z + .array(z.string().min(1)) + .default([]) + .describe('Default tag names applied to ingested Metabase artifacts.'), + }) + .describe('Metabase database-to-warehouse mapping and sync configuration.'); -const lookerMappingsSchema = z +export const lookerMappingsSchema = z .object({ - connectionMappings: z.record(z.string().min(1), stringTargetSchema).default({}), - }); + connectionMappings: z + .record(z.string().min(1), stringTargetSchema) + .default({}) + .describe('Map of Looker connection name to KTX connection ID. Use null to explicitly unmap.'), + }) + .describe('Looker connection-to-warehouse mapping configuration.'); -const lookmlMappingsSchema = z +export const lookmlMappingsSchema = z .object({ - expectedLookerConnectionName: z.string().min(1).nullable().default(null), - }); + expectedLookerConnectionName: z + .string() + .min(1) + .nullable() + .default(null) + .describe('Looker connection name that LookML models must declare; mismatches block sl_write_source at ingest time.'), + }) + .describe('LookML connection-name expectation for ingest gating.'); export type MetabaseMappingBootstrap = { adapter: 'metabase'; @@ -54,6 +77,11 @@ export type LookmlMappingBootstrap = { export type ConnectionMappingBootstrap = MetabaseMappingBootstrap | LookerMappingBootstrap | LookmlMappingBootstrap; +type MappingConnectionInput = Record & { + driver?: unknown; + mappings?: unknown; +}; + function recordValue(value: unknown): Record { return typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record) : {}; } @@ -66,13 +94,13 @@ function assertPositiveIntegerKeys(field: string, record: Record { const result = await initKtxProject({ projectDir, - projectName: 'warehouse', authorName: 'Agent', authorEmail: 'agent@example.com', }); expect(result.projectDir).toBe(projectDir); - expect(result.config.project).toBe('warehouse'); expect(result.commitHash).toMatch(/^[0-9a-f]{40}$/); - await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('project: warehouse'); + await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.not.toContain('project:'); const gitignore = await readFile(join(projectDir, '.ktx/.gitignore'), 'utf-8'); expect(gitignore).toContain('cache/'); expect(gitignore).toContain('db.sqlite'); @@ -46,7 +44,7 @@ describe('KTX local project runtime', () => { it('loads an initialized project with a working file store', async () => { const projectDir = join(tempDir, 'warehouse'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const loaded = await loadKtxProject({ projectDir }); await loaded.fileStore.writeFile( @@ -57,7 +55,6 @@ describe('KTX local project runtime', () => { 'Add revenue page', ); - expect(loaded.config.project).toBe('warehouse'); await expect(loaded.fileStore.readFile('wiki/global/revenue.md')).resolves.toMatchObject({ content: '# Revenue\n', }); @@ -65,16 +62,12 @@ describe('KTX local project runtime', () => { it('rejects reinitializing an existing project unless force is set', async () => { const projectDir = join(tempDir, 'warehouse'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); - await expect(initKtxProject({ projectDir, projectName: 'warehouse' })).rejects.toThrow( - 'Project already contains ktx.yaml', - ); + await expect(initKtxProject({ projectDir })).rejects.toThrow('Project already contains ktx.yaml'); - await expect(initKtxProject({ projectDir, projectName: 'warehouse-v2', force: true })).resolves.toMatchObject({ - config: { - project: 'warehouse-v2', - }, + await expect(initKtxProject({ projectDir, force: true })).resolves.toMatchObject({ + configPath: join(projectDir, 'ktx.yaml'), }); }); }); diff --git a/packages/context/src/project/project.ts b/packages/context/src/project/project.ts index 50f89262..572ac325 100644 --- a/packages/context/src/project/project.ts +++ b/packages/context/src/project/project.ts @@ -7,7 +7,6 @@ import { LocalGitFileStore } from './local-git-file-store.js'; export interface InitKtxProjectOptions { projectDir: string; - projectName?: string; force?: boolean; authorName?: string; authorEmail?: string; @@ -101,7 +100,7 @@ async function createRuntime( export async function initKtxProject(options: InitKtxProjectOptions): Promise { const projectDir = resolve(options.projectDir); - const projectName = options.projectName?.trim() || basename(projectDir) || 'ktx-project'; + const projectName = basename(projectDir) || 'ktx-project'; const authorName = options.authorName ?? 'ktx'; const authorEmail = options.authorEmail ?? 'ktx@example.com'; const logger = options.logger ?? noopLogger; @@ -112,7 +111,7 @@ export async function initKtxProject(options: InitKtxProjectOptions): Promise { }); it('sets setup database connection ids without duplicates', () => { - const config = buildDefaultKtxProjectConfig('warehouse'); + const config = buildDefaultKtxProjectConfig(); const withDatabases = setKtxSetupDatabaseConnectionIds(config, ['warehouse', 'analytics', 'warehouse']); diff --git a/packages/context/src/scan/local-enrichment-artifacts.test.ts b/packages/context/src/scan/local-enrichment-artifacts.test.ts index 8e0c25fd..068e6cb1 100644 --- a/packages/context/src/scan/local-enrichment-artifacts.test.ts +++ b/packages/context/src/scan/local-enrichment-artifacts.test.ts @@ -231,7 +231,6 @@ describe('writeLocalScanEnrichmentArtifacts', () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-enrichment-artifacts-')); project = await initKtxProject({ projectDir: join(tempDir, 'project'), - projectName: 'warehouse', }); }); diff --git a/packages/context/src/scan/local-enrichment.test.ts b/packages/context/src/scan/local-enrichment.test.ts index 62a3903d..f0ddd448 100644 --- a/packages/context/src/scan/local-enrichment.test.ts +++ b/packages/context/src/scan/local-enrichment.test.ts @@ -371,7 +371,7 @@ describe('local scan enrichment', () => { }, }, relationshipSettings: { - ...buildDefaultKtxProjectConfig('warehouse').scan.relationships, + ...buildDefaultKtxProjectConfig().scan.relationships, llmProposals: false, maxLlmTablesPerBatch: 40, }, @@ -383,7 +383,7 @@ describe('local scan enrichment', () => { it('skips relationship detection when scan relationships are disabled', async () => { const settings = { - ...buildDefaultKtxProjectConfig('warehouse').scan.relationships, + ...buildDefaultKtxProjectConfig().scan.relationships, enabled: false, }; const result = await runLocalScanEnrichment({ @@ -509,7 +509,7 @@ describe('local scan enrichment', () => { })), }; const settings = { - ...buildDefaultKtxProjectConfig('test').scan.relationships, + ...buildDefaultKtxProjectConfig().scan.relationships, enabled: false, }; diff --git a/packages/context/src/scan/local-enrichment.ts b/packages/context/src/scan/local-enrichment.ts index c820b8a7..e6a9976b 100644 --- a/packages/context/src/scan/local-enrichment.ts +++ b/packages/context/src/scan/local-enrichment.ts @@ -538,8 +538,7 @@ export async function runLocalScanEnrichment( const now = input.now ?? (() => new Date()); const state = completedKtxScanEnrichmentStateSummary(); const syncId = input.syncId ?? input.context.runId; - const relationshipSettings = - input.relationshipSettings ?? buildDefaultKtxProjectConfig(input.connectionId).scan.relationships; + const relationshipSettings = input.relationshipSettings ?? buildDefaultKtxProjectConfig().scan.relationships; const inputHash = computeKtxScanEnrichmentInputHash({ snapshot, mode: input.mode, diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/context/src/scan/local-scan.test.ts index 6e9076c6..40c8e225 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/context/src/scan/local-scan.test.ts @@ -105,7 +105,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -123,7 +122,6 @@ async function writeDatabaseConfigWithoutIngestAdapters(projectDir: string): Pro await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -184,7 +182,7 @@ describe('local scan', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-scan-')); const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeLiveDatabaseConfig(projectDir); project = await loadKtxProject({ projectDir }); }); @@ -1037,7 +1035,6 @@ describe('local scan', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: postgres', @@ -1393,7 +1390,6 @@ describe('local scan', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -1425,7 +1421,6 @@ describe('local scan', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: mysql', @@ -1457,7 +1452,6 @@ describe('local scan', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: clickhouse', @@ -1492,7 +1486,6 @@ describe('local scan', () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlserver', diff --git a/packages/context/src/scan/local-structural-artifacts.test.ts b/packages/context/src/scan/local-structural-artifacts.test.ts index 653c3b53..e8089158 100644 --- a/packages/context/src/scan/local-structural-artifacts.test.ts +++ b/packages/context/src/scan/local-structural-artifacts.test.ts @@ -13,7 +13,6 @@ describe('readLocalScanStructuralSnapshot', () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-structural-artifacts-')); project = await initKtxProject({ projectDir: join(tempDir, 'project'), - projectName: 'warehouse', }); }); diff --git a/packages/context/src/scan/relationship-artifacts.test.ts b/packages/context/src/scan/relationship-artifacts.test.ts index c366f1b2..f66de0c2 100644 --- a/packages/context/src/scan/relationship-artifacts.test.ts +++ b/packages/context/src/scan/relationship-artifacts.test.ts @@ -19,7 +19,6 @@ async function writeWarehouseConfig(projectDir: string): Promise { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', @@ -67,7 +66,7 @@ function liveDatabaseAdapter(): SourceAdapter { } async function createLiveDatabaseRun(projectDir: string, runId: string) { - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeWarehouseConfig(projectDir); const project = await loadKtxProject({ projectDir }); await runLocalStageOnlyIngest({ @@ -283,7 +282,7 @@ describe('local scan relationship artifact reader', () => { it('returns null when the scan run has no report', async () => { const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-run-')); try { - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); const project = await loadKtxProject({ projectDir }); await expect(readLocalScanRelationshipArtifacts(project, 'missing-run')).resolves.toBeNull(); diff --git a/packages/context/src/scan/relationship-discovery.test.ts b/packages/context/src/scan/relationship-discovery.test.ts index 5d958b2d..4e83227b 100644 --- a/packages/context/src/scan/relationship-discovery.test.ts +++ b/packages/context/src/scan/relationship-discovery.test.ts @@ -243,7 +243,7 @@ function llmProvider(): KtxLlmProvider { } function relationshipSettings() { - return buildDefaultKtxProjectConfig('warehouse').scan.relationships; + return buildDefaultKtxProjectConfig().scan.relationships; } function llmOnlyRelationshipSnapshot(): KtxSchemaSnapshot { @@ -557,7 +557,7 @@ describe('production relationship discovery', () => { `); const settings = { - ...buildDefaultKtxProjectConfig('warehouse').scan.relationships, + ...buildDefaultKtxProjectConfig().scan.relationships, acceptThreshold: 0.99, reviewThreshold: 0.55, }; @@ -633,7 +633,7 @@ describe('production relationship discovery', () => { schema: snapshotToKtxEnrichedSchema(richSnapshot), context: { runId: 'candidate-cap' }, settings: { - ...buildDefaultKtxProjectConfig('warehouse').scan.relationships, + ...buildDefaultKtxProjectConfig().scan.relationships, maxCandidatesPerColumn: 1, }, }); diff --git a/packages/context/src/scan/relationship-review-apply.test.ts b/packages/context/src/scan/relationship-review-apply.test.ts index 7a8e597f..268c9598 100644 --- a/packages/context/src/scan/relationship-review-apply.test.ts +++ b/packages/context/src/scan/relationship-review-apply.test.ts @@ -202,7 +202,6 @@ async function projectWithDecisions( const tempDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-apply-')); const project = await initKtxProject({ projectDir: join(tempDir, 'project'), - projectName: 'warehouse', }); await project.fileStore.writeFile( 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', diff --git a/packages/context/src/scan/relationship-review-decisions.test.ts b/packages/context/src/scan/relationship-review-decisions.test.ts index 30277c57..979c863b 100644 --- a/packages/context/src/scan/relationship-review-decisions.test.ts +++ b/packages/context/src/scan/relationship-review-decisions.test.ts @@ -19,11 +19,10 @@ async function writeProjectFile(projectDir: string, relativePath: string, conten } async function createProject(projectDir: string): Promise { - await initKtxProject({ projectDir, projectName: 'warehouse' }); + await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite', diff --git a/packages/context/src/search/backend-conformance.test.ts b/packages/context/src/search/backend-conformance.test.ts index 95858486..31519c8b 100644 --- a/packages/context/src/search/backend-conformance.test.ts +++ b/packages/context/src/search/backend-conformance.test.ts @@ -241,7 +241,7 @@ describe('SQLite hybrid search backend conformance', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-search-conformance-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); dbPath = join(tempDir, '.ktx', 'db.sqlite'); }); diff --git a/packages/context/src/sl/local-query.test.ts b/packages/context/src/sl/local-query.test.ts index 4105c9f6..2852b35a 100644 --- a/packages/context/src/sl/local-query.test.ts +++ b/packages/context/src/sl/local-query.test.ts @@ -13,7 +13,7 @@ describe('compileLocalSlQuery', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-query-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', diff --git a/packages/context/src/sl/local-sl.test.ts b/packages/context/src/sl/local-sl.test.ts index 4c9dfe1a..00c00874 100644 --- a/packages/context/src/sl/local-sl.test.ts +++ b/packages/context/src/sl/local-sl.test.ts @@ -51,7 +51,7 @@ describe('local semantic-layer helpers', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-sl-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); }); afterEach(async () => { diff --git a/packages/context/src/sl/pglite-sl-search-prototype.test.ts b/packages/context/src/sl/pglite-sl-search-prototype.test.ts index 0c599dca..29c81062 100644 --- a/packages/context/src/sl/pglite-sl-search-prototype.test.ts +++ b/packages/context/src/sl/pglite-sl-search-prototype.test.ts @@ -169,7 +169,7 @@ describe('PGlite semantic-layer search prototype', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-pglite-sl-prototype-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); project.config.ingest.embeddings.dimensions = 3; pgliteDataDir = join(tempDir, 'pglite-search'); port = await allocatePort(); diff --git a/packages/context/src/sl/sl-dictionary-profile.test.ts b/packages/context/src/sl/sl-dictionary-profile.test.ts index 756cb81a..21400a71 100644 --- a/packages/context/src/sl/sl-dictionary-profile.test.ts +++ b/packages/context/src/sl/sl-dictionary-profile.test.ts @@ -11,7 +11,7 @@ describe('loadLatestSlDictionaryEntries', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-sl-dictionary-profile-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); }); afterEach(async () => { diff --git a/packages/context/src/wiki/local-knowledge.test.ts b/packages/context/src/wiki/local-knowledge.test.ts index 09d61a3c..122e56b5 100644 --- a/packages/context/src/wiki/local-knowledge.test.ts +++ b/packages/context/src/wiki/local-knowledge.test.ts @@ -28,7 +28,7 @@ describe('local knowledge helpers', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-knowledge-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + project = await initKtxProject({ projectDir: join(tempDir, 'project') }); }); afterEach(async () => { diff --git a/python/ktx-sl/AGENTS.md b/python/ktx-sl/AGENTS.md index 591ed9da..b9b54f18 100644 --- a/python/ktx-sl/AGENTS.md +++ b/python/ktx-sl/AGENTS.md @@ -1,6 +1,6 @@ # Semantic Layer Engine -Python semantic layer that generates SQL from structured JSON queries. No `from` clause — sources are inferred from fully-qualified field names (`source.column`). +Python semantic layer that generates SQL from structured JSON queries. No `from` clause - sources are inferred from fully-qualified field names (`source.column`). ## Quick Start @@ -16,7 +16,7 @@ Use `--model` to pass a self-contained YAML model (list of source definitions) i ### 1. Create an inline model file ```yaml -# /tmp/model.yaml — a YAML list of source definitions +# /tmp/model.yaml - a YAML list of source definitions - name: orders table: public.orders grain: [id] @@ -119,9 +119,9 @@ uv run python -m semantic_layer.cli --model /tmp/model.yaml \ ## Coding Guidelines -### Expression handling — always use sqlglot AST, never regex on SQL +### Expression handling - always use sqlglot AST, never regex on SQL -- **Parse expressions** with `sqlglot.parse_one(f"SELECT {expr}")` and walk/transform the AST. Never use `str.replace()`, `re.sub()`, or string splitting on SQL fragments — these corrupt string literals, aliases, and nested expressions. +- **Parse expressions** with `sqlglot.parse_one(f"SELECT {expr}")` and walk/transform the AST. Never use `str.replace()`, `re.sub()`, or string splitting on SQL fragments - these corrupt string literals, aliases, and nested expressions. - **Quote reserved words first**: always call `quote_reserved_identifiers(expr)` before passing to `sqlglot.parse_one()`. Column/source names like `group`, `key`, `order` will fail to parse otherwise. - **Use the parse cache** in `parser.py` (`ExpressionParser._parse_as_select()`) for read-only AST walks. Direct `sqlglot.parse_one()` calls are fine when you need to `.transform()` the tree. - **Regex is fine for non-SQL tasks**: sanitizing alias names, masking string literals before parse, etc. The rule is: don't use regex to interpret SQL structure. diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 0ae3371a..5e6a93d7 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -43,7 +43,7 @@ describe('standalone example docs', () => { assert.match(readme, /Accepted: 9/); assert.match(readme, /Review: 0/); assert.match(readme, /Rejected: 0/); - assert.match(config, /project: orbit-relationship-verification/); + assert.doesNotMatch(config, /^project:/m); assert.match(config, /orbit:/); assert.match(config, /driver: sqlite/); assert.match( @@ -181,8 +181,9 @@ describe('standalone example docs', () => { assert.match(connectionReference, /ktx connection list/); assert.match(connectionReference, /ktx connection test my-warehouse/); + assert.match(connectionReference, /ktx connection test --all/); assert.match(quickstart, /Connection test passed/); - assert.match(quickstart, /Driver: PostgreSQL .* Tables: 42/); + assert.match(quickstart, /Driver: PostgreSQL .* Status: ok/); }); it('documents public npm and managed runtime usage', async () => { diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index fae97850..6337a582 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -87,7 +87,6 @@ export function buildSeedSql() { export function buildKtxYaml(postgresUrl) { return [ - 'project: artifact-live-database', 'connections:', ' warehouse:', ' driver: postgres', diff --git a/scripts/installed-live-database-smoke.test.mjs b/scripts/installed-live-database-smoke.test.mjs index 3eda6cf0..ef618725 100644 --- a/scripts/installed-live-database-smoke.test.mjs +++ b/scripts/installed-live-database-smoke.test.mjs @@ -53,7 +53,6 @@ describe('installed live-database artifact smoke helpers', () => { assert.equal( buildKtxYaml('postgresql://ktx:postgres@127.0.0.1:15432/warehouse'), // pragma: allowlist secret [ - 'project: artifact-live-database', 'connections:', ' warehouse:', ' driver: postgres', diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 219f3e3c..efe7f6f3 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -642,7 +642,6 @@ try { await writeFile( join(projectDir, 'ktx.yaml'), [ - 'project: warehouse', 'connections:', ' warehouse:', ' driver: sqlite',