diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d70d495..3da14c7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Install TypeScript dependencies run: pnpm install --frozen-lockfile + - name: Run TypeScript dead-code checks + run: pnpm run dead-code + - name: Run TypeScript checks run: pnpm run check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8908b532..167681a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,6 +33,19 @@ repos: name: ruff format (python) files: ^python/ + - repo: local + hooks: + - id: biome-dead-code + name: biome dead-code check + entry: pnpm exec biome ci . --formatter-enabled=false --assist-enabled=false + language: system + pass_filenames: false + - id: knip-dead-code + name: knip dead-code check + entry: pnpm exec knip --reporter compact + language: system + pass_filenames: false + - repo: https://github.com/Yelp/detect-secrets rev: v1.5.0 hooks: diff --git a/AGENTS.md b/AGENTS.md index 96940fdf..4a235864 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,7 @@ pnpm run build pnpm run type-check pnpm run test pnpm run check +pnpm run dead-code pnpm --filter @ktx/cli run smoke pnpm --filter './packages/*' run build pnpm --filter './packages/*' run test @@ -130,6 +131,7 @@ shared contracts or package exports are affected. - Build/export changes: `pnpm run build` - Workspace scripts: `node --test scripts/*.test.mjs` or the specific script test file +- TypeScript dead-code tooling/config changes: `pnpm run dead-code` - Python semantic layer: `uv run pytest python/ktx-sl/tests -q` - Python daemon: `uv run pytest python/ktx-daemon/tests -q` - Python files: also run `uv run pre-commit run --files [FILES]` when @@ -159,6 +161,23 @@ pnpm run test 2>&1 | tee /tmp/ktx-test-output.log - Do not manually edit generated or built output under `dist/`; edit source and rebuild. +### Dead TypeScript Code Checks + +KTX uses Biome for local unused-code linting and Knip for workspace graph +analysis. These checks are intentionally part of CI and pre-commit because the +normal development workflow is agent-based. + +- Run `pnpm run dead-code` after TypeScript changes. +- Treat Knip findings as investigation prompts, not automatic deletion orders. +- Remove private dead code when you confirm there are no imports, dynamic + references, generated references, or tests that still need it. +- Preserve public package exports unless the task explicitly includes API + pruning. +- Add narrow `knip.json` ignores only for intentional dynamic or public cases. + Do not add broad package-level ignores to silence unrelated findings. +- Update `knip.json` when adding dynamic entrypoints, generated files, package + exports, CLI bins, or framework files that Knip cannot infer. + ### CLI Standards - Use Commander for CLI command trees, arguments, options, help text, custom diff --git a/README.md b/README.md index 1cd20080..b52a31f6 100644 --- a/README.md +++ b/README.md @@ -152,8 +152,6 @@ ktx dev runtime install --yes ktx dev runtime status ktx dev runtime start ktx dev runtime stop -ktx dev runtime prune --dry-run -ktx dev runtime prune --yes ``` The release artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` diff --git a/biome.json b/biome.json new file mode 100644 index 00000000..35c6d596 --- /dev/null +++ b/biome.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.4.15/schema.json", + "assist": { + "enabled": false + }, + "formatter": { + "enabled": false + }, + "files": { + "includes": [ + "scripts/**/*.mjs", + "packages/**/*.ts", + "packages/**/*.tsx", + "docs-site/**/*.ts", + "docs-site/**/*.tsx", + "docs-site/**/*.mjs", + "!**/dist/**", + "!**/coverage/**", + "!**/.next/**", + "!**/node_modules/**", + "!**/*.gen.ts", + "!**/*.generated.ts" + ] + }, + "linter": { + "enabled": true, + "rules": { + "recommended": false, + "correctness": { + "noUnusedImports": "error", + "noUnusedVariables": "error", + "noUnusedPrivateClassMembers": "error" + } + } + } +} diff --git a/docs-site/components/terminal-preview.tsx b/docs-site/components/terminal-preview.tsx index a1f950c8..d430c4ac 100644 --- a/docs-site/components/terminal-preview.tsx +++ b/docs-site/components/terminal-preview.tsx @@ -47,7 +47,7 @@ export function TerminalPreview() {
${" "} - ktx agent context --json + ktx status --json
diff --git a/docs-site/content/docs/ai-resources/agent-quickstart.mdx b/docs-site/content/docs/ai-resources/agent-quickstart.mdx index 40983224..6fd6e5ac 100644 --- a/docs-site/content/docs/ai-resources/agent-quickstart.mdx +++ b/docs-site/content/docs/ai-resources/agent-quickstart.mdx @@ -22,7 +22,7 @@ Agents should start with the smallest source that answers the task: | How to check project readiness | [ktx status](/docs/cli-reference/ktx-status) | [Quickstart](/docs/getting-started/quickstart) | | How context gets built | [Building Context](/docs/guides/building-context) | [ktx ingest](/docs/cli-reference/ktx-ingest) | | How semantic YAML works | [Writing Context](/docs/guides/writing-context) | [ktx sl](/docs/cli-reference/ktx-sl) | -| How machine-readable CLI output is shaped | [ktx agent](/docs/cli-reference/ktx-agent) | [Markdown Access](/docs/ai-resources/markdown-access) | +| How machine-readable CLI output is shaped | [ktx sl](/docs/cli-reference/ktx-sl) | [ktx wiki](/docs/cli-reference/ktx-wiki) | ## Operating workflow diff --git a/docs-site/content/docs/ai-resources/markdown-access.mdx b/docs-site/content/docs/ai-resources/markdown-access.mdx index c363a215..12bb7456 100644 --- a/docs-site/content/docs/ai-resources/markdown-access.mdx +++ b/docs-site/content/docs/ai-resources/markdown-access.mdx @@ -31,7 +31,8 @@ Every docs page has a Markdown route: ```text https://docs.kaelio.com/ktx/docs/getting-started/quickstart.md -https://docs.kaelio.com/ktx/docs/cli-reference/ktx-agent.md +https://docs.kaelio.com/ktx/docs/cli-reference/ktx-sl.md +https://docs.kaelio.com/ktx/docs/cli-reference/ktx-wiki.md https://docs.kaelio.com/ktx/docs/guides/building-context.md ``` diff --git a/docs-site/content/docs/cli-reference/ktx-agent.mdx b/docs-site/content/docs/cli-reference/ktx-agent.mdx deleted file mode 100644 index cdc4ceac..00000000 --- a/docs-site/content/docs/cli-reference/ktx-agent.mdx +++ /dev/null @@ -1,148 +0,0 @@ ---- -title: "ktx agent" -description: "Machine-readable commands for coding agents." ---- - -Hidden commands that provide machine-readable JSON output for coding agents. These are the commands that agent integrations (Claude Code, Cursor, Codex, OpenCode) call under the hood — you typically won't use them directly. - -All `ktx agent` subcommands require `--json` and produce structured JSON output on stdout. - -## Command signature - -```bash -ktx agent --json [options] -``` - -## Subcommands - -| Subcommand | Description | -|-----------|-------------| -| `tools` | Print available agent-facing KTX tools | -| `context` | Print project context for agent planning | -| `sl list` | List semantic-layer sources | -| `sl read ` | Read one semantic-layer source | -| `sl query` | Run a semantic-layer query from a JSON file | -| `wiki search ` | Search KTX wiki pages | -| `wiki read ` | Read one KTX wiki page | -| `sql execute` | Execute read-only SQL with a row limit | - -## Options - -### `agent tools` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | - -### `agent context` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | - -### `agent sl list` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | -| `--connection-id ` | Filter by connection id | — | -| `--query ` | Search source names and descriptions | — | - -### `agent sl read` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | -| `--connection-id ` | Connection id containing the source | — | - -### `agent sl query` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | -| `--connection-id ` | Connection id for execution (required) | — | -| `--query-file ` | JSON semantic-layer query file (required) | — | -| `--execute` | Execute the compiled query against the connection | `false` | -| `--max-rows ` | Maximum rows to return when executing (1-1000) | — | - -### `agent wiki search` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | -| `--limit ` | Maximum search results | `10` | - -### `agent wiki read` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | - -### `agent sql execute` - -| Flag | Description | Default | -|------|-------------|---------| -| `--json` | Print JSON output (required) | — | -| `--connection-id ` | Connection id for execution (required) | — | -| `--sql-file ` | SQL file to execute (required) | — | -| `--max-rows ` | Maximum rows to return, 1-1000 (required) | — | - -## Examples - -```bash -# List available tools -ktx agent tools --json - -# Get project context for planning -ktx agent context --json - -# List semantic sources -ktx agent sl list --json - -# Search semantic sources by name -ktx agent sl list --json --query "revenue" - -# Read a semantic source -ktx agent sl read orders --json --connection-id my-warehouse - -# Run a semantic-layer query from a file -ktx agent sl query --json \ - --connection-id my-warehouse \ - --query-file /tmp/query.json \ - --execute \ - --max-rows 100 - -# Search wiki pages -ktx agent wiki search "churn definition" --json - -# Read a specific wiki page -ktx agent wiki read page-abc123 --json - -# Execute read-only SQL -ktx agent sql execute --json \ - --connection-id my-warehouse \ - --sql-file /tmp/query.sql \ - --max-rows 500 -``` - -## Output - -Every `ktx agent` command writes JSON to stdout and diagnostic text to stderr. Agents should parse stdout as JSON and treat a non-zero exit code as a failed tool call. - -```json -{ - "ok": true, - "data": { - "type": "agent-response" - } -} -``` - -## Common errors - -| Error | Cause | Recovery | -|-------|-------|----------| -| Missing JSON output | `--json` was omitted | Re-run the same subcommand with `--json` | -| Unknown connection id | The requested connection is not configured in `ktx.yaml` | Call `ktx agent context --json` or `ktx connection list` to discover valid ids | -| Query file cannot be read | `--query-file` points to a missing or invalid JSON file | Write the query payload to a real file and pass its absolute path | -| SQL execution rejected | SQL is not read-only or `--max-rows` is missing | Use semantic-layer queries first; for direct SQL, pass read-only SQL and an explicit row limit | diff --git a/docs-site/content/docs/cli-reference/ktx-dev.mdx b/docs-site/content/docs/cli-reference/ktx-dev.mdx index eea02562..e00a4585 100644 --- a/docs-site/content/docs/cli-reference/ktx-dev.mdx +++ b/docs-site/content/docs/cli-reference/ktx-dev.mdx @@ -16,7 +16,7 @@ ktx dev [options] | Subcommand | Description | |-----------|-------------| | `init [directory]` | Initialize a Git-backed KTX project directory | -| `runtime` | Install, start, stop, inspect, and prune the KTX-managed Python runtime | +| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime | ## `dev init` @@ -27,15 +27,14 @@ ktx dev [options] ## `dev runtime` -`ktx dev runtime` supports `install`, `start`, `stop`, `status`, and `prune`. +`ktx dev runtime` supports `install`, `start`, `stop`, and `status`. | Flag | Description | Default | |------|-------------|---------| -| `--feature ` | Runtime feature level for `install`, `start`, and `status` (`core` or `local-embeddings`) | `core` | +| `--feature ` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` | | `--json` | Print JSON output for `status` | `false` | -| `--yes` | Confirm runtime install or prune actions where supported | `false` | +| `--yes` | Confirm runtime install actions where supported | `false` | | `--force` | Reinstall or restart where supported | `false` | -| `--dry-run` | Preview runtime pruning without removing files | `false` | ## Examples @@ -48,8 +47,6 @@ ktx dev runtime install --yes ktx dev runtime status ktx dev runtime start ktx dev runtime stop -ktx dev runtime prune --dry-run -ktx dev runtime prune --yes ``` ## Common errors diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index 4ec7bdd1..f5a31b27 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -28,6 +28,7 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| | `--connection-id ` | Filter by KTX connection id | — | +| `--query ` | Search source names and descriptions | — | | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | @@ -36,6 +37,7 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| | `--connection-id ` | KTX connection id (required) | — | +| `--json` | Print JSON output | `false` | ### `sl validate` @@ -55,6 +57,7 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| | `--connection-id ` | KTX connection id | — | +| `--query-file ` | JSON semantic-layer query file | — | | `--measure ` | Measure to query; repeatable (at least one required) | — | | `--dimension ` | Dimension to include; repeatable | — | | `--filter ` | Filter expression; repeatable | — | @@ -78,9 +81,15 @@ ktx sl list --connection-id my-warehouse # List sources as JSON ktx sl list --json +# Search sources as JSON +ktx sl list --json --query "revenue" + # Read a source definition ktx sl read orders --connection-id my-warehouse +# Read a source definition as JSON +ktx sl read orders --connection-id my-warehouse --json + # Validate a source against the live schema ktx sl validate orders --connection-id my-warehouse @@ -119,6 +128,13 @@ ktx sl query \ --dimension orders.created_date \ --execute \ --max-rows 1000 + +# Execute a query from a JSON file +ktx sl query \ + --connection-id my-warehouse \ + --query-file query.json \ + --execute \ + --max-rows 100 ``` ## Output diff --git a/docs-site/content/docs/cli-reference/ktx-wiki.mdx b/docs-site/content/docs/cli-reference/ktx-wiki.mdx index a709ac07..7e45420e 100644 --- a/docs-site/content/docs/cli-reference/ktx-wiki.mdx +++ b/docs-site/content/docs/cli-reference/ktx-wiki.mdx @@ -26,19 +26,23 @@ ktx wiki [options] | Flag | Description | Default | |------|-------------|---------| +| `--json` | Print JSON output | `false` | | `--user-id ` | Local user id | `local` | ### `wiki read` | Flag | Description | Default | |------|-------------|---------| +| `--json` | Print JSON output | `false` | | `--user-id ` | Local user id | `local` | ### `wiki search` | Flag | Description | Default | |------|-------------|---------| +| `--json` | Print JSON output | `false` | | `--user-id ` | Local user id | `local` | +| `--limit ` | Maximum search results | — | ### `wiki write` @@ -58,12 +62,21 @@ ktx wiki [options] # List all wiki pages ktx wiki list +# List all wiki pages as JSON +ktx wiki list --json + # Read a specific wiki page ktx wiki read revenue-definitions +# Read a specific wiki page as JSON +ktx wiki read revenue-definitions --json + # Search wiki pages ktx wiki search "monthly recurring revenue" +# Search wiki pages as JSON +ktx wiki search "monthly recurring revenue" --json --limit 10 + # Write a global knowledge page ktx wiki write revenue-definitions \ --summary "Canonical revenue metric definitions" \ @@ -97,13 +110,16 @@ Wiki commands print local knowledge pages and search results. Agents should sear ```json { - "results": [ - { - "key": "revenue-definitions", - "summary": "Canonical revenue metric definitions", - "score": 0.92 - } - ] + "kind": "list", + "data": { + "items": [ + { + "key": "revenue-definitions", + "summary": "Canonical revenue metric definitions", + "score": 0.92 + } + ] + } } ``` diff --git a/docs-site/content/docs/cli-reference/meta.json b/docs-site/content/docs/cli-reference/meta.json index a5d7a95f..bed3f98c 100644 --- a/docs-site/content/docs/cli-reference/meta.json +++ b/docs-site/content/docs/cli-reference/meta.json @@ -9,7 +9,6 @@ "ktx-sl", "ktx-wiki", "ktx-status", - "ktx-agent", "ktx-dev" ] } diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 13b973e3..6aef2b14 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -211,7 +211,7 @@ KTX writes project state as plain files so agents can inspect and edit changes i | `semantic-layer//*.yaml` | context build, ingestion, or `ktx sl write` | Semantic source definitions agents use for SQL generation | | `knowledge/global/*.md` | ingestion or `ktx wiki write --scope global` | Shared business context and metric definitions | | `knowledge/user//*.md` | `ktx wiki write --scope user` | User-scoped notes for one agent/user context | -| `.claude/skills/ktx/SKILL.md`, `.agents/skills/ktx/SKILL.md` | CLI-mode agent integration setup | Agent instructions for calling `ktx agent` commands | +| `.claude/skills/ktx/SKILL.md`, `.agents/skills/ktx/SKILL.md` | CLI-mode agent integration setup | Agent instructions for calling public `ktx` commands | ## Verify it worked diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 4285611b..b6f073b8 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -3,37 +3,36 @@ title: Serving Agents description: Expose your context to Claude Code, Cursor, Codex, and other coding agents. --- -Once you've built and refined your context, the final step is exposing it to -coding agents. KTX provides machine-readable CLI commands for direct terminal -access from Claude Code, Cursor, Codex, OpenCode, and custom agent workflows. +Once you've built and refined your context, expose it to coding agents through +the public KTX CLI. Claude Code, Cursor, Codex, OpenCode, and custom agent +workflows can call the same commands you use at a terminal. ## CLI Commands -KTX provides a set of machine-readable commands under `ktx agent`. These return -JSON output designed for programmatic consumption. +KTX public commands support JSON output for the context reads that agents use +most often. Use `--project-dir` when the agent is not already running inside the +KTX project directory. ### Available commands ```bash -# List available tools and their descriptions -ktx agent tools --json - -# Get project context for planning -ktx agent context --json +# Check setup and context readiness +ktx status --json ``` **Semantic layer:** ```bash # List sources -ktx agent sl list --json -ktx agent sl list --json --connection-id my-postgres +ktx sl list --json +ktx sl list --json --connection-id my-postgres +ktx sl list --json --query "revenue" # Read a source -ktx agent sl read orders --json --connection-id my-postgres +ktx sl read orders --json --connection-id my-postgres # Run a query from a JSON file -ktx agent sl query --json \ +ktx sl query --json \ --connection-id my-postgres \ --query-file query.json \ --execute \ @@ -44,20 +43,10 @@ ktx agent sl query --json \ ```bash # Search knowledge pages -ktx agent wiki search "revenue recognition" --json --limit 10 +ktx wiki search "revenue recognition" --json --limit 10 # Read a specific page -ktx agent wiki read order-status-definitions --json -``` - -**SQL execution:** - -```bash -# Execute read-only SQL with a row limit -ktx agent sql execute --json \ - --connection-id my-postgres \ - --sql-file query.sql \ - --max-rows 500 +ktx wiki read order-status-definitions --json ``` ## Setting Up Your Agent diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 1c105e1f..8a055fda 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -3,7 +3,9 @@ title: Agent Clients description: Set up KTX with Claude Code, Cursor, Codex, and OpenCode. --- -KTX integrates with coding agents through CLI skills and command files. These files teach agents to call `ktx agent ...` commands directly from the terminal for semantic-layer context, wiki knowledge, and safe SQL execution. +KTX integrates with coding agents through CLI skills and command files. These +files teach agents to call public `ktx` commands directly from the terminal for +semantic-layer context and wiki knowledge. Run `ktx setup` and select your agent targets, or configure manually using the snippets below. @@ -26,17 +28,17 @@ Create `.claude/skills/ktx/SKILL.md`: ```markdown title=".claude/skills/ktx/SKILL.md" --- name: ktx -description: Use local KTX semantic context, wiki knowledge, and safe SQL execution for this project. +description: Use local KTX semantic context and wiki knowledge for this project. --- Available commands: -- `ktx agent context --json --project-dir /path/to/project` -- `ktx agent sl list --json --project-dir /path/to/project` -- `ktx agent sl read '' --json --project-dir /path/to/project` -- `ktx agent sl query --json --project-dir /path/to/project --connection-id '' --query-file '' --execute --max-rows 100` -- `ktx agent wiki search '' --json --project-dir /path/to/project` -- `ktx agent wiki read '' --json --project-dir /path/to/project` -- `ktx agent sql execute --json --project-dir /path/to/project --connection-id '' --sql-file '' --max-rows 100` +- `ktx status --json --project-dir /path/to/project` +- `ktx sl list --json --project-dir /path/to/project` +- `ktx sl list --json --project-dir /path/to/project --query ''` +- `ktx sl read '' --json --project-dir /path/to/project --connection-id ''` +- `ktx sl query --json --project-dir /path/to/project --connection-id '' --query-file '' --execute --max-rows 100` +- `ktx wiki search '' --json --project-dir /path/to/project --limit 10` +- `ktx wiki read '' --json --project-dir /path/to/project` ``` ### Workflow tips @@ -123,22 +125,19 @@ All supported agent clients call the same KTX CLI commands: | Command | Description | |---------|-------------| -| `ktx agent context --json` | Return a compact project context summary | -| `ktx agent tools --json` | List available agent-facing commands | -| `ktx agent wiki search --json` | Search knowledge pages | -| `ktx agent wiki read --json` | Read a knowledge page | -| `ktx agent wiki write --json` | Write or update a knowledge page | -| `ktx agent sl list --json` | List semantic layer sources | -| `ktx agent sl read --json` | Read a semantic source definition | -| `ktx agent sl write --json` | Write or update a semantic source | -| `ktx agent sl validate --json` | Validate semantic source definitions | -| `ktx agent sl query --json` | Execute a semantic layer query when semantic compute is configured | -| `ktx agent sql execute --json` | Execute read-only SQL with an explicit row limit | +| `ktx status --json` | Return project setup and context readiness | +| `ktx wiki search --json` | Search knowledge pages | +| `ktx wiki read --json` | Read a knowledge page | +| `ktx wiki write ` | Write or update a knowledge page | +| `ktx sl list --json` | List semantic-layer sources | +| `ktx sl list --query --json` | Search semantic-layer sources | +| `ktx sl read --json --connection-id ` | Read a semantic source definition | +| `ktx sl write --connection-id ` | Write or update a semantic source | +| `ktx sl validate --connection-id ` | Validate semantic source definitions | +| `ktx sl query --json` | Execute a semantic-layer query when semantic compute is configured | ### Security constraints -- SQL execution is always read-only. -- Agent SQL execution requires an explicit `--max-rows` limit from 1 to 1000. - Secrets and credentials are never exposed in command output. - Commands resolve the project from `--project-dir`, `KTX_PROJECT_DIR`, or the nearest `ktx.yaml`. diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 49200d47..94dc4e44 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -511,4 +511,4 @@ No authentication required — SQLite is file-based. The file must be readable b | Scan returns no tables | Schema/database/project filter is wrong or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | | Historic SQL is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun scan or setup | | Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on structural scan output | -| SQL execution fails through agents | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the agent command flags | +| Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags | diff --git a/docs-site/lib/llm-docs.ts b/docs-site/lib/llm-docs.ts index 9d9b5c74..69aac698 100644 --- a/docs-site/lib/llm-docs.ts +++ b/docs-site/lib/llm-docs.ts @@ -67,12 +67,12 @@ ${link("/docs/guides/writing-context", "Writing Context", "Write semantic source - [Full documentation](${absoluteUrl("/llms-full.txt")}): All docs pages in one plain-text markdown response - [Markdown access guide](${absoluteUrl("/docs/ai-resources/markdown-access.md")}): How to fetch llms.txt, llms-full.txt, and per-page Markdown - [Quickstart markdown](${absoluteUrl("/docs/getting-started/quickstart.md")}): Human setup walkthrough -- [Agent CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-agent.md")}): Machine-readable agent commands +- [Semantic-layer CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-sl.md")}): Semantic-layer commands and JSON output +- [Wiki CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-wiki.md")}): Knowledge page commands and JSON output ## CLI Reference ${link("/docs/cli-reference/ktx-setup", "ktx setup", "Interactive project setup")} -${link("/docs/cli-reference/ktx-agent", "ktx agent", "Machine-readable commands for coding agents")} ${link("/docs/cli-reference/ktx-sl", "ktx sl", "Semantic-layer commands")} ${link("/docs/cli-reference/ktx-wiki", "ktx wiki", "Knowledge page commands")} ${link("/docs/cli-reference/ktx-connection", "ktx connection", "Connection management commands")} diff --git a/docs/superpowers/plans/2026-05-12-notion-warehouse-verification-gap-closure.md b/docs/superpowers/plans/2026-05-12-notion-warehouse-verification-gap-closure.md new file mode 100644 index 00000000..3cfdc843 --- /dev/null +++ b/docs/superpowers/plans/2026-05-12-notion-warehouse-verification-gap-closure.md @@ -0,0 +1,785 @@ +# Notion Warehouse Verification Gap Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the remaining v1 gaps that prevent ingest agents, especially +Notion WorkUnits, from reliably verifying warehouse table and column +identifiers before writing wiki or semantic-layer output. + +**Architecture:** Keep the existing warehouse verification tool module and +runner wiring. Add Notion target-warehouse scoping through the local adapter +factory, make the active WorkUnit prompt name the shipped tools, enforce +`allowedConnectionNames` in `discover_data`, and teach `entity_details` to +resolve and reject column-level display targets. + +**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX local +ingest adapters, KTX file store. + +--- + +## Audit summary + +The previous implementation plan landed the main tool module and prompt +protocol, but four v1-blocking gaps remain: + +- Notion ingest sessions still allow only the Notion connection unless a + specific adapter supplies target IDs. `NotionSourceAdapter` does not supply + target warehouse IDs, so the original Notion hallucination case cannot use + `entity_details` or raw-schema `discover_data` for the warehouse connection. +- The active WorkUnit framing prompt still tells agents to call + `wiki_sl_search` and `sl_describe_table`, which are not shipped KTX tools. +- `discover_data` accepts an explicit out-of-scope `connectionName` and still + searches raw schema for that connection. +- `entity_details({ targets: [{ display: "schema.table.column" }] })` does not + resolve column display strings and does not fail explicit missing-column + targets. + +Non-blocking gaps remain out of scope for this plan: + +- Full DDL-style `entity_details` formatting with FK and profile summaries. +- AST-backed SQL read-only validation for data-modifying CTEs. +- Search over `enrichment/descriptions.json` for generated descriptions. +- Lexicographic latest-sync edge cases for non-timestamp sync IDs. +- Hard write-time validation in `wiki_write` and `emit_unmapped_fallback`. + +## File structure + +Modify these files: + +- `packages/context/src/ingest/adapters/notion/notion.adapter.ts`: add + configured target warehouse IDs and implement `listTargetConnectionIds()`. +- `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`: cover + Notion target connection ID fan-out. +- `packages/context/src/ingest/local-adapters.ts`: pass primary warehouse IDs + into `NotionSourceAdapter`. +- `packages/context/src/ingest/local-adapters.test.ts`: cover local Notion + adapter target IDs. +- `packages/context/src/ingest/adapters/notion/chunk.ts`: update Notion + WorkUnit notes to prefer the warehouse verification tools. +- `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`: update + Notion note expectations. +- `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md`: replace + stale tool names in the active WorkUnit prompt. +- `packages/context/src/ingest/ingest-prompts.test.ts`: guard the WorkUnit + prompt against stale tool names. +- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: + refuse explicit out-of-scope connection names. +- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: + cover `discover_data` scoping. +- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: + add column-aware display-target resolution. +- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: + cover column display resolution. +- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: + use column-aware resolution and report missing columns. +- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: + cover column display and missing-column behavior. + +### Task 1: Give Notion ingest access to target warehouses + +**Files:** +- Modify: `packages/context/src/ingest/adapters/notion/notion.adapter.ts` +- Modify: `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts` +- Modify: `packages/context/src/ingest/local-adapters.ts` +- Modify: `packages/context/src/ingest/local-adapters.test.ts` + +- [ ] **Step 1: Write the failing Notion adapter test** + +Add this test inside `describe('NotionSourceAdapter', ...)` in +`packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`: + +```ts +it('returns configured target warehouse connection ids', async () => { + const adapter = new NotionSourceAdapter({ + targetConnectionIds: ['warehouse', 'warehouse', 'analytics'], + }); + + await expect(adapter.listTargetConnectionIds?.(stagedDir)).resolves.toEqual([ + 'analytics', + 'warehouse', + ]); +}); +``` + +- [ ] **Step 2: Run the failing Notion adapter test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/adapters/notion/notion.adapter.test.ts -t "target warehouse connection ids" +``` + +Expected: FAIL because `NotionSourceAdapterDeps` has no +`targetConnectionIds` option and `NotionSourceAdapter` does not implement +`listTargetConnectionIds()`. + +- [ ] **Step 3: Implement Notion target connection IDs** + +Modify `packages/context/src/ingest/adapters/notion/notion.adapter.ts`: + +```ts +export interface NotionSourceAdapterDeps { + onPullSucceeded?: (ctx: NotionPullSucceededContext) => Promise; + logger?: NotionFetchLogger; + targetConnectionIds?: string[]; +} + +function uniqueSorted(values: readonly string[] | undefined): string[] { + return [...new Set(values ?? [])].sort((left, right) => + left.localeCompare(right), + ); +} +``` + +Add this method to `NotionSourceAdapter`: + +```ts + async listTargetConnectionIds(_stagedDir: string): Promise { + return uniqueSorted(this.deps.targetConnectionIds); + } +``` + +- [ ] **Step 4: Pass primary warehouses into the local Notion adapter** + +Modify the Notion adapter construction in +`packages/context/src/ingest/local-adapters.ts`: + +```ts + new NotionSourceAdapter({ + targetConnectionIds: primaryWarehouseConnectionIds(project), + ...(options.logger ? { logger: options.logger } : {}), + }), +``` + +- [ ] **Step 5: Write the local adapter fan-out test** + +Add this test to `packages/context/src/ingest/local-adapters.test.ts`: + +```ts +it('passes primary warehouse connection ids to the local Notion adapter', async () => { + const adapters = createDefaultLocalIngestAdapters( + projectWithConnections({ + notion: { + driver: 'notion', + auth_token: 'secret', + crawl_mode: 'selected_roots', + root_page_ids: ['page-1'], + }, + warehouse: { + driver: 'postgres', + url: 'postgresql://readonly@db.example.test/analytics', + }, + docs: { + driver: 'dbt', + source_dir: './dbt', + }, + } as never), + ); + + const notion = adapters.find((adapter) => adapter.source === 'notion'); + + await expect(notion?.listTargetConnectionIds?.('/tmp/staged-notion')).resolves.toEqual([ + 'warehouse', + ]); +}); +``` + +- [ ] **Step 6: Run the Notion target tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/adapters/notion/notion.adapter.test.ts -t "target warehouse connection ids" \ + src/ingest/local-adapters.test.ts -t "local Notion adapter" +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/adapters/notion/notion.adapter.ts \ + packages/context/src/ingest/adapters/notion/notion.adapter.test.ts \ + packages/context/src/ingest/local-adapters.ts \ + packages/context/src/ingest/local-adapters.test.ts +git commit -m "fix(context): expose target warehouses to Notion ingest" +``` + +### Task 2: Remove stale tool names from active ingest prompts + +**Files:** +- Modify: `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md` +- Modify: `packages/context/src/ingest/ingest-prompts.test.ts` +- Modify: `packages/context/src/ingest/adapters/notion/chunk.ts` +- Modify: `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts` + +- [ ] **Step 1: Add failing prompt guards** + +Add this test to `packages/context/src/ingest/ingest-prompts.test.ts`: + +```ts +it('uses shipped warehouse verification tools in the WorkUnit prompt', async () => { + const prompt = await readFile( + new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), + 'utf-8', + ); + + expect(prompt).toContain('discover_data'); + expect(prompt).toContain('entity_details'); + expect(prompt).not.toContain('wiki_sl_search'); + expect(prompt).not.toContain('sl_describe_table'); +}); +``` + +- [ ] **Step 2: Run the failing prompt guard** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/ingest-prompts.test.ts -t "warehouse verification tools" +``` + +Expected: FAIL because the WorkUnit prompt still contains `wiki_sl_search` and +`sl_describe_table`. + +- [ ] **Step 3: Update the WorkUnit framing prompt** + +In `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md`, replace +the first `` paragraph with: + +```md +You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, Metabase card JSONs, Notion pages, or similar) and you must translate that slice into KTX semantic-layer sources and/or knowledge wiki pages, in one pass. Prior WorkUnits in this same job may have already written SL sources and wiki pages; their writes are visible on the working branch and discoverable with `discover_data`. +``` + +In workflow step 2, replace the final sentence with: + +```md +The triage skill tells you how to react when `discover_data` reveals that a prior WU already wrote something overlapping. +``` + +In workflow step 4, replace the sentence that starts +`For each raw file:` with: + +```md +4. For each raw file: call `read_raw_file` (or `read_raw_span` for slicing large files) to load content. Before writing a new SL source or wiki page, call `discover_data` for each candidate source, table, metric, or topic name to find prior-WU writes, existing wiki pages, SL sources, and raw warehouse matches; apply `ingest_triage` when you hit one, and apply any matching canonical pin before deciding whether to edit, rename, or skip. +``` + +In the `` block, replace the physical-column rule with: + +```md +- Do not invent physical column names or grain keys. For table-backed SL sources, every `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr` column must come from raw-file column declarations or warehouse-backed discovery (`discover_data`, `sl_discover`, `entity_details`). If column names are not confirmed, capture the business context in wiki instead of writing a full SL source. +``` + +- [ ] **Step 4: Update Notion WorkUnit notes** + +In `packages/context/src/ingest/adapters/notion/chunk.ts`, replace +`NOTION_SL_WRITE_GUIDANCE` with: + +```ts +const NOTION_SL_WRITE_GUIDANCE = + 'Write wiki entries with wiki_write. Wiki keys must be flat slugs like orbit-company-overview, not orbit/company-overview. Search existing wiki pages, SL sources, and raw warehouse schema for the same tables or sl_refs with discover_data before creating a new page. Only write or edit SL sources after discover_data plus sl_discover/sl_read_source or entity_details confirms a mapped non-Notion target source; if no mapped target exists, emit_unmapped_fallback and keep the fact wiki-only. Notion dataSourceCount counts Notion databases/data sources only, not warehouse/dbt mappings. If a warehouse/dbt connection exists but the named table or source is absent, use reason no_physical_table rather than no_connection_mapping. Do not create SL sources under the Notion connection just because a page mentions a warehouse table.'; +``` + +In the `reconcileNotes` array in the same file, replace: + +```ts + 'Notion dataSourceCount is Notion-only; use sl_discover for warehouse/dbt mapping decisions.', +``` + +with: + +```ts + 'Notion dataSourceCount is Notion-only; use discover_data/entity_details for warehouse/dbt mapping decisions.', +``` + +- [ ] **Step 5: Update Notion note expectations** + +In `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`, +update the note expectations in `it('chunks changed Notion pages...')`: + +```ts +expect(result.workUnits[0].notes).toContain('discover_data'); +expect(result.workUnits[0].notes).toContain('entity_details'); +``` + +Update the exact `reconcileNotes` expectation to: + +```ts +expect(result.reconcileNotes).toEqual([ + 'Notion maxKnowledgeCreatesPerRun=25', + 'Notion maxKnowledgeUpdatesPerRun=20', + 'Notion dataSourceCount is Notion-only; use discover_data/entity_details for warehouse/dbt mapping decisions.', + 'Reconcile Notion wiki pages sharing tables/sl_refs before creating distinct artifacts.', +]); +``` + +- [ ] **Step 6: Run prompt and Notion note tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/ingest-prompts.test.ts \ + src/ingest/adapters/notion/notion.adapter.test.ts +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add \ + packages/context/prompts/memory_agent_bundle_ingest_work_unit.md \ + packages/context/src/ingest/ingest-prompts.test.ts \ + packages/context/src/ingest/adapters/notion/chunk.ts \ + packages/context/src/ingest/adapters/notion/notion.adapter.test.ts +git commit -m "fix(context): update ingest prompts for warehouse verification tools" +``` + +### Task 3: Enforce allowed connection scope in discover_data + +**Files:** +- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` + +- [ ] **Step 1: Write the failing scoping test** + +Add this test to +`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: + +```ts +it('refuses explicit out-of-scope connection names', async () => { + const result = await tool.call({ query: 'orders', connectionName: 'billing' }, context); + + expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.'); + expect(result.structured).toEqual({ wiki: null, sl: null, raw: null }); + expect(wikiSearchTool.call).not.toHaveBeenCalled(); + expect(slDiscoverTool.call).not.toHaveBeenCalled(); + expect(catalog.searchByName).not.toHaveBeenCalled(); +}); +``` + +- [ ] **Step 2: Run the failing scoping test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -t "out-of-scope" +``` + +Expected: FAIL because `discover_data` currently searches raw schema for an +explicit `connectionName` even when it is not in `allowedConnectionNames`. + +- [ ] **Step 3: Add the scope guard** + +In +`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`, +add this helper near `totalSources()`: + +```ts +function allowedConnectionNames(context: ToolContext): ReadonlySet | null { + return context.session?.allowedConnectionNames ?? null; +} +``` + +At the top of `DiscoverDataTool.call()`, before the `sourceName` branch and +before calling any child tool, add: + +```ts + const allowed = allowedConnectionNames(context); + if (input.connectionName && allowed && !allowed.has(input.connectionName)) { + return { + markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, + structured: { wiki: null, sl: null, raw: null }, + }; + } +``` + +Then replace the raw connection-list construction with: + +```ts + const connections = input.connectionName ? [input.connectionName] : [...(allowed ?? [])].sort(); +``` + +- [ ] **Step 4: Run discover_data tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +git commit -m "fix(context): scope raw schema discovery to allowed connections" +``` + +### Task 4: Fix column-level entity_details verification + +**Files:** +- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` + +- [ ] **Step 1: Write failing catalog column-target tests** + +First update `seedLiveDatabaseScan()` in that test file so BigQuery tables have +a project/catalog. Replace the repeated inline table refs with: + +```ts +const tableRef = { + catalog: driver === 'bigquery' ? 'analytics' : null, + db: driver === 'sqlite' ? null : 'public', + name: 'orders', +}; +``` + +Use `tableRef.catalog`, `tableRef.db`, and `tableRef.name` for the seeded +table and profile table references. + +Then add these tests to +`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: + +```ts +it('resolves postgres column display strings without treating the column as a table', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplayTarget('warehouse', 'public.orders.status')).resolves.toMatchObject({ + resolved: { catalog: null, db: 'public', name: 'orders', column: 'status' }, + candidates: [], + dialect: 'postgres', + }); +}); + +it('resolves BigQuery column display strings with four parts', async () => { + await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplayTarget('warehouse', 'analytics.public.orders.status')).resolves.toMatchObject({ + resolved: { catalog: 'analytics', db: 'public', name: 'orders', column: 'status' }, + candidates: [], + dialect: 'bigquery', + }); +}); +``` + +- [ ] **Step 2: Run the failing catalog tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts -t "column display" +``` + +Expected: FAIL because `resolveDisplayTarget()` does not exist. + +- [ ] **Step 3: Implement column-aware display resolution** + +In +`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`, +add this exported interface near `RawSchemaHit`: + +```ts +export interface DisplayTargetResolution { + resolved: (KtxTableRef & { column?: string }) | null; + candidates: KtxTableRef[]; + dialect: string; +} +``` + +Add these helpers near `parseDisplay()`: + +```ts +function expectedDisplayPartCount(driver: CatalogDriver): number { + if (driver === 'sqlite' || driver === 'sqlite3') { + return 1; + } + if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { + return 3; + } + return 2; +} + +function parseColumnDisplay(driver: CatalogDriver, display: string): (KtxTableRef & { column: string }) | null { + const parts = splitDisplay(display); + const tablePartCount = expectedDisplayPartCount(driver); + if (parts.length !== tablePartCount + 1) { + return null; + } + const column = parts.at(-1); + if (!column) { + return null; + } + const table = parseDisplay(driver, parts.slice(0, -1).join('.')); + return table ? { ...table, column } : null; +} +``` + +Add this method to `WarehouseCatalogService` after `resolveDisplay()`: + +```ts + async resolveDisplayTarget(connectionName: string, display: string): Promise { + const catalog = await this.loadCatalog(connectionName); + if (!catalog) { + return { resolved: null, candidates: [], dialect: 'unknown' }; + } + + const dialect = getDialectForDriver(catalog.driver).type; + const tableResolution = await this.resolveDisplay(connectionName, display); + if (tableResolution.resolved) { + return tableResolution; + } + + const parsedColumn = parseColumnDisplay(catalog.driver, display); + if (!parsedColumn) { + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + } + + const table = catalog.tables.find((candidate) => refsEqual(candidate, parsedColumn)); + if (!table) { + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + } + + return { + resolved: { + catalog: table.catalog, + db: table.db, + name: table.name, + column: parsedColumn.column, + }, + candidates: [], + dialect, + }; + } +``` + +- [ ] **Step 4: Write failing entity_details column tests** + +Add these tests to +`packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: + +```ts +it('resolves display targets that include a column name', async () => { + const result = await tool.call( + { connectionName: 'warehouse', targets: [{ display: 'public.orders.status' }] }, + context, + ); + + expect(result.markdown).toContain('### public.orders'); + expect(result.markdown).toContain('- status (text, nullable=false)'); + expect(result.markdown).not.toContain('- id (integer'); + expect(result.structured.resolved).toHaveLength(1); + expect(result.structured.resolved[0]?.columns.map((column) => column.name)).toEqual(['status']); +}); + +it('reports missing explicit columns instead of returning an empty column list', async () => { + const result = await tool.call( + { connectionName: 'warehouse', targets: [{ display: 'public.orders.plan_tier' }] }, + context, + ); + + expect(result.markdown).toContain('Column not found in scan: public.orders.plan_tier'); + expect(result.markdown).toContain('Available columns: id, status'); + expect(result.structured.resolved).toHaveLength(0); + expect(result.structured.missing).toHaveLength(1); +}); +``` + +- [ ] **Step 5: Run the failing entity_details tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -t "column" +``` + +Expected: FAIL because display column targets are treated as table names and +missing columns are not reported. + +- [ ] **Step 6: Use column-aware resolution in entity_details** + +In +`packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`, +add this helper near `appendTableMarkdown()`: + +```ts +function findColumn(detail: TableDetail, columnName: string): TableDetail['columns'][number] | null { + const normalized = columnName.toLowerCase(); + return detail.columns.find((column) => column.name.toLowerCase() === normalized) ?? null; +} +``` + +Replace the display resolution block inside the `for (const target of +input.targets)` loop with: + +```ts + const resolution = + 'display' in target + ? await catalog.resolveDisplayTarget(input.connectionName, target.display) + : { + resolved: { catalog: target.catalog, db: target.db, name: target.name, column: target.column }, + candidates: [], + dialect: '', + }; +``` + +After `const detail = await catalog.getTable(...)`, replace the existing +`resolved.push(detail); appendTableMarkdown(...)` lines with: + +```ts + const requestedColumn = resolution.resolved.column; + if (requestedColumn) { + const column = findColumn(detail, requestedColumn); + if (!column) { + missing.push({ + target, + candidates: [{ catalog: detail.catalog, db: detail.db, name: detail.name }], + }); + parts.push(`Column not found in scan: ${detail.display}.${requestedColumn}`); + parts.push(`Available columns: ${detail.columns.map((candidate) => candidate.name).join(', ')}`); + continue; + } + const scopedDetail = { ...detail, columns: [column] }; + resolved.push(scopedDetail); + appendTableMarkdown(parts, scopedDetail, column.name); + continue; + } + + resolved.push(detail); + appendTableMarkdown(parts, detail); +``` + +- [ ] **Step 7: Run warehouse verification tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +git commit -m "fix(context): verify warehouse column display targets" +``` + +### Task 5: Verify the v1 gap closure + +**Files:** +- Verify all files changed by Tasks 1-4. + +- [ ] **Step 1: Run focused tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/adapters/notion/notion.adapter.test.ts \ + src/ingest/local-adapters.test.ts \ + src/ingest/ingest-prompts.test.ts \ + src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ + src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run package type-check** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +``` + +Expected: PASS. + +- [ ] **Step 3: Run package tests** + +Run: + +```bash +pnpm --filter @ktx/context run test +``` + +Expected: PASS. + +- [ ] **Step 4: Run pre-commit on changed files when configured** + +Run: + +```bash +uv run pre-commit run --files \ + packages/context/src/ingest/adapters/notion/notion.adapter.ts \ + packages/context/src/ingest/adapters/notion/notion.adapter.test.ts \ + packages/context/src/ingest/local-adapters.ts \ + packages/context/src/ingest/local-adapters.test.ts \ + packages/context/src/ingest/adapters/notion/chunk.ts \ + packages/context/prompts/memory_agent_bundle_ingest_work_unit.md \ + packages/context/src/ingest/ingest-prompts.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +``` + +Expected: PASS. If the repo has no pre-commit config or the local `uv` version +cannot satisfy the project pin, record the exact error and rely on focused +tests plus type-check. + +- [ ] **Step 5: Inspect final git status** + +Run: + +```bash +git status --short +``` + +Expected: only intentional files are modified. Commit any formatter-driven +changes with: + +```bash +git add packages/context +git commit -m "chore(context): verify warehouse verification v1 gaps" +``` + +## Self-review checklist + +- Spec coverage: this plan closes the remaining v1 paths for Notion warehouse + verification, active WorkUnit prompt correctness, raw discovery scoping, and + column-level identifier verification. +- Placeholder scan: no task relies on future-work markers, unnamed edge-case + handling, or cross-task shorthand. +- Type consistency: `discover_data` continues to use `connectionName`, + `sl_discover` still receives `connectionId` internally, and + `resolveDisplayTarget()` returns the same table identity plus optional + `column`. diff --git a/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md b/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md new file mode 100644 index 00000000..f48fea36 --- /dev/null +++ b/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md @@ -0,0 +1,957 @@ +# Warehouse Verification Final V1 Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the remaining v1 gaps that still prevent ingest agents from +reliably following warehouse verification results through to `entity_details` +and `sql_execution`. + +**Architecture:** Keep the existing warehouse verification module and runner +session scoping. Add connection names to raw discovery hits, expose primary +warehouse targets from the remaining source adapters, and make local ingest +SQL probes use the same scan connector read-only execution path as schema scan. + +**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX local +ingest runtime, KTX scan connectors. + +--- + +## Audit summary + +The first two implementation plans landed the warehouse verification tools, +prompt protocol, Notion warehouse scoping, and stale prompt-name cleanup. The +focused audit on May 12, 2026, found three remaining v1-blocking gaps: + +- `discover_data` searches multiple allowed raw warehouse scans, but raw hits do + not carry or render `connectionName`. The tool tells the agent to call + `entity_details({connectionName, targets: [...]})`, then omits the required + `connectionName` from the follow-up evidence. +- Local LookML and MetricFlow adapters do not expose primary warehouse target + IDs. The runner only adds adapter-provided targets to `allowedConnectionNames`, + so those WorkUnits cannot use raw warehouse verification unless their source + connection is itself the warehouse. +- `sql_execution` calls the local ingest connection catalog, but the catalog + either has no query executor in normal CLI ingest or calls an injected + executor without `projectDir` and connection config. The default local query + executor cannot dispatch without that config. + +Non-blocking gaps remain out of scope for this v1 plan: + +- Full DDL-style `entity_details` formatting with FK profile summaries. +- AST-backed SQL read-only validation for data-modifying CTE bodies. +- Search over generated `enrichment/descriptions.json`. +- Lexicographic latest-sync edge cases for non-timestamp sync IDs. +- Hard write-time validation in `wiki_write` and `emit_unmapped_fallback`. + +## File structure + +Modify these files: + +- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: + add `connectionName` to raw schema hit records. +- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: + render raw hit connection names and preserve them in structured output. +- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: + cover multi-connection raw discovery follow-up data. +- `packages/context/src/ingest/adapters/lookml/lookml.adapter.ts`: + accept and return configured target warehouse connection IDs. +- `packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts`: + cover LookML target warehouse IDs. +- `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts`: + accept and return configured target warehouse connection IDs. +- `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts`: + cover MetricFlow target warehouse IDs. +- `packages/context/src/ingest/local-adapters.ts`: + pass primary warehouse IDs into LookML and MetricFlow adapters. +- `packages/context/src/ingest/local-adapters.test.ts`: + cover local adapter warehouse target fan-out. +- `packages/context/src/ingest/local-bundle-runtime.ts`: + pass full project connection config to local ingest query executors. +- `packages/context/src/ingest/local-bundle-runtime.test.ts`: + cover the local ingest query executor call shape. +- `packages/context/src/ingest/local-ingest.ts`: + use the shared query executor port type. +- `packages/context/src/mcp/local-project-ports.ts`: + no behavior change expected, but type-checks against the updated local ingest + query executor type. +- `packages/cli/src/ingest.ts`: + provide a read-only scan-connector-backed query executor for normal local + ingest runs. + +Create these files: + +- `packages/cli/src/ingest-query-executor.ts`: CLI query executor that adapts + scan connectors' `executeReadOnly()` method to `KtxSqlQueryExecutorPort`. +- `packages/cli/src/ingest-query-executor.test.ts`: unit coverage for the CLI + ingest query executor. + +### Task 1: Preserve raw discovery connection names + +**Files:** +- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` + +- [ ] **Step 1: Write the failing multi-connection discovery test** + +Add this test to +`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: + +```ts + it('includes connectionName on raw schema hits so entity_details can follow up', async () => { + const multiConnectionContext: ToolContext = { + ...context, + session: { allowedConnectionNames: new Set(['warehouse', 'analytics']) } as any, + }; + catalog.searchByName.mockImplementation(async (connectionName: string, query: string) => [ + { + kind: 'table', + connectionName, + ref: { catalog: null, db: 'public', name: `${connectionName}_${query}` }, + display: `public.${connectionName}_${query}`, + matchedOn: 'name', + }, + ]); + + const result = await tool.call({ query: 'orders', limit: 10 }, multiConnectionContext); + + expect(catalog.searchByName).toHaveBeenCalledWith('analytics', 'orders', 10); + expect(catalog.searchByName).toHaveBeenCalledWith('warehouse', 'orders', 10); + expect(result.markdown).toContain('connectionName=analytics'); + expect(result.markdown).toContain('connectionName=warehouse'); + expect(result.markdown).toContain( + 'entity_details({connectionName: "analytics", targets: [{display: "public.analytics_orders"}]})', + ); + expect(result.structured.raw?.hits.map((hit) => hit.connectionName)).toEqual([ + 'analytics', + 'warehouse', + ]); + }); +``` + +- [ ] **Step 2: Run the failing discovery test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -t "connectionName on raw schema hits" +``` + +Expected: FAIL because `RawSchemaHit` has no `connectionName` property and the +markdown only renders the display string. + +- [ ] **Step 3: Add `connectionName` to raw schema hits** + +Modify the raw hit type and hit construction in +`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: + +```ts +export type RawSchemaHit = + | { + kind: 'table'; + connectionName: string; + ref: KtxTableRef; + display: string; + matchedOn: 'name' | 'db' | 'comment' | 'description'; + } + | { + kind: 'column'; + connectionName: string; + ref: KtxTableRef & { column: string }; + display: string; + matchedOn: 'name' | 'comment' | 'description'; + }; +``` + +In the table hit block, add `connectionName`: + +```ts + hits.push({ + kind: 'table', + connectionName, + ref: { catalog: table.catalog, db: table.db, name: table.name }, + display: formatDisplay(catalog.driver, table), + matchedOn: tableMatch, + }); +``` + +In the column hit block, add `connectionName`: + +```ts + hits.push({ + kind: 'column', + connectionName, + ref: { catalog: table.catalog, db: table.db, name: table.name, column: column.name }, + display: `${formatDisplay(catalog.driver, table)}.${column.name}`, + matchedOn: columnMatch, + }); +``` + +- [ ] **Step 4: Render follow-up-ready raw hits** + +Modify the raw schema markdown in +`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: + +```ts + parts.push('## Raw Warehouse Schema', '> use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values'); + parts.push( + rawHits + .slice(0, limit) + .map( + (hit) => + `- ${hit.kind}: ${hit.display} [connectionName=${hit.connectionName}] (matched on ${hit.matchedOn}) — ` + + `follow up with \`entity_details({connectionName: "${hit.connectionName}", targets: [{display: "${hit.display}"}]})\``, + ) + .join('\n'), + ); +``` + +- [ ] **Step 5: Run the discovery test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +git commit -m "fix(context): include raw discovery connection names" +``` + +### Task 2: Expose LookML and MetricFlow warehouse targets + +**Files:** +- Modify: `packages/context/src/ingest/adapters/lookml/lookml.adapter.ts` +- Modify: `packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts` +- Modify: `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts` +- Modify: `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts` +- Modify: `packages/context/src/ingest/local-adapters.ts` +- Modify: `packages/context/src/ingest/local-adapters.test.ts` + +- [ ] **Step 1: Write failing adapter target tests** + +Add this test to +`packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts`: + +```ts + it('returns configured target warehouse connection ids', async () => { + const adapter = new LookmlSourceAdapter({ + homeDir: join(tmpRoot, 'home'), + targetConnectionIds: ['warehouse', 'analytics', 'warehouse'], + }); + + await expect(adapter.listTargetConnectionIds?.(join(tmpRoot, 'staged'))).resolves.toEqual([ + 'analytics', + 'warehouse', + ]); + }); +``` + +Add this test to +`packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts`: + +```ts + it('returns configured target warehouse connection ids', async () => { + const metricflow = new MetricflowSourceAdapter({ + homeDir: join(tmpRoot, 'cache-home'), + targetConnectionIds: ['warehouse', 'analytics', 'warehouse'], + }); + + await expect(metricflow.listTargetConnectionIds?.(stagedDir)).resolves.toEqual([ + 'analytics', + 'warehouse', + ]); + }); +``` + +- [ ] **Step 2: Run the failing adapter tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/adapters/lookml/lookml.adapter.test.ts -t "target warehouse connection ids" \ + src/ingest/adapters/metricflow/metricflow.adapter.test.ts -t "target warehouse connection ids" +``` + +Expected: FAIL because neither adapter accepts `targetConnectionIds` or +implements `listTargetConnectionIds()`. + +- [ ] **Step 3: Implement target ID support in LookML** + +Modify `packages/context/src/ingest/adapters/lookml/lookml.adapter.ts`: + +```ts +export interface LookmlSourceAdapterDeps { + homeDir: string; + targetConnectionIds?: string[]; +} + +function uniqueSorted(values: readonly string[] | undefined): string[] { + return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); +} +``` + +Add this method to `LookmlSourceAdapter`: + +```ts + async listTargetConnectionIds(_stagedDir: string): Promise { + return uniqueSorted(this.deps.targetConnectionIds); + } +``` + +- [ ] **Step 4: Implement target ID support in MetricFlow** + +Modify `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts`: + +```ts +export interface MetricflowSourceAdapterDeps { + homeDir: string; + targetConnectionIds?: string[]; +} + +function uniqueSorted(values: readonly string[] | undefined): string[] { + return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); +} +``` + +Add this method to `MetricflowSourceAdapter`: + +```ts + async listTargetConnectionIds(_stagedDir: string): Promise { + return uniqueSorted(this.deps.targetConnectionIds); + } +``` + +- [ ] **Step 5: Pass primary warehouses from the local adapter factory** + +Modify the LookML and MetricFlow adapter construction in +`packages/context/src/ingest/local-adapters.ts`: + +```ts + new LookmlSourceAdapter({ + homeDir: join(project.projectDir, '.ktx/cache'), + targetConnectionIds: primaryWarehouseConnectionIds(project), + }), +``` + +```ts + new MetricflowSourceAdapter({ + homeDir: join(project.projectDir, '.ktx/cache'), + targetConnectionIds: primaryWarehouseConnectionIds(project), + }), +``` + +- [ ] **Step 6: Write the local adapter fan-out test** + +Add this test to `packages/context/src/ingest/local-adapters.test.ts`: + +```ts + it('passes primary warehouse connection ids to local LookML and MetricFlow adapters', async () => { + const adapters = createDefaultLocalIngestAdapters( + projectWithConnections({ + warehouse: { + driver: 'postgres', + url: 'postgresql://readonly@db.example.test/analytics', + }, + lookml_docs: { + driver: 'lookml', + lookml: { + repoUrl: 'https://github.com/acme/lookml.git', + }, + }, + metrics_repo: { + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/metrics.git', + }, + }, + } as never), + ); + + const lookml = adapters.find((adapter) => adapter.source === 'lookml'); + const metricflow = adapters.find((adapter) => adapter.source === 'metricflow'); + + await expect(lookml?.listTargetConnectionIds?.('/tmp/staged-lookml')).resolves.toEqual([ + 'warehouse', + ]); + await expect(metricflow?.listTargetConnectionIds?.('/tmp/staged-metricflow')).resolves.toEqual([ + 'warehouse', + ]); + }); +``` + +- [ ] **Step 7: Run the target fan-out tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/adapters/lookml/lookml.adapter.test.ts \ + src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ + src/ingest/local-adapters.test.ts +``` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/adapters/lookml/lookml.adapter.ts \ + packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts \ + packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts \ + packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ + packages/context/src/ingest/local-adapters.ts \ + packages/context/src/ingest/local-adapters.test.ts +git commit -m "fix(context): expose warehouse targets for LookML and MetricFlow" +``` + +### Task 3: Pass full connection config to local ingest SQL execution + +**Files:** +- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` +- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` +- Modify: `packages/context/src/ingest/local-ingest.ts` + +- [ ] **Step 1: Write the failing local connection catalog test** + +In `packages/context/src/ingest/local-bundle-runtime.test.ts`, change the +Vitest import to include `vi`: + +```ts +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +``` + +Extend `RuntimeWithConnectionDeps`: + +```ts +type RuntimeWithConnectionDeps = { + deps: { + connections: { + listEnabledConnections(ids: string[]): Promise>; + getConnectionById(connectionId: string): Promise<{ id: string; name: string; connectionType: string } | null>; + executeQuery(connectionId: string, sql: string): Promise; + }; + }; +}; +``` + +Add this test: + +```ts + it('passes project connection config to local ingest query executors', async () => { + const agentRunner = new AgentRunnerService({ llmProvider: { getModel: () => ({}) as never } as any }); + const queryExecutor = { + execute: vi.fn(async () => ({ + headers: ['answer'], + rows: [[1]], + totalRows: 1, + command: 'SELECT', + rowCount: 1, + })), + }; + + const runtime = createLocalBundleIngestRuntime({ + project, + adapters: [new FakeSourceAdapter()], + agentRunner, + queryExecutor, + }); + const connections = (runtime.runner as unknown as RuntimeWithConnectionDeps).deps.connections; + + await expect(connections.executeQuery('warehouse', 'select 1')).resolves.toMatchObject({ + headers: ['answer'], + }); + expect(queryExecutor.execute).toHaveBeenCalledWith({ + connectionId: 'warehouse', + projectDir: project.projectDir, + connection: project.config.connections.warehouse, + sql: 'select 1', + }); + }); +``` + +- [ ] **Step 2: Run the failing local runtime test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -t "project connection config" +``` + +Expected: FAIL because `LocalConnectionCatalog.executeQuery()` only passes +`connectionId` and `sql`. + +- [ ] **Step 3: Update local ingest query executor types** + +In `packages/context/src/ingest/local-bundle-runtime.ts`, import the shared +query executor type: + +```ts +import { localConnectionInfoFromConfig, type KtxSqlQueryExecutorPort } from '../connections/index.js'; +``` + +Change `CreateLocalBundleIngestRuntimeOptions.queryExecutor` to: + +```ts + queryExecutor?: KtxSqlQueryExecutorPort; +``` + +Change `LocalConnectionCatalog` to store that type: + +```ts +class LocalConnectionCatalog implements SlConnectionCatalogPort { + constructor( + private readonly project: KtxLocalProject, + private readonly queryExecutor?: KtxSqlQueryExecutorPort, + ) {} +``` + +Change `executeQuery()`: + +```ts + async executeQuery(connectionId: string, sql: string): Promise { + if (!this.queryExecutor) { + throw new Error('Local ingest has no query executor configured'); + } + return this.queryExecutor.execute({ + connectionId, + projectDir: this.project.projectDir, + connection: this.project.config.connections[connectionId], + sql, + }); + } +``` + +In `packages/context/src/ingest/local-ingest.ts`, replace the local query +executor object type with the shared port: + +```ts +import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; +``` + +```ts + queryExecutor?: KtxSqlQueryExecutorPort; +``` + +- [ ] **Step 4: Run the local runtime test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -t "project connection config" +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/local-bundle-runtime.ts \ + packages/context/src/ingest/local-bundle-runtime.test.ts \ + packages/context/src/ingest/local-ingest.ts +git commit -m "fix(context): pass connection config to ingest query executors" +``` + +### Task 4: Supply a scan-connector query executor to CLI ingest + +**Files:** +- Create: `packages/cli/src/ingest-query-executor.ts` +- Create: `packages/cli/src/ingest-query-executor.test.ts` +- Modify: `packages/cli/src/ingest.ts` + +- [ ] **Step 1: Write the CLI query executor tests** + +Create `packages/cli/src/ingest-query-executor.test.ts`: + +```ts +import type { KtxLocalProject } from '@ktx/context/project'; +import { createKtxConnectorCapabilities, type KtxScanConnector } from '@ktx/context/scan'; +import { describe, expect, it, vi } from 'vitest'; +import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; + +function project(): KtxLocalProject { + return { + projectDir: '/tmp/ktx-query-project', + config: { + project: 'warehouse', + connections: { + warehouse: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' }, + }, + }, + } as unknown as KtxLocalProject; +} + +function connector(overrides: Partial = {}): KtxScanConnector { + return { + id: 'warehouse', + driver: 'postgres', + capabilities: createKtxConnectorCapabilities({ readOnlySql: true }), + async introspect() { + throw new Error('introspect is not used by this test'); + }, + executeReadOnly: vi.fn(async () => ({ + headers: ['answer'], + rows: [[1]], + totalRows: 1, + rowCount: 1, + })), + cleanup: vi.fn(async () => {}), + ...overrides, + }; +} + +describe('createKtxCliIngestQueryExecutor', () => { + it('executes read-only SQL through the scan connector and cleans it up', async () => { + const scanConnector = connector(); + const createConnector = vi.fn(async () => scanConnector); + const executor = createKtxCliIngestQueryExecutor(project(), { createConnector }); + + await expect( + executor.execute({ + connectionId: 'warehouse', + connection: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' }, + projectDir: '/tmp/ktx-query-project', + sql: 'select 1', + maxRows: 5, + }), + ).resolves.toMatchObject({ + headers: ['answer'], + rows: [[1]], + totalRows: 1, + command: 'SELECT', + rowCount: 1, + }); + + expect(createConnector).toHaveBeenCalledWith(project(), 'warehouse'); + expect(scanConnector.executeReadOnly).toHaveBeenCalledWith( + { connectionId: 'warehouse', sql: 'select 1', maxRows: 5 }, + { runId: 'ingest-sql-execution' }, + ); + expect(scanConnector.cleanup).toHaveBeenCalledTimes(1); + }); + + it('rejects connectors without read-only SQL support', async () => { + const scanConnector = connector({ + capabilities: createKtxConnectorCapabilities({ readOnlySql: false }), + executeReadOnly: undefined, + }); + const executor = createKtxCliIngestQueryExecutor(project(), { + createConnector: vi.fn(async () => scanConnector), + }); + + await expect( + executor.execute({ + connectionId: 'warehouse', + connection: { driver: 'postgres' }, + projectDir: '/tmp/ktx-query-project', + sql: 'select 1', + }), + ).rejects.toThrow('Connection "warehouse" driver "postgres" does not support read-only SQL execution.'); + expect(scanConnector.cleanup).toHaveBeenCalledTimes(1); + }); +}); +``` + +- [ ] **Step 2: Run the failing CLI query executor test** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts +``` + +Expected: FAIL because `ingest-query-executor.ts` does not exist. + +- [ ] **Step 3: Add the scan-connector-backed query executor** + +Create `packages/cli/src/ingest-query-executor.ts`: + +```ts +import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from '@ktx/context/connections'; +import type { KtxLocalProject } from '@ktx/context/project'; +import type { KtxScanConnector, KtxScanContext } from '@ktx/context/scan'; +import { createKtxCliScanConnector } from './local-scan-connectors.js'; + +type CreateConnector = typeof createKtxCliScanConnector; + +export interface KtxCliIngestQueryExecutorDeps { + createConnector?: CreateConnector; +} + +async function cleanupConnector(connector: KtxScanConnector | null): Promise { + await connector?.cleanup?.(); +} + +export function createKtxCliIngestQueryExecutor( + project: KtxLocalProject, + deps: KtxCliIngestQueryExecutorDeps = {}, +): KtxSqlQueryExecutorPort { + const createConnector = deps.createConnector ?? createKtxCliScanConnector; + return { + async execute(input: KtxSqlQueryExecutionInput) { + let connector: KtxScanConnector | null = null; + try { + connector = await createConnector(project, input.connectionId); + if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) { + throw new Error( + `Connection "${input.connectionId}" driver "${connector.driver}" does not support read-only SQL execution.`, + ); + } + + const ctx: KtxScanContext = { runId: 'ingest-sql-execution' }; + const result = await connector.executeReadOnly( + { connectionId: input.connectionId, sql: input.sql, maxRows: input.maxRows }, + ctx, + ); + return { + headers: result.headers, + rows: result.rows, + totalRows: result.totalRows, + command: 'SELECT', + rowCount: result.rowCount, + }; + } finally { + await cleanupConnector(connector); + } + }, + }; +} +``` + +- [ ] **Step 4: Wire the CLI executor into local ingest runs** + +In `packages/cli/src/ingest.ts`, import the executor and type: + +```ts +import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections'; +import type { KtxLocalProject } from '@ktx/context/project'; +import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; +``` + +Extend `KtxIngestDeps`: + +```ts + createQueryExecutor?: (project: KtxLocalProject) => KtxSqlQueryExecutorPort; +``` + +Inside the `args.command === 'run'` branch, after `localIngestOptions` is +defined, add: + +```ts + const queryExecutor = + localIngestOptions.queryExecutor ?? + (deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(project); +``` + +Pass `queryExecutor` to both local ingest execution paths. In the Metabase +fan-out call: + +```ts + ...localIngestOptions, + queryExecutor, + trigger: 'manual_resync', +``` + +In the normal local ingest call: + +```ts + ...localIngestOptions, + queryExecutor, + pullConfigOptions: adapterOptions, +``` + +- [ ] **Step 5: Add CLI wiring coverage** + +Add this test to `packages/cli/src/ingest.test.ts`: + +```ts + it('supplies a scan-connector query executor to local ingest runs', async () => { + const io = makeIo(); + const projectDir = join(tempDir, 'query-executor-project'); + await writeWarehouseConfig(projectDir); + const queryExecutor = { + execute: vi.fn(async () => ({ + headers: [], + rows: [], + totalRows: 0, + command: 'SELECT', + rowCount: 0, + })), + }; + const runLocalIngest = vi.fn(async (input: RunLocalIngestOptions): Promise => + completedLocalBundleRun(input, 'query-executor-run'), + ); + + await expect( + runKtxIngest( + { + command: 'run', + projectDir, + connectionId: 'warehouse', + adapter: 'fake', + outputMode: 'json', + }, + io.io, + { + runLocalIngest, + createAdapters: () => [], + createQueryExecutor: () => queryExecutor, + }, + ), + ).resolves.toBe(0); + + expect(runLocalIngest).toHaveBeenCalledWith(expect.objectContaining({ queryExecutor })); + }); +``` + +- [ ] **Step 6: Run CLI query executor tests** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "query executor" +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add \ + packages/cli/src/ingest-query-executor.ts \ + packages/cli/src/ingest-query-executor.test.ts \ + packages/cli/src/ingest.ts \ + packages/cli/src/ingest.test.ts +git commit -m "fix(cli): enable read-only SQL probes for local ingest" +``` + +### Task 5: Final verification + +**Files:** +- Verify: all files changed by Tasks 1-4. + +- [ ] **Step 1: Run focused context tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ + src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ + src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ + src/ingest/local-bundle-runtime.test.ts \ + src/ingest/local-adapters.test.ts \ + src/ingest/adapters/lookml/lookml.adapter.test.ts \ + src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ + src/ingest/ingest-bundle.runner.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run focused CLI tests** + +Run: + +```bash +pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Run type checks** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +pnpm --filter @ktx/cli run type-check +``` + +Expected: both commands pass. + +- [ ] **Step 4: Run pre-commit on changed files if configured** + +Run: + +```bash +uv run pre-commit run --files \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ + packages/context/src/ingest/adapters/lookml/lookml.adapter.ts \ + packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts \ + packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts \ + packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ + packages/context/src/ingest/local-adapters.ts \ + packages/context/src/ingest/local-adapters.test.ts \ + packages/context/src/ingest/local-bundle-runtime.ts \ + packages/context/src/ingest/local-bundle-runtime.test.ts \ + packages/context/src/ingest/local-ingest.ts \ + packages/cli/src/ingest-query-executor.ts \ + packages/cli/src/ingest-query-executor.test.ts \ + packages/cli/src/ingest.ts \ + packages/cli/src/ingest.test.ts \ + docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md +``` + +Expected: PASS. If the repository has no pre-commit config or the local `uv` +version cannot satisfy the configured toolchain, record the exact error and use +the focused test and type-check results as the closest verification. + +- [ ] **Step 5: Commit final verification fixes if any were needed** + +If verification required edits, run: + +```bash +git add +git commit -m "test: cover warehouse verification v1 closure" +``` + +If verification required no edits, do not create an empty commit. + +## Self-review + +Spec coverage: + +- Raw warehouse discovery still covers wiki, semantic-layer, and raw schema + results, and now raw hits include the connection name needed by the required + `entity_details` follow-up. +- Every local synthesis adapter with an external source connection now has a + path to target warehouse IDs: dbt and Notion already had it, Looker resolves + staged mappings, Metabase fan-out runs under target warehouse IDs, and this + plan adds LookML and MetricFlow. +- `sql_execution` remains scoped by `allowedConnectionNames`, retains the + read-only SQL wrapper, and gains a normal local ingest execution backend. + +Placeholder scan: + +- This plan contains no deferred implementation placeholders. +- Every code-changing step includes the exact test or implementation snippet to + add. + +Type consistency: + +- `connectionName` is added to `RawSchemaHit` and used by `DiscoverDataTool`. +- `targetConnectionIds` and `listTargetConnectionIds()` match the existing dbt + and Notion adapter pattern. +- Local ingest uses `KtxSqlQueryExecutorPort` consistently from CLI to context. diff --git a/docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md b/docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md new file mode 100644 index 00000000..42bb7f44 --- /dev/null +++ b/docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md @@ -0,0 +1,1617 @@ +# Warehouse Verification Tools Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add synthesis-time warehouse verification tools so ingest agents can verify raw warehouse tables, columns, and sample values before writing wiki pages, SL sources, `tables:` frontmatter, `sl_refs`, or unmapped fallback records. + +**Architecture:** Add a raw scan catalog service over `raw-sources//live-database//`, three BaseTool-backed ingest tools, and runner/tool-session scoping for allowed warehouse connections. Register the tools in the local ingest toolset so both WorkUnit and reconcile stages receive them through the existing `toAiSdkTools()` path. + +**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX file store, KTX semantic layer and wiki tools. + +--- + +## Audit summary + +The current repo has the original spec file only; no matching plan or implementation exists under `docs/superpowers/plans`. The following v1-blocking gaps remain: + +- `packages/context/src/connections/dialects.ts` does not exist. +- `packages/context/src/ingest/tools/warehouse-verification/` does not exist. +- `entity_details`, `sql_execution`, and `discover_data` are not available to ingest WU or reconcile toolsets. +- `ToolSession` does not carry the ingest stage's allowed warehouse connection IDs. +- Prompt updates are absent from the 11 writer skills named in the spec. +- Cleanup strings remain: `orbit_analytics.customer`, `wiki_sl_search`, and `sl_describe_table`. +- Prompt-bundling and warehouse-tool tests are absent. + +Non-blocking gaps remain out of scope for this plan: + +- Hard write-time validation in `wiki_write` and `emit_unmapped_fallback`. +- `dictionary_search`. +- `semantic_query` in synthesis toolsets. +- A raw-schema FTS index. +- A UUID identity layer for tables and columns. + +One repo-specific adjustment is required: do not import `@ktx/connector-*` +dialect classes into `@ktx/context`, because every connector package already +depends on `@ktx/context`. Add a minimal context-local dialect dispatch instead. + +## File structure + +Create these files: + +- `packages/context/src/connections/dialects.ts`: Context-local driver dispatch for identifier quoting and display formatting. +- `packages/context/src/connections/dialects.test.ts`: Driver dispatch and display-format tests. +- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: Reads the latest live-database scan, resolves display identifiers, and searches table and column metadata. +- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: Fixture-backed catalog tests. +- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: `entity_details` ingest tool. +- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: Tool contract tests. +- `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`: `sql_execution` ingest tool. +- `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts`: Read-only SQL and output tests. +- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: `discover_data` ingest tool composing wiki, SL, and raw-schema search. +- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: Discovery composition tests. +- `packages/context/src/ingest/tools/warehouse-verification/index.ts`: Exports tool classes and `createWarehouseVerificationTools()`. +- `packages/context/skills/_shared/identifier-verification.md`: Shared protocol text kept in the tree for review even though writer skills inline it. + +Modify these files: + +- `packages/context/src/connections/index.ts`: Export the dialect helper. +- `packages/context/src/tools/tool-session.ts`: Add `allowedConnectionNames`. +- `packages/context/src/ingest/ingest-bundle.runner.ts`: Populate `allowedConnectionNames` for WU and reconcile sessions. +- `packages/context/src/ingest/local-bundle-runtime.ts`: Register the warehouse verification tools in `LocalIngestToolsetFactory`. +- `packages/context/src/ingest/ingest-bundle.runner.test.ts`: Assert the runner scopes allowed warehouse connections. +- `packages/context/src/memory/memory-runtime-assets.test.ts`: Assert writer skills contain the protocol and banned strings are gone. +- `packages/context/src/ingest/ingest-runtime-assets.test.ts`: Assert ingest skill packaging includes the protocol. +- `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`: Replace the fictional table example. +- `packages/context/src/sl/tools/sl-warehouse-validation.ts`: Replace the stale `sl_describe_table` hint. +- `packages/context/skills/*/SKILL.md`: Inline protocol updates for the writer skills listed in the spec. + +### Task 1: Add context-local dialect dispatch + +**Files:** +- Create: `packages/context/src/connections/dialects.ts` +- Create: `packages/context/src/connections/dialects.test.ts` +- Modify: `packages/context/src/connections/index.ts` + +- [ ] **Step 1: Write the failing dialect tests** + +Create `packages/context/src/connections/dialects.test.ts`: + +```ts +import { describe, expect, it } from 'vitest'; +import { getDialectForDriver } from './dialects.js'; + +describe('getDialectForDriver', () => { + it.each([ + ['postgres', '"public"."orders"'], + ['postgresql', '"public"."orders"'], + ['mysql', '`public`.`orders`'], + ['clickhouse', '`public`.`orders`'], + ['sqlite', '"orders"'], + ['snowflake', '"analytics"."public"."orders"'], + ['bigquery', '`analytics`.`public`.`orders`'], + ['sqlserver', '[analytics].[public].[orders]'], + ] as const)('formats table names for %s', (driver, expected) => { + const dialect = getDialectForDriver(driver); + expect( + dialect.formatTableName({ + catalog: driver === 'snowflake' || driver === 'bigquery' || driver === 'sqlserver' ? 'analytics' : null, + db: driver === 'sqlite' ? null : 'public', + name: 'orders', + }), + ).toBe(expected); + }); + + it('throws with a supported-driver list for unknown drivers', () => { + expect(() => getDialectForDriver('oracle')).toThrow( + 'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver', + ); + }); +}); +``` + +- [ ] **Step 2: Run the failing test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts +``` + +Expected: FAIL because `./dialects.js` does not exist. + +- [ ] **Step 3: Add the minimal dialect implementation** + +Create `packages/context/src/connections/dialects.ts`: + +```ts +import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; + +export type SupportedDriver = + | 'postgres' + | 'postgresql' + | 'mysql' + | 'sqlserver' + | 'snowflake' + | 'bigquery' + | 'clickhouse' + | 'sqlite' + | 'sqlite3'; + +export interface KtxDialect { + readonly type: SupportedDriver; + quoteIdentifier(identifier: string): string; + formatTableName(table: KtxTableRef): string; + mapToDimensionType(nativeType: string): KtxSchemaDimensionType; +} + +const supportedDrivers: SupportedDriver[] = [ + 'bigquery', + 'clickhouse', + 'mysql', + 'postgres', + 'postgresql', + 'sqlite', + 'sqlite3', + 'snowflake', + 'sqlserver', +]; + +function doubleQuoted(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; +} + +function backtickQuoted(identifier: string): string { + return `\`${identifier.replace(/`/g, '``')}\``; +} + +function bigQueryQuoted(identifier: string): string { + return `\`${identifier.replace(/`/g, '\\`')}\``; +} + +function bracketQuoted(identifier: string): string { + return `[${identifier.replace(/\]/g, ']]')}]`; +} + +function inferDimensionType(nativeType: string): KtxSchemaDimensionType { + const normalized = nativeType.toLowerCase().trim(); + if (normalized.includes('date') || normalized.includes('time')) { + return 'time'; + } + if ( + normalized.includes('int') || + normalized.includes('num') || + normalized.includes('dec') || + normalized.includes('float') || + normalized.includes('double') || + normalized.includes('real') + ) { + return 'number'; + } + if (normalized.includes('bool') || normalized === 'bit') { + return 'boolean'; + } + return 'string'; +} + +function formatWithParts(table: KtxTableRef, quote: (identifier: string) => string, sqlite = false): string { + const parts = sqlite ? [table.name] : [table.catalog, table.db, table.name].filter((part): part is string => !!part); + return parts.map(quote).join('.'); +} + +function createDialect(type: SupportedDriver, quote: (identifier: string) => string, sqlite = false): KtxDialect { + return { + type, + quoteIdentifier: quote, + formatTableName: (table) => formatWithParts(table, quote, sqlite), + mapToDimensionType: inferDimensionType, + }; +} + +const dialects: Record = { + postgres: createDialect('postgres', doubleQuoted), + postgresql: createDialect('postgresql', doubleQuoted), + mysql: createDialect('mysql', backtickQuoted), + clickhouse: createDialect('clickhouse', backtickQuoted), + sqlite: createDialect('sqlite', doubleQuoted, true), + sqlite3: createDialect('sqlite3', doubleQuoted, true), + snowflake: createDialect('snowflake', doubleQuoted), + bigquery: createDialect('bigquery', bigQueryQuoted), + sqlserver: createDialect('sqlserver', bracketQuoted), +}; + +export function getDialectForDriver(driver: string): KtxDialect { + const normalized = driver.toLowerCase().trim(); + if (normalized in dialects) { + return dialects[normalized as SupportedDriver]; + } + throw new Error(`Unsupported warehouse driver "${driver}". Supported drivers: ${supportedDrivers.join(', ')}`); +} +``` + +Modify `packages/context/src/connections/index.ts`: + +```ts +export type { KtxDialect, SupportedDriver } from './dialects.js'; +export { getDialectForDriver } from './dialects.js'; +``` + +- [ ] **Step 4: Run the dialect tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/src/connections/dialects.ts packages/context/src/connections/dialects.test.ts packages/context/src/connections/index.ts +git commit -m "feat(context): add warehouse dialect dispatch" +``` + +### Task 2: Add the raw scan warehouse catalog service + +**Files:** +- Create: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` +- Create: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` + +- [ ] **Step 1: Write failing catalog tests** + +Create `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: + +```ts +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { initKtxProject, type KtxLocalProject } from '../../../project/index.js'; +import { WarehouseCatalogService } from './warehouse-catalog.service.js'; + +describe('WarehouseCatalogService', () => { + let tempDir: string; + let project: KtxLocalProject; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-warehouse-catalog-')); + project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + async function seedLiveDatabaseScan(connectionName = 'warehouse', syncId = 'sync-2', driver = 'postgres') { + const root = `raw-sources/${connectionName}/live-database/${syncId}`; + await project.fileStore.writeFile( + `${root}/connection.json`, + JSON.stringify({ connectionId: connectionName, driver, extractedAt: '2026-05-12T00:00:00.000Z' }, null, 2), + 'ktx', + 'ktx@example.com', + 'seed connection', + ); + await project.fileStore.writeFile( + `${root}/tables/orders.json`, + JSON.stringify( + { + catalog: null, + db: driver === 'sqlite' ? null : 'public', + name: 'orders', + kind: 'table', + comment: 'Customer orders', + estimatedRows: 12, + columns: [ + { + name: 'id', + nativeType: 'integer', + normalizedType: 'integer', + dimensionType: 'number', + nullable: false, + primaryKey: true, + comment: 'Order id', + }, + { + name: 'status', + nativeType: 'text', + normalizedType: 'text', + dimensionType: 'string', + nullable: false, + primaryKey: false, + comment: 'Order status', + }, + ], + foreignKeys: [], + }, + null, + 2, + ), + 'ktx', + 'ktx@example.com', + 'seed orders', + ); + await project.fileStore.writeFile( + `${root}/enrichment/relationship-profile.json`, + JSON.stringify( + { + connectionId: connectionName, + driver, + sqlAvailable: true, + queryCount: 3, + tables: [{ table: { catalog: null, db: driver === 'sqlite' ? null : 'public', name: 'orders' }, rowCount: 12 }], + columns: { + 'orders.status': { + table: { catalog: null, db: driver === 'sqlite' ? null : 'public', name: 'orders' }, + column: 'status', + nativeType: 'text', + normalizedType: 'text', + rowCount: 12, + nullCount: 0, + distinctCount: 2, + uniquenessRatio: 0.1667, + nullRate: 0, + sampleValues: ['paid', 'refunded'], + minTextLength: 4, + maxTextLength: 8, + }, + }, + warnings: [], + }, + null, + 2, + ), + 'ktx', + 'ktx@example.com', + 'seed profile', + ); + } + + it('finds the latest sync and merges table schema with relationship profile values', async () => { + await seedLiveDatabaseScan('warehouse', 'sync-1'); + await seedLiveDatabaseScan('warehouse', 'sync-2'); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.getLatestSyncId('warehouse')).resolves.toBe('sync-2'); + const detail = await catalog.getTable({ connectionName: 'warehouse', catalog: null, db: 'public', name: 'orders' }); + + expect(detail).toMatchObject({ + connectionName: 'warehouse', + display: 'public.orders', + rowCount: 12, + columns: [ + { name: 'id', nativeType: 'integer', primaryKey: true }, + { name: 'status', nativeType: 'text', sampleValues: ['paid', 'refunded'], distinctCount: 2 }, + ], + }); + }); + + it('returns scanAvailable=false when no live-database scan exists', async () => { + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + await expect(catalog.getTable({ connectionName: 'missing', catalog: null, db: 'public', name: 'orders' })).resolves.toBeNull(); + await expect(catalog.hasScan('missing')).resolves.toBe(false); + }); + + it('resolves postgres display strings and returns closest candidates for missing tables', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplay('warehouse', 'public.orders')).resolves.toMatchObject({ + resolved: { catalog: null, db: 'public', name: 'orders' }, + candidates: [], + dialect: 'postgres', + }); + await expect(catalog.resolveDisplay('warehouse', 'public.orderz')).resolves.toMatchObject({ + resolved: null, + candidates: [{ name: 'orders' }], + }); + }); + + it('treats two-part BigQuery identifiers as ambiguous instead of guessing', async () => { + await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplay('warehouse', 'public.orders')).resolves.toMatchObject({ + resolved: null, + dialect: 'bigquery', + }); + }); + + it('searches table names, column names, comments, and descriptions', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.searchByName('warehouse', 'status', 10)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'column', + ref: expect.objectContaining({ db: 'public', name: 'orders', column: 'status' }), + matchedOn: 'name', + }), + ]), + ); + }); +}); +``` + +- [ ] **Step 2: Run the failing catalog tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts +``` + +Expected: FAIL because the service file does not exist. + +- [ ] **Step 3: Add the catalog service** + +Create `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` with these exported shapes and behavior: + +```ts +import type { KtxFileStorePort } from '../../../core/index.js'; +import { getDialectForDriver } from '../../../connections/index.js'; +import type { KtxConnectionDriver, KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaTable, KtxTableRef } from '../../../scan/types.js'; + +export interface WarehouseCatalogServiceDeps { + fileStore: KtxFileStorePort; +} + +export interface WarehouseColumnDetail extends KtxSchemaColumn { + descriptions: Record; + rowCount: number | null; + nullCount: number | null; + distinctCount: number | null; + nullRate: number | null; + sampleValues: string[]; +} + +export interface TableDetail { + connectionName: string; + catalog: string | null; + db: string | null; + name: string; + display: string; + kind: string; + comment: string | null; + description: string | null; + rowCount: number | null; + columns: WarehouseColumnDetail[]; + foreignKeys: KtxSchemaForeignKey[]; +} + +export type RawSchemaHit = + | { kind: 'table'; ref: KtxTableRef; display: string; matchedOn: 'name' | 'db' | 'comment' | 'description' } + | { kind: 'column'; ref: KtxTableRef & { column: string }; display: string; matchedOn: 'name' | 'comment' | 'description' }; + +interface ConnectionArtifact { + driver?: KtxConnectionDriver; +} + +interface RelationshipProfileColumn { + table?: KtxTableRef; + column?: string; + rowCount?: number; + nullCount?: number; + distinctCount?: number; + nullRate?: number; + sampleValues?: unknown[]; +} + +interface RelationshipProfileArtifact { + driver?: KtxConnectionDriver; + tables?: Array<{ table?: KtxTableRef; rowCount?: number }>; + columns?: Record; +} + +interface ConnectionCatalog { + connectionName: string; + syncId: string; + driver: KtxConnectionDriver; + tables: KtxSchemaTable[]; + profile: RelationshipProfileArtifact | null; +} +``` + +The implementation must: + +- Use `fileStore.listFiles("raw-sources//live-database")` and choose the lexicographically latest path ending in `/connection.json`. +- Read every JSON file under `/tables/` rather than reconstructing a path from the table ref. This supports encoded and simple table filenames already present in tests. +- Parse display strings by driver: + - Postgres, MySQL, and ClickHouse: `schema.table`. + - SQL Server, Snowflake, and BigQuery: `catalog.schema.table`. + - SQLite: `table`. + - For BigQuery, a two-part display must return `resolved: null` and candidate matches. +- Match table refs case-insensitively, while preserving stored casing in outputs. +- Merge relationship-profile fields by `(catalog, db, name, column)`, with fallback matching on `table.name + "." + column`. +- Cache a loaded connection catalog per `connectionName` within the service instance. +- Return `null` from `getTable()` when the scan is absent or the table ref is not found. + +Use these method signatures: + +```ts +export class WarehouseCatalogService { + constructor(private readonly deps: WarehouseCatalogServiceDeps) {} + + async hasScan(connectionName: string): Promise; + async getLatestSyncId(connectionName: string): Promise; + async listTables(connectionName: string): Promise; + async getTable(ref: { connectionName: string } & KtxTableRef): Promise; + async resolveDisplay(connectionName: string, display: string): Promise<{ + resolved: KtxTableRef | null; + candidates: KtxTableRef[]; + dialect: string; + }>; + async searchByName(connectionName: string, query: string, limit: number): Promise; +} +``` + +- [ ] **Step 4: Run the catalog tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts +git commit -m "feat(context): read warehouse scan catalog" +``` + +### Task 3: Add `entity_details` + +**Files:** +- Create: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` +- Create: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` + +- [ ] **Step 1: Write failing `entity_details` tests** + +Create tests that instantiate the tool with a seeded `WarehouseCatalogService` and a `ToolContext` whose session has `allowedConnectionNames: new Set(['warehouse'])`. Test these cases: + +```ts +it('returns scoped table detail for a display target', async () => { + const result = await tool.call( + { connectionName: 'warehouse', targets: [{ display: 'public.orders' }] }, + context, + ); + expect(result.markdown).toContain('### public.orders'); + expect(result.markdown).toContain('- status (text, nullable=false)'); + expect(result.markdown).toContain('sample: ["paid","refunded"]'); + expect(result.structured.scanAvailable).toBe(true); + expect(result.structured.resolved).toHaveLength(1); +}); + +it('returns a no-scan state distinct from not found', async () => { + const result = await tool.call( + { connectionName: 'empty', targets: [{ display: 'public.orders' }] }, + { ...context, session: { ...context.session!, allowedConnectionNames: new Set(['empty']) } }, + ); + expect(result.markdown).toContain('No live-database scan available for connection "empty"; run `ktx scan` first.'); + expect(result.structured.scanAvailable).toBe(false); +}); + +it('refuses out-of-scope connections', async () => { + const result = await tool.call( + { connectionName: 'billing', targets: [{ display: 'public.orders' }] }, + context, + ); + expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.'); + expect(result.structured.scanAvailable).toBe(false); +}); +``` + +- [ ] **Step 2: Run the failing tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +``` + +Expected: FAIL because the tool file does not exist. + +- [ ] **Step 3: Implement the tool** + +Create `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: + +```ts +import { z } from 'zod'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import type { KtxTableRef } from '../../../scan/types.js'; +import { WarehouseCatalogService, type TableDetail } from './warehouse-catalog.service.js'; + +const targetSchema = z.union([ + z.object({ display: z.string().min(1) }), + z.object({ + catalog: z.string().nullable(), + db: z.string().nullable(), + name: z.string().min(1), + column: z.string().optional(), + }), +]); + +const entityDetailsInputSchema = z.object({ + connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + targets: z.array(targetSchema).min(1).max(50), +}); + +type EntityDetailsInput = z.infer; + +export interface EntityDetailsStructured { + resolved: TableDetail[]; + missing: Array<{ target: unknown; candidates: KtxTableRef[] }>; + scanAvailable: boolean; +} + +function allowedConnectionNames(context: ToolContext): ReadonlySet | null { + return context.session?.allowedConnectionNames ?? null; +} + +function sampleText(values: string[]): string { + return values.length > 0 ? ` - sample: ${JSON.stringify(values.slice(0, 10))}` : ''; +} + +function appendTableMarkdown(parts: string[], detail: TableDetail, columnName?: string): void { + const columns = columnName ? detail.columns.filter((column) => column.name === columnName) : detail.columns; + parts.push(`### ${detail.display}`); + parts.push(`Type: ${detail.kind} | Native columns: ${detail.columns.length}`); + if (detail.description || detail.comment) { + parts.push(`Description: ${detail.description ?? detail.comment}`); + } + parts.push('', 'Columns:'); + for (const column of columns) { + const pk = column.primaryKey ? ', PK' : ''; + parts.push(`- ${column.name} (${column.nativeType}, nullable=${column.nullable}${pk})${sampleText(column.sampleValues)}`); + } + parts.push(''); +} + +export class EntityDetailsTool extends BaseTool { + readonly name = 'entity_details'; + + constructor(private readonly catalogFactory: (context: ToolContext) => WarehouseCatalogService) { + super(); + } + + get description(): string { + return 'Verify warehouse tables and columns from the latest live-database scan before writing them into wiki or semantic-layer output.'; + } + + get inputSchema() { + return entityDetailsInputSchema; + } + + async call(input: EntityDetailsInput, context: ToolContext): Promise> { + const allowed = allowedConnectionNames(context); + if (allowed && !allowed.has(input.connectionName)) { + return { + markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, + structured: { resolved: [], missing: [], scanAvailable: false }, + }; + } + + const catalog = this.catalogFactory(context); + const scanAvailable = await catalog.hasScan(input.connectionName); + if (!scanAvailable) { + return { + markdown: `No live-database scan available for connection "${input.connectionName}"; run \`ktx scan\` first.`, + structured: { resolved: [], missing: [], scanAvailable: false }, + }; + } + + const parts: string[] = []; + const resolved: TableDetail[] = []; + const missing: EntityDetailsStructured['missing'] = []; + + for (const target of input.targets) { + const resolution = + 'display' in target + ? await catalog.resolveDisplay(input.connectionName, target.display) + : { resolved: { catalog: target.catalog, db: target.db, name: target.name }, candidates: [], dialect: '' }; + if (!resolution.resolved) { + missing.push({ target, candidates: resolution.candidates }); + parts.push(`Not found in scan: ${'display' in target ? target.display : target.name}`); + if (resolution.candidates.length > 0) { + parts.push(`Closest matches: ${resolution.candidates.map((candidate) => candidate.name).join(', ')}`); + } + continue; + } + const detail = await catalog.getTable({ connectionName: input.connectionName, ...resolution.resolved }); + if (!detail) { + missing.push({ target, candidates: resolution.candidates }); + continue; + } + resolved.push(detail); + appendTableMarkdown(parts, detail, 'column' in target ? target.column : undefined); + } + + return { + markdown: parts.join('\n').trim(), + structured: { resolved, missing, scanAvailable: true }, + }; + } +} +``` + +- [ ] **Step 4: Run the `entity_details` tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +git commit -m "feat(context): add entity details verification tool" +``` + +### Task 4: Add `sql_execution` + +**Files:** +- Create: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts` +- Create: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts` + +- [ ] **Step 1: Write failing `sql_execution` tests** + +Create tests for: + +```ts +it('wraps read-only SQL with a capped row limit', async () => { + connections.executeQuery.mockResolvedValue({ headers: ['status'], rows: [['paid']], totalRows: 1 }); + const result = await tool.call( + { connectionName: 'warehouse', sql: 'select status from public.orders', rowLimit: 5 }, + context, + ); + expect(connections.executeQuery).toHaveBeenCalledWith( + 'warehouse', + 'select * from (select status from public.orders) as ktx_query_result limit 5', + ); + expect(result.markdown).toContain('| status |'); + expect(result.structured.wrappedSql).toContain('limit 5'); +}); + +it.each(['insert into x values (1)', 'drop table x', 'vacuum'])('rejects mutating SQL: %s', async (sql) => { + const result = await tool.call({ connectionName: 'warehouse', sql }, context); + expect(result.markdown).toContain('Only read-only SELECT/WITH queries can be executed locally.'); + expect(connections.executeQuery).not.toHaveBeenCalled(); +}); + +it('surfaces connector errors verbatim', async () => { + connections.executeQuery.mockRejectedValue(new Error('relation "orbit_analytics.customer" does not exist')); + const result = await tool.call( + { connectionName: 'warehouse', sql: 'select 1 from orbit_analytics.customer', rowLimit: 1 }, + context, + ); + expect(result.markdown).toContain('relation "orbit_analytics.customer" does not exist'); + expect(result.structured.error).toContain('relation "orbit_analytics.customer" does not exist'); +}); +``` + +- [ ] **Step 2: Run the failing tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts +``` + +Expected: FAIL because the tool file does not exist. + +- [ ] **Step 3: Implement the tool** + +Create `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`: + +```ts +import { z } from 'zod'; +import { assertReadOnlySql, limitSqlForExecution } from '../../../connections/index.js'; +import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; + +const sqlExecutionInputSchema = z.object({ + connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + sql: z.string().min(1), + rowLimit: z.number().int().positive().max(1000).optional().default(100), +}); + +type SqlExecutionInput = z.infer; + +export interface SqlExecutionStructured { + headers: string[]; + rows: unknown[][]; + rowCount: number; + truncated: boolean; + sql: string; + wrappedSql: string; + error?: string; +} + +function markdownTable(headers: string[], rows: unknown[][], totalRows: number): string { + if (headers.length === 0) { + return rows.length === 0 ? 'Query returned no rows.' : JSON.stringify(rows.slice(0, 20)); + } + const visible = rows.slice(0, 20); + const lines = [ + `| ${headers.join(' | ')} |`, + `| ${headers.map(() => '---').join(' | ')} |`, + ...visible.map((row) => `| ${row.map((value) => String(value ?? '')).join(' | ')} |`), + ]; + if (totalRows > visible.length) { + lines.push(`... +${totalRows - visible.length} more rows`); + } + return lines.join('\n'); +} + +export class SqlExecutionTool extends BaseTool { + readonly name = 'sql_execution'; + + constructor(private readonly connections: SlConnectionCatalogPort) { + super(); + } + + get description(): string { + return 'Run a single read-only SELECT or WITH probe against an allowed warehouse connection and return a capped markdown table or the warehouse error.'; + } + + get inputSchema() { + return sqlExecutionInputSchema; + } + + async call(input: SqlExecutionInput, context: ToolContext): Promise> { + const allowed = context.session?.allowedConnectionNames; + if (allowed && !allowed.has(input.connectionName)) { + return { + markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, + structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql: input.sql, wrappedSql: '', error: 'connection_not_allowed' }, + }; + } + + let sql: string; + let wrappedSql: string; + try { + sql = assertReadOnlySql(input.sql); + wrappedSql = limitSqlForExecution(sql, input.rowLimit); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + markdown: message, + structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql: input.sql, wrappedSql: '', error: message }, + }; + } + + try { + const result = await this.connections.executeQuery(input.connectionName, wrappedSql); + const headers = result.headers ?? []; + const rows = result.rows ?? []; + const rowCount = result.totalRows ?? rows.length; + return { + markdown: markdownTable(headers, rows, rowCount), + structured: { headers, rows, rowCount, truncated: rowCount > rows.length, sql, wrappedSql }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + markdown: `SQL execution failed: ${message}`, + structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql, wrappedSql, error: message }, + }; + } + } +} +``` + +- [ ] **Step 4: Run the `sql_execution` tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts +git commit -m "feat(context): add ingest SQL verification tool" +``` + +### Task 5: Add `discover_data` + +**Files:** +- Create: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` +- Create: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` +- Create: `packages/context/src/ingest/tools/warehouse-verification/index.ts` + +- [ ] **Step 1: Write failing `discover_data` tests** + +Create tests with fake `wikiSearchTool.call`, `slDiscoverTool.call`, and `WarehouseCatalogService.searchByName`. Cover: + +```ts +it('groups wiki, semantic layer, and raw schema hits with routing hints', async () => { + const result = await tool.call({ query: 'orders', connectionName: 'warehouse', limit: 5 }, context); + expect(result.markdown).toContain('## Wiki Pages'); + expect(result.markdown).toContain('use `wiki_read(blockKey)` for full content'); + expect(result.markdown).toContain('## Semantic Layer Sources'); + expect(result.markdown).toContain('use `sl_read_source(sourceName)` for the YAML'); + expect(result.markdown).toContain('## Raw Warehouse Schema'); + expect(result.markdown).toContain('use `entity_details({connectionName, targets: [{display}]})`'); + expect(result.structured.raw?.hits).toHaveLength(1); +}); + +it('delegates sourceName inspect mode to sl_discover only', async () => { + const result = await tool.call({ sourceName: 'orders', connectionName: 'warehouse' }, context); + expect(slDiscoverTool.call).toHaveBeenCalledWith({ sourceName: 'orders', connectionId: 'warehouse' }, context); + expect(wikiSearchTool.call).not.toHaveBeenCalled(); + expect(catalog.searchByName).not.toHaveBeenCalled(); + expect(result.markdown).toContain('source detail'); +}); + +it('returns the empty-state message when all sections are empty', async () => { + const result = await tool.call({ query: 'customer source', connectionName: 'warehouse' }, emptyContext); + expect(result.markdown).toContain('No matches for "customer source" across wiki, semantic layer, or raw warehouse schema.'); +}); +``` + +- [ ] **Step 2: Run the failing tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +``` + +Expected: FAIL because the tool file does not exist. + +- [ ] **Step 3: Implement the tool and index export** + +Create `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: + +```ts +import { z } from 'zod'; +import type { BaseTool, ToolContext, ToolOutput } from '../../../tools/index.js'; +import { BaseTool as ToolBase } from '../../../tools/index.js'; +import { WarehouseCatalogService, type RawSchemaHit } from './warehouse-catalog.service.js'; + +const discoverDataInputSchema = z.object({ + query: z.string().optional(), + connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(), + limit: z.number().int().positive().max(50).optional().default(10), + sourceName: z.string().optional(), +}); + +type DiscoverDataInput = z.infer; + +export interface DiscoverDataStructured { + wiki: unknown | null; + sl: unknown | null; + raw: { hits: RawSchemaHit[] } | null; +} + +interface DiscoverDataDeps { + wikiSearchTool: BaseTool; + slDiscoverTool: BaseTool; + catalogFactory: (context: ToolContext) => WarehouseCatalogService; +} + +export class DiscoverDataTool extends ToolBase { + readonly name = 'discover_data'; + + constructor(private readonly deps: DiscoverDataDeps) { + super(); + } + + get description(): string { + return 'Discover existing wiki pages, semantic layer sources, and raw warehouse schema hits before writing ingest output.'; + } + + get inputSchema() { + return discoverDataInputSchema; + } + + async call(input: DiscoverDataInput, context: ToolContext): Promise> { + if (input.sourceName) { + const sl = await this.deps.slDiscoverTool.call( + { sourceName: input.sourceName, connectionId: input.connectionName }, + context, + ); + return { markdown: sl.markdown, structured: { wiki: null, sl: sl.structured, raw: null } }; + } + + const query = input.query?.trim() || ''; + const limit = input.limit ?? 10; + const parts: string[] = []; + let wiki: unknown | null = null; + let sl: unknown | null = null; + let raw: DiscoverDataStructured['raw'] = null; + + if (query) { + const wikiResult = await this.deps.wikiSearchTool.call({ query, limit }, context); + if (wikiResult.structured?.totalFound > 0) { + parts.push('## Wiki Pages', '> use `wiki_read(blockKey)` for full content', wikiResult.markdown, ''); + wiki = wikiResult.structured; + } + } + + const slResult = await this.deps.slDiscoverTool.call( + { query: query || undefined, connectionId: input.connectionName }, + context, + ); + if (slResult.structured?.totalSources > 0) { + parts.push('## Semantic Layer Sources', '> use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details', slResult.markdown, ''); + sl = slResult.structured; + } + + const catalog = this.deps.catalogFactory(context); + const connections = input.connectionName + ? [input.connectionName] + : [...(context.session?.allowedConnectionNames ?? [])].sort(); + const rawHits: RawSchemaHit[] = []; + for (const connectionName of connections) { + rawHits.push(...(await catalog.searchByName(connectionName, query, limit))); + } + if (rawHits.length > 0) { + parts.push('## Raw Warehouse Schema', '> use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values'); + parts.push( + rawHits + .slice(0, limit) + .map((hit) => `- ${hit.kind}: ${hit.display} (matched on ${hit.matchedOn})`) + .join('\n'), + ); + raw = { hits: rawHits.slice(0, limit) }; + } + + if (parts.length === 0) { + return { + markdown: `No matches for "${query}" across wiki, semantic layer, or raw warehouse schema. Try broader terms; this concept may not exist yet.`, + structured: { wiki, sl, raw }, + }; + } + + return { markdown: parts.join('\n'), structured: { wiki, sl, raw } }; + } +} +``` + +Create `packages/context/src/ingest/tools/warehouse-verification/index.ts`: + +```ts +import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import type { KtxFileStorePort } from '../../../core/index.js'; +import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import { DiscoverDataTool } from './discover-data.tool.js'; +import { EntityDetailsTool } from './entity-details.tool.js'; +import { SqlExecutionTool } from './sql-execution.tool.js'; +import { WarehouseCatalogService } from './warehouse-catalog.service.js'; + +export { DiscoverDataTool } from './discover-data.tool.js'; +export { EntityDetailsTool } from './entity-details.tool.js'; +export { SqlExecutionTool } from './sql-execution.tool.js'; +export { WarehouseCatalogService } from './warehouse-catalog.service.js'; +export type { TableDetail, WarehouseColumnDetail, RawSchemaHit } from './warehouse-catalog.service.js'; + +export function createWarehouseVerificationTools(deps: { + connections: SlConnectionCatalogPort; + fallbackFileStore: KtxFileStorePort; + wikiSearchTool: BaseTool; + slDiscoverTool: BaseTool; +}): BaseTool[] { + const catalogFactory = (context: ToolContext) => + new WarehouseCatalogService({ + fileStore: context.session?.configService ?? deps.fallbackFileStore, + }); + return [ + new EntityDetailsTool(catalogFactory), + new SqlExecutionTool(deps.connections), + new DiscoverDataTool({ + wikiSearchTool: deps.wikiSearchTool, + slDiscoverTool: deps.slDiscoverTool, + catalogFactory, + }), + ]; +} +``` + +- [ ] **Step 4: Run the `discover_data` tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts packages/context/src/ingest/tools/warehouse-verification/index.ts +git commit -m "feat(context): add raw warehouse discovery tool" +``` + +### Task 6: Wire tools into ingest sessions + +**Files:** +- Modify: `packages/context/src/tools/tool-session.ts` +- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` +- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` +- Modify: `packages/context/src/ingest/ingest-bundle.runner.test.ts` + +- [ ] **Step 1: Write failing scoping test** + +Add to `packages/context/src/ingest/ingest-bundle.runner.test.ts`: + +```ts +it('threads target warehouse connection names into WorkUnit and reconcile tool sessions', async () => { + const deps = makeDeps(); + const sessions: any[] = []; + deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse']); + deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession: any) => { + sessions.push(toolSession); + return { + toAiSdkTools: vi.fn().mockReturnValue({}), + getAllTools: vi.fn().mockReturnValue([]), + getToolNames: vi.fn().mockReturnValue([]), + }; + }); + deps.agentRunner.runLoop.mockResolvedValue({ stopReason: 'natural' }); + + const runner = buildRunner(deps); + (runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({ + currentHashes: new Map([['a.yml', 'h1']]), + rawDirInWorktree: 'raw-sources/notion/fake/s', + }); + (runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x'); + + await runner.run({ + jobId: 'j1', + connectionId: 'notion', + sourceKey: 'fake', + trigger: 'upload', + bundleRef: { kind: 'upload', uploadId: 'upload-x' }, + }); + + expect([...sessions[0].allowedConnectionNames].sort()).toEqual(['notion', 'warehouse']); +}); +``` + +- [ ] **Step 2: Run the failing runner test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.test.ts -t "threads target warehouse connection names" +``` + +Expected: FAIL because `allowedConnectionNames` is absent. + +- [ ] **Step 3: Thread allowed connection names** + +Modify `packages/context/src/tools/tool-session.ts`: + +```ts + allowedRawPaths?: ReadonlySet; + allowedConnectionNames?: ReadonlySet; + semanticLayerService: SemanticLayerService; +``` + +Modify WU session creation in `packages/context/src/ingest/ingest-bundle.runner.ts`: + +```ts + allowedRawPaths: new Set(wu.rawFiles), + allowedConnectionNames: new Set(slConnectionIds), + semanticLayerService: scopedSemanticLayerService, +``` + +Modify reconcile session creation in the same file: + +```ts + allowedRawPaths: reconciliationAllowedRawPaths, + allowedConnectionNames: new Set(slConnectionIds), + semanticLayerService: rcScopedSl, +``` + +- [ ] **Step 4: Register the tools in the local ingest toolset** + +Modify `packages/context/src/ingest/local-bundle-runtime.ts`: + +```ts +import { + createWarehouseVerificationTools, +} from './tools/warehouse-verification/index.js'; +``` + +Refactor the existing inline wiki and SL tool instances in `LocalIngestToolsetFactory` so `wikiSearchTool` and `slDiscoverTool` are named constants, then add the warehouse tools: + +```ts + const wikiSearchTool = new WikiSearchTool({ + search: async (input) => { + const results = await searchLocalKnowledgePages(deps.project, { + userId: input.userId, + query: input.query, + limit: input.limit, + embeddingService: deps.embedding, + }); + return { + results: results.slice(0, input.limit).map((result) => ({ + key: result.key, + path: result.path, + summary: result.summary, + score: result.score, + matchReasons: result.matchReasons, + lanes: result.lanes, + })), + totalFound: results.length, + }; + }, + }); + const slDiscoverTool = new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 }); + const warehouseVerificationTools = createWarehouseVerificationTools({ + connections: deps.connections, + fallbackFileStore: deps.project.fileStore, + wikiSearchTool, + slDiscoverTool, + }); + + this.baseTools = [ + new WikiReadTool(deps.wikiService, deps.knowledgeIndex), + wikiSearchTool, + new WikiListTagsTool(deps.wikiService, deps.knowledgeIndex), + new WikiWriteTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), + new WikiRemoveTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), + slDiscoverTool, + new SlEditSourceTool(slDeps), + new SlReadSourceTool(slDeps), + new SlWriteSourceTool(slDeps), + new SlValidateTool(slDeps), + new SlRollbackTool(deps.slSourcesRepository, deps.connections, 0), + ...warehouseVerificationTools, + ]; +``` + +- [ ] **Step 5: Run integration and toolset tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.test.ts -t "threads target warehouse connection names" +pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +Run: + +```bash +git add packages/context/src/tools/tool-session.ts packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/local-bundle-runtime.ts packages/context/src/ingest/ingest-bundle.runner.test.ts +git commit -m "feat(context): expose warehouse verification tools to ingest" +``` + +### Task 7: Update writer prompts and cleanup stale references + +**Files:** +- Create: `packages/context/skills/_shared/identifier-verification.md` +- Modify: `packages/context/skills/notion_synthesize/SKILL.md` +- Modify: `packages/context/skills/dbt_ingest/SKILL.md` +- Modify: `packages/context/skills/lookml_ingest/SKILL.md` +- Modify: `packages/context/skills/looker_ingest/SKILL.md` +- Modify: `packages/context/skills/metabase_ingest/SKILL.md` +- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` +- Modify: `packages/context/skills/live_database_ingest/SKILL.md` +- Modify: `packages/context/skills/historic_sql_table_digest/SKILL.md` +- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md` +- Modify: `packages/context/skills/knowledge_capture/SKILL.md` +- Modify: `packages/context/skills/sl_capture/SKILL.md` +- Modify: `packages/context/skills/sl/SKILL.md` +- Modify: `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts` +- Modify: `packages/context/src/sl/tools/sl-warehouse-validation.ts` + +- [ ] **Step 1: Add the shared protocol file** + +Create `packages/context/skills/_shared/identifier-verification.md`: + +```md +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe: + `SELECT DISTINCT FROM LIMIT 50`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution` with `SELECT 1 FROM LIMIT 0`. If it errors, the + identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.` placeholder strings from these instructions + into output. +``` + +- [ ] **Step 2: Inline the protocol into writer skills** + +Add the same protocol block to these skills: + +```text +packages/context/skills/notion_synthesize/SKILL.md +packages/context/skills/dbt_ingest/SKILL.md +packages/context/skills/lookml_ingest/SKILL.md +packages/context/skills/looker_ingest/SKILL.md +packages/context/skills/metabase_ingest/SKILL.md +packages/context/skills/metricflow_ingest/SKILL.md +packages/context/skills/live_database_ingest/SKILL.md +packages/context/skills/historic_sql_patterns/SKILL.md +packages/context/skills/knowledge_capture/SKILL.md +packages/context/skills/sl_capture/SKILL.md +``` + +For `packages/context/skills/historic_sql_table_digest/SKILL.md`, add this shorter block: + +```md +## Identifier Verification Protocol + +Only mention columns visible in the table's scan record. Use +`entity_details({connectionName, targets: [{display: ""}]})` if +the table or column attribution is uncertain. Do not infer join columns or +filters from neighboring SQL unless the scan record confirms the column exists +on the named table. +``` + +For `packages/context/skills/sl/SKILL.md`, add this cross-reference: + +```md +For capture-time identifier verification, load `sl_capture`. Synthesis writer +skills must verify warehouse identifiers with `discover_data`, +`entity_details`, and `sql_execution` before emitting table or column names. +``` + +- [ ] **Step 3: Apply per-skill edits** + +Make these exact content changes: + +- In `notion_synthesize`, add `discover_data`, `entity_details`, and `sql_execution` to the `Allowed:` line. Replace `tableRef: "orbit_analytics.customer"` with `tableRef: ".
"`. +- In `dbt_ingest`, replace `wiki_sl_search` with `discover_data` and `sl_describe_table` with `entity_details`. +- In `lookml_ingest`, add: `Verify each sql_table_name from the LookML view with entity_details before mapping to an SL source.` +- In `looker_ingest`, add: `For every Looker field reference, call entity_details on the underlying schema.table.column before promoting it to sl_refs or quoting it in wiki body.` +- In `metabase_ingest`, add: `Before writing a wiki page derived from a Metabase question SQL, verify each schema.table.column mentioned with entity_details.` +- In `metricflow_ingest`, add: `Verify each MetricFlow model source table with entity_details before producing the corresponding sl_write_source.` +- In `live_database_ingest`, add: `Sample values come from the scan record; do not invent values not present in relationship-profile.json.` +- In `historic_sql_patterns`, add: `Every join column mentioned in pattern descriptions must be verified via entity_details for both sides of the join.` +- In `knowledge_capture`, update the workflow to call `discover_data` first when a page relates to data or SL concepts. +- In `sl_capture`, add: `Before sl_write_source, call entity_details on the target table to confirm column names and types match the YAML being written.` + +- [ ] **Step 4: Remove stale code and prompt strings** + +Modify `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`: + +```ts +.describe('The fully-qualified table or source reference that triggered the fallback (e.g. ".
"). Used to generate canonical detail text.'), +``` + +Modify `packages/context/src/sl/tools/sl-warehouse-validation.ts`: + +```ts + `that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`, +``` + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add packages/context/skills packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts packages/context/src/sl/tools/sl-warehouse-validation.ts +git commit -m "docs(context): add ingest identifier verification protocol" +``` + +### Task 8: Add prompt-bundling and banned-string tests + +**Files:** +- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` +- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` + +- [ ] **Step 1: Add failing asset tests** + +Add to `packages/context/src/memory/memory-runtime-assets.test.ts`: + +```ts +const verificationWriterSkills = [ + 'notion_synthesize', + 'dbt_ingest', + 'lookml_ingest', + 'looker_ingest', + 'metabase_ingest', + 'metricflow_ingest', + 'live_database_ingest', + 'historic_sql_table_digest', + 'historic_sql_patterns', + 'knowledge_capture', + 'sl_capture', +] as const; + +it('ships identifier verification protocol in every synthesis writer skill', async () => { + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + expect(body).toContain('## Identifier Verification Protocol'); + expect(body).toMatch(/discover_data|entity_details/); + } +}); + +it('does not ship stale warehouse verification tool names or fictional identifiers', async () => { + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + expect(body).not.toContain('orbit_analytics.customer'); + expect(body).not.toContain('wiki_sl_search'); + expect(body).not.toContain('sl_describe_table'); + } +}); +``` + +Add to `packages/context/src/ingest/ingest-runtime-assets.test.ts`: + +```ts +it('packages identifier verification prompt assets', async () => { + const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); + expect(shared).toContain('## Identifier Verification Protocol'); + expect(shared).toContain('discover_data'); + expect(shared).toContain('entity_details'); + expect(shared).toContain('sql_execution'); +}); +``` + +- [ ] **Step 2: Run the asset tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts +``` + +Expected: PASS after Task 7. + +- [ ] **Step 3: Commit** + +Run: + +```bash +git add packages/context/src/memory/memory-runtime-assets.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts +git commit -m "test(context): guard ingest identifier verification prompts" +``` + +### Task 9: Run the full v1 verification set + +**Files:** +- Verify all files changed by Tasks 1-8. + +- [ ] **Step 1: Run focused tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run \ + src/connections/dialects.test.ts \ + src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ + src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ + src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ + src/ingest/ingest-bundle.runner.test.ts \ + src/memory/memory-runtime-assets.test.ts \ + src/ingest/ingest-runtime-assets.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run package type-check** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +``` + +Expected: PASS. + +- [ ] **Step 3: Run package tests** + +Run: + +```bash +pnpm --filter @ktx/context run test +``` + +Expected: PASS. + +- [ ] **Step 4: Run pre-commit on changed files when configured** + +Run: + +```bash +uv run pre-commit run --files \ + packages/context/src/connections/dialects.ts \ + packages/context/src/connections/dialects.test.ts \ + packages/context/src/connections/index.ts \ + packages/context/src/tools/tool-session.ts \ + packages/context/src/ingest/ingest-bundle.runner.ts \ + packages/context/src/ingest/local-bundle-runtime.ts \ + packages/context/src/ingest/ingest-bundle.runner.test.ts \ + packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts \ + packages/context/src/sl/tools/sl-warehouse-validation.ts \ + packages/context/src/memory/memory-runtime-assets.test.ts \ + packages/context/src/ingest/ingest-runtime-assets.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ + packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ + packages/context/src/ingest/tools/warehouse-verification/index.ts \ + packages/context/skills/_shared/identifier-verification.md \ + packages/context/skills/notion_synthesize/SKILL.md \ + packages/context/skills/dbt_ingest/SKILL.md \ + packages/context/skills/lookml_ingest/SKILL.md \ + packages/context/skills/looker_ingest/SKILL.md \ + packages/context/skills/metabase_ingest/SKILL.md \ + packages/context/skills/metricflow_ingest/SKILL.md \ + packages/context/skills/live_database_ingest/SKILL.md \ + packages/context/skills/historic_sql_table_digest/SKILL.md \ + packages/context/skills/historic_sql_patterns/SKILL.md \ + packages/context/skills/knowledge_capture/SKILL.md \ + packages/context/skills/sl_capture/SKILL.md \ + packages/context/skills/sl/SKILL.md +``` + +Expected: PASS. If the repo has no pre-commit config or the local `uv` version cannot satisfy the project pin, record the exact error and rely on the focused tests plus type-check. + +- [ ] **Step 5: Commit final verification notes if any files changed during checks** + +Run: + +```bash +git status --short +``` + +Expected: only intentional files are modified. Commit any formatter-driven edits with: + +```bash +git add packages/context +git commit -m "chore(context): verify warehouse verification tools" +``` + +## Self-review checklist + +- Spec coverage: the plan covers dialect dispatch, raw scan catalog reads, `entity_details`, `sql_execution`, `discover_data`, WU and reconcile availability, prompt updates, cleanups, and tests. +- Placeholder scan: no task relies on unnamed future work. +- Type consistency: tool inputs use `connectionName`; existing `sl_discover` calls receive `connectionId` internally; raw SQL execution uses `SlConnectionCatalogPort.executeQuery()` because `SemanticLayerService.executeQuery()` currently accepts semantic-layer query input, not raw SQL. diff --git a/docs/superpowers/plans/2026-05-13-warehouse-verification-prompt-shape-closure.md b/docs/superpowers/plans/2026-05-13-warehouse-verification-prompt-shape-closure.md new file mode 100644 index 00000000..05223b93 --- /dev/null +++ b/docs/superpowers/plans/2026-05-13-warehouse-verification-prompt-shape-closure.md @@ -0,0 +1,345 @@ +# Warehouse Verification Prompt Shape Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make every warehouse-verification prompt use KTX's shipped +`sql_execution` input shape so ingest agents include `connectionName` when they +probe warehouse identifiers. + +**Architecture:** Keep the warehouse verification tool code unchanged. Add +prompt-asset tests that reject Kaelio's old session-only SQL examples, then +update the shared identifier protocol and the three remaining per-skill SQL +probe examples that still show the legacy shape. + +**Tech Stack:** Markdown skill prompts, TypeScript, Vitest, pnpm workspace +commands. + +--- + +## Audit Summary + +The warehouse verification tools, runner wiring, adapter target fan-out, and +focused tests are present. Focused verification passed: + +```bash +pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/ingest/local-adapters.test.ts src/ingest/adapters/notion/notion.adapter.test.ts src/ingest/adapters/lookml/lookml.adapter.test.ts src/ingest/adapters/metricflow/metricflow.adapter.test.ts +pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "supplies a scan-connector query executor" +``` + +Remaining v1-blocking gap: + +- `packages/context/skills/lookml_ingest/SKILL.md`, + `packages/context/skills/metricflow_ingest/SKILL.md`, and + `packages/context/skills/sl_capture/SKILL.md` still contain + `sql_execution({ sql ... })` / "session shape" guidance inherited from + Kaelio. KTX's tool contract is + `sql_execution({connectionName, sql, rowLimit?})`, so these examples can make + agents call the shipped tool with invalid input. + +Non-blocking gaps remain out of scope for this v1 plan: + +- Full DDL-style `entity_details` formatting with FK profile summaries. +- AST-backed SQL validation for data-modifying CTE bodies. +- Search over generated `enrichment/descriptions.json`. +- Per-WorkUnit reuse of a single `WarehouseCatalogService` instance for cache + hits across separate tool calls. +- A deterministic fake-LLM end-to-end Notion hallucination regression. Prompt + guards and tool contract tests cover the v1 contract; a broader behavior + regression can land as follow-up. + +## File Structure + +Modify these files: + +- `packages/context/src/memory/memory-runtime-assets.test.ts`: add a prompt + guard that rejects the legacy session-only `sql_execution` shape. +- `packages/context/src/ingest/ingest-runtime-assets.test.ts`: strengthen the + shared prompt asset assertion for the KTX `connectionName` SQL shape. +- `packages/context/skills/_shared/identifier-verification.md`: make both SQL + probe instructions show the KTX `connectionName` argument. +- `packages/context/skills/notion_synthesize/SKILL.md`: inline the updated + protocol block. +- `packages/context/skills/dbt_ingest/SKILL.md`: inline the updated protocol + block. +- `packages/context/skills/lookml_ingest/SKILL.md`: inline the updated protocol + block and fix the legacy SQL fallback example. +- `packages/context/skills/looker_ingest/SKILL.md`: inline the updated + protocol block. +- `packages/context/skills/metabase_ingest/SKILL.md`: inline the updated + protocol block. +- `packages/context/skills/metricflow_ingest/SKILL.md`: inline the updated + protocol block and fix the legacy SQL fallback example. +- `packages/context/skills/live_database_ingest/SKILL.md`: inline the updated + protocol block. +- `packages/context/skills/historic_sql_table_digest/SKILL.md`: inline the + updated protocol block. +- `packages/context/skills/historic_sql_patterns/SKILL.md`: inline the updated + protocol block. +- `packages/context/skills/knowledge_capture/SKILL.md`: inline the updated + protocol block. +- `packages/context/skills/sl_capture/SKILL.md`: inline the updated protocol + block and fix the join-discovery SQL example. + +### Task 1: Add Prompt Guards For The KTX SQL Tool Shape + +**Files:** +- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` +- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` + +- [ ] **Step 1: Add the failing memory asset guard** + +In `packages/context/src/memory/memory-runtime-assets.test.ts`, add this test +after `does not ship stale warehouse verification tool names or fictional +identifiers`: + +```ts + it('ships only the KTX connectionName sql_execution call shape in writer guidance', async () => { + const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); + + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); + + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + expect(body).toContain('sql_execution({connectionName'); + expect(body).not.toContain('sql_execution({ sql'); + expect(body).not.toContain('session shape'); + expect(body).not.toContain('connection is already pinned by the ingest session'); + } + }); +``` + +- [ ] **Step 2: Strengthen the shared ingest asset guard** + +In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, update +`packages identifier verification prompt assets` so the final assertions are: + +```ts + expect(shared).toContain('discover_data'); + expect(shared).toContain('entity_details'); + expect(shared).toContain('sql_execution'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); +``` + +- [ ] **Step 3: Run the failing prompt guards** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts +``` + +Expected: FAIL. The failure must mention at least one current legacy string: +`sql_execution({ sql`, `session shape`, or missing +`sql_execution({connectionName`. + +### Task 2: Update The Shared Identifier Verification Protocol + +**Files:** +- Modify: `packages/context/skills/_shared/identifier-verification.md` +- Modify: `packages/context/skills/notion_synthesize/SKILL.md` +- Modify: `packages/context/skills/dbt_ingest/SKILL.md` +- Modify: `packages/context/skills/lookml_ingest/SKILL.md` +- Modify: `packages/context/skills/looker_ingest/SKILL.md` +- Modify: `packages/context/skills/metabase_ingest/SKILL.md` +- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` +- Modify: `packages/context/skills/live_database_ingest/SKILL.md` +- Modify: `packages/context/skills/historic_sql_table_digest/SKILL.md` +- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md` +- Modify: `packages/context/skills/knowledge_capture/SKILL.md` +- Modify: `packages/context/skills/sl_capture/SKILL.md` + +- [ ] **Step 1: Replace the shared protocol text** + +Replace the full `## Identifier Verification Protocol` block in +`packages/context/skills/_shared/identifier-verification.md` with: + +```md +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. +``` + +- [ ] **Step 2: Inline the same protocol in every writer skill** + +Replace the existing `## Identifier Verification Protocol` block in each writer +skill with the exact block from Step 1: + +```bash +packages/context/skills/notion_synthesize/SKILL.md +packages/context/skills/dbt_ingest/SKILL.md +packages/context/skills/lookml_ingest/SKILL.md +packages/context/skills/looker_ingest/SKILL.md +packages/context/skills/metabase_ingest/SKILL.md +packages/context/skills/metricflow_ingest/SKILL.md +packages/context/skills/live_database_ingest/SKILL.md +packages/context/skills/historic_sql_table_digest/SKILL.md +packages/context/skills/historic_sql_patterns/SKILL.md +packages/context/skills/knowledge_capture/SKILL.md +packages/context/skills/sl_capture/SKILL.md +``` + +- [ ] **Step 3: Run the shared prompt asset tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts +``` + +Expected: still FAIL because the per-skill legacy SQL examples in LookML, +MetricFlow, and `sl_capture` have not been fixed yet. + +### Task 3: Fix Legacy Per-Skill SQL Examples + +**Files:** +- Modify: `packages/context/skills/lookml_ingest/SKILL.md` +- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` +- Modify: `packages/context/skills/sl_capture/SKILL.md` + +- [ ] **Step 1: Fix the LookML fallback probe example** + +In `packages/context/skills/lookml_ingest/SKILL.md`, replace the current +Required flow item 2 with: + +```md +2. If the table isn't in the manifest, use the warehouse `connectionName` + returned by `discover_data` or the target connection chosen from + `sl_discover`, then call a dialect-appropriate SQL probe with that + connection name, for example: + `sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. + Replace `warehouse`, `analytics`, and `orders` with the verified connection, + schema or dataset, and table from the WorkUnit evidence. +``` + +- [ ] **Step 2: Fix the MetricFlow fallback probe example** + +In `packages/context/skills/metricflow_ingest/SKILL.md`, replace the paragraph +that begins `If \`sl_discover\` errors` with: + +```md +If `sl_discover` errors because no such table exists, use `discover_data` and +`entity_details` to find the warehouse target. If a SQL probe is still needed, +call `sql_execution` with the same warehouse connection name, for example: +`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. +**Never invent column names** - every column in `columns:`, `grain:`, and +`sql:` must be sourced from raw files, `entity_details`, or a successful SQL +probe. +``` + +- [ ] **Step 3: Fix the `sl_capture` join probe example** + +In `packages/context/skills/sl_capture/SKILL.md`, replace Tool sequence item 6 +with: + +```md +6. For join discovery: use `sql_execution({connectionName: "warehouse", sql: "SELECT count(*) FROM public.orders o JOIN public.customers c ON c.id = o.customer_id LIMIT 20"})` with the target warehouse connection name and dialect-correct table names to verify the join key exists in both tables and assess cardinality before declaring the join. +``` + +- [ ] **Step 4: Run the prompt asset tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts +``` + +Expected: PASS. The tests must report 2 files passed. + +### Task 4: Final Verification + +**Files:** +- No new files. + +- [ ] **Step 1: Run focused warehouse prompt and tool tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts +``` + +Expected: PASS. + +- [ ] **Step 2: Run package type-check** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +``` + +Expected: PASS. + +- [ ] **Step 3: Inspect final diff** + +Run: + +```bash +git diff -- packages/context/src/memory/memory-runtime-assets.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/skills/_shared/identifier-verification.md packages/context/skills/notion_synthesize/SKILL.md packages/context/skills/dbt_ingest/SKILL.md packages/context/skills/lookml_ingest/SKILL.md packages/context/skills/looker_ingest/SKILL.md packages/context/skills/metabase_ingest/SKILL.md packages/context/skills/metricflow_ingest/SKILL.md packages/context/skills/live_database_ingest/SKILL.md packages/context/skills/historic_sql_table_digest/SKILL.md packages/context/skills/historic_sql_patterns/SKILL.md packages/context/skills/knowledge_capture/SKILL.md packages/context/skills/sl_capture/SKILL.md +``` + +Expected: only prompt wording and prompt-asset guards changed. No tool +implementation files changed. + +- [ ] **Step 4: Commit** + +Run: + +```bash +git add packages/context/src/memory/memory-runtime-assets.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/skills/_shared/identifier-verification.md packages/context/skills/notion_synthesize/SKILL.md packages/context/skills/dbt_ingest/SKILL.md packages/context/skills/lookml_ingest/SKILL.md packages/context/skills/looker_ingest/SKILL.md packages/context/skills/metabase_ingest/SKILL.md packages/context/skills/metricflow_ingest/SKILL.md packages/context/skills/live_database_ingest/SKILL.md packages/context/skills/historic_sql_table_digest/SKILL.md packages/context/skills/historic_sql_patterns/SKILL.md packages/context/skills/knowledge_capture/SKILL.md packages/context/skills/sl_capture/SKILL.md +git commit -m "fix(context): align warehouse sql probe prompt shape" +``` + +Expected: one focused commit. + +## Self-Review + +Spec coverage: + +- The original spec requires `sql_execution` inputs to include + `connectionName`; this plan removes contradictory session-only examples from + all active writer guidance. +- The shared protocol remains in `_shared` and inlined in every synthesis + writer skill named by the original spec. +- The tool implementation remains unchanged because the shipped schema already + enforces the v1 contract. + +Placeholder scan: + +- The plan has no deferred implementation markers. +- Prompt examples use concrete `warehouse`, `analytics`, and `orders` example + names only to demonstrate JSON shape, and each example tells the worker to + replace them with discovered evidence. + +Type consistency: + +- Tests assert the exact KTX tool call shape: + `sql_execution({connectionName, sql: ...})`. +- Prompt wording consistently uses `connectionName`, matching + `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`. diff --git a/docs/superpowers/plans/2026-05-13-warehouse-verification-sql-example-closure.md b/docs/superpowers/plans/2026-05-13-warehouse-verification-sql-example-closure.md new file mode 100644 index 00000000..2d1b1779 --- /dev/null +++ b/docs/superpowers/plans/2026-05-13-warehouse-verification-sql-example-closure.md @@ -0,0 +1,215 @@ +# Warehouse Verification SQL Example Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove the last connectionless `sql_execution` prompt example so +warehouse-verification writer guidance always matches KTX's shipped tool +contract. + +**Architecture:** Keep the warehouse verification tool code unchanged. Tighten +the prompt asset guard so multiline `sql_execution({ sql: ... })` examples +fail tests, then update the stale `sl_capture` worked example to pass +`connectionName` explicitly. + +**Tech Stack:** Markdown skill prompts, TypeScript, Vitest, pnpm workspace +commands. + +--- + +## Audit summary + +The warehouse verification tools, runner wiring, source-adapter target fan-out, +CLI query executor, and focused tests are present. Focused verification passed: + +```bash +pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/ingest/local-adapters.test.ts src/ingest/adapters/notion/notion.adapter.test.ts src/ingest/adapters/lookml/lookml.adapter.test.ts src/ingest/adapters/metricflow/metricflow.adapter.test.ts +pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "supplies a scan-connector query executor" +``` + +Remaining v1-blocking gap: + +- `packages/context/skills/sl_capture/SKILL.md` still contains a worked example + with a multiline `sql_execution({ sql: ... })` call. KTX's tool contract is + `sql_execution({connectionName, sql, rowLimit?})`, so this example can teach + agents to call the shipped tool with invalid input. + +Non-blocking gaps remain out of scope for this v1 plan: + +- Full DDL-style `entity_details` formatting with FK profile summaries. +- AST-backed SQL validation for data-modifying CTE bodies. +- Search over generated `enrichment/descriptions.json`. +- Per-WorkUnit reuse of a single `WarehouseCatalogService` instance for cache + hits across separate tool calls. +- A deterministic fake-LLM end-to-end Notion hallucination regression. +- Tokenized or embedding-backed raw schema search ranking in `discover_data`. + +## File structure + +Modify these files: + +- `packages/context/src/memory/memory-runtime-assets.test.ts`: add a prompt + guard that catches multiline `sql_execution` calls without `connectionName`. +- `packages/context/skills/sl_capture/SKILL.md`: update the stale worked + example to include the target warehouse `connectionName`. + +### Task 1: Add a multiline SQL prompt guard + +**Files:** +- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` + +- [ ] **Step 1: Add a helper that extracts `sql_execution` call examples** + +In `packages/context/src/memory/memory-runtime-assets.test.ts`, add this helper +after `forbiddenProductPattern()`: + +```ts +function sqlExecutionCallBlocks(body: string): string[] { + const blocks: string[] = []; + const marker = 'sql_execution({'; + let offset = 0; + + while (offset < body.length) { + const start = body.indexOf(marker, offset); + if (start === -1) { + break; + } + const end = body.indexOf('})', start + marker.length); + blocks.push(body.slice(start, end === -1 ? start + marker.length : end + 2)); + offset = start + marker.length; + } + + return blocks; +} +``` + +- [ ] **Step 2: Strengthen the existing SQL-shape test** + +Replace the body of +`ships only the KTX connectionName sql_execution call shape in writer guidance` +with: + +```ts + const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); + const bodies = [{ name: '_shared/identifier-verification.md', body: shared }]; + + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); + + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + bodies.push({ name: `${skillName}/SKILL.md`, body }); + expect(body).toContain('sql_execution({connectionName'); + expect(body).not.toContain('sql_execution({ sql'); + expect(body).not.toContain('session shape'); + expect(body).not.toContain('connection is already pinned by the ingest session'); + } + + for (const { name, body } of bodies) { + const calls = sqlExecutionCallBlocks(body); + expect(calls.length, `${name} should contain sql_execution guidance`).toBeGreaterThan(0); + expect( + calls.filter((call) => !call.includes('connectionName')), + `${name} has sql_execution calls without connectionName`, + ).toEqual([]); + expect(body, `${name} has a connectionless multiline sql_execution call`).not.toMatch( + /sql_execution\(\{\s*sql\s*:/, + ); + } +``` + +- [ ] **Step 3: Run the failing prompt guard** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts -t "connectionName sql_execution" +``` + +Expected: FAIL. The failure must identify +`sl_capture/SKILL.md` as having a `sql_execution` call without +`connectionName` or a connectionless multiline `sql_execution` call. + +- [ ] **Step 4: Commit the failing guard** + +Run: + +```bash +git add packages/context/src/memory/memory-runtime-assets.test.ts +git commit -m "test(context): catch connectionless sql execution prompt examples" +``` + +### Task 2: Fix the stale `sl_capture` SQL example + +**Files:** +- Modify: `packages/context/skills/sl_capture/SKILL.md` +- Test: `packages/context/src/memory/memory-runtime-assets.test.ts` +- Test: `packages/context/src/ingest/ingest-runtime-assets.test.ts` + +- [ ] **Step 1: Update the worked example** + +In `packages/context/skills/sl_capture/SKILL.md`, replace the `sql_execution` +block in "Worked example - new join" with: + +```md +sql_execution({ + connectionName: "warehouse", + sql: "SELECT COUNT(*), COUNT(DISTINCT a.admin_user_id) FROM public.fct_orders a JOIN public.fct_mau_multiprotocol b ON a.admin_user_id = b.admin_user_id LIMIT 1" +}) +``` + +- [ ] **Step 2: Run the prompt guards** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts +``` + +Expected: PASS. + +- [ ] **Step 3: Run a direct stale-shape scan** + +Run: + +```bash +rg -n -U "sql_execution\\(\\{\\s*\\n\\s*sql:" packages/context/skills packages/context/prompts +``` + +Expected: no matches and exit code 1. + +- [ ] **Step 4: Run the context type-check** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +``` + +Expected: PASS. + +- [ ] **Step 5: Commit the prompt fix** + +Run: + +```bash +git add packages/context/skills/sl_capture/SKILL.md +git commit -m "fix(context): include connection name in sl capture sql example" +``` + +## Self-review + +Spec coverage: + +- The only remaining v1-blocking prompt-shape gap has a failing test and a + direct prompt edit. +- Tool implementation, runner wiring, adapter scoping, and CLI execution + remain covered by the focused suites listed in the audit summary. + +Placeholder scan: + +- This plan contains no deferred implementation placeholders. + +Type consistency: + +- The plan uses the shipped KTX tool shape: + `sql_execution({connectionName, sql, rowLimit?})`. diff --git a/docs/superpowers/plans/2026-05-13-warehouse-verification-structured-target-miss-closure.md b/docs/superpowers/plans/2026-05-13-warehouse-verification-structured-target-miss-closure.md new file mode 100644 index 00000000..48983c4a --- /dev/null +++ b/docs/superpowers/plans/2026-05-13-warehouse-verification-structured-target-miss-closure.md @@ -0,0 +1,236 @@ +# Warehouse Verification Structured Target Miss Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `entity_details` return model-visible not-found evidence for every documented target shape, including structured `{catalog, db, name, column?}` targets. + +**Architecture:** Keep the existing warehouse verification module. Add focused tests for missing structured table and column targets, then route structured target labels through the same candidate lookup used by display targets while preserving exact structured resolution. + +**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX ingest tools. + +--- + +## Audit Summary + +The implemented plans have landed the warehouse verification tools, ingest +runner wiring, adapter warehouse target fan-out, CLI read-only query executor, +and prompt-shape closures. Focused verification passed on May 13, 2026: + +```bash +pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/ingest/local-adapters.test.ts src/ingest/adapters/notion/notion.adapter.test.ts src/ingest/adapters/lookml/lookml.adapter.test.ts src/ingest/adapters/metricflow/metricflow.adapter.test.ts +pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "supplies a scan-connector query executor" +rg -n -U "sql_execution\\(\\{\\s*\\n\\s*sql:" packages/context/skills packages/context/prompts +rg -n "wiki_sl_search|sl_describe_table|orbit_analytics\\.customer" packages/context/skills packages/context/prompts packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts packages/context/src/sl/tools/sl-warehouse-validation.ts +``` + +Remaining v1-blocking gap: + +- `entity_details` accepts structured targets, but if a structured table target + does not exist, it records `structured.missing` and emits no markdown. Tool + outputs are sent to the model as markdown only, so the synthesis agent gets + an empty response instead of the required "Not found in scan" verification + signal. + +Non-blocking gaps remain out of scope for this v1 plan: + +- Full DDL-style `entity_details` formatting with FK and profile summaries. +- AST-backed SQL validation for data-modifying CTE bodies. +- Dialect-specific row-limit wrapping for SQL Server probes. +- Search over generated `enrichment/descriptions.json`. +- Per-WorkUnit reuse of a single `WarehouseCatalogService` instance for cache + hits across separate tool calls. +- A deterministic fake-LLM end-to-end Notion hallucination regression. +- Cleanup of legacy demo Orbit wiki fixtures that still mention + `orbit_analytics.customer`. + +## File Structure + +Modify these files: + +- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: add failing coverage for missing structured targets. +- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: render missing structured targets into markdown and reuse candidate lookup. + +### Task 1: Report Structured Target Misses In `entity_details` + +**Files:** +- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` +- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` + +- [ ] **Step 1: Add failing structured miss tests** + +In `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`, add these tests after `reports missing explicit columns instead of returning an empty column list`: + +```ts + it('reports missing structured table targets in model-visible markdown', async () => { + const result = await tool.call( + { + connectionName: 'warehouse', + targets: [{ catalog: null, db: 'public', name: 'orderz' }], + }, + context, + ); + + expect(result.markdown).toContain('Not found in scan: public.orderz'); + expect(result.markdown).toContain('Closest matches: orders'); + expect(result.structured.resolved).toHaveLength(0); + expect(result.structured.missing).toHaveLength(1); + }); + + it('reports missing structured column targets in model-visible markdown', async () => { + const result = await tool.call( + { + connectionName: 'warehouse', + targets: [{ catalog: null, db: 'public', name: 'orders', column: 'plan_tier' }], + }, + context, + ); + + expect(result.markdown).toContain('Column not found in scan: public.orders.plan_tier'); + expect(result.markdown).toContain('Available columns: id, status'); + expect(result.structured.resolved).toHaveLength(0); + expect(result.structured.missing).toHaveLength(1); + }); +``` + +- [ ] **Step 2: Run the failing focused test** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -t "structured" +``` + +Expected: FAIL. The first new test must fail because `result.markdown` does not contain `Not found in scan: public.orderz`. + +- [ ] **Step 3: Add structured target labels and candidate lookup** + +In `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`, add this type alias after `type EntityDetailsInput = z.infer;`: + +```ts +type EntityDetailsTarget = EntityDetailsInput['targets'][number]; +``` + +Add these helpers after `function allowedConnectionNames(context: ToolContext): ReadonlySet | null { ... }`: + +```ts +function targetLabel(target: EntityDetailsTarget): string { + if ('display' in target) { + return target.display; + } + return [target.catalog, target.db, target.name, target.column].filter((part): part is string => !!part).join('.'); +} + +function appendMissingTargetMarkdown(parts: string[], target: EntityDetailsTarget, candidates: KtxTableRef[]): void { + parts.push(`Not found in scan: ${targetLabel(target)}`); + if (candidates.length > 0) { + parts.push(`Closest matches: ${candidates.map((candidate) => candidate.name).join(', ')}`); + } +} + +async function resolveTarget( + catalog: WarehouseCatalogService, + connectionName: string, + target: EntityDetailsTarget, +): Promise<{ resolved: (KtxTableRef & { column?: string }) | null; candidates: KtxTableRef[] }> { + if ('display' in target) { + return catalog.resolveDisplayTarget(connectionName, target.display); + } + + const candidateResolution = await catalog.resolveDisplayTarget(connectionName, targetLabel(target)); + return { + resolved: { + catalog: target.catalog, + db: target.db, + name: target.name, + column: target.column, + }, + candidates: candidateResolution.candidates, + }; +} +``` + +Then replace the `const resolution = ...` block inside the `for (const target of input.targets)` loop with: + +```ts + const resolution = await resolveTarget(catalog, input.connectionName, target); +``` + +Replace the missing-resolution block with: + +```ts + if (!resolution.resolved) { + missing.push({ target, candidates: resolution.candidates }); + appendMissingTargetMarkdown(parts, target, resolution.candidates); + continue; + } +``` + +Replace the missing-detail block with: + +```ts + if (!detail) { + missing.push({ target, candidates: resolution.candidates }); + appendMissingTargetMarkdown(parts, target, resolution.candidates); + continue; + } +``` + +- [ ] **Step 4: Run the focused entity-details tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 5: Run warehouse verification regression tests** + +Run: + +```bash +pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Run context type-check** + +Run: + +```bash +pnpm --filter @ktx/context run type-check +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ + packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +git commit -m "fix(context): report structured entity detail misses" +``` + +## Self-review + +Spec coverage: + +- The original `entity_details` contract says structured and display targets + are mixed shapes and unresolved targets must produce `Not found in scan` with + candidates. This plan adds that model-visible behavior for structured table + misses and preserves the existing column-miss behavior. + +Placeholder scan: + +- This plan contains no deferred implementation placeholders. + +Type consistency: + +- The plan uses the existing `WarehouseCatalogService`, `KtxTableRef`, + `EntityDetailsStructured`, and `ToolOutput` types without adding public API + compatibility wrappers. diff --git a/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md b/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md new file mode 100644 index 00000000..074f00e5 --- /dev/null +++ b/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md @@ -0,0 +1,331 @@ +# Warehouse Verification Tools for Ingestion Synthesis + +**Date:** 2026-05-12 +**Author:** Andrey Avtomonov +**Status:** Design — pending implementation plan + +## Background and motivation + +KTX's ingest pipeline synthesises wiki pages and semantic-layer (SL) sources from third-party content (Notion, LookML, Looker, Metabase, dbt, MetricFlow, historic SQL, live-database scans, and chat). The synthesis stage is an LLM call that runs once per WorkUnit, governed by a skill prompt (e.g. `notion_synthesize`) and a set of allowed tools. + +A real-world inspection (project `/tmp/ktx-proj-1`) surfaced two failure modes the synthesis stage produces: + +1. **Fictional identifiers laundered into wiki output.** A Notion page mentioned `orbit_analytics.customer` as a legacy "customer source" table with a `plan_tier in {free, pro, enterprise}` column. Neither the table, the column, nor those values exist in the configured warehouse. The synthesis LLM faithfully copied them into `knowledge/global/orbit/customers-source.md` as a "Conflict Note", giving the fabricated names full wiki frontmatter, a `Source:` citation, and apparent authority. +2. **Column attribution drift.** The same wiki page documents columns under `orbit_raw.accounts` but states the `paying_account_count` measure filters on `normalized_plan_code` and `contract_status`. Those columns live on `orbit_analytics.mart_account_segments`, not on `accounts`. A reader (or a downstream agent) following the page will write `accounts.normalized_plan_code` and get a `column does not exist` error. + +Root cause analysis (`packages/context/skills/notion_synthesize/SKILL.md`, `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`, `packages/context/src/wiki/tools/wiki-write.tool.ts`) showed three contributing factors: + +- The synthesis LLM has no verification primitive that distinguishes a real warehouse identifier from a fabricated one. `sl_discover` only finds objects already promoted into the semantic layer; raw warehouse scans (which already exist on disk under `raw-sources//live-database//`) are not surfaced to the LLM at all. +- `wiki_write` performs no body-text validation — anything the LLM emits is written. +- The skill prompt itself uses `orbit_analytics.customer` as a canonical example string (`SKILL.md:70`), reinforcing the same fictional name the LLM ends up emitting. + +Kaelio's server-side ingest WU agent (`/Users/andrey/conductor/workspaces/kaelio-main2/douala/server/src/tools/toolset-factory.service.ts`) had four verification tools that KTX dropped during the open-source extraction: `discover_data`, `entity_details`, `dictionary_search`, and `sql_execution`. The underlying connector infrastructure (`KtxScanConnector`, dialect classes, `assertReadOnlySql`, `SemanticLayerService.executeQuery`) is present in KTX, so the gap is at the tool layer, not the platform layer. + +## Goal + +Give every ingest adapter's synthesis-time LLM call the tools and skill-prompt instructions needed to verify warehouse identifiers (`schema.table`, `schema.table.column`) and sample values before emitting them into wiki pages, SL sources, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback` records. + +## Non-goals + +- Not changing `wiki_write` itself. A complementary spec covers hard write-time validation; this spec focuses on giving the LLM the tools to self-validate. +- Not modifying any Notion fetch/chunk/cluster behaviour. +- Not changing the `_schema/*.yaml` format. +- Not introducing a UUID layer for tables or columns; KTX keeps `(connection, catalog, db, name)` as the canonical table identity. +- Not adding `semantic_query` to the synthesis toolset. `semantic_query` is a future tool for the research/chat-time agent; synthesis creates SL sources rather than queries them, so the wrong shape. +- Not adding `dictionary_search`. `entity_details` already returns per-column `sampleValues` from the relationship-profile, and `sql_execution` covers the rarer "where does this literal live?" case more accurately than a sampled-JSON full-text scan. + +## What already exists in KTX + +The dialect/driver/connection architecture is fully ported from Kaelio. The new tools sit on top of three already-shipping primitives: + +| Primitive | Location | +|---|---| +| `KtxTableRef = { catalog: string\|null, db: string\|null, name: string }` | `packages/context/src/scan/types.ts:168` | +| `SemanticLayerService.executeQuery(connectionId, sql)` | `packages/context/src/sl/semantic-layer.service.ts:1004`, used today by `sl_validate` | +| `assertReadOnlySql` / `limitSqlForExecution` | `packages/context/src/connections/read-only-sql.ts` | +| 7 connectors with parallel layout (postgres, mysql, sqlserver, snowflake, bigquery, clickhouse, sqlite), each exporting a dialect class | `packages/connector-*` | +| Raw scan artefacts: `tables/...json` and `enrichment/relationship-profile.json` (with `nativeType`, `nullable`, `primaryKey`, `foreignKeys`, `rowCount`, `nullCount`, `distinctCount`, `sampleValues`, descriptions) | `raw-sources//live-database//` | +| `wiki_search`, `sl_discover`, `sl_read_source`, `sl_validate`, `emit_unmapped_fallback` | already wired into synthesis stages | + +The only meaningfully new code is `WarehouseCatalogService`, a small `getDialectForDriver` dispatch, the three tool files, and the wiring in `ingest-bundle.runner.ts`. + +## Architecture + +### Module layout + +``` +packages/context/src/ingest/tools/warehouse-verification/ + discover-data.tool.ts + entity-details.tool.ts + sql-execution.tool.ts + warehouse-catalog.service.ts + index.ts # exports createWarehouseVerificationTools() +packages/context/src/connections/ + dialects.ts # adds getDialectForDriver() +packages/context/skills/_shared/ + identifier-verification.md # the protocol snippet referenced from every synthesis skill +``` + +### Canonical table identity + +Every tool that names a warehouse object uses the tuple `(connectionName, catalog, db, name[, column])`. `connectionName` is the slug from `ktx.yaml` (e.g., `"warehouse"`), validated against `^[a-zA-Z0-9][a-zA-Z0-9_-]*$`. There is no UUID layer. + +`display` strings the LLM picks up from source pages (e.g., `"orbit_raw.accounts"` for Postgres or `"project.dataset.table"` for BigQuery) are parsed by `WarehouseCatalogService.resolveDisplay`, which knows the connection's driver via `getDialectForDriver`. Ambiguous parses (e.g., a 2-part display on BigQuery) return a candidates list instead of guessing. + +Dialect mapping: + +| Driver | catalog | db | name | Display | +|---|---|---|---|---| +| postgres | `null` | schema | table | `schema.table` | +| mysql | `null` | schema | table | `schema.table` | +| sqlserver | catalog | schema | table | `catalog.schema.table` | +| snowflake | database | schema | table | `db.schema.table` | +| bigquery | project | dataset | table | `project.dataset.table` | +| clickhouse | `null` | database | table | `database.table` | +| sqlite | `null` | `null` | table | `table` | + +### `WarehouseCatalogService` + +Stateless except for a per-WorkUnit cache. Reads raw scan files under `raw-sources//live-database//`. + +```ts +class WarehouseCatalogService { + getTable(ref: { connectionName: string } & KtxTableRef): Promise; + listTables(connectionName: string): Promise; + resolveDisplay(connectionName: string, display: string): Promise<{ + resolved: KtxTableRef | null; + candidates: KtxTableRef[]; // ranked by edit distance when resolved is null + dialect: string; + }>; + searchByName(connectionName: string, query: string, limit: number): Promise>; + getLatestSyncId(connectionName: string): Promise; +} +``` + +`getTable` merges the raw schema file (native types, PK, FK, nullable) with the enrichment profile (row counts, null rates, distinct counts, sample values, AI-generated descriptions). When no scan exists for the connection, every read returns `null`; tools surface this as a distinct "no scan available" state rather than as "identifier not found", so the LLM doesn't conclude a real table is fictional just because a scan hasn't run yet. + +### `getDialectForDriver` + +```ts +// packages/context/src/connections/dialects.ts +export type SupportedDriver = 'postgres'|'postgresql'|'mysql'|'sqlserver'|'snowflake'|'bigquery'|'clickhouse'|'sqlite'|'sqlite3'; +export function getDialectForDriver(driver: SupportedDriver): KtxDialect; +``` + +Sync dispatch. The connectors' existing dialect classes already expose the same shape — `formatTableName(KtxTableRef)`, `quoteIdentifier(string)`, `mapToDimensionType(nativeType)`. The implementation plan introduces a minimal `KtxDialect` interface that these classes already satisfy structurally; no connector-internal changes required. Used by tools only for display-string parsing and error-message formatting; tools never construct executable SQL. + +## Tool contracts + +### `entity_details` + +```ts +input = { + connectionName: string, + targets: Array< // 1..50, mixed shapes allowed + | { display: string } // "orbit_raw.accounts" or "orbit_raw.accounts.account_id" + | { catalog: string|null, db: string, name: string, column?: string } + >, +} +``` + +Output (markdown, per target): + +``` +### orbit_raw.accounts +Type: table | Native columns: 11 | PK: account_id | FKs: parent_account_id → orbit_raw.accounts.account_id +Description: One row per customer account… + +Columns: +- account_id (text, nullable=false, PK) — sample: ["acct_001","acct_002",…] +- parent_account_id (text, nullable=true, FK → orbit_raw.accounts.account_id) +- account_name (text, nullable=false) +- … + +Profile: rowCount=4321 distinctCount(account_id)=4321 nullRate(parent_account_id)=0.62 +``` + +When `column` is provided in a target, output is scoped to that one column. When a target doesn't resolve, output is `Not found in scan. Closest matches: …` with up to 5 candidates from `searchByName`. When the connection has no `live-database` scan, output is `No live-database scan available for connection ""; run \`ktx scan\` first.` — distinct from the "not found" state. + +Structured output: `{ resolved: TableDetail[], missing: Array<{target, candidates}>, scanAvailable: boolean }`. + +Refuses `connectionName` values not in the WU-stage's `allowedConnectionNames` set. + +### `sql_execution` + +```ts +input = { + connectionName: string, + sql: string, // single SELECT or WITH only + rowLimit?: number, // default 100, hard cap 1000 +} +``` + +Pipeline: + +1. `assertReadOnlySql(sql)` — regex rejects anything starting with `insert|update|delete|merge|alter|drop|create|truncate|grant|revoke|copy|call|do|vacuum|analyze|refresh`. +2. `limitSqlForExecution(sql, rowLimit)` — wraps as `select * from () as ktx_query_result limit N`. +3. `SemanticLayerService.executeQuery(connectionName, wrappedSql)`. +4. Format as markdown table; first ~20 rows inline; if truncated, append `… +N more rows`. + +Structured output: `{ headers, rows, rowCount, truncated, sql, wrappedSql }`. + +Connector errors surface verbatim (e.g., Postgres `relation "orbit_analytics.customer" does not exist`). That error message is the most valuable verification signal — it tells the LLM the identifier is fictional. + +Refuses `connectionName` not in `allowedConnectionNames`. Each connector's driver-level read-only enforcement (Postgres read-only transaction, BigQuery query-only jobs) is a second defence under the regex gate. + +### `discover_data` + +```ts +input = { + query: string, + connectionName?: string, // omit to search all configured warehouse connections + limit?: number, // default 10 per section + sourceName?: string, // SL source detail mode (delegates to sl_discover) +} +``` + +Composes three searches and groups output into three sections, omitting empty sections: + +1. **Wiki Pages** — `wiki_search({query, limit})`. Routing hint: *use `wiki_read(blockKey)` for full content*. +2. **Semantic Layer Sources** — `sl_discover({query, connectionName})`. Routing hint: *use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details*. +3. **Raw Warehouse Schema** — `WarehouseCatalogService.searchByName(connectionName, query, limit)`. Routing hint: *use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values*. + +When `sourceName` is set, delegates entirely to `sl_discover` inspect mode and skips other sections. When all three sections are empty, output is `No matches for "" across wiki, semantic layer, or raw warehouse schema. Try broader terms; this concept may not exist yet.` + +Structured output: `{ wiki: WikiSearchStructured|null, sl: SlDiscoverStructured|null, raw: RawSchemaHits|null }`. + +## Wiring + +`packages/context/src/ingest/ingest-bundle.runner.ts` already plumbs `emit_unmapped_fallback` into both the WorkUnit stage (`createEmitUnmappedFallbackTool` around line 726) and the reconcile stage (around line 962), with merging done via `packages/context/src/ingest/stages/build-wu-context.ts` and `build-reconcile-context.ts`. + +Add a parallel factory next to those existing calls: + +```ts +const warehouseTools = createWarehouseVerificationTools({ + semanticLayerService: scopedSemanticLayerService, + warehouseCatalog: new WarehouseCatalogService({ fileStore, projectDir }), + dialects: getDialectForDriver, + allowedConnectionNames: slConnectionIds, // reuse existing scoping + sqlExecutionRowLimit: 100, +}); +// Merge `entity_details`, `sql_execution`, `discover_data` into both stage tool maps +// alongside emit_unmapped_fallback. +``` + +`createWarehouseVerificationTools` returns `Record` with three keys. The set is wired into every adapter's synthesis stage — no per-adapter opt-in. + +## Skill-prompt updates + +### Shared protocol + +`packages/context/skills/_shared/identifier-verification.md`: + +```md +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: +1. `discover_data({query: ""})` — see what wikis, SL sources, and raw tables + already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: +2. `entity_details({connectionName, targets: [{display: ""}]})` — + confirm the identifier resolves; inspect native types, FK/PK, and sampleValues. +3. For literal values from the source (status codes, plan tiers): check whether + they appear in `entity_details`' `sampleValues` for the relevant column. + If `sampleValues` is short or you suspect the sample missed real values, run + a `sql_execution` probe: `SELECT DISTINCT FROM LIMIT 50`. +4. If the candidate identifier still doesn't resolve, do one of: + (a) Use `sql_execution` with `SELECT 1 FROM LIMIT 0`. If it errors, + the identifier is fictional. + (b) Wrap the identifier in `[unverified — from ]` in the wiki body, + citing the exact raw path that mentioned it. + (c) When recording `emit_unmapped_fallback` with `no_physical_table`, + include the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. +``` + +Each affected skill inlines this block verbatim (skill files are independent prompts; KTX has no cross-skill include mechanism today). + +### Per-skill diffs + +Two skills are deliberately excluded from updates: `ingest_triage` (read-only triage; produces no wiki or SL output) and `sl` (umbrella reference doc; cross-links to the protocol but doesn't need its own copy). + +| Skill | Changes | +|---|---| +| `notion_synthesize` | Inline protocol; append `discover_data`, `entity_details`, `sql_execution` to `Allowed:` (line 74); replace `orbit_analytics.customer` example on line 70 with `.
` | +| `dbt_ingest` | Inline protocol; line 24: replace `wiki_sl_search` → `discover_data` and `sl_describe_table` → `entity_details`; strengthen the "not permission to invent physical columns" paragraph by naming `entity_details` as the verification call | +| `lookml_ingest` | Inline protocol; add: "Verify each `sql_table_name` from the LookML view with `entity_details` before mapping to an SL source" | +| `looker_ingest` | Inline protocol; add: "For every Looker field reference, call `entity_details` on the underlying `(schema, table, column)` before promoting to `sl_refs` or quoting in wiki body" | +| `metabase_ingest` | Inline protocol; add: "Before writing a wiki page derived from a Metabase question's SQL, verify each `schema.table.column` mentioned with `entity_details`" | +| `metricflow_ingest` | Inline protocol; add: "Verify each MetricFlow model's source table with `entity_details` before producing the corresponding `sl_write_source`" | +| `live_database_ingest` | Inline protocol; add: "Sample values come from the scan record; do not invent values not present in `relationship-profile.json`" | +| `historic_sql_table_digest` | Shortened protocol focused on column attribution: "Only mention columns visible in the table's scan record. Use `entity_details({display})` if uncertain" | +| `historic_sql_patterns` | Inline protocol; add: "Every join column mentioned in pattern descriptions must be verified via `entity_details` for both sides of the join" | +| `knowledge_capture` | Inline protocol; update line 44: "First call `discover_data` to find existing wiki pages, SL sources, and raw tables on the topic" | +| `sl_capture` | Inline protocol; add: "Before `sl_write_source`, call `entity_details` on the target table to confirm column names and types match the YAML being written" | + +### Cleanups beyond the four-tool addition + +- `notion_synthesize/SKILL.md:70` — remove `orbit_analytics.customer` (placeholder). +- `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts:67` — same example string in the Zod `.describe()` — replace with `.
`. +- `dbt_ingest/SKILL.md:24` — fix `wiki_sl_search` and `sl_describe_table` (neither tool exists in KTX). +- `packages/context/src/sl/tools/sl-warehouse-validation.ts:93` — inline error message references the non-existent `sl_describe_table`. Replace with `sl_read_source`. + +## Testing strategy + +### Unit tests + +| Component | Tests | +|---|---| +| `getDialectForDriver` | Every supported driver returns a dialect; unknown driver throws with a clear list of supported drivers | +| `WarehouseCatalogService.getTable` | Reads and merges `tables/.json` and `relationship-profile.json`; returns `null` when no sync exists; returns `null` for unknown `(catalog, db, name)` | +| `WarehouseCatalogService.resolveDisplay` | Postgres 2-part display → `{catalog: null, db, name}`; BigQuery 3-part display → `{catalog, db, name}`; ambiguous 2-part on BigQuery returns candidates list; unknown displays produce closest-match candidates ordered by edit distance | +| `WarehouseCatalogService.searchByName` | Substring and token match; tiers (exact-name → token-match) ordered correctly; cache hit on second call within same instance | +| `entity_details` | Resolves `{display}` and structured inputs; reports "Not found" with candidates for unknown ref; reports "no scan available" distinctly when scan dir missing; truncates above 50 targets | +| `discover_data` | Three sections present when all three have hits; sections omitted when empty; `sourceName` inspect mode delegates to `sl_discover` and skips other sections; `allowedConnectionNames` scope honoured | +| `sql_execution` | `assertReadOnlySql` rejects each mutating verb; row-limit wrap visible in `wrappedSql`; connector errors surface verbatim with the failing SQL; rejects `connectionName` not in `allowedConnectionNames` | + +### Integration tests + +- Extend `packages/context/src/ingest/ingest-bundle.runner.test.ts` to verify the three new tools are present in both WU-stage and reconcile-stage tool maps and refuse out-of-scope `connectionName` values. +- New fixture-based test: stage a small `raw-sources//live-database//` directory with 2 tables + 1 enrichment profile, then call each tool through the runner's tool map and assert the markdown contains the expected fields. Uses the same fake-LLM harness as `notion.adapter.test.ts`. +- One end-to-end regression test reproducing the `orbit_analytics.customer` hallucination: a fake Notion page mentioning the fictional table is fed to the synthesis stage; the run produces a wiki page where the fictional name is wrapped in `[unverified — …]` or omitted, not promoted to `tables:` frontmatter. + +### Prompt-bundling tests + +Extend `packages/context/src/memory/memory-runtime-assets.test.ts`: + +- Every skill in the synthesis-writers list embeds the verification-protocol block (assert by stable header text). +- Every such skill lists the three new tools when it has a `## Tools / Allowed` section, or mentions them inline in a workflow step otherwise. +- No skill file contains any of the banned strings: `orbit_analytics.customer`, `wiki_sl_search`, `sl_describe_table`. + +### Performance guards + +`WarehouseCatalogService` caches the per-connection table list per stage (one WorkUnit's lifetime). Tests assert second call is a cache hit. No DB index for `searchByName` in this iteration — linear scan over scan artefacts is acceptable up to ~50K columns. If volume warrants it later, a follow-up PR adds a SQLite FTS index. + +## Rollout + +Four mergeable PRs: + +| PR | Lands | +|---|---| +| 1 | `getDialectForDriver` + `WarehouseCatalogService` + `entity_details` tool + wiring in `ingest-bundle.runner.ts` + unit/integration tests | +| 2 | `sql_execution` tool + tests + the `orbit_analytics.customer` regression test (which exercises protocol steps 4a/4c) | +| 3 | `discover_data` tool + tests | +| 4 | All 11 skill prompts updated with the verification protocol + the three cleanups + extended `memory-runtime-assets.test.ts` | + +Skill prompts land last so they can reference the tools that already exist. + +## Out of scope + +- **Hard write-time validation in `wiki_write` / `emit_unmapped_fallback`.** A complementary spec covers regex-based identifier validation at the write boundary. Defence-in-depth — separate concern. +- **SQLite FTS index for `searchByName`.** Deferred until the linear scan benchmark fails. +- **`raw_schema_search` as a standalone tool.** `discover_data`'s raw section covers the concept-search case. +- **`semantic_query` in the synthesis toolset.** `semantic_query` will exist in KTX for the research/chat-time agent; it is deliberately excluded from synthesis because synthesis creates SL sources rather than queries them. +- **`dictionary_search`.** `entity_details` already returns per-column `sampleValues`; for the rarer "where does this literal live?" case, `sql_execution` is more accurate than a sampled-JSON scan. +- **UUID layer for tables/columns.** KTX deliberately stays string-keyed on `(connection, catalog, db, name)`. diff --git a/examples/package-artifacts/README.md b/examples/package-artifacts/README.md index 8c92f84e..22ecaf92 100644 --- a/examples/package-artifacts/README.md +++ b/examples/package-artifacts/README.md @@ -14,9 +14,7 @@ generated local project. The managed Python runtime smoke requires `uv` on `PATH`, isolates `KTX_RUNTIME_ROOT`, verifies `ktx dev runtime status`, runs `ktx sl query --yes` to install the core runtime from the bundled wheel, checks `ktx dev runtime status`, -starts and reuses the managed daemon, stops it, previews a stale runtime with -`ktx dev runtime prune --dry-run`, verifies confirmation is required, and removes -the stale runtime with `ktx dev runtime prune --yes`. +starts and reuses the managed daemon, and stops it. The artifact manifest contains the public `@kaelio/ktx` npm tarball and the bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone diff --git a/knip.json b/knip.json new file mode 100644 index 00000000..a6bf5d26 --- /dev/null +++ b/knip.json @@ -0,0 +1,114 @@ +{ + "$schema": "https://unpkg.com/knip@6/schema.json", + "workspaces": { + ".": { + "entry": ["scripts/**/*.mjs"], + "project": ["scripts/**/*.mjs"] + }, + "packages/cli": { + "entry": [ + "src/index.ts", + "src/bin.ts", + "src/**/*.test.ts", + "src/**/*.test.tsx", + "scripts/**/*.mjs" + ], + "project": ["src/**/*.{ts,tsx}", "scripts/**/*.mjs", "vitest.config.ts"] + }, + "packages/context": { + "entry": [ + "src/index.ts", + "src/agent/index.ts", + "src/core/index.ts", + "src/connections/index.ts", + "src/daemon/index.ts", + "src/ingest/index.ts", + "src/ingest/memory-flow/index.ts", + "src/ingest/metabase-mapping.ts", + "src/scan/index.ts", + "src/search/index.ts", + "src/sql-analysis/index.ts", + "src/memory/index.ts", + "src/mcp/index.ts", + "src/project/index.ts", + "src/prompts/index.ts", + "src/skills/index.ts", + "src/sl/index.ts", + "src/sl/descriptions.ts", + "src/tools/index.ts", + "src/wiki/index.ts", + "src/**/*.test.ts", + "scripts/**/*.mjs" + ], + "project": ["src/**/*.ts", "scripts/**/*.mjs", "vitest.config.ts"] + }, + "packages/llm": { + "entry": ["src/index.ts", "src/**/*.test.ts"], + "project": ["src/**/*.ts", "vitest.config.ts"] + }, + "packages/connector-*": { + "entry": ["src/index.ts", "src/**/*.test.ts"], + "project": ["src/**/*.ts"] + }, + "docs-site": { + "entry": [ + "app/**/*.{ts,tsx}", + "components/**/*.{ts,tsx}", + "lib/**/*.{ts,tsx}", + "middleware.ts", + "next.config.mjs", + "source.config.ts", + "tests/**/*.mjs" + ], + "project": [ + "app/**/*.{ts,tsx}", + "components/**/*.{ts,tsx}", + "lib/**/*.{ts,tsx}", + "*.ts", + "*.mjs", + "tests/**/*.mjs" + ], + "ignoreDependencies": ["tailwindcss"] + } + }, + "ignore": [ + "**/dist/**", + "**/coverage/**", + "**/.next/**", + "**/node_modules/**", + "**/*.gen.ts", + "**/*.generated.ts" + ], + "ignoreIssues": { + "packages/cli/src/clack.ts": ["exports"], + "packages/cli/src/commands/connection-metabase-setup.ts": ["exports", "types"], + "packages/cli/src/ingest.test-utils.ts": ["exports"], + "packages/cli/src/io/symbols.ts": ["exports"], + "packages/cli/src/managed-python-command.ts": ["types"], + "packages/cli/src/managed-python-daemon.ts": ["types"], + "packages/cli/src/managed-python-http.ts": ["exports", "types"], + "packages/cli/src/managed-python-runtime.ts": ["types"], + "packages/cli/src/memory-flow-tui.tsx": ["types"], + "packages/cli/src/next-steps.ts": ["exports"], + "packages/cli/src/print-command-tree.ts": ["exports"], + "packages/cli/src/setup-agents.ts": ["exports", "types"], + "packages/cli/src/setup-context.ts": ["types"], + "packages/cli/src/setup-demo-tour.ts": ["exports"], + "packages/cli/src/setup-models.ts": ["exports"], + "packages/cli/src/setup-project.ts": ["types"], + "packages/cli/src/setup-ready-menu.ts": ["types"], + "packages/cli/src/setup-sources.ts": ["types"], + "packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts": ["exports", "types"], + "packages/context/src/ingest/adapters/lookml/pull-config.ts": ["exports"], + "packages/context/src/ingest/adapters/metabase/serialize-card.ts": ["types"], + "packages/context/src/ingest/adapters/metabase/types.ts": ["exports"], + "packages/context/src/ingest/adapters/metricflow/parse.ts": ["types"], + "packages/context/src/ingest/ports.ts": ["types"], + "packages/context/src/ingest/stages/stage-3-work-units.ts": ["types"], + "packages/context/src/ingest/stages/stage-index.types.ts": ["types"], + "packages/context/src/project/config.ts": ["types"], + "packages/context/src/scan/relationship-candidates.ts": ["types"], + "packages/context/src/scan/relationship-diagnostics.ts": ["types"], + "packages/context/src/tools/context-evidence-tool-store.ts": ["types"] + } +} diff --git a/package.json b/package.json index 159bd709..b105784b 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,10 @@ "artifacts:verify-manifest": "node scripts/package-artifacts.mjs verify-manifest", "build": "pnpm --filter './packages/*' run build", "check": "node scripts/check-boundaries.mjs && node --test scripts/*.test.mjs && pnpm --filter './packages/*' run build && pnpm --filter './packages/*' run test", + "dead-code": "pnpm run dead-code:biome && pnpm run dead-code:knip", + "dead-code:biome": "biome ci . --formatter-enabled=false --assist-enabled=false", + "dead-code:fix": "biome check . --formatter-enabled=false --assist-enabled=false --write && knip --fix --format", + "dead-code:knip": "knip --reporter compact", "ktx": "node scripts/run-ktx.mjs", "link:dev": "node scripts/link-dev-cli.mjs", "native:rebuild": "pnpm -r rebuild better-sqlite3", @@ -36,9 +40,12 @@ "type-check": "pnpm --filter './packages/*' run type-check" }, "devDependencies": { + "@biomejs/biome": "^2.4.15", "@types/node": "^25.7.0", + "better-sqlite3": "^12.10.0", + "knip": "^6.12.2", "typescript": "^6.0.3", - "vitest": "^4.1.6" + "yaml": "^2.9.0" }, "pnpm": { "onlyBuiltDependencies": [ diff --git a/packages/cli/src/agent-runtime.test.ts b/packages/cli/src/agent-runtime.test.ts deleted file mode 100644 index 808ddac3..00000000 --- a/packages/cli/src/agent-runtime.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - KTX_AGENT_MAX_ROWS_CAP, - createKtxAgentRuntime, - parseAgentMaxRows, - readAgentJsonFile, - writeAgentJson, - writeAgentJsonError, -} from './agent-runtime.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { write: (chunk: string) => (stdout += chunk) }, - stderr: { write: (chunk: string) => (stderr += chunk) }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -describe('agent runtime helpers', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-agent-runtime-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('writes JSON success and error envelopes without color or spinners', () => { - const successIo = makeIo(); - const errorIo = makeIo(); - - writeAgentJson(successIo.io, { ok: true }); - writeAgentJsonError(errorIo.io, 'missing source', { code: 'NOT_FOUND' }); - - expect(JSON.parse(successIo.stdout())).toEqual({ ok: true }); - expect(successIo.stderr()).toBe(''); - expect(JSON.parse(errorIo.stderr())).toEqual({ - ok: false, - error: { message: 'missing source', code: 'NOT_FOUND' }, - }); - expect(errorIo.stdout()).toBe(''); - }); - - it('reads JSON query files as objects', async () => { - const path = join(tempDir, 'query.json'); - await writeFile(path, '{"measures":["revenue"],"limit":50}', 'utf-8'); - - await expect(readAgentJsonFile(path)).resolves.toEqual({ measures: ['revenue'], limit: 50 }); - }); - - it('rejects non-object JSON query files', async () => { - const path = join(tempDir, 'query.json'); - await writeFile(path, '["revenue"]', 'utf-8'); - - await expect(readAgentJsonFile(path)).rejects.toThrow('must contain a JSON object'); - }); - - it('requires positive row limits and enforces the agent cap', () => { - expect(parseAgentMaxRows(100)).toBe(100); - expect(() => parseAgentMaxRows(undefined)).toThrow('maxRows is required'); - expect(() => parseAgentMaxRows(0)).toThrow('positive integer'); - expect(() => parseAgentMaxRows(KTX_AGENT_MAX_ROWS_CAP + 1)).toThrow(String(KTX_AGENT_MAX_ROWS_CAP)); - }); - - it('constructs local context ports with semantic compute and query executor', async () => { - const project = { - projectDir: tempDir, - configPath: join(tempDir, 'ktx.yaml'), - config: { project: 'revenue', connections: {} }, - coreConfig: {}, - git: {}, - fileStore: {}, - } as never; - const ports = { knowledge: {}, semanticLayer: {} } as never; - const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; - const queryExecutor = { execute: vi.fn() }; - const loadProject = vi.fn(async () => project); - const createContextTools = vi.fn(() => ports); - - await expect( - createKtxAgentRuntime( - { projectDir: tempDir, enableSemanticCompute: true, enableQueryExecution: true }, - { - loadProject, - createContextTools, - createSemanticLayerCompute: () => semanticLayerCompute, - createQueryExecutor: () => queryExecutor, - }, - ), - ).resolves.toMatchObject({ project, ports, queryExecutor }); - - expect(loadProject).toHaveBeenCalledWith({ projectDir: tempDir }); - expect(createContextTools).toHaveBeenCalledWith(project, { - semanticLayerCompute, - queryExecutor, - }); - }); - - it('creates managed semantic compute when no test override is injected', async () => { - const project = { - projectDir: tempDir, - configPath: join(tempDir, 'ktx.yaml'), - config: { project: 'revenue', connections: {} }, - coreConfig: {}, - git: {}, - fileStore: {}, - } as never; - const ports = { semanticLayer: {} } as never; - const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; - const loadProject = vi.fn(async () => project); - const createContextTools = vi.fn(() => ports); - const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); - const { io } = makeIo(); - - await expect( - createKtxAgentRuntime( - { - projectDir: tempDir, - enableSemanticCompute: true, - enableQueryExecution: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - io, - }, - { - loadProject, - createContextTools, - createManagedSemanticLayerCompute, - }, - ), - ).resolves.toMatchObject({ project, ports, semanticLayerCompute }); - - expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io, - }); - expect(createContextTools).toHaveBeenCalledWith(project, { - semanticLayerCompute, - }); - }); -}); diff --git a/packages/cli/src/agent-runtime.ts b/packages/cli/src/agent-runtime.ts deleted file mode 100644 index feccae7c..00000000 --- a/packages/cli/src/agent-runtime.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { readFile } from 'node:fs/promises'; -import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; -import { createLocalProjectMcpContextPorts, type KtxMcpContextPorts } from '@ktx/context/mcp'; -import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project'; -import type { KtxCliIo } from './cli-runtime.js'; -import { - createManagedPythonSemanticLayerComputePort, - type KtxManagedPythonInstallPolicy, -} from './managed-python-command.js'; - -export const KTX_AGENT_MAX_ROWS_CAP = 1000; - -export interface KtxAgentRuntimeOptions { - projectDir: string; - enableSemanticCompute: boolean; - enableQueryExecution: boolean; - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; - io?: KtxCliIo; -} - -export interface KtxAgentRuntime { - project: KtxLocalProject; - ports: KtxMcpContextPorts; - semanticLayerCompute?: KtxSemanticLayerComputePort; - queryExecutor?: KtxSqlQueryExecutorPort; -} - -export interface KtxAgentRuntimeDeps { - loadProject?: typeof loadKtxProject; - createContextTools?: typeof createLocalProjectMcpContextPorts; - createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; - createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; - createQueryExecutor?: () => KtxSqlQueryExecutorPort; -} - -export function writeAgentJson(io: KtxCliIo, value: unknown): void { - io.stdout.write(`${JSON.stringify(value, null, 2)}\n`); -} - -export function writeAgentJsonError( - io: KtxCliIo, - message: string, - detail: Record = {}, -): void { - io.stderr.write(`${JSON.stringify({ ok: false, error: { message, ...detail } }, null, 2)}\n`); -} - -export async function readAgentJsonFile(path: string): Promise> { - const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error(`${path} must contain a JSON object.`); - } - return parsed as Record; -} - -export function parseAgentMaxRows(value: number | undefined): number { - if (!Number.isInteger(value) || value === undefined || value <= 0) { - throw new Error('maxRows is required and must be a positive integer.'); - } - if (value > KTX_AGENT_MAX_ROWS_CAP) { - throw new Error(`maxRows must be less than or equal to ${KTX_AGENT_MAX_ROWS_CAP}.`); - } - return value; -} - -async function createAgentSemanticLayerCompute( - options: KtxAgentRuntimeOptions, - deps: KtxAgentRuntimeDeps, -): Promise { - if (!options.enableSemanticCompute) { - return undefined; - } - if (deps.createSemanticLayerCompute) { - return deps.createSemanticLayerCompute(); - } - if (!options.cliVersion || !options.runtimeInstallPolicy || !options.io) { - throw new Error('Managed Python semantic compute requires cliVersion, runtimeInstallPolicy, and io.'); - } - const createManagedSemanticLayerCompute = - deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; - return createManagedSemanticLayerCompute({ - cliVersion: options.cliVersion, - installPolicy: options.runtimeInstallPolicy, - io: options.io, - }); -} - -export async function createKtxAgentRuntime( - options: KtxAgentRuntimeOptions, - deps: KtxAgentRuntimeDeps = {}, -): Promise { - const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir }); - const semanticLayerCompute = await createAgentSemanticLayerCompute(options, deps); - const queryExecutor = options.enableQueryExecution - ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() - : undefined; - const ports = (deps.createContextTools ?? createLocalProjectMcpContextPorts)(project, { - ...(semanticLayerCompute ? { semanticLayerCompute } : {}), - ...(queryExecutor ? { queryExecutor } : {}), - }); - return { - project, - ports, - ...(semanticLayerCompute ? { semanticLayerCompute } : {}), - ...(queryExecutor ? { queryExecutor } : {}), - }; -} diff --git a/packages/cli/src/agent-search-readiness.test.ts b/packages/cli/src/agent-search-readiness.test.ts deleted file mode 100644 index 432afa90..00000000 --- a/packages/cli/src/agent-search-readiness.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - isMissingProjectConfigError, - missingConnectionSlSearchReadiness, - missingProjectSlSearchReadiness, - noConnectionsSlSearchReadiness, - noIndexedSourcesSlSearchReadiness, -} from './agent-search-readiness.js'; - -describe('agent semantic-layer search readiness guidance', () => { - it('formats missing project guidance with exact recovery commands', () => { - expect(missingProjectSlSearchReadiness('/tmp/ktx-search', 'gross revenue')).toEqual({ - code: 'agent_sl_search_missing_project', - message: 'Semantic-layer search needs an initialized KTX project at /tmp/ktx-search.', - nextSteps: [ - 'ktx setup --project-dir /tmp/ktx-search', - 'ktx status --project-dir /tmp/ktx-search', - 'ktx ingest run --connection-id --adapter ', - 'ktx agent sl list --json --query "gross revenue" --project-dir /tmp/ktx-search', - ], - }); - }); - - it('formats no-connection and no-index guidance without hiding the project path', () => { - expect(noConnectionsSlSearchReadiness('/tmp/ktx-search', 'revenue')).toMatchObject({ - code: 'agent_sl_search_no_connections', - message: 'Semantic-layer search found no configured connections in /tmp/ktx-search.', - }); - expect(noIndexedSourcesSlSearchReadiness('/tmp/ktx-search', 'orders')).toMatchObject({ - code: 'agent_sl_search_no_indexed_sources', - message: 'Semantic-layer search found no indexed semantic-layer sources in /tmp/ktx-search.', - }); - }); - - it('formats unknown connection guidance', () => { - expect(missingConnectionSlSearchReadiness('/tmp/ktx-search', 'warehouse', 'revenue')).toMatchObject({ - code: 'agent_sl_search_unknown_connection', - message: 'Semantic-layer search connection "warehouse" is not configured in /tmp/ktx-search.', - }); - }); - - it('detects missing ktx.yaml read errors', () => { - const error = Object.assign(new Error('ENOENT: no such file or directory'), { - code: 'ENOENT', - path: '/tmp/ktx-search/ktx.yaml', - }); - - expect(isMissingProjectConfigError(error)).toBe(true); - expect(isMissingProjectConfigError(new Error('other'))).toBe(false); - }); -}); diff --git a/packages/cli/src/agent-search-readiness.ts b/packages/cli/src/agent-search-readiness.ts deleted file mode 100644 index e4de7318..00000000 --- a/packages/cli/src/agent-search-readiness.ts +++ /dev/null @@ -1,94 +0,0 @@ -export type KtxAgentSlSearchReadinessCode = - | 'agent_sl_search_missing_project' - | 'agent_sl_search_no_connections' - | 'agent_sl_search_unknown_connection' - | 'agent_sl_search_no_indexed_sources'; - -export interface KtxAgentSlSearchReadinessDetail { - code: KtxAgentSlSearchReadinessCode; - message: string; - nextSteps: string[]; -} - -function queryForCommand(query: string | undefined): string { - const trimmed = query?.trim(); - return trimmed && trimmed.length > 0 ? trimmed : 'revenue'; -} - -function projectSearchCommand(projectDir: string, query: string | undefined): string { - return `ktx agent sl list --json --query ${JSON.stringify(queryForCommand(query))} --project-dir ${projectDir}`; -} - -function baseNextSteps(projectDir: string, query: string | undefined): string[] { - return [ - `ktx setup --project-dir ${projectDir}`, - `ktx status --project-dir ${projectDir}`, - 'ktx ingest run --connection-id --adapter ', - projectSearchCommand(projectDir, query), - ]; -} - -export function missingProjectSlSearchReadiness( - projectDir: string, - query: string | undefined, -): KtxAgentSlSearchReadinessDetail { - return { - code: 'agent_sl_search_missing_project', - message: `Semantic-layer search needs an initialized KTX project at ${projectDir}.`, - nextSteps: baseNextSteps(projectDir, query), - }; -} - -export function noConnectionsSlSearchReadiness( - projectDir: string, - query: string | undefined, -): KtxAgentSlSearchReadinessDetail { - return { - code: 'agent_sl_search_no_connections', - message: `Semantic-layer search found no configured connections in ${projectDir}.`, - nextSteps: baseNextSteps(projectDir, query), - }; -} - -export function missingConnectionSlSearchReadiness( - projectDir: string, - connectionId: string, - query: string | undefined, -): KtxAgentSlSearchReadinessDetail { - return { - code: 'agent_sl_search_unknown_connection', - message: `Semantic-layer search connection "${connectionId}" is not configured in ${projectDir}.`, - nextSteps: baseNextSteps(projectDir, query), - }; -} - -export function noIndexedSourcesSlSearchReadiness( - projectDir: string, - query: string | undefined, -): KtxAgentSlSearchReadinessDetail { - return { - code: 'agent_sl_search_no_indexed_sources', - message: `Semantic-layer search found no indexed semantic-layer sources in ${projectDir}.`, - nextSteps: baseNextSteps(projectDir, query), - }; -} - -function errorCode(error: unknown): string | undefined { - if (typeof error !== 'object' || error === null || !('code' in error)) { - return undefined; - } - const code = (error as { code?: unknown }).code; - return typeof code === 'string' ? code : undefined; -} - -function errorPath(error: unknown): string | undefined { - if (typeof error !== 'object' || error === null || !('path' in error)) { - return undefined; - } - const path = (error as { path?: unknown }).path; - return typeof path === 'string' ? path : undefined; -} - -export function isMissingProjectConfigError(error: unknown): boolean { - return errorCode(error) === 'ENOENT' && (errorPath(error)?.endsWith('ktx.yaml') ?? false); -} diff --git a/packages/cli/src/agent.test.ts b/packages/cli/src/agent.test.ts deleted file mode 100644 index 566f5763..00000000 --- a/packages/cli/src/agent.test.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { buildDefaultKtxProjectConfig } from '@ktx/context/project'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxAgent } from './agent.js'; -import type { KtxAgentRuntime } from './agent-runtime.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { write: (chunk: string) => (stdout += chunk) }, - stderr: { write: (chunk: string) => (stderr += chunk) }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -function runtime(overrides: Record = {}): KtxAgentRuntime { - const config = buildDefaultKtxProjectConfig('revenue'); - return { - project: { - projectDir: '/tmp/revenue', - configPath: '/tmp/revenue/ktx.yaml', - config: { - ...config, - connections: { - warehouse: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true as const }, - }, - }, - coreConfig: {} as KtxAgentRuntime['project']['coreConfig'], - git: {} as KtxAgentRuntime['project']['git'], - fileStore: {} as KtxAgentRuntime['project']['fileStore'], - }, - ports: { - connections: { list: vi.fn(async () => [{ id: 'warehouse', name: 'warehouse', connectionType: 'sqlite' }]) }, - semanticLayer: { - listSources: vi.fn(async () => ({ - sources: [ - { - connectionId: 'warehouse', - connectionName: 'warehouse', - name: 'orders', - columnCount: 2, - measureCount: 1, - joinCount: 0, - }, - ], - totalSources: 1, - })), - readSource: vi.fn(async () => ({ sourceName: 'orders', yaml: 'name: orders\n' })), - writeSource: vi.fn(async () => ({ success: true, sourceName: 'orders' })), - validate: vi.fn(async () => ({ success: true, errors: [], warnings: [] })), - query: vi.fn(async () => ({ sql: 'select 1', headers: ['x'], rows: [[1]], totalRows: 1, plan: {} })), - }, - knowledge: { - search: vi.fn(async () => ({ - results: [ - { - key: 'page-1', - path: 'knowledge/global/page-1.md', - scope: 'GLOBAL' as const, - summary: 'Revenue logic', - score: 0.9, - matchReasons: ['lexical' as const], - }, - ], - totalFound: 1, - })), - read: vi.fn(async () => ({ - key: 'page-1', - scope: 'GLOBAL' as const, - summary: 'Revenue logic', - content: 'Use net revenue.', - })), - write: vi.fn(async () => ({ success: true, key: 'page-1', action: 'created' as const })), - }, - }, - queryExecutor: { - execute: vi.fn(async () => ({ headers: ['x'], rows: [[1]], totalRows: 1, command: 'SELECT', rowCount: 1 })), - }, - ...overrides, - }; -} - -function runtimeWithoutConnections(): KtxAgentRuntime { - const base = runtime(); - return { - ...base, - project: { - ...base.project, - config: { - ...base.project.config, - connections: {}, - }, - }, - ports: { - ...base.ports, - semanticLayer: { - ...base.ports.semanticLayer!, - listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })), - }, - }, - }; -} - -describe('runKtxAgent', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-agent-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('prints tool discovery with every stable command', async () => { - const io = makeIo(); - - await expect(runKtxAgent({ command: 'tools', projectDir: tempDir, json: true }, io.io)).resolves.toBe(0); - - const body = JSON.parse(io.stdout()); - expect(body.projectDir).toBe(tempDir); - expect(body.tools.map((tool: { name: string }) => tool.name)).toEqual([ - 'context', - 'sl.list', - 'sl.read', - 'sl.query', - 'wiki.search', - 'wiki.read', - 'sql.execute', - ]); - expect(io.stderr()).toBe(''); - }); - - it('prints project context from setup status, connections, and SL summaries', async () => { - const io = makeIo(); - const createRuntime = vi.fn(async () => runtime()); - const readSetupStatus = vi.fn(async () => ({ project: { path: tempDir, ready: true }, agents: [] })); - - await expect( - runKtxAgent({ command: 'context', projectDir: tempDir, json: true }, io.io, { createRuntime, readSetupStatus }), - ).resolves.toBe(0); - - expect(JSON.parse(io.stdout())).toMatchObject({ - projectDir: tempDir, - status: { project: { ready: true } }, - connections: [{ id: 'warehouse' }], - semanticLayer: { totalSources: 1 }, - }); - }); - - it('dispatches SL list, SL read, wiki search, and wiki read through local ports', async () => { - for (const args of [ - { command: 'sl-list' as const, projectDir: tempDir, json: true as const, connectionId: 'warehouse' }, - { - command: 'sl-read' as const, - projectDir: tempDir, - json: true as const, - connectionId: 'warehouse', - sourceName: 'orders', - }, - { command: 'wiki-search' as const, projectDir: tempDir, json: true as const, query: 'revenue', limit: 10 }, - { command: 'wiki-read' as const, projectDir: tempDir, json: true as const, pageId: 'page-1' }, - ]) { - const io = makeIo(); - await expect(runKtxAgent(args, io.io, { createRuntime: async () => runtime() })).resolves.toBe(0); - expect(JSON.parse(io.stdout())).toBeTruthy(); - expect(io.stderr()).toBe(''); - } - }); - - it('prints wiki hybrid search metadata from the hidden agent wiki search command', async () => { - const fakeRuntime = runtime(); - const knowledge = fakeRuntime.ports.knowledge; - if (!knowledge) { - throw new Error('Expected runtime knowledge port'); - } - fakeRuntime.ports.knowledge = { - ...knowledge, - search: vi.fn(async () => ({ - results: [ - { - key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', - scope: 'GLOBAL' as const, - summary: 'Revenue metric definition', - score: 0.02459016393442623, - matchReasons: ['lexical' as const, 'token' as const], - }, - ], - totalFound: 1, - })), - }; - const io = makeIo(); - - await expect( - runKtxAgent({ command: 'wiki-search', projectDir: tempDir, json: true, query: 'paid order', limit: 5 }, io.io, { - createRuntime: async () => fakeRuntime, - }), - ).resolves.toBe(0); - - expect(JSON.parse(io.stdout())).toEqual({ - results: [ - expect.objectContaining({ - key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', - matchReasons: ['lexical', 'token'], - }), - ], - totalFound: 1, - }); - }); - - it('executes SL queries from a JSON query file', async () => { - const queryFile = join(tempDir, 'sl-query.json'); - const io = makeIo(); - await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8'); - - await expect( - runKtxAgent( - { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile, - execute: true, - maxRows: 100, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'never', - }, - io.io, - { createRuntime: async () => runtime() }, - ), - ).resolves.toBe(0); - - expect(JSON.parse(io.stdout())).toMatchObject({ sql: 'select 1', rows: [[1]] }); - }); - - it('passes managed runtime options into default SL query runtime creation', async () => { - const queryFile = join(tempDir, 'sl-query.json'); - const io = makeIo(); - const createRuntime = vi.fn(async () => runtime()); - await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8'); - - await expect( - runKtxAgent( - { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile, - execute: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - }, - io.io, - { createRuntime }, - ), - ).resolves.toBe(0); - - expect(createRuntime).toHaveBeenCalledWith({ - projectDir: tempDir, - enableSemanticCompute: true, - enableQueryExecution: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - io: io.io, - }); - }); - - it('executes read-only SQL from a SQL file with an explicit row limit', async () => { - const sqlFile = join(tempDir, 'query.sql'); - const fakeRuntime = runtime(); - const io = makeIo(); - await writeFile(sqlFile, 'select 1', 'utf-8'); - - await expect( - runKtxAgent( - { - command: 'sql-execute', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - sqlFile, - maxRows: 100, - }, - io.io, - { createRuntime: async () => fakeRuntime as never }, - ), - ).resolves.toBe(0); - - expect(fakeRuntime.queryExecutor?.execute).toHaveBeenCalledWith({ - connectionId: 'warehouse', - projectDir: '/tmp/revenue', - connection: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true }, - sql: 'select 1', - maxRows: 100, - }); - }); - - it('prints guided JSON when semantic-layer search runs outside a project', async () => { - const io = makeIo(); - const missingProjectError = Object.assign(new Error('ENOENT: no such file or directory'), { - code: 'ENOENT', - path: join(tempDir, 'ktx.yaml'), - }); - - await expect( - runKtxAgent( - { command: 'sl-list', projectDir: tempDir, json: true, query: 'gross revenue' }, - io.io, - { createRuntime: vi.fn(async () => Promise.reject(missingProjectError)) }, - ), - ).resolves.toBe(1); - - expect(JSON.parse(io.stderr())).toEqual({ - ok: false, - error: { - code: 'agent_sl_search_missing_project', - message: `Semantic-layer search needs an initialized KTX project at ${tempDir}.`, - nextSteps: [ - `ktx setup --project-dir ${tempDir}`, - `ktx status --project-dir ${tempDir}`, - 'ktx ingest run --connection-id --adapter ', - `ktx agent sl list --json --query "gross revenue" --project-dir ${tempDir}`, - ], - }, - }); - expect(io.stdout()).toBe(''); - }); - - it('prints guided JSON when semantic-layer search has no configured connections', async () => { - const io = makeIo(); - - await expect( - runKtxAgent( - { command: 'sl-list', projectDir: tempDir, json: true, query: 'revenue' }, - io.io, - { createRuntime: async () => runtimeWithoutConnections() }, - ), - ).resolves.toBe(1); - - expect(JSON.parse(io.stderr())).toMatchObject({ - ok: false, - error: { - code: 'agent_sl_search_no_connections', - message: `Semantic-layer search found no configured connections in ${tempDir}.`, - nextSteps: [ - `ktx setup --project-dir ${tempDir}`, - `ktx status --project-dir ${tempDir}`, - 'ktx ingest run --connection-id --adapter ', - `ktx agent sl list --json --query "revenue" --project-dir ${tempDir}`, - ], - }, - }); - }); - - it('prints guided JSON when semantic-layer search asks for an unknown connection', async () => { - const io = makeIo(); - - await expect( - runKtxAgent( - { command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'missing', query: 'revenue' }, - io.io, - { createRuntime: async () => runtime() }, - ), - ).resolves.toBe(1); - - expect(JSON.parse(io.stderr())).toMatchObject({ - ok: false, - error: { - code: 'agent_sl_search_unknown_connection', - message: `Semantic-layer search connection "missing" is not configured in ${tempDir}.`, - }, - }); - }); - - it('prints guided JSON when semantic-layer search has no indexed sources', async () => { - const fakeRuntime = runtime(); - const semanticLayer = fakeRuntime.ports.semanticLayer!; - fakeRuntime.ports.semanticLayer = { - ...semanticLayer, - listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })), - }; - const io = makeIo(); - - await expect( - runKtxAgent( - { command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'warehouse', query: 'revenue' }, - io.io, - { createRuntime: async () => fakeRuntime }, - ), - ).resolves.toBe(1); - - expect(JSON.parse(io.stderr())).toMatchObject({ - ok: false, - error: { - code: 'agent_sl_search_no_indexed_sources', - message: `Semantic-layer search found no indexed semantic-layer sources in ${tempDir}.`, - }, - }); - }); - - it('returns JSON errors when required ports or records are missing', async () => { - const io = makeIo(); - - await expect( - runKtxAgent({ command: 'wiki-read', projectDir: tempDir, json: true, pageId: 'missing' }, io.io, { - createRuntime: async () => - runtime({ - ports: { knowledge: { read: vi.fn(async () => null) } }, - }) as never, - }), - ).resolves.toBe(1); - - expect(JSON.parse(io.stderr())).toMatchObject({ - ok: false, - error: { message: expect.stringContaining('missing') }, - }); - }); -}); diff --git a/packages/cli/src/agent.ts b/packages/cli/src/agent.ts deleted file mode 100644 index 61d85b8c..00000000 --- a/packages/cli/src/agent.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { readFile } from 'node:fs/promises'; -import type { KtxCliIo } from './cli-runtime.js'; -import { - createKtxAgentRuntime, - parseAgentMaxRows, - readAgentJsonFile, - writeAgentJson, - writeAgentJsonError, - type KtxAgentRuntime, - type KtxAgentRuntimeDeps, -} from './agent-runtime.js'; -import { - isMissingProjectConfigError, - missingConnectionSlSearchReadiness, - missingProjectSlSearchReadiness, - noConnectionsSlSearchReadiness, - noIndexedSourcesSlSearchReadiness, - type KtxAgentSlSearchReadinessDetail, -} from './agent-search-readiness.js'; -import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -import { readKtxSetupStatus, type KtxSetupStatus } from './setup.js'; - -export type KtxAgentArgs = - | { command: 'tools'; projectDir: string; json: true } - | { command: 'context'; projectDir: string; json: true } - | { command: 'sl-list'; projectDir: string; json: true; connectionId?: string; query?: string } - | { command: 'sl-read'; projectDir: string; json: true; connectionId?: string; sourceName: string } - | { - command: 'sl-query'; - projectDir: string; - json: true; - connectionId: string; - queryFile: string; - execute: boolean; - maxRows?: number; - cliVersion: string; - runtimeInstallPolicy: KtxManagedPythonInstallPolicy; - } - | { command: 'wiki-search'; projectDir: string; json: true; query: string; limit: number } - | { command: 'wiki-read'; projectDir: string; json: true; pageId: string } - | { command: 'sql-execute'; projectDir: string; json: true; connectionId: string; sqlFile: string; maxRows?: number }; - -export interface KtxAgentDeps extends KtxAgentRuntimeDeps { - createRuntime?: (options: { - projectDir: string; - enableSemanticCompute: boolean; - enableQueryExecution: boolean; - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; - io?: KtxCliIo; - }) => Promise; - readSetupStatus?: ( - projectDir: string, - ) => Promise; -} - -const AGENT_TOOLS = [ - { name: 'context', command: 'ktx agent context --json' }, - { name: 'sl.list', command: 'ktx agent sl list --json [--connection-id ] [--query ]' }, - { name: 'sl.read', command: 'ktx agent sl read --json [--connection-id ]' }, - { - name: 'sl.query', - command: 'ktx agent sl query --json --connection-id --query-file --execute --max-rows 100', - }, - { name: 'wiki.search', command: 'ktx agent wiki search --json [--limit 10]' }, - { name: 'wiki.read', command: 'ktx agent wiki read --json' }, - { - name: 'sql.execute', - command: 'ktx agent sql execute --json --connection-id --sql-file --max-rows 100', - }, -] as const; - -function writeAgentSlSearchReadinessError(io: KtxCliIo, detail: KtxAgentSlSearchReadinessDetail): void { - writeAgentJsonError(io, detail.message, { code: detail.code, nextSteps: detail.nextSteps }); -} - -async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps, io: KtxCliIo): Promise { - const needsSemanticCompute = args.command === 'sl-query'; - const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute); - const runtimeOptions = { - projectDir: args.projectDir, - enableSemanticCompute: needsSemanticCompute, - enableQueryExecution: needsQueryExecution, - ...(args.command === 'sl-query' - ? { - cliVersion: args.cliVersion, - runtimeInstallPolicy: args.runtimeInstallPolicy, - io, - } - : {}), - }; - return deps.createRuntime ? deps.createRuntime(runtimeOptions) : createKtxAgentRuntime(runtimeOptions, deps); -} - -function connectionIdForSource(runtime: KtxAgentRuntime, requested: string | undefined): string { - if (requested) return requested; - const ids = Object.keys(runtime.project.config.connections ?? {}); - if (ids.length === 1) return ids[0] as string; - throw new Error('Use --connection-id when the project has zero or multiple connections.'); -} - -export async function runKtxAgent(args: KtxAgentArgs, io: KtxCliIo, deps: KtxAgentDeps = {}): Promise { - try { - if (args.command === 'tools') { - writeAgentJson(io, { projectDir: args.projectDir, tools: AGENT_TOOLS }); - return 0; - } - - const runtime = await runtimeFor(args, deps, io); - - if (args.command === 'context') { - const [status, connections, semanticLayer] = await Promise.all([ - (deps.readSetupStatus ?? readKtxSetupStatus)(args.projectDir), - runtime.ports.connections?.list() ?? [], - runtime.ports.semanticLayer?.listSources({}) ?? { sources: [], totalSources: 0 }, - ]); - writeAgentJson(io, { projectDir: args.projectDir, status, connections, semanticLayer, tools: AGENT_TOOLS }); - return 0; - } - - if (args.command === 'sl-list') { - const semanticLayer = runtime.ports.semanticLayer; - if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.'); - if (args.query) { - const connectionIds = Object.keys(runtime.project.config.connections ?? {}); - if (args.connectionId && !runtime.project.config.connections[args.connectionId]) { - writeAgentSlSearchReadinessError( - io, - missingConnectionSlSearchReadiness(args.projectDir, args.connectionId, args.query), - ); - return 1; - } - if (connectionIds.length === 0) { - writeAgentSlSearchReadinessError(io, noConnectionsSlSearchReadiness(args.projectDir, args.query)); - return 1; - } - } - - const listed = await semanticLayer.listSources({ connectionId: args.connectionId, query: args.query }); - if (args.query && listed.sources.length === 0) { - const allSources = await semanticLayer.listSources({ connectionId: args.connectionId }); - if (allSources.totalSources === 0) { - writeAgentSlSearchReadinessError(io, noIndexedSourcesSlSearchReadiness(args.projectDir, args.query)); - return 1; - } - } - - writeAgentJson(io, listed); - return 0; - } - - if (args.command === 'sl-read') { - const semanticLayer = runtime.ports.semanticLayer; - if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.'); - const source = await semanticLayer.readSource({ - connectionId: connectionIdForSource(runtime, args.connectionId), - sourceName: args.sourceName, - }); - if (!source) throw new Error(`Semantic-layer source "${args.sourceName}" was not found.`); - writeAgentJson(io, source); - return 0; - } - - if (args.command === 'sl-query') { - const semanticLayer = runtime.ports.semanticLayer; - if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.'); - const query = await readAgentJsonFile(args.queryFile); - const maxRows = args.execute ? parseAgentMaxRows(args.maxRows) : args.maxRows; - writeAgentJson( - io, - await semanticLayer.query({ - connectionId: args.connectionId, - query: { ...query, ...(maxRows !== undefined ? { limit: maxRows } : {}) } as never, - }), - ); - return 0; - } - - if (args.command === 'wiki-search') { - const knowledge = runtime.ports.knowledge; - if (!knowledge) throw new Error('Wiki tools are not available for this project.'); - writeAgentJson(io, await knowledge.search({ userId: 'agent', query: args.query, limit: args.limit })); - return 0; - } - - if (args.command === 'wiki-read') { - const knowledge = runtime.ports.knowledge; - if (!knowledge) throw new Error('Wiki tools are not available for this project.'); - const page = await knowledge.read({ userId: 'agent', key: args.pageId }); - if (!page) throw new Error(`Wiki page "${args.pageId}" was not found.`); - writeAgentJson(io, page); - return 0; - } - - const queryExecutor = runtime.queryExecutor; - if (!queryExecutor) throw new Error('SQL execution is not available for this project.'); - const connection = runtime.project.config.connections[args.connectionId]; - if (!connection) throw new Error(`Connection "${args.connectionId}" was not found.`); - const maxRows = parseAgentMaxRows(args.maxRows); - writeAgentJson( - io, - await queryExecutor.execute({ - connectionId: args.connectionId, - projectDir: runtime.project.projectDir, - connection, - sql: await readFile(args.sqlFile, 'utf-8'), - maxRows, - }), - ); - return 0; - } catch (error) { - if (args.command === 'sl-list' && args.query && isMissingProjectConfigError(error)) { - writeAgentSlSearchReadinessError(io, missingProjectSlSearchReadiness(args.projectDir, args.query)); - return 1; - } - writeAgentJsonError(io, error instanceof Error ? error.message : String(error)); - return 1; - } -} diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index 682c027a..7d6a98f3 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -1,6 +1,5 @@ import { Command, InvalidArgumentError } from '@commander-js/extra-typings'; import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; -import { registerAgentCommands } from './commands/agent-commands.js'; import { registerConnectionCommands } from './commands/connection-commands.js'; import { registerIngestCommands } from './commands/ingest-commands.js'; import { registerWikiCommands } from './commands/knowledge-commands.js'; @@ -321,7 +320,6 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { registerWikiCommands(program, context); registerSlCommands(program, context); registerStatusCommands(program, context); - registerAgentCommands(program, context); registerDevCommands(program, context); return program; diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index 8fc06589..f303309a 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -2,7 +2,6 @@ import { createRequire } from 'node:module'; import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js'; import type { KtxConnectionNotionArgs } from './commands/connection-notion.js'; -import type { KtxAgentArgs } from './agent.js'; import type { KtxConnectionArgs } from './connection.js'; import type { KtxDoctorArgs } from './doctor.js'; import type { KtxIngestArgs } from './ingest.js'; @@ -30,7 +29,6 @@ export interface KtxCliIo { export interface KtxCliDeps { setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise; - agent?: (args: KtxAgentArgs, io: KtxCliIo) => Promise; connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; connectionNotion?: (args: KtxConnectionNotionArgs, io: KtxCliIo) => Promise; connectionMetabaseSetup?: (args: KtxConnectionMetabaseSetupArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/command-schemas.ts b/packages/cli/src/command-schemas.ts index 9ffe6de3..cb11f2eb 100644 --- a/packages/cli/src/command-schemas.ts +++ b/packages/cli/src/command-schemas.ts @@ -53,15 +53,18 @@ export const slQueryCommandSchema = z.object({ command: z.literal('query'), projectDir: projectDirSchema, connectionId: z.string().min(1).optional(), - query: z.object({ - measures: z.array(z.string().min(1)).min(1), - dimensions: stringArraySchema, - filters: stringArraySchema.optional(), - segments: stringArraySchema.optional(), - order_by: z.array(orderBySchema).optional(), - limit: z.number().int().positive().optional(), - include_empty: z.literal(true).optional(), - }), + query: z + .object({ + measures: z.array(z.string().min(1)).min(1), + dimensions: stringArraySchema, + filters: stringArraySchema.optional(), + segments: stringArraySchema.optional(), + order_by: z.array(orderBySchema).optional(), + limit: z.number().int().positive().optional(), + include_empty: z.literal(true).optional(), + }) + .optional(), + queryFile: z.string().min(1).optional(), format: z.enum(['json', 'sql']), execute: z.boolean(), cliVersion: z.string().min(1), diff --git a/packages/cli/src/commands/agent-commands.ts b/packages/cli/src/commands/agent-commands.ts deleted file mode 100644 index 2593991a..00000000 --- a/packages/cli/src/commands/agent-commands.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { Option, type Command } from '@commander-js/extra-typings'; -import type { KtxAgentArgs } from '../agent.js'; -import type { KtxCliCommandContext } from '../cli-program.js'; -import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js'; -import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; - -async function runAgent(context: KtxCliCommandContext, args: KtxAgentArgs): Promise { - const runner = context.deps.agent ?? (await import('../agent.js')).runKtxAgent; - context.setExitCode(await runner(args, context.io)); -} - -function jsonOption(): Option { - return new Option('--json', 'Print JSON output').makeOptionMandatory(); -} - -export function registerAgentCommands(program: Command, context: KtxCliCommandContext): void { - const agent = program - .command('agent', { hidden: true }) - .description('Machine-readable KTX commands for coding agents') - .showHelpAfterError(); - - agent.hook('preAction', (_thisCommand, actionCommand) => { - context.writeDebug?.('agent', actionCommand); - }); - - agent - .command('tools') - .description('Print available agent-facing KTX tools') - .addOption(jsonOption()) - .action(async (_options, command) => { - await runAgent(context, { command: 'tools', projectDir: resolveCommandProjectDir(command), json: true }); - }); - - agent - .command('context') - .description('Print project context for agent planning') - .addOption(jsonOption()) - .action(async (_options, command) => { - await runAgent(context, { command: 'context', projectDir: resolveCommandProjectDir(command), json: true }); - }); - - const sl = agent.command('sl').description('Semantic-layer agent commands'); - sl.command('list') - .description('List semantic-layer sources') - .addOption(jsonOption()) - .option('--connection-id ', 'Filter by connection id') - .option('--query ', 'Search source names and descriptions') - .action(async (options: { connectionId?: string; query?: string }, command) => { - await runAgent(context, { - command: 'sl-list', - projectDir: resolveCommandProjectDir(command), - json: true, - ...(options.connectionId ? { connectionId: options.connectionId } : {}), - ...(options.query ? { query: options.query } : {}), - }); - }); - sl.command('read') - .description('Read one semantic-layer source') - .argument('') - .addOption(jsonOption()) - .option('--connection-id ', 'Connection id containing the source') - .action(async (sourceName: string, options: { connectionId?: string }, command) => { - await runAgent(context, { - command: 'sl-read', - projectDir: resolveCommandProjectDir(command), - json: true, - sourceName, - ...(options.connectionId ? { connectionId: options.connectionId } : {}), - }); - }); - sl.command('query') - .description('Run a semantic-layer query JSON file') - .addOption(jsonOption()) - .requiredOption('--connection-id ', 'Connection id for execution') - .requiredOption('--query-file ', 'JSON semantic-layer query file') - .option('--execute', 'Execute the compiled query against the connection', false) - .option('--yes', 'Install the managed Python runtime without prompting when required', false) - .option('--no-input', 'Disable interactive managed runtime installation') - .option('--max-rows ', 'Maximum rows to return when executing', parsePositiveIntegerOption) - .action( - async ( - options: { - connectionId: string; - queryFile: string; - execute: boolean; - maxRows?: number; - yes?: boolean; - input?: boolean; - }, - command, - ) => { - await runAgent(context, { - command: 'sl-query', - projectDir: resolveCommandProjectDir(command), - json: true, - connectionId: options.connectionId, - queryFile: options.queryFile, - execute: options.execute, - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), - ...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}), - }); - }, - ); - - const wiki = agent.command('wiki').description('KTX wiki agent commands'); - wiki - .command('search') - .description('Search KTX wiki pages') - .argument('') - .addOption(jsonOption()) - .option('--limit ', 'Maximum search results', parsePositiveIntegerOption, 10) - .action(async (query: string, options: { limit: number }, command) => { - await runAgent(context, { - command: 'wiki-search', - projectDir: resolveCommandProjectDir(command), - json: true, - query, - limit: options.limit, - }); - }); - wiki - .command('read') - .description('Read one KTX wiki page') - .argument('') - .addOption(jsonOption()) - .action(async (pageId: string, _options, command) => { - await runAgent(context, { command: 'wiki-read', projectDir: resolveCommandProjectDir(command), json: true, pageId }); - }); - - const sql = agent.command('sql').description('Safe SQL execution commands'); - sql - .command('execute') - .description('Execute read-only SQL with a row limit') - .addOption(jsonOption()) - .requiredOption('--connection-id ', 'Connection id for execution') - .requiredOption('--sql-file ', 'SQL file to execute') - .requiredOption('--max-rows ', 'Maximum rows to return', parsePositiveIntegerOption) - .action(async (options: { connectionId: string; sqlFile: string; maxRows: number }, command) => { - await runAgent(context, { - command: 'sql-execute', - projectDir: resolveCommandProjectDir(command), - json: true, - connectionId: options.connectionId, - sqlFile: options.sqlFile, - maxRows: options.maxRows, - }); - }); -} diff --git a/packages/cli/src/commands/connection-commands.ts b/packages/cli/src/commands/connection-commands.ts index f3c87709..4ce75057 100644 --- a/packages/cli/src/commands/connection-commands.ts +++ b/packages/cli/src/commands/connection-commands.ts @@ -188,7 +188,7 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm registerConnectionNotionCommands(connection, context); } -export function registerConnectionMappingCommands(connection: Command, context: KtxCliCommandContext): void { +function registerConnectionMappingCommands(connection: Command, context: KtxCliCommandContext): void { const mapping = connection .command('mapping') .description('Manage Metabase warehouse mappings') diff --git a/packages/cli/src/commands/connection-notion-tree.ts b/packages/cli/src/commands/connection-notion-tree.ts index 57a8f335..379ac938 100644 --- a/packages/cli/src/commands/connection-notion-tree.ts +++ b/packages/cli/src/commands/connection-notion-tree.ts @@ -369,14 +369,6 @@ function setExpanded(state: PickerState, nodeId: string, value: boolean | 'toggl return cloneState(state, { expanded }); } -function expandPath(state: PickerState, nodeId: string): PickerState { - const expanded = new Set(state.expanded); - for (const ancestorId of ancestorsOf(nodeId, state.byId)) { - expanded.add(ancestorId); - } - return cloneState(state, { expanded }); -} - export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'right'): PickerState { const node = state.byId.get(state.cursorId); if (!node) { diff --git a/packages/cli/src/commands/connection-notion-tui.test.tsx b/packages/cli/src/commands/connection-notion-tui.test.tsx index 0b7efc82..dc394688 100644 --- a/packages/cli/src/commands/connection-notion-tui.test.tsx +++ b/packages/cli/src/commands/connection-notion-tui.test.tsx @@ -1,6 +1,6 @@ /* @jsxImportSource react */ import { render as renderInkTest } from 'ink-testing-library'; -import React, { act, type ReactNode } from 'react'; +import { act, type ReactNode } from 'react'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js'; import { diff --git a/packages/cli/src/commands/connection-notion-tui.tsx b/packages/cli/src/commands/connection-notion-tui.tsx index c0a4746a..b2a47036 100644 --- a/packages/cli/src/commands/connection-notion-tui.tsx +++ b/packages/cli/src/commands/connection-notion-tui.tsx @@ -1,6 +1,6 @@ /* @jsxImportSource react */ import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink'; -import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; +import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; import { filterTree, flattenSelection, diff --git a/packages/cli/src/commands/knowledge-commands.ts b/packages/cli/src/commands/knowledge-commands.ts index c85a118c..f8d716f7 100644 --- a/packages/cli/src/commands/knowledge-commands.ts +++ b/packages/cli/src/commands/knowledge-commands.ts @@ -1,5 +1,10 @@ import { type Command, Option } from '@commander-js/extra-typings'; -import { collectOption, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js'; +import { + collectOption, + type KtxCliCommandContext, + parsePositiveIntegerOption, + resolveCommandProjectDir, +} from '../cli-program.js'; import { wikiWriteCommandSchema } from '../command-schemas.js'; import type { KtxKnowledgeArgs } from '../knowledge.js'; import { profileMark } from '../startup-profile.js'; @@ -24,12 +29,14 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon wiki .command('list') .description('List local wiki pages') + .option('--json', 'Print JSON output', false) .option('--user-id ', 'Local user id', 'local') - .action(async (options: { userId: string }, command) => { + .action(async (options: { userId: string; json?: boolean }, command) => { await runKnowledgeArgs(context, { command: 'list', projectDir: resolveCommandProjectDir(command), userId: options.userId, + json: options.json, }); }); @@ -37,13 +44,15 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon .command('read') .description('Read one local wiki page') .argument('', 'Wiki page key') + .option('--json', 'Print JSON output', false) .option('--user-id ', 'Local user id', 'local') - .action(async (key: string, options: { userId: string }, command) => { + .action(async (key: string, options: { userId: string; json?: boolean }, command) => { await runKnowledgeArgs(context, { command: 'read', projectDir: resolveCommandProjectDir(command), key, userId: options.userId, + json: options.json, }); }); @@ -51,13 +60,17 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon .command('search') .description('Search local wiki pages') .argument('', 'Search query') + .option('--json', 'Print JSON output', false) .option('--user-id ', 'Local user id', 'local') - .action(async (query: string, options: { userId: string }, command) => { + .option('--limit ', 'Maximum search results', parsePositiveIntegerOption) + .action(async (query: string, options: { userId: string; json?: boolean; limit?: number }, command) => { await runKnowledgeArgs(context, { command: 'search', projectDir: resolveCommandProjectDir(command), query, userId: options.userId, + json: options.json, + ...(options.limit !== undefined ? { limit: options.limit } : {}), }); }); diff --git a/packages/cli/src/commands/runtime-commands.ts b/packages/cli/src/commands/runtime-commands.ts index b57eae86..cf0abb42 100644 --- a/packages/cli/src/commands/runtime-commands.ts +++ b/packages/cli/src/commands/runtime-commands.ts @@ -18,7 +18,7 @@ async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArg export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void { const runtime = program .command('runtime') - .description('Install, inspect, and prune the KTX-managed Python runtime') + .description('Install, start, stop, and inspect the KTX-managed Python runtime') .showHelpAfterError(); runtime @@ -64,7 +64,7 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand runtime .command('status') - .description('Show managed Python runtime status') + .description('Show managed Python runtime status and readiness checks') .option('--json', 'Print JSON output', false) .action(async (options: { json?: boolean }) => { await runRuntimeArgs(context, { @@ -73,18 +73,4 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand json: options.json === true, }); }); - - runtime - .command('prune') - .description('Remove stale managed Python runtimes for older CLI versions') - .option('--dry-run', 'List stale runtimes without deleting them', false) - .option('--yes', 'Confirm deletion of stale runtime directories', false) - .action(async (options: { dryRun?: boolean; yes?: boolean }) => { - await runRuntimeArgs(context, { - command: 'prune', - cliVersion: context.packageInfo.version, - dryRun: options.dryRun === true, - yes: options.yes === true, - }); - }); } diff --git a/packages/cli/src/commands/sl-commands.ts b/packages/cli/src/commands/sl-commands.ts index 36d75fac..e1b985a3 100644 --- a/packages/cli/src/commands/sl-commands.ts +++ b/packages/cli/src/commands/sl-commands.ts @@ -51,6 +51,7 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte sl.command('list') .description('List semantic-layer sources') .option('--connection-id ', 'KTX connection id') + .option('--query ', 'Search source names and descriptions') .addOption( new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ 'pretty', @@ -59,26 +60,34 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte ]), ) .option('--json', 'Shortcut for --output=json (overrides --output)', false) - .action(async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => { + .action( + async ( + options: { connectionId?: string; query?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, + command, + ) => { await runSlArgs(context, { command: 'list', projectDir: resolveCommandProjectDir(command), connectionId: options.connectionId, + query: options.query, output: options.output, json: options.json, }); - }); + }, + ); sl.command('read') .description('Read a semantic-layer source') .argument('', 'Semantic-layer source name') .requiredOption('--connection-id ', 'KTX connection id') - .action(async (sourceName: string, options: { connectionId: string }, command) => { + .option('--json', 'Print JSON output', false) + .action(async (sourceName: string, options: { connectionId: string; json?: boolean }, command) => { await runSlArgs(context, { command: 'read', projectDir: resolveCommandProjectDir(command), connectionId: options.connectionId, sourceName, + json: options.json, }); }); @@ -113,6 +122,7 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte sl.command('query') .description('Compile or execute a semantic-layer query') .option('--connection-id ', 'KTX connection id') + .option('--query-file ', 'JSON semantic-layer query file') .option('--measure ', 'Measure to query; repeatable', collectOption, []) .option('--dimension ', 'Dimension to include; repeatable', collectOption, []) .option('--filter ', 'Filter expression; repeatable', collectOption, []) @@ -126,22 +136,26 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte .option('--no-input', 'Disable interactive managed runtime installation') .option('--max-rows ', 'Maximum rows to return when executing', parsePositiveIntegerOption) .action(async (options, command) => { - if (options.measure.length === 0) { + if (options.measure.length === 0 && !options.queryFile) { throw new Error('sl query requires at least one --measure'); } const args = slQueryCommandSchema.parse({ command: 'query', projectDir: resolveCommandProjectDir(command), connectionId: options.connectionId, - query: { - measures: options.measure, - dimensions: options.dimension, - ...(options.filter.length > 0 ? { filters: options.filter } : {}), - ...(options.segment.length > 0 ? { segments: options.segment } : {}), - ...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}), - ...(options.limit !== undefined ? { limit: options.limit } : {}), - ...(options.includeEmpty === true ? { include_empty: true } : {}), - }, + ...(options.queryFile + ? { queryFile: options.queryFile } + : { + query: { + measures: options.measure, + dimensions: options.dimension, + ...(options.filter.length > 0 ? { filters: options.filter } : {}), + ...(options.segment.length > 0 ? { segments: options.segment } : {}), + ...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}), + ...(options.limit !== undefined ? { limit: options.limit } : {}), + ...(options.includeEmpty === true ? { include_empty: true } : {}), + }, + }), format: options.format, execute: options.execute === true, cliVersion: context.packageInfo.version, diff --git a/packages/cli/src/dev.test.ts b/packages/cli/src/dev.test.ts index 5aca4201..fe75d1af 100644 --- a/packages/cli/src/dev.test.ts +++ b/packages/cli/src/dev.test.ts @@ -106,6 +106,7 @@ describe('dev Commander tree', () => { for (const argv of [ ['dev', 'doctor', 'setup'], ['dev', 'runtime', 'doctor'], + ['dev', 'runtime', 'prune', '--dry-run'], ['dev', 'scan', 'warehouse'], ['dev', 'ingest', 'run'], ['dev', 'mapping', 'list'], @@ -126,7 +127,7 @@ describe('dev Commander tree', () => { it.each([ { argv: ['dev', 'runtime', '--help'], - expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status', 'prune'], + expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status'], }, { argv: ['scan', '--help'], @@ -147,6 +148,10 @@ describe('dev Commander tree', () => { for (const text of expected) { expect(io.stdout()).toContain(text); } + if (argv.join(' ') === 'dev runtime --help') { + expect(io.stdout()).not.toContain('prune'); + expect(io.stdout()).not.toContain('doctor'); + } expect(io.stderr()).toBe(''); expect(doctor).not.toHaveBeenCalled(); expect(ingest).not.toHaveBeenCalled(); diff --git a/packages/cli/src/example-smoke.test.ts b/packages/cli/src/example-smoke.test.ts index f5b70bfc..221c20f2 100644 --- a/packages/cli/src/example-smoke.test.ts +++ b/packages/cli/src/example-smoke.test.ts @@ -73,26 +73,27 @@ describe('standalone local warehouse example', () => { const projectDir = await copyExampleProject(tempDir); const sourceDir = join(projectDir, 'source'); - const knowledgeList = await runBuiltCli(['agent', 'wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]); + const knowledgeList = await runBuiltCli(['wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]); expect(knowledgeList).toMatchObject({ code: 0, stderr: '' }); - expect(parseJsonOutput<{ results: Array<{ key: string; summary: string }> }>(knowledgeList.stdout).results).toContainEqual( - expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' }), - ); + expect( + parseJsonOutput<{ data: { items: Array<{ key: string; summary: string }> } }>(knowledgeList.stdout).data.items, + ).toContainEqual(expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' })); - const knowledgeRead = await runBuiltCli(['agent', 'wiki', 'read', 'revenue', '--json', '--project-dir', projectDir]); + const knowledgeRead = await runBuiltCli(['wiki', 'read', 'revenue', '--json', '--project-dir', projectDir]); expect(knowledgeRead).toMatchObject({ code: 0, stderr: '' }); - expect(parseJsonOutput<{ content: string }>(knowledgeRead.stdout).content).toContain( + expect(parseJsonOutput<{ data: { content: string } }>(knowledgeRead.stdout).data.content).toContain( 'Revenue is paid order amount after refund adjustments.', ); - const slList = await runBuiltCli(['agent', 'sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']); + const slList = await runBuiltCli(['sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']); expect(slList).toMatchObject({ code: 0, stderr: '' }); - expect(parseJsonOutput<{ sources: Array<{ connectionId: string; name: string; columnCount: number }> }>(slList.stdout).sources).toContainEqual( - expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 }), - ); + expect( + parseJsonOutput<{ data: { items: Array<{ connectionId: string; name: string; columnCount: number }> } }>( + slList.stdout, + ).data.items, + ).toContainEqual(expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 })); const slRead = await runBuiltCli([ - 'agent', 'sl', 'read', 'orders', @@ -103,7 +104,7 @@ describe('standalone local warehouse example', () => { projectDir, ]); expect(slRead).toMatchObject({ code: 0, stderr: '' }); - expect(parseJsonOutput<{ yaml: string }>(slRead.stdout).yaml).toContain('name: orders'); + expect(parseJsonOutput<{ data: { yaml: string } }>(slRead.stdout).data.yaml).toContain('name: orders'); const ingest = await runBuiltCli([ 'ingest', diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 9c08e58a..1e69c590 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -159,7 +159,7 @@ describe('runKtxCli', () => { await expect(runKtxCli(['dev', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); await expect(runKtxCli(['dev', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0); await expect(runKtxCli(['dev', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['dev', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1); expect(runtime).toHaveBeenNthCalledWith( 1, @@ -208,19 +208,11 @@ describe('runKtxCli', () => { }, statusIo.io, ); - expect(runtime).toHaveBeenNthCalledWith( - 6, - { - command: 'prune', - cliVersion: '0.0.0-private', - dryRun: true, - yes: false, - }, - pruneIo.io, - ); - for (const io of [installIo, startIo, stopIo, stopAllIo, statusIo, pruneIo]) { + expect(runtime).toHaveBeenCalledTimes(5); + for (const io of [installIo, startIo, stopIo, stopAllIo, statusIo]) { expect(io.stderr()).toBe(''); } + expect(pruneIo.stderr()).toMatch(/unknown command|error:/); }); it('prints the resolved project directory for ordinary project commands', async () => { @@ -1149,136 +1141,28 @@ describe('runKtxCli', () => { expect(setupIo.stderr()).toContain('Choose only one Historic SQL action'); }); - it('registers hidden agent help and tools discovery without showing agent in root help', async () => { - const helpIo = makeIo(); - const toolsIo = makeIo(); - const agent = vi.fn(async () => 0); + it('rejects the removed hidden agent command', async () => { + const io = makeIo(); - await expect(runKtxCli(['agent', '--help'], helpIo.io, { agent })).resolves.toBe(0); - await expect( - runKtxCli(['--project-dir', tempDir, 'agent', 'tools', '--json'], toolsIo.io, { agent }), - ).resolves.toBe(0); + await expect(runKtxCli(['agent'], io.io)).resolves.toBe(1); - expect(helpIo.stdout()).toContain('Usage: ktx agent'); - expect(toolsIo.stderr()).toBe(''); - expect(agent).toHaveBeenCalledWith({ command: 'tools', projectDir: tempDir, json: true }, toolsIo.io); + expect(io.stderr()).toContain("unknown command 'agent'"); + expect(io.stdout()).toBe(''); }); - it('dispatches full hidden agent commands without exposing agent in root help', async () => { - const agent = vi.fn(async () => 0); - const cases = [ - { - argv: ['--project-dir', tempDir, 'agent', 'context', '--json'], - args: { command: 'context', projectDir: tempDir, json: true }, - }, - { - argv: [ - '--project-dir', - tempDir, - 'agent', - 'sl', - 'list', - '--json', - '--connection-id', - 'warehouse', - '--query', - 'orders', - ], - args: { command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'warehouse', query: 'orders' }, - }, - { - argv: ['--project-dir', tempDir, 'agent', 'sl', 'read', 'orders', '--json', '--connection-id', 'warehouse'], - args: { command: 'sl-read', projectDir: tempDir, json: true, sourceName: 'orders', connectionId: 'warehouse' }, - }, - { - argv: [ - '--project-dir', - tempDir, - 'agent', - 'sl', - 'query', - '--json', - '--connection-id', - 'warehouse', - '--query-file', - '/tmp/query.json', - '--execute', - '--max-rows', - '100', - ], - args: { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile: '/tmp/query.json', - execute: true, - maxRows: 100, - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'prompt', - }, - }, - { - argv: ['--project-dir', tempDir, 'agent', 'wiki', 'search', 'revenue', '--json', '--limit', '5'], - args: { command: 'wiki-search', projectDir: tempDir, json: true, query: 'revenue', limit: 5 }, - }, - { - argv: ['--project-dir', tempDir, 'agent', 'wiki', 'read', 'page-1', '--json'], - args: { command: 'wiki-read', projectDir: tempDir, json: true, pageId: 'page-1' }, - }, - { - argv: [ - '--project-dir', - tempDir, - 'agent', - 'sql', - 'execute', - '--json', - '--connection-id', - 'warehouse', - '--sql-file', - '/tmp/query.sql', - '--max-rows', - '100', - ], - args: { - command: 'sql-execute', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - sqlFile: '/tmp/query.sql', - maxRows: 100, - }, - }, - ]; - - for (const entry of cases) { - const io = makeIo(); - await expect(runKtxCli(entry.argv, io.io, { agent })).resolves.toBe(0); - expect(agent).toHaveBeenLastCalledWith(entry.args, io.io); - expect(io.stderr()).toBe(''); - } - - const helpIo = makeIo(); - await expect(runKtxCli(['--help'], helpIo.io, { agent })).resolves.toBe(0); - expect(helpIo.stdout()).not.toContain('agent '); - }); - - it('routes hidden agent SL query managed runtime policies', async () => { + it('routes public SL query files with managed runtime policies', async () => { const autoIo = makeIo(); const neverIo = makeIo(); const conflictIo = makeIo(); - const agent = vi.fn(async () => 0); + const sl = vi.fn(async () => 0); await expect( runKtxCli( [ '--project-dir', tempDir, - 'agent', 'sl', 'query', - '--json', '--connection-id', 'warehouse', '--query-file', @@ -1286,7 +1170,7 @@ describe('runKtxCli', () => { '--yes', ], autoIo.io, - { agent }, + { sl }, ), ).resolves.toBe(0); @@ -1295,10 +1179,8 @@ describe('runKtxCli', () => { [ '--project-dir', tempDir, - 'agent', 'sl', 'query', - '--json', '--connection-id', 'warehouse', '--query-file', @@ -1306,7 +1188,7 @@ describe('runKtxCli', () => { '--no-input', ], neverIo.io, - { agent }, + { sl }, ), ).resolves.toBe(0); @@ -1315,10 +1197,8 @@ describe('runKtxCli', () => { [ '--project-dir', tempDir, - 'agent', 'sl', 'query', - '--json', '--connection-id', 'warehouse', '--query-file', @@ -1327,33 +1207,33 @@ describe('runKtxCli', () => { '--no-input', ], conflictIo.io, - { agent }, + { sl }, ), ).resolves.toBe(1); - expect(agent).toHaveBeenNthCalledWith( + expect(sl).toHaveBeenNthCalledWith( 1, { - command: 'sl-query', + command: 'query', projectDir: tempDir, - json: true, connectionId: 'warehouse', queryFile: '/tmp/query.json', execute: false, + format: 'json', cliVersion: '0.0.0-private', runtimeInstallPolicy: 'auto', }, autoIo.io, ); - expect(agent).toHaveBeenNthCalledWith( + expect(sl).toHaveBeenNthCalledWith( 2, { - command: 'sl-query', + command: 'query', projectDir: tempDir, - json: true, connectionId: 'warehouse', queryFile: '/tmp/query.json', execute: false, + format: 'json', cliVersion: '0.0.0-private', runtimeInstallPolicy: 'never', }, @@ -1362,112 +1242,6 @@ describe('runKtxCli', () => { expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); }); - it('prints semantic-layer hybrid search metadata from the hidden agent sl list command', async () => { - const agent = vi.fn(async (args, io) => { - expect(args).toEqual({ - command: 'sl-list', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - query: 'paid', - }); - io.stdout.write( - `${JSON.stringify( - { - sources: [ - { - connectionId: 'warehouse', - connectionName: 'warehouse', - name: 'orders', - columnCount: 2, - measureCount: 1, - joinCount: 0, - score: 0.03278688524590164, - matchReasons: ['dictionary'], - dictionaryMatches: [{ column: 'status', values: ['paid'] }], - }, - ], - totalSources: 1, - }, - null, - 2, - )}\n`, - ); - return 0; - }); - const io = makeIo(); - - await expect( - runKtxCli( - ['--project-dir', tempDir, 'agent', 'sl', 'list', '--json', '--connection-id', 'warehouse', '--query', 'paid'], - io.io, - { agent }, - ), - ).resolves.toBe(0); - - expect(JSON.parse(io.stdout())).toEqual({ - sources: [ - expect.objectContaining({ - connectionId: 'warehouse', - name: 'orders', - matchReasons: ['dictionary'], - dictionaryMatches: [{ column: 'status', values: ['paid'] }], - }), - ], - totalSources: 1, - }); - }); - - it('prints wiki hybrid search metadata from the hidden agent wiki search command', async () => { - const agent = vi.fn(async (args, io) => { - expect(args).toEqual({ - command: 'wiki-search', - projectDir: tempDir, - json: true, - query: 'paid order', - limit: 5, - }); - io.stdout.write( - `${JSON.stringify( - { - results: [ - { - key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', - scope: 'GLOBAL', - summary: 'Revenue metric definition', - score: 0.02459016393442623, - matchReasons: ['lexical', 'token'], - }, - ], - totalFound: 1, - }, - null, - 2, - )}\n`, - ); - return 0; - }); - const io = makeIo(); - - await expect( - runKtxCli(['--project-dir', tempDir, 'agent', 'wiki', 'search', 'paid order', '--json', '--limit', '5'], io.io, { - agent, - }), - ).resolves.toBe(0); - - expect(JSON.parse(io.stdout())).toEqual({ - results: [ - expect.objectContaining({ - key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', - matchReasons: ['lexical', 'token'], - }), - ], - totalFound: 1, - }); - }); - it('dispatches public connection subcommands through the existing connection implementation', async () => { const tempDir = await mkdtemp(join(tmpdir(), 'ktx-connection-dispatch-')); const connection = vi.fn(async () => 0); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index de906ece..2cf9d5b2 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,17 +9,6 @@ export { type KtxCliIo, type KtxCliPackageInfo, } from './cli-runtime.js'; -export { runKtxAgent, type KtxAgentArgs } from './agent.js'; -export { - KTX_AGENT_MAX_ROWS_CAP, - createKtxAgentRuntime, - parseAgentMaxRows, - readAgentJsonFile, - writeAgentJson, - writeAgentJsonError, - type KtxAgentRuntime, - type KtxAgentRuntimeDeps, -} from './agent-runtime.js'; export { runKtxSetup, type KtxSetupArgs, type KtxSetupStatus } from './setup.js'; export type { KtxSetupDatabaseDriver, diff --git a/packages/cli/src/ingest-query-executor.test.ts b/packages/cli/src/ingest-query-executor.test.ts new file mode 100644 index 00000000..343202a1 --- /dev/null +++ b/packages/cli/src/ingest-query-executor.test.ts @@ -0,0 +1,86 @@ +import type { KtxLocalProject } from '@ktx/context/project'; +import { createKtxConnectorCapabilities, type KtxScanConnector } from '@ktx/context/scan'; +import { describe, expect, it, vi } from 'vitest'; +import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; + +function project(): KtxLocalProject { + return { + projectDir: '/tmp/ktx-query-project', + config: { + project: 'warehouse', + connections: { + warehouse: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' }, + }, + }, + } as unknown as KtxLocalProject; +} + +function connector(overrides: Partial = {}): KtxScanConnector { + return { + id: 'warehouse', + driver: 'postgres', + capabilities: createKtxConnectorCapabilities({ readOnlySql: true }), + async introspect() { + throw new Error('introspect is not used by this test'); + }, + executeReadOnly: vi.fn(async () => ({ + headers: ['answer'], + rows: [[1]], + totalRows: 1, + rowCount: 1, + })), + cleanup: vi.fn(async () => {}), + ...overrides, + }; +} + +describe('createKtxCliIngestQueryExecutor', () => { + it('executes read-only SQL through the scan connector and cleans it up', async () => { + const scanConnector = connector(); + const createConnector = vi.fn(async () => scanConnector); + const executor = createKtxCliIngestQueryExecutor(project(), { createConnector }); + + await expect( + executor.execute({ + connectionId: 'warehouse', + connection: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' }, + projectDir: '/tmp/ktx-query-project', + sql: 'select 1', + maxRows: 5, + }), + ).resolves.toMatchObject({ + headers: ['answer'], + rows: [[1]], + totalRows: 1, + command: 'SELECT', + rowCount: 1, + }); + + expect(createConnector).toHaveBeenCalledWith(project(), 'warehouse'); + expect(scanConnector.executeReadOnly).toHaveBeenCalledWith( + { connectionId: 'warehouse', sql: 'select 1', maxRows: 5 }, + { runId: 'ingest-sql-execution' }, + ); + expect(scanConnector.cleanup).toHaveBeenCalledTimes(1); + }); + + it('rejects connectors without read-only SQL support', async () => { + const scanConnector = connector({ + capabilities: createKtxConnectorCapabilities({ readOnlySql: false }), + executeReadOnly: undefined, + }); + const executor = createKtxCliIngestQueryExecutor(project(), { + createConnector: vi.fn(async () => scanConnector), + }); + + await expect( + executor.execute({ + connectionId: 'warehouse', + connection: { driver: 'postgres' }, + projectDir: '/tmp/ktx-query-project', + sql: 'select 1', + }), + ).rejects.toThrow('Connection "warehouse" driver "postgres" does not support read-only SQL execution.'); + expect(scanConnector.cleanup).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli/src/ingest-query-executor.ts b/packages/cli/src/ingest-query-executor.ts new file mode 100644 index 00000000..197119be --- /dev/null +++ b/packages/cli/src/ingest-query-executor.ts @@ -0,0 +1,49 @@ +import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from '@ktx/context/connections'; +import type { KtxLocalProject } from '@ktx/context/project'; +import type { KtxScanConnector, KtxScanContext } from '@ktx/context/scan'; +import { createKtxCliScanConnector } from './local-scan-connectors.js'; + +type CreateConnector = typeof createKtxCliScanConnector; + +export interface KtxCliIngestQueryExecutorDeps { + createConnector?: CreateConnector; +} + +async function cleanupConnector(connector: KtxScanConnector | null): Promise { + await connector?.cleanup?.(); +} + +export function createKtxCliIngestQueryExecutor( + project: KtxLocalProject, + deps: KtxCliIngestQueryExecutorDeps = {}, +): KtxSqlQueryExecutorPort { + const createConnector = deps.createConnector ?? createKtxCliScanConnector; + return { + async execute(input: KtxSqlQueryExecutionInput) { + let connector: KtxScanConnector | null = null; + try { + connector = await createConnector(project, input.connectionId); + if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) { + throw new Error( + `Connection "${input.connectionId}" driver "${connector.driver}" does not support read-only SQL execution.`, + ); + } + + const ctx: KtxScanContext = { runId: 'ingest-sql-execution' }; + const result = await connector.executeReadOnly( + { connectionId: input.connectionId, sql: input.sql, maxRows: input.maxRows }, + ctx, + ); + return { + headers: result.headers, + rows: result.rows, + totalRows: result.totalRows, + command: 'SELECT', + rowCount: result.rowCount, + }; + } finally { + await cleanupConnector(connector); + } + }, + }; +} diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index 09a1fe17..3596d215 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -1,10 +1,8 @@ import { EventEmitter } from 'node:events'; -import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { AgentRunnerService, type RunLoopParams } from '@ktx/context/agent'; import { - LocalLookerRuntimeStore, KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache, MetabaseSourceAdapter, @@ -13,12 +11,10 @@ import { type FetchContext, type IngestReportSnapshot, type LocalIngestResult, - type LocalMetabaseFanoutProgress, type LookerMappingClient, type LookerRuntimeClient, type LookerTableIdentifierParser, type MemoryFlowEventSink, - type MemoryFlowReplayInput, type MetabaseCard, type MetabaseCardSummary, type MetabaseClientFactory, @@ -29,7 +25,7 @@ import { } from '@ktx/context/ingest'; import { ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; import { expect, vi } from 'vitest'; -import { type KtxIngestArgs, runKtxIngest } from './ingest.js'; +import { runKtxIngest } from './ingest.js'; export function makeIo( options: { @@ -266,6 +262,18 @@ export class CliLookerSlWritingAgentRunner extends AgentRunnerService { params.telemetryTags?.operationName === 'ingest-bundle-wu' && params.telemetryTags?.unitKey === 'looker-explore-ecommerce-orders' ) { + const ledger = params.toolSet.record_verification_ledger; + if (!ledger?.execute) { + throw new Error('record_verification_ledger tool was not available to the Looker WorkUnit'); + } + await ledger.execute( + { + summary: 'Test fixture verified Looker explore target identifiers before writing SL.', + verifiedIdentifiers: ['prod-warehouse', 'public.orders'], + unverifiedIdentifiers: [], + }, + { toolCallId: 'cli-looker-verification-ledger', messages: [] }, + ); const slWrite = params.toolSet.sl_write_source; if (!slWrite?.execute) { throw new Error('sl_write_source tool was not available to the Looker WorkUnit'); diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index ab126991..24f8c1ca 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -4,10 +4,8 @@ import { join } from 'node:path'; import { LocalLookerRuntimeStore, LocalMetabaseDiscoveryCache, - getLocalIngestStatus, type LocalIngestResult, type LocalMetabaseFanoutProgress, - type MemoryFlowReplayInput, type RunLocalIngestOptions, type SourceAdapter, } from '@ktx/context/ingest'; @@ -20,7 +18,6 @@ import { CliMetabaseAgentRunner, CliMetabaseSourceAdapter, completedLocalBundleRun, - emitLiveLocalMemoryFlow, failedLocalBundleRun, localFakeBundleReport, makeCliLookerParser, @@ -28,7 +25,6 @@ import { makeIo, persistLocalBundleReport, runPublicMetabaseSyncModeCase, - writeBundleReportFile, writeMetabaseConfig, writeWarehouseConfig, } from './ingest.test-utils.js'; @@ -803,6 +799,44 @@ describe('runKtxIngest', () => { expect(runLocalIngest).toHaveBeenCalledWith(expect.objectContaining({ llmDebugRequestFile: debugFile })); }); + it('supplies a scan-connector query executor to local ingest runs', async () => { + const io = makeIo(); + const projectDir = join(tempDir, 'query-executor-project'); + await writeWarehouseConfig(projectDir); + const queryExecutor = { + execute: vi.fn(async () => ({ + headers: [], + rows: [], + totalRows: 0, + command: 'SELECT', + rowCount: 0, + })), + }; + const runLocalIngest = vi.fn(async (input: RunLocalIngestOptions): Promise => + completedLocalBundleRun(input, 'query-executor-run'), + ); + + await expect( + runKtxIngest( + { + command: 'run', + projectDir, + connectionId: 'warehouse', + adapter: 'fake', + outputMode: 'json', + }, + io.io, + { + runLocalIngest, + createAdapters: () => [], + createQueryExecutor: () => queryExecutor, + }, + ), + ).resolves.toBe(0); + + expect(runLocalIngest).toHaveBeenCalledWith(expect.objectContaining({ queryExecutor })); + }); + it('passes daemon database introspection URL to default local ingest adapters', async () => { const projectDir = join(tempDir, 'project'); await writeWarehouseConfig(projectDir); diff --git a/packages/cli/src/ingest.ts b/packages/cli/src/ingest.ts index 4a68edf3..6e0648b5 100644 --- a/packages/cli/src/ingest.ts +++ b/packages/cli/src/ingest.ts @@ -16,7 +16,9 @@ import { runLocalMetabaseIngest, savedMemoryCountsForReport, } from '@ktx/context/ingest'; -import { loadKtxProject } from '@ktx/context/project'; +import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections'; +import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; import { readIngestReportSnapshotFile } from './ingest-report-file.js'; import { createCliOperationalLogger } from './io/logger.js'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; @@ -69,6 +71,7 @@ interface KtxIngestDeps { jobIdFactory?: () => string; now?: () => Date; createAdapters?: typeof createKtxCliLocalIngestAdapters; + createQueryExecutor?: (project: KtxLocalProject) => KtxSqlQueryExecutorPort; runLocalIngest?: typeof runLocalIngest; runLocalMetabaseIngest?: typeof runLocalMetabaseIngest; readReportFile?: typeof readIngestReportSnapshotFile; @@ -532,6 +535,9 @@ export async function runKtxIngest( ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}), logger: operationalLogger, }; + const queryExecutor = + localIngestOptions.queryExecutor ?? + (deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(project); if (args.adapter === 'metabase' && args.sourceDir) { throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter'); } @@ -544,6 +550,7 @@ export async function runKtxIngest( adapters: createAdapters(project, adapterOptions), metabaseConnectionId: args.connectionId, ...localIngestOptions, + queryExecutor, trigger: 'manual_resync', jobIdFactory: deps.jobIdFactory, ...(progress ? { progress } : {}), @@ -604,6 +611,7 @@ export async function runKtxIngest( trigger: 'manual_resync', jobId, ...localIngestOptions, + queryExecutor, pullConfigOptions: adapterOptions, ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}), ...(memoryFlow ? { memoryFlow } : {}), diff --git a/packages/cli/src/knowledge.test.ts b/packages/cli/src/knowledge.test.ts index d7e17605..db794289 100644 --- a/packages/cli/src/knowledge.test.ts +++ b/packages/cli/src/knowledge.test.ts @@ -93,6 +93,65 @@ describe('runKtxKnowledge', () => { expect(searchIo.stdout()).toContain('metrics-revenue'); }); + it('prints wiki list, search, and read as public JSON envelopes', async () => { + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir, projectName: 'warehouse' }); + + await expect( + runKtxKnowledge( + { + command: 'write', + projectDir, + key: 'metrics-revenue', + scope: 'GLOBAL', + userId: 'local', + summary: 'Revenue', + content: 'Revenue is paid order value.', + tags: ['finance'], + refs: [], + slRefs: ['orders'], + }, + makeIo().io, + ), + ).resolves.toBe(0); + + const listIo = makeIo(); + await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local', json: true }, listIo.io)).resolves.toBe( + 0, + ); + expect(JSON.parse(listIo.stdout())).toMatchObject({ + kind: 'list', + data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] }, + meta: { command: 'wiki list' }, + }); + + const searchIo = makeIo(); + await expect( + runKtxKnowledge( + { command: 'search', projectDir, query: 'paid order', userId: 'local', json: true, limit: 5 }, + searchIo.io, + ), + ).resolves.toBe(0); + expect(JSON.parse(searchIo.stdout())).toMatchObject({ + kind: 'list', + data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] }, + meta: { command: 'wiki search' }, + }); + + const readIo = makeIo(); + await expect( + runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local', json: true }, readIo.io), + ).resolves.toBe(0); + expect(JSON.parse(readIo.stdout())).toMatchObject({ + kind: 'wiki.page', + data: { + key: 'metrics-revenue', + summary: 'Revenue', + content: 'Revenue is paid order value.', + }, + }); + }); + it('rejects slash-delimited write keys with a flat-key suggestion', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index 40cc5372..5c5df1ea 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -11,11 +11,12 @@ import { searchLocalKnowledgePages, writeLocalKnowledgePage, } from '@ktx/context/wiki'; +import { writeJsonResult } from './io/print-list.js'; export type KtxKnowledgeArgs = - | { command: 'list'; projectDir: string; userId: string } - | { command: 'read'; projectDir: string; key: string; userId: string } - | { command: 'search'; projectDir: string; query: string; userId: string } + | { command: 'list'; projectDir: string; userId: string; json?: boolean } + | { command: 'read'; projectDir: string; key: string; userId: string; json?: boolean } + | { command: 'search'; projectDir: string; query: string; userId: string; json?: boolean; limit?: number } | { command: 'write'; projectDir: string; @@ -61,6 +62,14 @@ export async function runKtxKnowledge( const project = await loadKtxProject({ projectDir: args.projectDir }); if (args.command === 'list') { const pages = await listLocalKnowledgePages(project, { userId: args.userId }); + if (args.json) { + writeJsonResult(io, { + kind: 'list', + data: { items: pages }, + meta: { command: 'wiki list' }, + }); + return 0; + } for (const page of pages) { io.stdout.write(`${page.scope}\t${page.key}\t${page.summary}\n`); } @@ -71,6 +80,14 @@ export async function runKtxKnowledge( if (!page) { throw new Error(`Knowledge page "${args.key}" was not found`); } + if (args.json) { + writeJsonResult(io, { + kind: 'wiki.page', + data: page, + meta: { command: 'wiki read' }, + }); + return 0; + } io.stdout.write(`# ${page.key}\n\n`); io.stdout.write(`Scope: ${page.scope}\n`); io.stdout.write(`Summary: ${page.summary}\n\n`); @@ -82,7 +99,16 @@ export async function runKtxKnowledge( query: args.query, userId: args.userId, embeddingService: wikiSearchEmbeddingService(project, deps), + limit: args.limit, }); + if (args.json) { + writeJsonResult(io, { + kind: 'list', + data: { items: results }, + meta: { command: 'wiki search' }, + }); + return 0; + } if (results.length === 0) { const pages = await listLocalKnowledgePages(project, { userId: args.userId }); if (pages.length === 0) { diff --git a/packages/cli/src/managed-python-runtime.test.ts b/packages/cli/src/managed-python-runtime.test.ts index f2d9e784..63755ad1 100644 --- a/packages/cli/src/managed-python-runtime.test.ts +++ b/packages/cli/src/managed-python-runtime.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -8,7 +8,6 @@ import { doctorManagedPythonRuntime, installManagedPythonRuntime, managedPythonRuntimeLayout, - pruneManagedPythonRuntimes, readManagedPythonRuntimeStatus, verifyRuntimeAsset, type ManagedPythonRuntimeExec, @@ -471,41 +470,3 @@ describe('doctorManagedPythonRuntime', () => { }); }); }); - -describe('pruneManagedPythonRuntimes', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-prune-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('removes stale version directories and keeps the current version', async () => { - const runtimeRoot = join(tempDir, 'runtime'); - await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); - await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); - await writeFile(join(runtimeRoot, 'README.txt'), 'not a runtime directory\n'); - - const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot }); - - expect(result.removed).toEqual([join(runtimeRoot, '0.1.0')]); - expect(result.kept).toEqual([join(runtimeRoot, '0.2.0')]); - await expect(stat(join(runtimeRoot, '0.1.0'))).rejects.toThrow(); - expect(await readdir(runtimeRoot)).toEqual(['0.2.0', 'README.txt']); - }); - - it('supports dry-run without deleting stale directories', async () => { - const runtimeRoot = join(tempDir, 'runtime'); - await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); - await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); - - const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot, dryRun: true }); - - expect(result.removed).toEqual([]); - expect(result.stale).toEqual([join(runtimeRoot, '0.1.0')]); - expect(await readdir(runtimeRoot)).toEqual(['0.1.0', '0.2.0']); - }); -}); diff --git a/packages/cli/src/managed-python-runtime.ts b/packages/cli/src/managed-python-runtime.ts index 251ac7da..563b62f7 100644 --- a/packages/cli/src/managed-python-runtime.ts +++ b/packages/cli/src/managed-python-runtime.ts @@ -1,6 +1,6 @@ import { execFile } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { access, appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { access, appendFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { homedir } from 'node:os'; import { basename, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -107,13 +107,6 @@ export interface ManagedPythonRuntimeDoctorCheck { fix?: string; } -export interface ManagedPythonRuntimePruneResult { - runtimeRoot: string; - stale: string[]; - kept: string[]; - removed: string[]; -} - export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx dev runtime install --yes'; @@ -441,36 +434,3 @@ export async function doctorManagedPythonRuntime( ); return checks; } - -export async function pruneManagedPythonRuntimes(options: { - cliVersion: string; - runtimeRoot: string; - dryRun?: boolean; -}): Promise { - if (!(await pathExists(options.runtimeRoot))) { - return { runtimeRoot: options.runtimeRoot, stale: [], kept: [], removed: [] }; - } - const entries = await readdir(options.runtimeRoot); - const stale: string[] = []; - const kept: string[] = []; - for (const entry of entries) { - const path = join(options.runtimeRoot, entry); - const info = await stat(path); - if (!info.isDirectory()) { - continue; - } - if (entry === options.cliVersion) { - kept.push(path); - } else { - stale.push(path); - } - } - const removed: string[] = []; - if (options.dryRun !== true) { - for (const path of stale) { - await rm(path, { recursive: true, force: true }); - removed.push(path); - } - } - return { runtimeRoot: options.runtimeRoot, stale, kept, removed }; -} diff --git a/packages/cli/src/memory-flow-hud.tsx b/packages/cli/src/memory-flow-hud.tsx index fbeaf219..5d2be9eb 100644 --- a/packages/cli/src/memory-flow-hud.tsx +++ b/packages/cli/src/memory-flow-hud.tsx @@ -1,7 +1,7 @@ /* @jsxImportSource react */ import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@ktx/context/ingest/memory-flow'; import { Box, Text } from 'ink'; -import React, { type ReactNode } from 'react'; +import { type ReactNode } from 'react'; import { buildDemoMetrics, formatCost, formatDuration } from './demo-metrics.js'; import { formatNextStepLines } from './next-steps.js'; import { profileMark } from './startup-profile.js'; @@ -38,45 +38,6 @@ function isPrepopulatedDemoReplay(input: MemoryFlowReplayInput): boolean { return input.metadata?.origin === 'packaged' || input.metadata?.timing === 'prebuilt'; } -function flowLine(width: number, frame: number, active: boolean): string { - if (!active) return '━'.repeat(width); - const pulse = ['░', '▒', '▓', '█', '█', '█', '▓', '▒', '░']; - const pw = pulse.length; - const chars: string[] = []; - const offset = (frame * 2) % (width + pw); - for (let i = 0; i < width; i += 1) { - const p = i - offset + pw; - chars.push(p >= 0 && p < pw ? (pulse[p] ?? '━') : '━'); - } - return chars.join(''); -} - -function brailleFlow(width: number, frame: number): string { - // Braille unicode: U+2800 + dot bitmask - // Dots: 1=0x01 2=0x02 3=0x04 4=0x08 5=0x10 6=0x20 7=0x40 8=0x80 - // Layout: col0=[1,2,3,7] col1=[4,5,6,8] - const chars: string[] = []; - for (let i = 0; i < width; i += 1) { - const density = (i + 1) / width; - const phase = (i * 3 + frame * 2) % 12; - let dots = 0; - - // Sparse diagonal streams on the left, dense on the right - // Each "stream" is a diagonal line of dots moving rightward - if ((phase + 0) % 4 < density * 4) dots |= 0x01; // dot 1 - if ((phase + 1) % 5 < density * 4) dots |= 0x08; // dot 4 - if ((phase + 2) % 4 < density * 3) dots |= 0x02; // dot 2 - if ((phase + 3) % 5 < density * 3) dots |= 0x10; // dot 5 - if ((phase + 4) % 4 < density * 2.5) dots |= 0x04; // dot 3 - if ((phase + 5) % 5 < density * 2.5) dots |= 0x20; // dot 6 - if ((phase + 1) % 6 < density * 2) dots |= 0x40; // dot 7 - if ((phase + 3) % 6 < density * 2) dots |= 0x80; // dot 8 - - chars.push(String.fromCharCode(0x2800 + dots)); - } - return chars.join(''); -} - function progressBarOverall( finishedCount: number, activeCount: number, @@ -104,43 +65,6 @@ function progressBarOverall( return finished + activeChars.join('') + '░'.repeat(queuedWidth); } -function sparkleWipe(width: number, frame: number, row: number): string { - const chars: string[] = []; - const sweepPos = (frame * 2 + row * 6) % (width + 8); - const sparkles = ['✨', '✦', '✧', '·']; - for (let i = 0; i < width; i += 1) { - const dist = i - sweepPos; - if (dist < -6) { - const t = (i * 11 + row * 5 + frame * 3) % 10; - chars.push(t === 0 ? sparkles[0]! : t === 3 ? sparkles[1]! : t === 7 ? sparkles[2]! : ' '); - } else if (dist < -3) { - const t = (i + frame) % 3; - chars.push(t === 0 ? sparkles[1]! : t === 1 ? sparkles[2]! : sparkles[3]!); - } else if (dist <= 0) { - const gradient = ['░', '▒', '▓', '█']; - chars.push(gradient[Math.min(3, dist + 3)] ?? '█'); - } else if (dist <= 2) { - chars.push(dist === 1 ? '▓' : '▒'); - } else { - const noise = (i * 31 + row * 17 + frame * 3) % 5; - const messy = ['░', '▒', '▓', '▒', '░']; - chars.push(messy[noise] ?? '▒'); - } - } - return chars.join(''); -} - -function activityWave(width: number, frame: number, offset: number): string { - const heights = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; - const chars: string[] = []; - for (let i = 0; i < width; i += 1) { - const wave = Math.sin(((i * 2 + frame + offset * 5) * Math.PI) / 6); - const idx = Math.round(((wave + 1) / 2) * (heights.length - 1)); - chars.push(heights[idx] ?? '▁'); - } - return chars.join(''); -} - function topicName(key: string): string { return (key.split('/').pop()?.replace(/\.md$/, '') ?? key).replace(/[_-]/g, ' '); } @@ -155,18 +79,9 @@ function humanizeInsight(key: string, target: 'sl' | 'wiki', summary: string | u return target === 'sl' ? `Query definition: ${name}` : `Knowledge page: ${name}`; } -const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_']; const INTERNAL_DEMO_CONNECTION_ID = 'orbit_demo'; const PUBLIC_DEMO_SOURCE_LABEL = 'Orbit Demo'; -function humanizeUnitKey(unitKey: string): string { - let key = unitKey.replace(/-/g, '_'); - for (const prefix of ADAPTER_PREFIXES) { - if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; } - } - return key.replace(/_/g, ' '); -} - interface SourceInfo { type: string; name: string; @@ -224,13 +139,6 @@ function sourceDescription(input: MemoryFlowReplayInput): SourceInfo { return { type: info.type, name: conn, sourceCount: count, itemNounPlural: info.plural, readingVerb: info.verb, ingestDescription: info.description }; } -function activeWorkUnit( - input: MemoryFlowReplayInput, -): { unitKey: string; stepIndex: number; stepBudget: number } | null { - const units = activeWorkUnits(input); - return units.at(-1) ?? null; -} - function activeWorkUnits( input: MemoryFlowReplayInput, ): Array<{ unitKey: string; stepIndex: number; stepBudget: number }> { @@ -299,22 +207,6 @@ function finishedUnits(input: MemoryFlowReplayInput): Array<{ unitKey: string; a return units; } -function artifactCounts(input: MemoryFlowReplayInput): { sl: number; wiki: number } { - let sl = 0; - let wiki = 0; - for (const e of input.events) { - if (e.type === 'candidate_action') { - if (e.target === 'sl') sl++; - else wiki++; - } - } - return { sl, wiki }; -} - -function pad(str: string, width: number): string { - return str.length >= width ? str : str + ' '.repeat(width - str.length); -} - const KTX_LOGO_SMALL = [ '██╗ ██╗████████╗██╗ ██╗', '██║ ██╔╝╚══██╔══╝╚██╗██╔╝', @@ -344,12 +236,7 @@ export function Hud(props: { width: number; now?: () => number; }): ReactNode { - const isRunning = props.input.status === 'running'; - const isDone = props.input.status === 'done'; - const isFlowing = isRunning && hasWorkStarted(props.input); - const src = sourceDescription(props.input); - const counts = artifactCounts(props.input); const metrics = buildDemoMetrics(props.input, props.now ? { now: props.now } : {}); const workStarted = hasWorkStarted(props.input); @@ -358,11 +245,6 @@ export function Hud(props: { const innerWidth = Math.max(60, props.width - 6); - const actives = activeWorkUnits(props.input); - const reconEvent = props.input.events.find((e) => e.type === 'reconciliation_finished'); - const allAnalyzed = isFlowing && actives.length === 0; - const isReconciling = allAnalyzed && !reconEvent && !isDone; - const hLine = '─'.repeat(innerWidth); const elapsed = formatDuration(metrics.elapsedMs); @@ -429,7 +311,6 @@ export function ActivityFeed(props: { const workStarted = hasWorkStarted(props.input); const totalChunks = planEvent?.chunkCount ?? 0; - const finishedWithArtifacts = finished.filter((u) => u.artifactCount > 0); const finishedAreas = totalChunks > 0 ? Math.min(finished.length, totalChunks) : finished.length; const allWorkDone = workStarted && actives.length === 0 && queued.length === 0; const isReconciling = allWorkDone && !reconEvent && !isDone && !isError; diff --git a/packages/cli/src/memory-flow-tui.test.tsx b/packages/cli/src/memory-flow-tui.test.tsx index 83a46795..b555c6c1 100644 --- a/packages/cli/src/memory-flow-tui.test.tsx +++ b/packages/cli/src/memory-flow-tui.test.tsx @@ -11,7 +11,6 @@ import { startLiveMemoryFlowTui, type KtxMemoryFlowTuiIo, type MemoryFlowInkInstance, - type MemoryFlowInkRenderOptions, } from './memory-flow-tui.js'; function replayInput(): MemoryFlowReplayInput { diff --git a/packages/cli/src/memory-flow-tui.tsx b/packages/cli/src/memory-flow-tui.tsx index 31a3aea4..37b9f2af 100644 --- a/packages/cli/src/memory-flow-tui.tsx +++ b/packages/cli/src/memory-flow-tui.tsx @@ -1,7 +1,6 @@ /* @jsxImportSource react */ import { buildMemoryFlowViewModel, - buildMemoryFlowVisualModel, createInitialMemoryFlowInteractionState, findMemoryFlowSearchMatches, type MemoryFlowColumnId, @@ -14,8 +13,7 @@ import { selectedMemoryFlowDetails, } from '@ktx/context/ingest'; import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink'; -import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; -import { buildDemoMetrics } from './demo-metrics.js'; +import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; import { ActivityFeed, Hud, @@ -201,14 +199,6 @@ function stageLabel(columnId: MemoryFlowColumnId): string { return STAGE_LABELS[columnId]; } -function statusLabel(status: string): 'OK' | 'RUN' | 'WARN' | 'FAIL' | 'WAIT' { - if (status === 'complete') return 'OK'; - if (status === 'active') return 'RUN'; - if (status === 'warning') return 'WARN'; - if (status === 'failed') return 'FAIL'; - return 'WAIT'; -} - function filterLabel(filter: MemoryFlowInteractionState['filter']): string { return filter === 'failed_or_flagged' ? 'issues' : 'all'; } @@ -325,7 +315,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode { const view = useMemo(() => buildMemoryFlowViewModel(pacedInput), [pacedInput]); const [state, setState] = useState(() => createInitialMemoryFlowInteractionState(view)); const [frame, setFrame] = useState(0); - const [thoughtFrame, setThoughtFrame] = useState(0); const [completionFrame, setCompletionFrame] = useState(0); const [holdComplete, setHoldComplete] = useState(false); const [userHasNavigated, setUserHasNavigated] = useState(false); @@ -346,7 +335,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode { useEffect(() => { const timer = setInterval(() => { setFrame((current) => current + 1); - setThoughtFrame((current) => current + 1); }, props.frameMs ?? DEFAULT_TUI_TIMING.frameMs); return () => clearInterval(timer); }, [props.frameMs]); @@ -354,7 +342,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode { useEffect(() => { if (lastEventCountRef.current !== pacedInput.events.length) { lastEventCountRef.current = pacedInput.events.length; - setThoughtFrame(0); } }, [pacedInput.events.length]); @@ -409,10 +396,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode { }); const isComplete = pacedInput.status === 'done' || pacedInput.status === 'error'; - const completionMetrics = useMemo( - () => buildDemoMetrics(pacedInput, pacedNow ? { now: pacedNow } : {}), - [pacedInput, pacedNow], - ); const termWidth = props.terminalWidth ?? 80; diff --git a/packages/cli/src/next-steps.test.ts b/packages/cli/src/next-steps.test.ts index fd8d8216..b4706d72 100644 --- a/packages/cli/src/next-steps.test.ts +++ b/packages/cli/src/next-steps.test.ts @@ -25,12 +25,8 @@ describe('KTX demo next steps', () => { it('uses supported final public commands', () => { expect(KTX_NEXT_STEP_COMMANDS).toEqual([ { - command: 'ktx agent context --json', - description: 'Verify the project context your agent can read', - }, - { - command: 'ktx agent tools --json', - description: 'List direct CLI tools available to agents', + command: 'ktx status --json', + description: 'Verify project setup and context readiness', }, { command: 'ktx sl list', @@ -46,8 +42,8 @@ describe('KTX demo next steps', () => { it('uses only the direct CLI route for agent verification', () => { const commands = KTX_NEXT_STEP_COMMANDS.map((step) => step.command); - expect(commands).toContain('ktx agent context --json'); - expect(commands).toContain('ktx agent tools --json'); + expect(commands).not.toContain('ktx agent context --json'); + expect(commands).toContain('ktx status --json'); expect(commands).not.toContain('ktx serve --mcp stdio --user-id local'); }); @@ -64,8 +60,8 @@ describe('KTX demo next steps', () => { it('does not advertise removed Commander migration commands', () => { const rendered = formatNextStepLines().join('\n'); - expect(rendered).toContain('ktx agent tools --json'); - expect(rendered).toContain('ktx agent context --json'); + expect(rendered).toContain('ktx status --json'); + expect(rendered).not.toContain('ktx agent'); expect(rendered).toContain('ktx sl list'); expect(rendered).toContain('ktx wiki list'); @@ -109,7 +105,8 @@ describe('KTX demo next steps', () => { }).join('\n'); expect(rendered).toContain('KTX context is ready for agents.'); - expect(rendered).toContain('ktx agent context --json'); + expect(rendered).toContain('ktx status --json'); + expect(rendered).not.toContain('ktx agent'); expect(rendered).not.toContain('ktx serve --mcp stdio --user-id local'); expect(rendered).not.toContain('Build KTX context next.'); }); diff --git a/packages/cli/src/next-steps.ts b/packages/cli/src/next-steps.ts index db85da66..ee7535d7 100644 --- a/packages/cli/src/next-steps.ts +++ b/packages/cli/src/next-steps.ts @@ -11,12 +11,8 @@ export const KTX_CONTEXT_BUILD_COMMANDS = [ export const KTX_NEXT_STEP_DIRECT_COMMANDS = [ { - command: 'ktx agent context --json', - description: 'Verify the project context your agent can read', - }, - { - command: 'ktx agent tools --json', - description: 'List direct CLI tools available to agents', + command: 'ktx status --json', + description: 'Verify project setup and context readiness', }, { command: 'ktx sl list', diff --git a/packages/cli/src/project-dir.test.ts b/packages/cli/src/project-dir.test.ts index c59172a6..c0022d4d 100644 --- a/packages/cli/src/project-dir.test.ts +++ b/packages/cli/src/project-dir.test.ts @@ -35,8 +35,7 @@ describe('project directory defaults', () => { const ingest = vi.fn(async () => 0); const scan = vi.fn(async () => 0); const setup = vi.fn(async () => 0); - const agent = vi.fn(async () => 0); - const deps: KtxCliDeps = { agent, connection, doctor, ingest, scan, setup }; + const deps: KtxCliDeps = { connection, doctor, ingest, scan, setup }; const cases: Array<{ argv: string[]; @@ -74,12 +73,6 @@ describe('project directory defaults', () => { expected: { command: 'run', projectDir: '/tmp/ktx-env-project', connectionId: 'warehouse' }, expectedStderr: 'Project: /tmp/ktx-env-project\n', }, - { - argv: ['agent', 'tools', '--json'], - spy: agent, - expected: { command: 'tools', projectDir: '/tmp/ktx-env-project' }, - expectedStderr: '', - }, ]; for (const item of cases) { diff --git a/packages/cli/src/project-dir.ts b/packages/cli/src/project-dir.ts deleted file mode 100644 index d8aef2e2..00000000 --- a/packages/cli/src/project-dir.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { resolve } from 'node:path'; - -export function resolveProjectDir(projectDir?: string, fallback = '.'): string { - return resolve(projectDir ?? fallback); -} diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index c9d9f7bb..f8296177 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -6,9 +6,9 @@ import { profileMark } from './startup-profile.js'; profileMark('module:public-ingest'); -export type KtxPublicIngestStepName = 'scan' | 'source-ingest' | 'enrich' | 'memory-update'; -export type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run'; -export type KtxPublicIngestInputMode = 'auto' | 'disabled'; +type KtxPublicIngestStepName = 'scan' | 'source-ingest' | 'enrich' | 'memory-update'; +type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run'; +type KtxPublicIngestInputMode = 'auto' | 'disabled'; export type KtxPublicIngestArgs = | { diff --git a/packages/cli/src/runtime.test.ts b/packages/cli/src/runtime.test.ts index 16e879cc..8151a4b3 100644 --- a/packages/cli/src/runtime.test.ts +++ b/packages/cli/src/runtime.test.ts @@ -5,6 +5,7 @@ import type { ManagedPythonDaemonStopResult, } from './managed-python-daemon.js'; import type { + ManagedPythonRuntimeDoctorCheck, ManagedPythonRuntimeInstallResult, ManagedPythonRuntimeStatus, } from './managed-python-runtime.js'; @@ -256,7 +257,7 @@ describe('runKtxRuntime', () => { expect(io.stderr()).toContain('process scan: ps failed'); }); - it('prints runtime status as JSON', async () => { + it('prints runtime status and doctor checks as JSON with doctor-style exit status', async () => { const io = makeIo(); const deps: KtxRuntimeDeps = { readStatus: vi.fn(async (): Promise => ({ @@ -278,38 +279,41 @@ describe('runKtxRuntime', () => { daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', }, })), + doctorRuntime: vi.fn(async (): Promise => [ + { id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' }, + { id: 'asset', label: 'Bundled Python wheel', status: 'pass', detail: '/assets/python/runtime.whl' }, + { + id: 'runtime', + label: 'Managed Python runtime', + status: 'fail', + detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + fix: 'Run: ktx dev runtime install --yes', + }, + ]), }; - await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: true }, io.io, deps)).resolves.toBe(0); + await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: true }, io.io, deps)).resolves.toBe(1); expect(JSON.parse(io.stdout())).toMatchObject({ kind: 'missing', detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', layout: { runtimeRoot: '/runtime' }, + checks: [ + { id: 'uv', status: 'pass' }, + { id: 'asset', status: 'pass' }, + { id: 'runtime', status: 'fail' }, + ], }); + expect(deps.readStatus).toHaveBeenCalledWith({ cliVersion: '0.2.0' }); + expect(deps.doctorRuntime).toHaveBeenCalledWith({ cliVersion: '0.2.0' }); }); - it('requires --yes before pruning stale runtime directories', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - pruneRuntime: vi.fn(async () => { - throw new Error('should not prune without --yes'); - }), - }; - - await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: false, yes: false }, io.io, deps)) - .resolves.toBe(1); - - expect(io.stderr()).toContain('Refusing to prune without --yes'); - expect(deps.pruneRuntime).not.toHaveBeenCalled(); - }); - - it('prints stale directories during prune dry-run', async () => { + it('prints runtime status and doctor checks in plain output', async () => { const io = makeIo(); const deps: KtxRuntimeDeps = { readStatus: vi.fn(async (): Promise => ({ - kind: 'missing', - detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', + kind: 'ready', + detail: 'Runtime ready at /runtime/0.2.0', layout: { cliVersion: '0.2.0', runtimeRoot: '/runtime', @@ -325,19 +329,43 @@ describe('runKtxRuntime', () => { daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', }, + manifest: { + schemaVersion: 1, + cliVersion: '0.2.0', + installedAt: '2026-05-11T00:00:00.000Z', + asset: { + schemaVersion: 1, + distributionName: 'kaelio-ktx', + normalizedName: 'kaelio_ktx', + version: '0.1.0', + wheel: { + file: 'kaelio_ktx-0.1.0-py3-none-any.whl', + sha256: 'a'.repeat(64), + bytes: 10, + }, + }, + features: ['core'], + python: { + executable: '/runtime/0.2.0/.venv/bin/python', + daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', + }, + installLog: '/runtime/0.2.0/install.log', + }, })), - pruneRuntime: vi.fn(async () => ({ - runtimeRoot: '/runtime', - stale: ['/runtime/0.1.0'], - kept: ['/runtime/0.2.0'], - removed: [], - })), + doctorRuntime: vi.fn(async (): Promise => [ + { id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' }, + { id: 'asset', label: 'Bundled Python wheel', status: 'pass', detail: '/assets/python/runtime.whl' }, + { id: 'runtime', label: 'Managed Python runtime', status: 'pass', detail: 'Runtime ready at /runtime/0.2.0' }, + ]), }; - await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: true, yes: false }, io.io, deps)) - .resolves.toBe(0); + await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: false }, io.io, deps)).resolves.toBe(0); - expect(io.stdout()).toContain('Stale KTX Python runtimes'); - expect(io.stdout()).toContain('/runtime/0.1.0'); + expect(io.stdout()).toContain('KTX Python runtime'); + expect(io.stdout()).toContain('status: ready'); + expect(io.stdout()).toContain('KTX Python runtime checks'); + expect(io.stdout()).toContain('PASS uv: uv 0.9.5'); + expect(io.stdout()).toContain('PASS Managed Python runtime: Runtime ready at /runtime/0.2.0'); + expect(io.stderr()).toBe(''); }); }); diff --git a/packages/cli/src/runtime.ts b/packages/cli/src/runtime.ts index d5f4b5cb..8bb3fc7c 100644 --- a/packages/cli/src/runtime.ts +++ b/packages/cli/src/runtime.ts @@ -8,14 +8,14 @@ import { type ManagedPythonDaemonStopResult, } from './managed-python-daemon.js'; import { + doctorManagedPythonRuntime, installManagedPythonRuntime, - pruneManagedPythonRuntimes, readManagedPythonRuntimeStatus, type KtxRuntimeFeature, + type ManagedPythonRuntimeDoctorCheck, type ManagedPythonRuntimeInstallOptions, type ManagedPythonRuntimeInstallResult, type ManagedPythonRuntimeLayoutOptions, - type ManagedPythonRuntimePruneResult, type ManagedPythonRuntimeStatus, } from './managed-python-runtime.js'; @@ -23,8 +23,7 @@ export type KtxRuntimeArgs = | { command: 'install'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } | { command: 'start'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } | { command: 'stop'; cliVersion: string; all: boolean } - | { command: 'status'; cliVersion: string; json: boolean } - | { command: 'prune'; cliVersion: string; dryRun: boolean; yes: boolean }; + | { command: 'status'; cliVersion: string; json: boolean }; export interface KtxRuntimeDeps { installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; @@ -36,11 +35,7 @@ export interface KtxRuntimeDeps { stopDaemon?: (options: { cliVersion: string }) => Promise; stopAllDaemons?: (options: { cliVersion: string }) => Promise; readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; - pruneRuntime?: (options: { - cliVersion: string; - runtimeRoot: string; - dryRun?: boolean; - }) => Promise; + doctorRuntime?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; } function writeJson(io: KtxCliIo, value: unknown): void { @@ -145,17 +140,20 @@ function writeStatus(io: KtxCliIo, status: ManagedPythonRuntimeStatus): void { } } -function writePrune(io: KtxCliIo, result: ManagedPythonRuntimePruneResult, dryRun: boolean): void { - if (result.stale.length === 0) { - io.stdout.write(`No stale KTX Python runtimes found under ${result.runtimeRoot}\n`); - return; - } - io.stdout.write(dryRun ? 'Stale KTX Python runtimes\n' : 'Removed stale KTX Python runtimes\n'); - for (const path of dryRun ? result.stale : result.removed) { - io.stdout.write(`${path}\n`); +function writeRuntimeChecks(io: KtxCliIo, checks: ManagedPythonRuntimeDoctorCheck[]): void { + io.stdout.write('KTX Python runtime checks\n'); + for (const check of checks) { + io.stdout.write(`${check.status.toUpperCase()} ${check.label}: ${check.detail}\n`); + if (check.fix) { + io.stdout.write(` Fix: ${check.fix}\n`); + } } } +function hasRuntimeCheckFailures(checks: ManagedPythonRuntimeDoctorCheck[]): boolean { + return checks.some((check) => check.status === 'fail'); +} + export async function runKtxRuntime( args: KtxRuntimeArgs, io: KtxCliIo = process, @@ -196,27 +194,19 @@ export async function runKtxRuntime( } if (args.command === 'status') { const readStatus = deps.readStatus ?? readManagedPythonRuntimeStatus; + const doctorRuntime = deps.doctorRuntime ?? doctorManagedPythonRuntime; const status = await readStatus({ cliVersion: args.cliVersion }); + const checks = await doctorRuntime({ cliVersion: args.cliVersion }); if (args.json) { - writeJson(io, status); + writeJson(io, { ...status, checks }); } else { writeStatus(io, status); + writeRuntimeChecks(io, checks); } - return 0; + return hasRuntimeCheckFailures(checks) ? 1 : 0; } - if (!args.dryRun && !args.yes) { - io.stderr.write('Refusing to prune without --yes. Preview with: ktx dev runtime prune --dry-run\n'); - return 1; - } - const status = await (deps.readStatus ?? readManagedPythonRuntimeStatus)({ cliVersion: args.cliVersion }); - const pruneRuntime = deps.pruneRuntime ?? pruneManagedPythonRuntimes; - const result = await pruneRuntime({ - cliVersion: args.cliVersion, - runtimeRoot: status.layout.runtimeRoot, - dryRun: args.dryRun, - }); - writePrune(io, result, args.dryRun); - return 0; + const _exhaustive: never = args; + return _exhaustive; } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return 1; diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index 322db2aa..19647a3f 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -84,7 +84,10 @@ describe('setup agents', () => { const skill = await readFile(join(tempDir, '.agents/skills/ktx/SKILL.md'), 'utf-8'); expect(skill).toContain(`--project-dir ${tempDir}`); expect(skill).toContain('must not print secrets'); - expect(skill).toContain('agent sql execute'); + expect(skill).toContain('status --json'); + expect(skill).toContain('sl list --json'); + expect(skill).not.toContain('agent '); + expect(skill).not.toContain('sql execute'); expect(await readKtxAgentInstallManifest(tempDir)).toMatchObject({ version: 1, projectDir: tempDir, @@ -115,8 +118,9 @@ describe('setup agents', () => { const skill = await readFile(join(tempDir, '.agents/skills/ktx/SKILL.md'), 'utf-8'); expect(skill).not.toContain('`ktx agent'); - expect(skill).toContain('agent context --json'); - expect(skill).toContain('agent sql execute'); + expect(skill).toContain('status --json'); + expect(skill).toContain('sl query'); + expect(skill).not.toContain('sql execute'); }); it('removes only manifest-listed files', async () => { diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 9c5c6fd5..3c7829c7 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -123,7 +123,7 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun return [ '---', 'name: ktx', - 'description: Use local KTX semantic context, wiki knowledge, and safe SQL execution for this project.', + 'description: Use local KTX semantic context and wiki knowledge for this project.', '---', '', '# KTX Local Context', @@ -136,11 +136,11 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun '', 'Available commands:', '', - `- \`${ktxCommandLine(input.launcher, ['agent', 'context', ...projectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, ['agent', 'sl', 'list', ...projectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, ['agent', 'sl', 'read', '', ...projectDirArgs])}\``, + `- \`${ktxCommandLine(input.launcher, ['status', ...projectDirArgs])}\``, + `- \`${ktxCommandLine(input.launcher, ['sl', 'list', ...projectDirArgs])}\``, + `- \`${ktxCommandLine(input.launcher, ['sl', 'list', ...projectDirArgs, '--query', ''])}\``, + `- \`${ktxCommandLine(input.launcher, ['sl', 'read', '', ...projectDirArgs, '--connection-id', ''])}\``, `- \`${ktxCommandLine(input.launcher, [ - 'agent', 'sl', 'query', ...projectDirArgs, @@ -152,29 +152,17 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun '--max-rows', '100', ])}\``, - `- \`${ktxCommandLine(input.launcher, ['agent', 'wiki', 'search', '', ...projectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, ['agent', 'wiki', 'read', '', ...projectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, [ - 'agent', - 'sql', - 'execute', - ...projectDirArgs, - '--connection-id', - '', - '--sql-file', - '', - '--max-rows', - '100', - ])}\``, + `- \`${ktxCommandLine(input.launcher, ['wiki', 'search', '', ...projectDirArgs, '--limit', '10'])}\``, + `- \`${ktxCommandLine(input.launcher, ['wiki', 'read', '', ...projectDirArgs])}\``, '', - 'SQL execution is read-only, requires an explicit row limit, and should use the smallest useful limit.', + 'Use semantic-layer queries before direct database access. Do not print secrets or credential references.', '', ].join('\n'); } function ruleInstructionContent(input: { projectDir: string }): string { return [ - `Use the \`ktx\` CLI to query local semantic context, wiki knowledge, and execute safe SQL for this project (\`--project-dir ${input.projectDir}\`).`, + `Use the \`ktx\` CLI to query local semantic context and wiki knowledge for this project (\`--project-dir ${input.projectDir}\`).`, '', 'Use when the user asks about data schemas, metrics, dimensions, database structure, or wants to run SQL queries.', '', diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index 8f6e5ce1..94589bdc 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -472,16 +472,6 @@ async function markContextComplete(projectDir: string): Promise { await markKtxSetupStateStepComplete(projectDir, 'context'); } -function writeBuildHeader(projectDir: string, runId: string, io: KtxCliIo): void { - const commands = contextBuildCommands(projectDir, runId); - io.stdout.write('\nKTX context build\n'); - io.stdout.write(`Run: ${runId}\n`); - io.stdout.write(`Project: ${resolve(projectDir)}\n\n`); - io.stdout.write('Detach: press d to leave this running.\n'); - io.stdout.write(`Resume: ${commands.watch}\n`); - io.stdout.write(`Status: ${commands.status}\n\n`); -} - function writeMissingCapabilities(missing: string[], io: KtxCliIo): void { io.stderr.write('KTX cannot build agent-ready context yet.\n\n'); io.stderr.write('Missing:\n'); diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index c04ec15d..8d360c58 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -84,6 +84,71 @@ describe('runKtxSl', () => { expect(listIo.stdout()).toContain('warehouse\torders\tcolumns=1\tmeasures=0\tjoins=0'); }); + it('prints semantic-layer reads and searched lists as public JSON envelopes', async () => { + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir, projectName: 'warehouse' }); + + await expect( + runKtxSl( + { + command: 'write', + projectDir, + connectionId: 'warehouse', + sourceName: 'orders', + yaml: [ + 'name: orders', + 'table: public.orders', + 'description: Paid order facts', + 'grain: [order_id]', + 'columns:', + ' - name: order_id', + ' type: string', + '', + ].join('\n'), + }, + makeIo().io, + ), + ).resolves.toBe(0); + + const readIo = makeIo(); + await expect( + runKtxSl( + { command: 'read', projectDir, connectionId: 'warehouse', sourceName: 'orders', json: true }, + readIo.io, + ), + ).resolves.toBe(0); + expect(JSON.parse(readIo.stdout())).toMatchObject({ + kind: 'sl.source', + data: { + connectionId: 'warehouse', + name: 'orders', + yaml: expect.stringContaining('name: orders'), + }, + }); + + const listIo = makeIo(); + await expect( + runKtxSl( + { command: 'list', projectDir, connectionId: 'warehouse', query: 'paid', json: true }, + listIo.io, + ), + ).resolves.toBe(0); + expect(JSON.parse(listIo.stdout())).toMatchObject({ + kind: 'list', + data: { + items: [ + expect.objectContaining({ + connectionId: 'warehouse', + name: 'orders', + score: expect.any(Number), + matchReasons: expect.arrayContaining(['token']), + }), + ], + }, + meta: { command: 'sl list' }, + }); + }); + it('fails validation when a table-backed source declares columns absent from a matching warehouse manifest', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); @@ -191,6 +256,73 @@ joins: [] expect(stderr.write).not.toHaveBeenCalled(); }); + it('runs sl query from a JSON query file', async () => { + const projectDir = join(tempDir, 'project'); + const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); + project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + `name: orders +table: public.orders +grain: [id] +columns: + - name: id + type: number +measures: + - name: order_count + expr: count(*) +joins: [] +`, + 'ktx', + 'ktx@example.com', + 'Add orders source', + ); + const queryFile = join(tempDir, 'query.json'); + await writeFile(queryFile, '{"measures":["orders.order_count"],"dimensions":[]}', 'utf-8'); + + const stdout = { write: vi.fn() }; + const stderr = { write: vi.fn() }; + const query = vi.fn(async () => ({ + sql: 'select count(*) as order_count from public.orders', + dialect: 'postgres', + columns: [{ name: 'orders.order_count' }], + plan: {}, + })); + const createSemanticLayerCompute = vi.fn(() => ({ + query, + validateSources: vi.fn(), + generateSources: vi.fn(), + })); + + await expect( + runKtxSl( + { + command: 'query', + projectDir, + connectionId: 'warehouse', + queryFile, + format: 'json', + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + { stdout, stderr }, + { createSemanticLayerCompute }, + ), + ).resolves.toBe(0); + + expect(query).toHaveBeenCalledWith( + expect.objectContaining({ + query: { measures: ['orders.order_count'], dimensions: [] }, + }), + ); + expect(JSON.parse(String(stdout.write.mock.calls[0][0]))).toMatchObject({ + sql: 'select count(*) as order_count from public.orders', + plan: { execution: { mode: 'compile_only' } }, + }); + expect(stderr.write).not.toHaveBeenCalled(); + }); + it('creates default sl query compute through the managed runtime helper', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index fb0f129e..ebf3eca7 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -1,14 +1,22 @@ +import { readFile } from 'node:fs/promises'; import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections'; +import { + createLocalKtxEmbeddingProviderFromConfig, + KtxIngestEmbeddingPortAdapter, + type KtxEmbeddingPort, +} from '@ktx/context'; import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; import { compileLocalSlQuery, listLocalSlSources, readLocalSlSource, + searchLocalSlSources, validateLocalSlSource, writeLocalSlSource, type SemanticLayerQueryInput, } from '@ktx/context/sl'; +import { writeJsonResult } from './io/print-list.js'; import { createManagedPythonSemanticLayerComputePort, type KtxManagedPythonInstallPolicy, @@ -20,15 +28,16 @@ profileMark('module:sl'); type SlQueryFormat = 'json' | 'sql'; export type KtxSlArgs = - | { command: 'list'; projectDir: string; connectionId?: string; output?: string; json?: boolean } - | { command: 'read'; projectDir: string; connectionId: string; sourceName: string } + | { command: 'list'; projectDir: string; connectionId?: string; query?: string; output?: string; json?: boolean } + | { command: 'read'; projectDir: string; connectionId: string; sourceName: string; json?: boolean } | { command: 'validate'; projectDir: string; connectionId: string; sourceName: string } | { command: 'write'; projectDir: string; connectionId: string; sourceName: string; yaml: string } | { command: 'query'; projectDir: string; connectionId?: string; - query: SemanticLayerQueryInput; + query?: SemanticLayerQueryInput; + queryFile?: string; format: SlQueryFormat; execute: boolean; maxRows?: number; @@ -43,6 +52,8 @@ interface KtxSlIo { interface KtxSlDeps { loadProject?: typeof loadKtxProject; + embeddingService?: KtxEmbeddingPort | null; + createEmbeddingProvider?: typeof createLocalKtxEmbeddingProviderFromConfig; createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; createManagedSemanticLayerCompute?: (options: { cliVersion: string; @@ -52,11 +63,35 @@ interface KtxSlDeps { createQueryExecutor?: () => KtxSqlQueryExecutorPort; } +function slSearchEmbeddingService(project: KtxLocalProject, deps: KtxSlDeps): KtxEmbeddingPort | null { + if ('embeddingService' in deps) { + return deps.embeddingService ?? null; + } + const provider = (deps.createEmbeddingProvider ?? createLocalKtxEmbeddingProviderFromConfig)( + project.config.ingest.embeddings, + ); + return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null; +} + +async function readSlQueryFile(path: string): Promise { + const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`${path} must contain a JSON object.`); + } + return parsed as SemanticLayerQueryInput; +} + export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: KtxSlDeps = {}): Promise { try { const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); if (args.command === 'list') { - const sources = await listLocalSlSources(project, { connectionId: args.connectionId }); + const sources = args.query + ? await searchLocalSlSources(project, { + connectionId: args.connectionId, + query: args.query, + embeddingService: slSearchEmbeddingService(project, deps), + }) + : await listLocalSlSources(project, { connectionId: args.connectionId }); const { resolveOutputMode } = await import('./io/mode.js'); const { printList } = await import('./io/print-list.js'); const mode = resolveOutputMode({ explicit: args.output, json: args.json, io }); @@ -86,6 +121,14 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx if (!source) { throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`); } + if (args.json) { + writeJsonResult(io, { + kind: 'sl.source', + data: source, + meta: { command: 'sl read' }, + }); + return 0; + } io.stdout.write(source.yaml); return 0; } @@ -108,6 +151,10 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx return 0; } if (args.command === 'query') { + const query = args.query ?? (args.queryFile ? await readSlQueryFile(args.queryFile) : undefined); + if (!query) { + throw new Error('sl query requires query input from --query-file or at least one --measure'); + } const compute = deps.createSemanticLayerCompute ? deps.createSemanticLayerCompute() : await (deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort)({ @@ -118,7 +165,7 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined; const result = await compileLocalSlQuery(project as KtxLocalProject, { connectionId: args.connectionId, - query: args.query, + query, compute, execute: args.execute, maxRows: args.maxRows, diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 9efa52cb..c2b7386c 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -126,10 +126,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich ); } -function parseJsonOutput(stdout: string): T { - return JSON.parse(stdout) as T; -} - function expectProjectStderr(result: CliResult, projectDir: string): void { expect(result).toMatchObject({ code: 0, stderr: `Project: ${projectDir}\n` }); } @@ -190,49 +186,21 @@ describe('standalone built ktx CLI smoke', () => { ); }); - it('prints guided JSON for agent semantic-layer search outside a project through the built binary', async () => { - const projectDir = join(tempDir, 'missing-search-project'); - await mkdir(projectDir, { recursive: true }); - - const result = await runBuiltCli([ - 'agent', - 'sl', - 'list', - '--json', - '--query', - 'revenue', - '--project-dir', - projectDir, - ]); + it('rejects the removed agent command through the built binary', async () => { + const result = await runBuiltCli(['agent']); expect(result.code).toBe(1); expect(result.stdout).toBe(''); - const errorJson = parseJsonOutput<{ - ok: false; - error: { code: string; message: string; nextSteps: string[] }; - }>(result.stderr); - expect(errorJson).toEqual({ - ok: false, - error: { - code: 'agent_sl_search_missing_project', - message: `Semantic-layer search needs an initialized KTX project at ${projectDir}.`, - nextSteps: [ - `ktx setup --project-dir ${projectDir}`, - `ktx status --project-dir ${projectDir}`, - 'ktx ingest run --connection-id --adapter ', - `ktx agent sl list --json --query "revenue" --project-dir ${projectDir}`, - ], - }, - }); + expect(result.stderr).toContain("unknown command 'agent'"); }); it('runs doctor setup through the built binary', async () => { const result = await runBuiltCli(['status', '--no-input']); - expect(result.stdout).toContain('KTX setup doctor'); + expect(result.stdout).toMatch(/KTX (setup|project) doctor/); expect(result.stdout).toContain('Node 22+'); expect(result.stdout).toContain('Workspace-local CLI'); - expect(result.stderr).toBe(''); + expect(result.stderr === '' || result.stderr.startsWith('Project: ')).toBe(true); expect([0, 1]).toContain(result.code); }); diff --git a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md index d076d4e5..5d2316fd 100644 --- a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md +++ b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md @@ -20,6 +20,7 @@ Parsimonious. Stage 3 WUs already loaded `ingest_triage` and handled conflicts t All wiki writes are GLOBAL (same as Stage 3). SL writes target the same session worktree Stage 3 used. +Wiki keys must be flat slugs, not directory paths. If a Stage 3 page used a path-like key and a flat retry exists, treat the flat key as the canonical page. diff --git a/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md b/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md index 9645ccc4..c7c9eb6d 100644 --- a/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md +++ b/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md @@ -1,5 +1,5 @@ -You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, Metabase card JSONs, or similar) and you must translate that slice into KTX semantic-layer sources and/or knowledge wiki pages, in one pass. Prior WorkUnits in this same job may have already written SL sources and wiki pages; their writes are visible on the working branch and searchable via `wiki_sl_search`. +You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, Metabase card JSONs, Notion pages, or similar) and you must translate that slice into KTX semantic-layer sources and/or knowledge wiki pages, in one pass. Prior WorkUnits in this same job may have already written SL sources and wiki pages; their writes are visible on the working branch and discoverable with `discover_data`. @@ -8,9 +8,9 @@ Assertive. The bundle was explicitly submitted for ingest. Default to capturing 1. Read this WorkUnit's section at the end of the user prompt. It lists your `rawFiles`, any unchanged `dependencyPaths` you may need to resolve references, the `peerFileIndex` (paths only; you CANNOT read them), the source's `skillNames`, and any `priorProvenance` rows telling you what earlier syncs produced from these files. -2. Load the per-source review skill first (e.g. `lookml_ingest`, `metricflow_ingest`, `dbt_ingest`), then `sl_capture` and `knowledge_capture`, and `ingest_triage` last. The triage skill tells you how to react when `wiki_sl_search` reveals that a prior WU already wrote something overlapping. +2. Load the per-source review skill first (e.g. `lookml_ingest`, `metricflow_ingest`, `dbt_ingest`), then `sl_capture` and `knowledge_capture`, and `ingest_triage` last. The triage skill tells you how to react when `discover_data` reveals that a prior WU already wrote something overlapping. 3. If the system prompt includes ``, read those pins before choosing artifact keys. A pin's `canonicalArtifactKey` is the preferred artifact for its `contestedKey`: prefer editing the pinned canonical artifact when it already exists or when this raw file clearly updates it. Do not create a duplicate contested artifact when a pin says another artifact is canonical; use a specific disambiguated key only when the raw file describes a genuinely different domain. -4. For each raw file: call `read_raw_file` (or `read_raw_span` for slicing large files) to load content. Before writing a new SL source or wiki page, call `wiki_sl_search` for each candidate name to find prior-WU writes; apply `ingest_triage` when you hit one, and apply any matching canonical pin before deciding whether to edit, rename, or skip. +4. For each raw file: call `read_raw_file` (or `read_raw_span` for slicing large files) to load content. Before writing a new SL source or wiki page, call `discover_data` for each candidate source, table, metric, or topic name to find prior-WU writes, existing wiki pages, SL sources, and raw warehouse matches; apply `ingest_triage` when you hit one, and apply any matching canonical pin before deciding whether to edit, rename, or skip. 5. For every `wiki_write`, `wiki_remove`, `sl_write_source`, or `sl_edit_source` call, include `rawPaths` with only the raw file paths that directly support that action. If one artifact synthesizes several files, list each contributing raw file. Do not include unrelated files from the same WorkUnit. 6. When `priorProvenance` names an existing artifact for one of your raw files, prefer `sl_edit` over `sl_write` for that artifact: the re-ingest change rule says expression-only changes replace silently, grain/column/filter changes replace and flag. 7. When a raw file cannot map to normal SL and you use a fallback path, call `emit_unmapped_fallback` exactly once for that raw file and reason. Use `fallback: "sql_standalone"` for a standalone SQL source, `fallback: "wiki_only"` for documentation-only capture, and `fallback: "flagged"` when no reliable artifact can be written. @@ -19,12 +19,13 @@ Assertive. The bundle was explicitly submitted for ingest. Default to capturing All wiki writes go to the GLOBAL scope. Bundle ingests are not personal. The `wiki_write` tool selects scope automatically for this caller. +Wiki keys must be flat slugs like `paid-order-lifecycle`, not directory paths like `historic-sql/paid-order-lifecycle`. Use `tags`, `source`, and page content to group related pages. - Do not read peer files; only files listed in `rawFiles` or `dependencyPaths` are accessible. `read_raw_file` will reject everything else. - Do not invent measures/joins/rules not declared in the raw files. -- Do not invent physical column names or grain keys. For table-backed SL sources, every `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr` column must come from raw-file column declarations or warehouse-backed discovery (`wiki_sl_search`, `sl_discover`, `sl_describe_table`). If column names are not confirmed, capture the business context in wiki instead of writing a full SL source. +- Do not invent physical column names or grain keys. For table-backed SL sources, every `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr` column must come from raw-file column declarations or warehouse-backed discovery (`discover_data`, `sl_discover`, `entity_details`). If column names are not confirmed, capture the business context in wiki instead of writing a full SL source. - Do not write context-source overlays into the context source connection just because that is the current WorkUnit connection. Use `sl_discover` across data sources and write the SL artifact to the warehouse/data-source connection that owns the matching manifest. If there is no confirmed target connection, use `emit_unmapped_fallback` and wiki capture. - Do not duplicate an artifact that prior provenance says you already produced; update it. - Do not silently accept a name collision with a prior WU's write when the formula differs. Trigger `ingest_triage`. diff --git a/packages/context/skills/_shared/identifier-verification.md b/packages/context/skills/_shared/identifier-verification.md new file mode 100644 index 00000000..775203bd --- /dev/null +++ b/packages/context/skills/_shared/identifier-verification.md @@ -0,0 +1,27 @@ +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. diff --git a/packages/context/skills/dbt_ingest/SKILL.md b/packages/context/skills/dbt_ingest/SKILL.md index 135dd2e5..6b332d8e 100644 --- a/packages/context/skills/dbt_ingest/SKILL.md +++ b/packages/context/skills/dbt_ingest/SKILL.md @@ -12,16 +12,16 @@ Use this skill for **uploaded** dbt projects (`dbt_project.yml` at stage root, ` | dbt | KTX | Notes | |-----|--------|--------| -| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `wiki_sl_search` / `sl_describe_table`) | One SL source per physical table; model name may differ from DB name — resolve with `read_raw_file` + warehouse context. | +| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `discover_data` / `entity_details`) | One SL source per physical table; model name may differ from DB name — resolve with `read_raw_file` + warehouse context. | | `sources:` → `tables:` | Same as models; use `identifier` when present instead of logical `name`. | Schema + name must match how the connection sees tables. | | Column `description` | `descriptions.user` or merged `descriptions` map on the column | Do not overwrite `dbt` description keys from sync. | | `data_tests: not_null` / `unique` | Short hint in column `descriptions` or notes: “dbt: not null”, “dbt: unique” | Full structured metadata lands in manifest via **sync**; the skill keeps bundle-time SL text useful for the agent. | -| `accepted_values` | Add a **brief** line in the column description: allowed values (truncate long lists) | Also mention enum-like use in `wiki_sl_search` / filters. | -| `relationships` | Add or confirm `joins:` on the overlay **only** when `to` resolves to a real table via `read_raw_file` + `wiki_sl_search` / `sl_describe_table` | If the ref cannot be resolved, capture the intent in a wiki page instead. | +| `accepted_values` | Add a **brief** line in the column description: allowed values (truncate long lists) | Also mention enum-like use in `discover_data` / filters. | +| `relationships` | Add or confirm `joins:` on the overlay **only** when `to` resolves to a real table via `read_raw_file` + `discover_data` / `entity_details` | If the ref cannot be resolved, capture the intent in a wiki page instead. | ## Physical schema grounding -dbt YAML is documentation and test metadata; it is not permission to invent physical columns. Before writing any table-backed SL source, confirm the real warehouse shape with `wiki_sl_search`, `sl_discover`, or `sl_describe_table` and use only confirmed column names in `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr`. +dbt YAML is documentation and test metadata; it is not permission to invent physical columns. Before writing any table-backed SL source, confirm the real warehouse shape with `discover_data`, `sl_discover`, or `entity_details` and use only confirmed column names in `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr`. For dbt context-source ingest, the dbt connection is usually not the warehouse connection. Call `sl_discover` without `connectionId` first, then write overlays to the connection that owns the matching manifest-backed source (for example `postgres-warehouse`), not to the dbt connection (for example `dbt-main`). If no matching manifest-backed source is visible on any warehouse connection, do not call `sl_write_source`; record `emit_unmapped_fallback` and keep the fact wiki-only. @@ -31,6 +31,34 @@ Include `rawPaths` on every `wiki_write`, `sl_write_source`, and `sl_edit_source After every `sl_write_source`, call `sl_validate`. A validation error saying a declared column or measure reference is absent from the physical table is a hard stop: re-read the warehouse-backed source and rewrite with confirmed names, or remove the invalid SL fields. +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## 1.1 test hints (descriptions / meta) When YAML shows `accepted_values` or `not_null`, add **short** hints into `columns[].descriptions` (e.g. under `user`) or freeform column notes so chat and validation see intent before the next git sync refreshes `constraints` / `enum_values` in `_schema`. Keep hints under a few words when possible. diff --git a/packages/context/skills/historic_sql_patterns/SKILL.md b/packages/context/skills/historic_sql_patterns/SKILL.md index 33eb6fe0..5e898c47 100644 --- a/packages/context/skills/historic_sql_patterns/SKILL.md +++ b/packages/context/skills/historic_sql_patterns/SKILL.md @@ -18,6 +18,37 @@ Use this skill when the WorkUnit raw file is a `patterns-input/part-0001.json` s 6. Set each evidence object's `rawPath` to the exact raw file path read in step 3. 7. Stop after all pattern evidence has been emitted. +Every join column mentioned in pattern descriptions must be verified via +entity_details for both sides of the join. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Evidence Shape Each call to `emit_historic_sql_evidence` must use this shape: diff --git a/packages/context/skills/historic_sql_table_digest/SKILL.md b/packages/context/skills/historic_sql_table_digest/SKILL.md index 34e49d27..0815e3dc 100644 --- a/packages/context/skills/historic_sql_table_digest/SKILL.md +++ b/packages/context/skills/historic_sql_table_digest/SKILL.md @@ -17,6 +17,34 @@ Use this skill when the WorkUnit raw file is one `tables/..json` f 5. Call `emit_historic_sql_evidence` exactly once with `kind: "table_usage"`. 6. Stop after the evidence tool succeeds. +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Evidence Shape Call `emit_historic_sql_evidence` with this shape: diff --git a/packages/context/skills/knowledge_capture/SKILL.md b/packages/context/skills/knowledge_capture/SKILL.md index 1e6a8f6c..2a111d90 100644 --- a/packages/context/skills/knowledge_capture/SKILL.md +++ b/packages/context/skills/knowledge_capture/SKILL.md @@ -40,6 +40,8 @@ If nothing is worth capturing, respond without calling any tool. 1. Read the wiki index (provided in the prompt) and decide whether the turn introduces durable knowledge. 2. **Before writing**, search for related content so cross-references are accurate: + - `discover_data` first when a page relates to data or SL concepts — find + existing wiki pages, SL sources, and raw warehouse schema together. - `wiki_search` with the topic — find related wiki pages to populate `refs`. - `sl_discover` with the concept — if the page defines a metric (revenue, churn, retention, LTV, ARR, MRR, CAC, attribution, etc.), find matching SL sources or measures to populate `sl_refs`. If no matches, pass `sl_refs: []` so future readers know you checked. 3. If updating an existing page, `wiki_read` it first. Use the returned `structured.content` or markdown body as the exact stored text for targeted replacements; current tags, refs, and sl_refs are returned in structured metadata. @@ -48,6 +50,34 @@ If nothing is worth capturing, respond without calling any tool. For bundle/external ingest, include `rawPaths` on every `wiki_write`/`wiki_remove` call with only the raw files that directly support that wiki action. This keeps ingest provenance tied to the actual source file, not every file in the WorkUnit. +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Keys, summaries, and content - **Keys** are short kebab-case topic identifiers: `leads-source-filter`, `revenue-definition`, `churn-calculation`. No namespacing, no prefixes. @@ -70,6 +100,10 @@ The `wiki_write` tool accepts three array fields that go into the page frontmatt - **`refs`**: keys of related wiki pages. Add when the new page materially depends on concepts from another (e.g., a churn definition that uses the paid-orders filter from a revenue definition). Don't add refs just because pages share a topic area. - **`sl_refs`**: names of SL sources or measures the page relates to. Format: `"source_name"` or `"source_name.measure_name"`. Discover via `sl_discover` → inspect with `sl_read_source` → include the confirmed matches. +Wiki page keys must be flat slugs. Use `large-contract-requesters`, not +`historic-sql/large-contract-requesters`. Use `tags`, `source`, and content +headings for grouping. + ### Replace semantics All three fields use REPLACE semantics on update: diff --git a/packages/context/skills/live_database_ingest/SKILL.md b/packages/context/skills/live_database_ingest/SKILL.md index 9db52484..2b9cb6d8 100644 --- a/packages/context/skills/live_database_ingest/SKILL.md +++ b/packages/context/skills/live_database_ingest/SKILL.md @@ -24,6 +24,37 @@ Use this skill when the ingest work unit contains raw files under or column comments. 9. Run `sl_validate` for the table source before the work unit completes. +Sample values come from the scan record; do not invent values not present in +relationship-profile.json. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Source shape For a raw table with this shape: diff --git a/packages/context/skills/looker_ingest/SKILL.md b/packages/context/skills/looker_ingest/SKILL.md index 462a5910..7a41fa6e 100644 --- a/packages/context/skills/looker_ingest/SKILL.md +++ b/packages/context/skills/looker_ingest/SKILL.md @@ -21,6 +21,37 @@ Looker runtime ingest turns API-staged dashboards, Looks, and explores into dura 9. Write SL from Looker runtime evidence only through the staged warehouse target contract. For explores and inherited dashboard/Look queries, branch on `targetTable.ok`; when it is true, write on `targetWarehouseConnectionId` and use `targetTable.canonicalTable` as `source.table`. When it is false or missing, write wiki knowledge candidates and record `emit_unmapped_fallback` with the staged reason. 10. Run `sl_validate` after every SL write. If validation fails, fix the source or roll it back before the WorkUnit ends. +For every Looker field reference, call entity_details on the underlying +schema.table.column before promoting it to sl_refs or quoting it in wiki body. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Explore WorkUnits Explore WUs have raw files like `explores//.json` and usually depend on `lookml_models.json`. diff --git a/packages/context/skills/lookml_ingest/SKILL.md b/packages/context/skills/lookml_ingest/SKILL.md index 18b43f3e..5a9c79a3 100644 --- a/packages/context/skills/lookml_ingest/SKILL.md +++ b/packages/context/skills/lookml_ingest/SKILL.md @@ -51,10 +51,47 @@ LookML's `dimension_group: date { type: time; timeframes: [raw, date, week, mont A prior replay hallucinated `date_date`, `date_week` into `sql:`, `columns:`, and `grain:` across 4+ standalones; every measure on each affected source returned `400 Unrecognized name: date_date` at query time. Preventable. +Verify each sql_table_name from the LookML view with entity_details before +mapping to an SL source. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + **Required flow before writing any overlay or standalone**: 1. Call `sl_discover()` for each base table you're about to touch. That returns the real columns. -2. If the table isn't in the manifest, fall back to `sql_execution({ sql: "SELECT column_name FROM .INFORMATION_SCHEMA.COLUMNS WHERE table_name = '
'" })` (session shape — a connection is already pinned by the ingest session). +2. If the table isn't in the manifest, use the warehouse `connectionName` + returned by `discover_data` or the target connection chosen from + `sl_discover`, then call a dialect-appropriate SQL probe with that + connection name, for example: + `sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. + Replace `warehouse`, `analytics`, and `orders` with the verified connection, + schema or dataset, and table from the WorkUnit evidence. 3. Use only those names in `sql:`, `columns:`, and `grain:`. Map each `dimension_group` to ONE `{ name: , type: time, role: time }` entry — never one per timeframe. | LookML input | KTX `columns:` entry | diff --git a/packages/context/skills/metabase_ingest/SKILL.md b/packages/context/skills/metabase_ingest/SKILL.md index 061760bf..f5aa00e2 100644 --- a/packages/context/skills/metabase_ingest/SKILL.md +++ b/packages/context/skills/metabase_ingest/SKILL.md @@ -44,6 +44,37 @@ Use `resultMetadata` to: - `lastRunAt`: ISO timestamp of the card's last execution. If null or very old, the card may be dead; prefer skipping over creating a source. - `dashboardCount`: number of dashboards referencing the card. Cards with `dashboardCount: 0` and a stale `lastRunAt` are strong skip signals. +Before writing a wiki page derived from a Metabase question SQL, verify each +schema.table.column mentioned with entity_details. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Decision tree For each card: diff --git a/packages/context/skills/metricflow_ingest/SKILL.md b/packages/context/skills/metricflow_ingest/SKILL.md index 5a24cda8..47187ffb 100644 --- a/packages/context/skills/metricflow_ingest/SKILL.md +++ b/packages/context/skills/metricflow_ingest/SKILL.md @@ -29,6 +29,37 @@ A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to Type map: MetricFlow `time` to KTX `time`; `categorical` to `string`; `number` to `number`; `boolean` to `boolean`. Follow `expr` over `name` when both differ — `expr` is the physical column. +Verify each MetricFlow model source table with entity_details before producing +the corresponding sl_write_source. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Flattening `extends:` Within one WorkUnit, multiple semantic_models linked by `extends:` are guaranteed to be present (the chunker groups them). Resolve inheritance **before** writing: @@ -49,7 +80,13 @@ The `model:` field on a semantic_model is a string like `ref('table_name')`, `so - `source('s','t')` → table name `t`. Verify via `sl_discover(t)`. - Literal (no `ref(...)` / `source(...)`) → treat as the table name directly. -If `sl_discover` errors (no such table), fall back to `sql_execution({ sql: "SELECT column_name FROM .INFORMATION_SCHEMA.COLUMNS WHERE table_name = ''" })` (session shape — a connection is already pinned by the ingest session). **Never invent column names** — every column in `columns:`, `grain:`, and `sql:` must be sourced from a real probe. +If `sl_discover` errors because no such table exists, use `discover_data` and +`entity_details` to find the warehouse target. If a SQL probe is still needed, +call `sql_execution` with the same warehouse connection name, for example: +`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. +**Never invent column names** - every column in `columns:`, `grain:`, and +`sql:` must be sourced from raw files, `entity_details`, or a successful SQL +probe. After every `sl_write_source`, call `sl_validate`. The warehouse will reject invented columns with `Unrecognized name: ` — treat as a hard failure and re-read the schema. diff --git a/packages/context/skills/notion_synthesize/SKILL.md b/packages/context/skills/notion_synthesize/SKILL.md index 933acc55..524c6832 100644 --- a/packages/context/skills/notion_synthesize/SKILL.md +++ b/packages/context/skills/notion_synthesize/SKILL.md @@ -67,10 +67,38 @@ Search existing wiki pages for the same `tables:` or `sl_refs:` frontmatter and - Do not create SL sources under the Notion connection just because a page mentions a warehouse, dbt, Looker, or Metabase object. Use the mapped warehouse/source connection after discovery, or emit an unmapped fallback and write wiki-only. - Distinguish fallback reasons precisely: if a non-Notion warehouse/dbt connection exists but `sl_discover` cannot find the named table/source, use `no_physical_table`; reserve `no_connection_mapping` for cases where there is no plausible non-Notion target connection at all. - If `sl_discover` resolves the table/source, do not call `emit_unmapped_fallback` for that table. Use the resolved source for `sl_refs`, overlay edits, or wiki-only documentation. -- When calling `emit_unmapped_fallback`, pass the table or source identifier as `tableRef` (e.g. `tableRef: "orbit_analytics.customer"`) — the tool generates the canonical detail string from the reason code and `tableRef`. Use the optional `clarification` field only to add context that does not contradict the reason. Do not restate the reason in `clarification`. +- When calling `emit_unmapped_fallback`, pass the table or source identifier as `tableRef` (e.g. `tableRef: ".
"`) — the tool generates the canonical detail string from the reason code and `tableRef`. Use the optional `clarification` field only to add context that does not contradict the reason. Do not restate the reason in `clarification`. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. ## Tools -Allowed: `read_raw_file`, `read_raw_span`, `wiki_search`, `wiki_read`, `wiki_write`, `sl_discover`, `sl_read_source`, `sl_write_source`, `sl_edit_source`, `sl_validate`, `context_evidence_search`, `context_evidence_read`, `context_evidence_neighbors`, `emit_unmapped_fallback`, `eviction_list`, `context_eviction_decision_write`. +Allowed: `read_raw_file`, `read_raw_span`, `wiki_search`, `wiki_read`, `wiki_write`, `discover_data`, `entity_details`, `sql_execution`, `sl_discover`, `sl_read_source`, `sl_write_source`, `sl_edit_source`, `sl_validate`, `context_evidence_search`, `context_evidence_read`, `context_evidence_neighbors`, `emit_unmapped_fallback`, `eviction_list`, `context_eviction_decision_write`. Not allowed: `context_candidate_write`, `context_candidate_mark`. diff --git a/packages/context/skills/sl/SKILL.md b/packages/context/skills/sl/SKILL.md index 2719b9d4..9cdb8b34 100644 --- a/packages/context/skills/sl/SKILL.md +++ b/packages/context/skills/sl/SKILL.md @@ -13,6 +13,10 @@ This skill covers two parts: Capture (when and how to add new patterns to the SL) is a separate concern handled by the memory-agent — see the `sl_capture` skill if you are running in capture mode. The research agent **reads** and **queries** the SL via the tools described here; it does not write to it. +For capture-time identifier verification, load `sl_capture`. Synthesis writer +skills must verify warehouse identifiers with `discover_data`, +`entity_details`, and `sql_execution` before emitting table or column names. + --- ## Part 1 — Schema reference diff --git a/packages/context/skills/sl_capture/SKILL.md b/packages/context/skills/sl_capture/SKILL.md index ffb1780d..4bc383eb 100644 --- a/packages/context/skills/sl_capture/SKILL.md +++ b/packages/context/skills/sl_capture/SKILL.md @@ -174,6 +174,37 @@ Wiki-only is correct when the user is documenting *about* the measure (definition in business terms, owner, policy, glossary, examples of when to use it) without changing its SQL expression or filters. +Before sl_write_source, call entity_details on the target table to confirm +column names and types match the YAML being written. + +## Identifier Verification Protocol + +Before writing a wiki page or SL source on any topic: + +1. `discover_data({query: ""})` - see what wikis, SL sources, and raw + tables already exist. Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, +SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionName, targets: [{display: ""}]})` - + confirm the identifier resolves; inspect native types, FK/PK, and + sampleValues. +3. For literal values from the source, such as status codes or plan tiers, + check whether they appear in `entity_details` sampleValues for the relevant + column. If sampleValues is short or the sample may have missed real values, + run a `sql_execution` probe with the same warehouse connection name: + `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. + If it errors, the identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, + citing the exact raw path that mentioned it. + - When recording `emit_unmapped_fallback` with `no_physical_table`, include + the failing probe error in `clarification`. +5. Never copy `.
` placeholder strings from these instructions + into output. + ## Tool sequence 1. `sl_discover` — see what source files exist. @@ -181,7 +212,7 @@ use it) without changing its SQL expression or filters. 3. `sl_read_source({ sourceName })` — read the raw YAML before editing. 4. For modifications: `sl_edit_source({ sourceName, old_string, new_string })` with exact-string replacements. `old_string` must match exactly and be unique in the file. 5. For new sources or full rewrites: `sl_write_source({ sourceName, content })` with the full YAML content. -6. For join discovery: `sql_execution({ sql })` to verify the join key exists in both tables and assess cardinality before declaring the join. +6. For join discovery: use `sql_execution({connectionName: "warehouse", sql: "SELECT count(*) FROM public.orders o JOIN public.customers c ON c.id = o.customer_id LIMIT 20"})` with the target warehouse connection name and dialect-correct table names to verify the join key exists in both tables and assess cardinality before declaring the join. 7. Cross-reference knowledge: author the edge once on the **wiki** side via `sl_refs: [source_name]` in the page's front-matter. The reverse edge (wiki pages that cite an SL source) is derived automatically by the reconciler — do not add a `knowledge_refs:` field to SL YAMLs. 8. `sl_validate` — run after writing or editing to surface schema issues, duplicate measure names, and cross-source validation errors. Read-only; the writes are already committed (the squash-at-end flow will collapse them into one commit). @@ -248,7 +279,8 @@ Prior turn: user asked to correlate LTV with protocol count; assistant joined `f sl_read_source({ sourceName: "fct_orders" }) → no joins section yet sql_execution({ - sql: "SELECT COUNT(*), COUNT(DISTINCT a.admin_user_id) FROM fct_orders a JOIN fct_mau_multiprotocol b ON a.admin_user_id = b.admin_user_id LIMIT 1" + connectionName: "warehouse", + sql: "SELECT COUNT(*), COUNT(DISTINCT a.admin_user_id) FROM public.fct_orders a JOIN public.fct_mau_multiprotocol b ON a.admin_user_id = b.admin_user_id LIMIT 1" }) → confirms cardinality (many orders per MAU row = many_to_one) sl_edit_source({ diff --git a/packages/context/src/connections/dialects.test.ts b/packages/context/src/connections/dialects.test.ts new file mode 100644 index 00000000..6c9b6c41 --- /dev/null +++ b/packages/context/src/connections/dialects.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { getDialectForDriver } from './dialects.js'; + +describe('getDialectForDriver', () => { + it.each([ + ['postgres', '"public"."orders"'], + ['postgresql', '"public"."orders"'], + ['mysql', '`public`.`orders`'], + ['clickhouse', '`public`.`orders`'], + ['sqlite', '"orders"'], + ['snowflake', '"analytics"."public"."orders"'], + ['bigquery', '`analytics`.`public`.`orders`'], + ['sqlserver', '[analytics].[public].[orders]'], + ] as const)('formats table names for %s', (driver, expected) => { + const dialect = getDialectForDriver(driver); + expect( + dialect.formatTableName({ + catalog: driver === 'snowflake' || driver === 'bigquery' || driver === 'sqlserver' ? 'analytics' : null, + db: driver === 'sqlite' ? null : 'public', + name: 'orders', + }), + ).toBe(expected); + }); + + it('throws with a supported-driver list for unknown drivers', () => { + expect(() => getDialectForDriver('oracle')).toThrow( + 'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver', + ); + }); +}); diff --git a/packages/context/src/connections/dialects.ts b/packages/context/src/connections/dialects.ts new file mode 100644 index 00000000..afac4bd2 --- /dev/null +++ b/packages/context/src/connections/dialects.ts @@ -0,0 +1,102 @@ +import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; + +export type SupportedDriver = + | 'postgres' + | 'postgresql' + | 'mysql' + | 'sqlserver' + | 'snowflake' + | 'bigquery' + | 'clickhouse' + | 'sqlite' + | 'sqlite3'; + +export interface KtxDialect { + readonly type: SupportedDriver; + quoteIdentifier(identifier: string): string; + formatTableName(table: KtxTableRef): string; + mapToDimensionType(nativeType: string): KtxSchemaDimensionType; +} + +const supportedDrivers: SupportedDriver[] = [ + 'bigquery', + 'clickhouse', + 'mysql', + 'postgres', + 'postgresql', + 'sqlite', + 'sqlite3', + 'snowflake', + 'sqlserver', +]; + +function doubleQuoted(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; +} + +function backtickQuoted(identifier: string): string { + return `\`${identifier.replace(/`/g, '``')}\``; +} + +function bigQueryQuoted(identifier: string): string { + return `\`${identifier.replace(/`/g, '\\`')}\``; +} + +function bracketQuoted(identifier: string): string { + return `[${identifier.replace(/\]/g, ']]')}]`; +} + +function inferDimensionType(nativeType: string): KtxSchemaDimensionType { + const normalized = nativeType.toLowerCase().trim(); + if (normalized.includes('date') || normalized.includes('time')) { + return 'time'; + } + if ( + normalized.includes('int') || + normalized.includes('num') || + normalized.includes('dec') || + normalized.includes('float') || + normalized.includes('double') || + normalized.includes('real') + ) { + return 'number'; + } + if (normalized.includes('bool') || normalized === 'bit') { + return 'boolean'; + } + return 'string'; +} + +function formatWithParts(table: KtxTableRef, quote: (identifier: string) => string, sqlite = false): string { + const parts = sqlite ? [table.name] : [table.catalog, table.db, table.name].filter((part): part is string => !!part); + return parts.map(quote).join('.'); +} + +function createDialect(type: SupportedDriver, quote: (identifier: string) => string, sqlite = false): KtxDialect { + return { + type, + quoteIdentifier: quote, + formatTableName: (table) => formatWithParts(table, quote, sqlite), + mapToDimensionType: inferDimensionType, + }; +} + +const dialects: Record = { + postgres: createDialect('postgres', doubleQuoted), + postgresql: createDialect('postgresql', doubleQuoted), + mysql: createDialect('mysql', backtickQuoted), + clickhouse: createDialect('clickhouse', backtickQuoted), + sqlite: createDialect('sqlite', doubleQuoted, true), + sqlite3: createDialect('sqlite3', doubleQuoted, true), + snowflake: createDialect('snowflake', doubleQuoted), + bigquery: createDialect('bigquery', bigQueryQuoted), + sqlserver: createDialect('sqlserver', bracketQuoted), +}; + +export function getDialectForDriver(driver: string): KtxDialect { + const normalized = driver.toLowerCase().trim(); + if (normalized in dialects) { + return dialects[normalized as SupportedDriver]; + } + throw new Error(`Unsupported warehouse driver "${driver}". Supported drivers: ${supportedDrivers.join(', ')}`); +} diff --git a/packages/context/src/connections/index.ts b/packages/context/src/connections/index.ts index 513818fa..0917a7ca 100644 --- a/packages/context/src/connections/index.ts +++ b/packages/context/src/connections/index.ts @@ -3,7 +3,9 @@ export type { KtxSqlQueryExecutionResult, KtxSqlQueryExecutorPort, } from './query-executor.js'; +export type { KtxDialect, SupportedDriver } from './dialects.js'; export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js'; +export { getDialectForDriver } from './dialects.js'; export { normalizeQueryRows } from './query-executor.js'; export { createPostgresQueryExecutor } from './postgres-query-executor.js'; export { assertReadOnlySql, limitSqlForExecution } from './read-only-sql.js'; diff --git a/packages/context/src/core/git.service.test.ts b/packages/context/src/core/git.service.test.ts index 14e93495..ba1d9e0f 100644 --- a/packages/context/src/core/git.service.test.ts +++ b/packages/context/src/core/git.service.test.ts @@ -379,5 +379,37 @@ describe('GitService', () => { await service.removeWorktree(wtDir).catch(() => undefined); await rm(wtDir, { recursive: true, force: true }).catch(() => undefined); }); + + it('reports untracked files that would be overwritten by the squash merge', async () => { + const { commitHash: baseSha } = await writeAndCommit('seed.md', 'seed'); + const parent = await realpath(join(tempDir, '..')); + const wtDir = join(parent, `wt-${Date.now()}-untracked`); + await service.addWorktree(wtDir, 'session/untracked', baseSha); + + const scoped = service.forWorktree(wtDir); + await writeFile(join(wtDir, 'knowledge.md'), 'session version\n', 'utf-8'); + await scoped.commitFile('knowledge.md', 'session write', 'System User', 'system@example.com'); + await writeFile(join(tempDir, 'knowledge.md'), 'untracked local version\n', 'utf-8'); + + const result = await service.squashMergeIntoMain( + 'session/untracked', + 'System User', + 'system@example.com', + 'Memory capture: 1 file [chat=untracked]', + ); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error('unreachable'); + } + expect(result.conflict).toBe(true); + expect(result.conflictPaths).toEqual(['knowledge.md']); + + const status = await (service as unknown as { git: import('simple-git').SimpleGit }).git.status(); + expect(status.not_added).toContain('knowledge.md'); + + await service.removeWorktree(wtDir).catch(() => undefined); + await rm(wtDir, { recursive: true, force: true }).catch(() => undefined); + }); }); }); diff --git a/packages/context/src/core/git.service.ts b/packages/context/src/core/git.service.ts index 6539f9fd..8d05a089 100644 --- a/packages/context/src/core/git.service.ts +++ b/packages/context/src/core/git.service.ts @@ -31,6 +31,40 @@ export type SquashMergeResult = | { ok: true; squashSha: string; touchedPaths: string[] } | { ok: false; conflict: true; conflictPaths: string[] }; +function mergeErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return String(error); +} + +function extractUntrackedOverwritePaths(message: string): string[] { + const marker = 'The following untracked working tree files would be overwritten by merge:'; + const markerIndex = message.indexOf(marker); + if (markerIndex === -1) { + return []; + } + + const afterMarker = message.slice(markerIndex + marker.length); + const abortIndex = afterMarker.indexOf('Please move or remove them before you merge.'); + const pathBlock = abortIndex === -1 ? afterMarker : afterMarker.slice(0, abortIndex); + return pathBlock + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0 && line !== 'Aborting') + .map((line) => line.replace(/^"(.+)"$/, '$1')); +} + +function mergeConflictPaths(unmergedPaths: string[], mergeError: unknown): string[] { + const paths = new Set(unmergedPaths); + if (mergeError !== null) { + for (const path of extractUntrackedOverwritePaths(mergeErrorMessage(mergeError))) { + paths.add(path); + } + } + return [...paths]; +} + export class GitService { private static readonly mutationQueues = new Map>(); @@ -639,10 +673,11 @@ export class GitService { } const unmergedOut = await this.git.raw(['diff', '--name-only', '--diff-filter=U']).catch(() => ''); - const conflictPaths = unmergedOut + const unmergedPaths = unmergedOut .split('\n') .map((l) => l.trim()) .filter(Boolean); + const conflictPaths = mergeConflictPaths(unmergedPaths, mergeError); if (conflictPaths.length > 0 || mergeError !== null) { // `merge --abort` only works for an in-progress merge; squash sets MERGE_MSG but not @@ -651,7 +686,7 @@ export class GitService { await this.git.raw(['reset', '--hard', 'HEAD']).catch(() => undefined); this.logger.warn( `squashMergeIntoMain: conflict merging ${branch} — aborted. conflictPaths=${conflictPaths.join(',')}` + - (mergeError ? ` error=${mergeError instanceof Error ? mergeError.message : String(mergeError)}` : ''), + (mergeError ? ` error=${mergeErrorMessage(mergeError)}` : ''), ); return { ok: false, conflict: true, conflictPaths }; } diff --git a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index 22f35cfc..c7a334bf 100644 --- a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -277,7 +277,7 @@ describe('historic-SQL local ingest retrieval acceptance', () => { await expect(readFile(join(project.projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves .toContain('Analysts repeatedly inspect paid order lifecycle by customer segment.'); - await expect(readFile(join(project.projectDir, 'knowledge/global/historic-sql/paid-order-lifecycle.md'), 'utf-8')) + await expect(readFile(join(project.projectDir, 'knowledge/global/historic-sql-paid-order-lifecycle.md'), 'utf-8')) .resolves.toContain('Paid Order Lifecycle'); const reloaded = await loadKtxProject({ projectDir: project.projectDir }); @@ -295,7 +295,7 @@ describe('historic-SQL local ingest retrieval acceptance', () => { searchLocalKnowledgePages(reloaded, { query: 'paid order lifecycle', userId: 'local', limit: 5 }), ).resolves.toEqual([ expect.objectContaining({ - key: 'historic-sql/paid-order-lifecycle', + key: 'historic-sql-paid-order-lifecycle', summary: 'Paid Order Lifecycle', matchReasons: expect.arrayContaining(['lexical']), }), diff --git a/packages/context/src/ingest/adapters/historic-sql/post-processor.ts b/packages/context/src/ingest/adapters/historic-sql/post-processor.ts index 815b6798..8d89d397 100644 --- a/packages/context/src/ingest/adapters/historic-sql/post-processor.ts +++ b/packages/context/src/ingest/adapters/historic-sql/post-processor.ts @@ -10,7 +10,7 @@ async function commitProjectionChanges(workdir: string): Promise { const status = await git.status(); const paths = status.files .map((file) => file.path) - .filter((path) => path.startsWith('semantic-layer/') || path.startsWith('knowledge/global/historic-sql/')); + .filter((path) => path.startsWith('semantic-layer/') || path.startsWith('knowledge/global/historic-sql')); if (paths.length === 0) { return; } diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts b/packages/context/src/ingest/adapters/historic-sql/projection.test.ts index e6cb736a..f2a5b068 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/projection.test.ts @@ -106,7 +106,7 @@ describe('projectHistoricSqlEvidence', () => { await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.customers.json', { table: 'public.customers' }); await writeText( workdir, - 'knowledge/global/historic-sql/old-order-lifecycle.md', + 'knowledge/global/historic-sql-old-order-lifecycle.md', [ '---', YAML.stringify({ @@ -127,7 +127,7 @@ describe('projectHistoricSqlEvidence', () => { ); await writeText( workdir, - 'knowledge/global/historic-sql/retired-pattern.md', + 'knowledge/global/historic-sql-retired-pattern.md', [ '---', YAML.stringify({ @@ -164,15 +164,15 @@ describe('projectHistoricSqlEvidence', () => { const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); expect(result.patternPagesWritten).toBe(1); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/old-order-lifecycle.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'knowledge/global/historic-sql-old-order-lifecycle.md'), 'utf-8')).resolves.toContain( 'Order Lifecycle Analysis', ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/retired-pattern.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'knowledge/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain( 'stale_since: "2026-05-11T00:00:00.000Z"', ); }); - it('writes a reappearing pattern to the active slug instead of reusing an archived page key', async () => { + it('rewrites a reappearing archived pattern at the flat slug', async () => { const workdir = await tempWorkdir(); await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { source: 'historic-sql', @@ -192,7 +192,7 @@ describe('projectHistoricSqlEvidence', () => { await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.customers.json', { table: 'public.customers' }); await writeText( workdir, - 'knowledge/global/historic-sql/_archived/order-lifecycle-analysis.md', + 'knowledge/global/historic-sql-order-lifecycle-analysis.md', [ '---', YAML.stringify({ @@ -230,15 +230,10 @@ describe('projectHistoricSqlEvidence', () => { const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); expect(result.patternPagesWritten).toBe(1); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/order-lifecycle-analysis.md'), 'utf-8')).resolves.toContain( - 'Order Lifecycle Analysis', - ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/_archived/order-lifecycle-analysis.md'), 'utf-8')).resolves.toContain( - 'Archived body', - ); - await expect( - readFile(join(workdir, 'knowledge/global/historic-sql/_archived/_archived/order-lifecycle-analysis.md'), 'utf-8'), - ).rejects.toMatchObject({ code: 'ENOENT' }); + const page = await readFile(join(workdir, 'knowledge/global/historic-sql-order-lifecycle-analysis.md'), 'utf-8'); + expect(page).toContain('Analysts compare order status with customer segment again.'); + expect(page).not.toContain('Archived body'); + expect(page).not.toContain('archived'); }); it('leaves already archived pattern pages stable when they are still absent', async () => { @@ -259,7 +254,7 @@ describe('projectHistoricSqlEvidence', () => { }); await writeText( workdir, - 'knowledge/global/historic-sql/_archived/retired-pattern.md', + 'knowledge/global/historic-sql-retired-pattern.md', [ '---', YAML.stringify({ @@ -284,12 +279,9 @@ describe('projectHistoricSqlEvidence', () => { expect(result.archivedPatternPages).toBe(0); expect(result.stalePatternPagesMarked).toBe(0); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/_archived/retired-pattern.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'knowledge/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain( 'Archived retired body', ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/_archived/_archived/retired-pattern.md'), 'utf-8')).rejects.toMatchObject({ - code: 'ENOENT', - }); }); it('marks missing table usage stale and deletes legacy historic SQL query pages', async () => { @@ -330,7 +322,7 @@ describe('projectHistoricSqlEvidence', () => { }); await writeText( workdir, - 'knowledge/global/historic-sql/legacy-template.md', + 'knowledge/global/historic-sql-legacy-template.md', [ '---', YAML.stringify({ @@ -365,7 +357,7 @@ describe('projectHistoricSqlEvidence', () => { commonJoins: [], staleSince: '2026-05-11T00:00:00.000Z', }); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/legacy-template.md'), 'utf-8')).rejects.toMatchObject({ + await expect(readFile(join(workdir, 'knowledge/global/historic-sql-legacy-template.md'), 'utf-8')).rejects.toMatchObject({ code: 'ENOENT', }); }); diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.ts b/packages/context/src/ingest/adapters/historic-sql/projection.ts index 366b98f3..25a317f3 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.ts +++ b/packages/context/src/ingest/adapters/historic-sql/projection.ts @@ -37,7 +37,7 @@ interface HistoricSqlPatternPage { } function safeKnowledgeSlug(value: string): string { - return value.toLowerCase().replace(/[^a-z0-9/-]+/g, '-').replace(/^-+|-+$/g, ''); + return value.toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, ''); } async function pathExists(path: string): Promise { @@ -159,7 +159,7 @@ function isLegacyQueryPage(page: HistoricSqlPatternPage): boolean { function isArchivedPatternPage(page: HistoricSqlPatternPage): boolean { const tags = Array.isArray(page.frontmatter.tags) ? page.frontmatter.tags : []; - return page.key.startsWith('_archived/') || tags.includes('archived'); + return tags.includes('archived'); } function stringArray(value: unknown): string[] { @@ -191,6 +191,9 @@ async function loadPatternPages(root: string): Promise const files = await walkFiles(root); const pages: HistoricSqlPatternPage[] = []; for (const file of files.filter((candidate) => candidate.endsWith('.md'))) { + if (file.includes('/')) { + continue; + } const key = file.replace(/\.md$/, ''); const path = join(root, file); const page = parseMarkdownPage(key, path, await readFile(path, 'utf-8')); @@ -201,6 +204,10 @@ async function loadPatternPages(root: string): Promise return pages; } +function historicSqlFlatKey(slug: string): string { + return `historic-sql-${safeKnowledgeSlug(slug)}`; +} + async function currentStagedTables(rawDir: string): Promise> { const tablesRoot = join(rawDir, 'tables'); const files = await walkFiles(tablesRoot); @@ -276,7 +283,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp } } - const wikiRoot = join(input.workdir, 'knowledge/global/historic-sql'); + const wikiRoot = join(input.workdir, 'knowledge/global'); await mkdir(wikiRoot, { recursive: true }); const allPages = await loadPatternPages(wikiRoot); const activePages = allPages.filter((page) => !isArchivedPatternPage(page)); @@ -286,7 +293,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp for (const pattern of patternEvidence) { const incomingSignals = [...pattern.pattern.tablesInvolved, ...pattern.pattern.constituentTemplateIds]; const reusable = patternPages.find((page) => overlapRatio(incomingSignals, existingPageSignals(page)) >= 0.6); - const key = reusable?.key ?? safeKnowledgeSlug(pattern.pattern.slug); + const key = reusable?.key ?? historicSqlFlatKey(pattern.pattern.slug); const pagePath = join(wikiRoot, `${key}.md`); const frontmatter = { summary: pattern.pattern.title, @@ -308,11 +315,12 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp for (const page of patternPages) { if (writtenKeys.has(page.key)) continue; if (shouldArchive(page.frontmatter.stale_since, manifest.fetchedAt, manifest.staleArchiveAfterDays)) { - const archivePath = join(wikiRoot, '_archived', `${page.key}.md`); const tags = [...new Set([...stringArray(page.frontmatter.tags), 'archived'])]; - await mkdir(dirname(archivePath), { recursive: true }); - await writeFile(archivePath, renderMarkdownPage({ ...page.frontmatter, tags }, page.content), 'utf-8'); - await rm(page.path, { force: true }); + await writeFile( + page.path, + renderMarkdownPage({ ...page.frontmatter, tags, archived_since: manifest.fetchedAt }, page.content), + 'utf-8', + ); result.archivedPatternPages += 1; continue; } diff --git a/packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts b/packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts index 32564da1..d22597b9 100644 --- a/packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts +++ b/packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts @@ -15,6 +15,18 @@ describe('LookmlSourceAdapter validation sidecars', () => { afterEach(async () => rm(tmpRoot, { recursive: true, force: true })); + it('returns configured target warehouse connection ids', async () => { + const adapter = new LookmlSourceAdapter({ + homeDir: join(tmpRoot, 'home'), + targetConnectionIds: ['warehouse', 'analytics', 'warehouse'], + }); + + await expect(adapter.listTargetConnectionIds?.(join(tmpRoot, 'staged'))).resolves.toEqual([ + 'analytics', + 'warehouse', + ]); + }); + it('writes a partial fetch report and marks mismatched chunks as SL-disallowed', async () => { const originRoot = join(tmpRoot, 'origin-src'); await mkdir(join(originRoot, 'views'), { recursive: true }); diff --git a/packages/context/src/ingest/adapters/lookml/lookml.adapter.ts b/packages/context/src/ingest/adapters/lookml/lookml.adapter.ts index 9978ddd4..3ea7b9a6 100644 --- a/packages/context/src/ingest/adapters/lookml/lookml.adapter.ts +++ b/packages/context/src/ingest/adapters/lookml/lookml.adapter.ts @@ -14,6 +14,11 @@ import { parseLookmlPullConfig } from './pull-config.js'; export interface LookmlSourceAdapterDeps { homeDir: string; + targetConnectionIds?: string[]; +} + +function uniqueSorted(values: readonly string[] | undefined): string[] { + return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); } export class LookmlSourceAdapter implements SourceAdapter { @@ -43,6 +48,10 @@ export class LookmlSourceAdapter implements SourceAdapter { return readLookmlFetchReport(stagedDir); } + async listTargetConnectionIds(_stagedDir: string): Promise { + return uniqueSorted(this.deps.targetConnectionIds); + } + async chunk(stagedDir: string, diffSet?: DiffSet): Promise { const project = await parseLookmlStagedDir(stagedDir); const mismatchedModelNames = await readLookmlMismatchedModelNames(stagedDir); diff --git a/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts b/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts index af2e409f..19bb6cdc 100644 --- a/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts @@ -42,6 +42,15 @@ describe('MetricflowSourceAdapter', () => { expect(adapter.skillNames).toEqual(['metricflow_ingest']); }); + it('returns configured target warehouse connection ids', async () => { + const metricflow = new MetricflowSourceAdapter({ + homeDir: join(tmpRoot, 'cache-home'), + targetConnectionIds: ['warehouse', 'analytics', 'warehouse'], + }); + + await expect(metricflow.listTargetConnectionIds?.(stagedDir)).resolves.toEqual(['analytics', 'warehouse']); + }); + it('detects a staged dir with a semantic_models YAML', async () => { await mkdir(join(stagedDir, 'models'), { recursive: true }); await writeFile( diff --git a/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts b/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts index a465a973..c8182ed8 100644 --- a/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts +++ b/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts @@ -9,6 +9,11 @@ import { parseMetricflowPullConfig } from './pull-config.js'; export interface MetricflowSourceAdapterDeps { homeDir: string; + targetConnectionIds?: string[]; +} + +function uniqueSorted(values: readonly string[] | undefined): string[] { + return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); } export class MetricflowSourceAdapter implements SourceAdapter { @@ -30,6 +35,10 @@ export class MetricflowSourceAdapter implements SourceAdapter { }); } + async listTargetConnectionIds(_stagedDir: string): Promise { + return uniqueSorted(this.deps.targetConnectionIds); + } + async chunk(stagedDir: string, diffSet?: DiffSet): Promise { const project = await parseMetricFlowStagedDir(stagedDir); const chunk = await chunkMetricFlowProject(project, { diffSet }); diff --git a/packages/context/src/ingest/adapters/notion/chunk.ts b/packages/context/src/ingest/adapters/notion/chunk.ts index 7d85fb76..bdae8622 100644 --- a/packages/context/src/ingest/adapters/notion/chunk.ts +++ b/packages/context/src/ingest/adapters/notion/chunk.ts @@ -8,7 +8,7 @@ const MAX_NOTION_WORK_UNIT_CHARS = 40_000; export const NOTION_ORG_KNOWLEDGE_WARNING = 'Anything accessible to this Notion integration can become organization knowledge.'; const NOTION_SL_WRITE_GUIDANCE = - 'Write wiki entries with wiki_write. Wiki keys must be flat slugs like orbit-company-overview, not orbit/company-overview. Search existing wiki pages for the same tables or sl_refs before creating a new page. Only write or edit SL sources after sl_discover/sl_read_source confirms a mapped non-Notion target source; if no mapped target exists, emit_unmapped_fallback and keep the fact wiki-only. Notion dataSourceCount counts Notion databases/data sources only, not warehouse/dbt mappings. If a warehouse/dbt connection exists but the named table or source is absent, use reason no_physical_table rather than no_connection_mapping. Do not create SL sources under the Notion connection just because a page mentions a warehouse table.'; + 'Write wiki entries with wiki_write. Wiki keys must be flat slugs like orbit-company-overview, not orbit/company-overview. Search existing wiki pages, SL sources, and raw warehouse schema for the same tables or sl_refs with discover_data before creating a new page. Only write or edit SL sources after discover_data plus sl_discover/sl_read_source or entity_details confirms a mapped non-Notion target source; if no mapped target exists, emit_unmapped_fallback and keep the fact wiki-only. Notion dataSourceCount counts Notion databases/data sources only, not warehouse/dbt mappings. If a warehouse/dbt connection exists but the named table or source is absent, use reason no_physical_table rather than no_connection_mapping. Do not create SL sources under the Notion connection just because a page mentions a warehouse table.'; async function walk(root: string): Promise { const entries = await readdir(root, { withFileTypes: true, recursive: true }); @@ -117,7 +117,7 @@ export async function chunkNotionStagedDir(stagedDir: string, diffSet?: DiffSet) reconcileNotes: [ `Notion maxKnowledgeCreatesPerRun=${manifest.maxKnowledgeCreatesPerRun}`, `Notion maxKnowledgeUpdatesPerRun=${manifest.maxKnowledgeUpdatesPerRun}`, - 'Notion dataSourceCount is Notion-only; use sl_discover for warehouse/dbt mapping decisions.', + 'Notion dataSourceCount is Notion-only; use discover_data/entity_details for warehouse/dbt mapping decisions.', 'Reconcile Notion wiki pages sharing tables/sl_refs before creating distinct artifacts.', ], contextReport: { diff --git a/packages/context/src/ingest/adapters/notion/notion.adapter.test.ts b/packages/context/src/ingest/adapters/notion/notion.adapter.test.ts index de989d02..0f500d5e 100644 --- a/packages/context/src/ingest/adapters/notion/notion.adapter.test.ts +++ b/packages/context/src/ingest/adapters/notion/notion.adapter.test.ts @@ -52,6 +52,14 @@ describe('NotionSourceAdapter', () => { expect(adapter.triageSupported).toBe(true); }); + it('returns configured target warehouse connection ids', async () => { + const adapter = new NotionSourceAdapter({ + targetConnectionIds: ['warehouse', 'warehouse', 'analytics'], + }); + + await expect(adapter.listTargetConnectionIds?.(stagedDir)).resolves.toEqual(['analytics', 'warehouse']); + }); + it('returns structural triage signals for a staged Notion page', async () => { await mkdir(join(stagedDir, 'pages', 'page-1'), { recursive: true }); await writeFile( @@ -242,6 +250,8 @@ describe('NotionSourceAdapter', () => { }); expect(result.workUnits[0].notes).toContain('Synthesize durable wiki and SL knowledge'); expect(result.workUnits[0].notes).toContain('emit_unmapped_fallback'); + expect(result.workUnits[0].notes).toContain('discover_data'); + expect(result.workUnits[0].notes).toContain('entity_details'); expect(result.workUnits[0].notes).toContain('use reason no_physical_table rather than no_connection_mapping'); expect(result.workUnits[0].notes).toContain('Do not create SL sources under the Notion connection'); expect(result.workUnits[0].notes).toContain( @@ -250,7 +260,7 @@ describe('NotionSourceAdapter', () => { expect(result.reconcileNotes).toEqual([ 'Notion maxKnowledgeCreatesPerRun=25', 'Notion maxKnowledgeUpdatesPerRun=20', - 'Notion dataSourceCount is Notion-only; use sl_discover for warehouse/dbt mapping decisions.', + 'Notion dataSourceCount is Notion-only; use discover_data/entity_details for warehouse/dbt mapping decisions.', 'Reconcile Notion wiki pages sharing tables/sl_refs before creating distinct artifacts.', ]); expect(result.contextReport).toEqual({ capped: false, warnings: [NOTION_ORG_KNOWLEDGE_WARNING] }); diff --git a/packages/context/src/ingest/adapters/notion/notion.adapter.ts b/packages/context/src/ingest/adapters/notion/notion.adapter.ts index fba68cee..b903d4b3 100644 --- a/packages/context/src/ingest/adapters/notion/notion.adapter.ts +++ b/packages/context/src/ingest/adapters/notion/notion.adapter.ts @@ -32,6 +32,11 @@ interface NotionPullSucceededContext { export interface NotionSourceAdapterDeps { onPullSucceeded?: (ctx: NotionPullSucceededContext) => Promise; logger?: NotionFetchLogger; + targetConnectionIds?: string[]; +} + +function uniqueSorted(values: readonly string[] | undefined): string[] { + return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); } export class NotionSourceAdapter implements SourceAdapter { @@ -73,6 +78,10 @@ export class NotionSourceAdapter implements SourceAdapter { return describeNotionScope(stagedDir); } + async listTargetConnectionIds(_stagedDir: string): Promise { + return uniqueSorted(this.deps.targetConnectionIds); + } + async getTriageSignals(stagedDir: string, externalId: string): Promise { const metadata = await this.findMetadataByExternalId(stagedDir, externalId); if (!metadata) { diff --git a/packages/context/src/ingest/ingest-bundle.runner.test.ts b/packages/context/src/ingest/ingest-bundle.runner.test.ts index a4513f63..6134fbe7 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.test.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.test.ts @@ -409,6 +409,38 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { ); }); + it('threads target warehouse connection names into WorkUnit and reconcile tool sessions', async () => { + const deps = makeDeps(); + const sessions: any[] = []; + deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse']); + deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession: any) => { + sessions.push(toolSession); + return { + toAiSdkTools: vi.fn().mockReturnValue({}), + getAllTools: vi.fn().mockReturnValue([]), + getToolNames: vi.fn().mockReturnValue([]), + }; + }); + deps.agentRunner.runLoop.mockResolvedValue({ stopReason: 'natural' }); + + const runner = buildRunner(deps); + (runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({ + currentHashes: new Map([['a.yml', 'h1']]), + rawDirInWorktree: 'raw-sources/notion/fake/s', + }); + (runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x'); + + await runner.run({ + jobId: 'j1', + connectionId: 'notion', + sourceKey: 'fake', + trigger: 'upload', + bundleRef: { kind: 'upload', uploadId: 'upload-x' }, + }); + + expect([...sessions[0].allowedConnectionNames].sort()).toEqual(['notion', 'warehouse']); + }); + it('reuses document evidence indexing and page triage for document WorkUnits', async () => { const deps = makeDeps(); deps.adapter.source = 'notion'; @@ -643,6 +675,14 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { }); } if (params.telemetryTags.operationName === 'ingest-bundle-reconcile') { + await params.toolSet.record_verification_ledger.execute( + { + summary: 'Reconciliation emits no warehouse identifiers before fallback recording.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 'ledger-1', messages: [] }, + ); await params.toolSet.emit_conflict_resolution.execute( { kind: 'near_duplicate', @@ -811,6 +851,14 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { { path: 'a.yml', startLine: 1, endLine: 2 }, { toolCallId: 'read-1', messages: [] }, ); + await params.toolSet.record_verification_ledger.execute( + { + summary: 'Wiki write contains no warehouse identifiers.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 'ledger-1', messages: [] }, + ); await params.toolSet.wiki_write.execute( { key: 'knowledge/a.md', content: 'safe summary' }, { toolCallId: 'wiki-1', messages: [] }, @@ -850,9 +898,9 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { { unitKey: 'u1', path: '/tmp/ktx-test/run/wu-transcripts/j1/u1.jsonl', - toolCallCount: 2, + toolCallCount: 3, errorCount: 0, - toolNames: ['read_raw_span', 'wiki_write'], + toolNames: ['read_raw_span', 'record_verification_ledger', 'wiki_write'], }, ], }), @@ -864,6 +912,14 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { const deps = makeDeps(); deps.agentRunner.runLoop.mockImplementation(async (params: any) => { if (params.telemetryTags.operationName === 'ingest-bundle-wu') { + await params.toolSet.record_verification_ledger.execute( + { + summary: 'Unmapped fallback records an unsupported conversion metric without verified warehouse identifiers.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 'ledger-1', messages: [] }, + ); await params.toolSet.emit_unmapped_fallback.execute( { rawPath: 'a.yml', @@ -920,6 +976,14 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { }); deps.agentRunner.runLoop.mockImplementation(async (params: any) => { if (params.telemetryTags.operationName === 'ingest-bundle-reconcile') { + await params.toolSet.record_verification_ledger.execute( + { + summary: 'Reconciliation records conflict, eviction, and fallback decisions without warehouse identifiers.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 'ledger-1', messages: [] }, + ); await params.toolSet.emit_conflict_resolution.execute( { kind: 'near_duplicate', diff --git a/packages/context/src/ingest/ingest-bundle.runner.ts b/packages/context/src/ingest/ingest-bundle.runner.ts index 31b444bf..d8f47c2a 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.ts @@ -53,6 +53,7 @@ import type { UnresolvedCardInfo, WorkUnit, } from './types.js'; +import { repairWikiSlRefs, type WikiSlRefRepairResult } from './wiki-sl-ref-repair.js'; function workUnitToMemoryFlowPlannedWorkUnit(workUnit: WorkUnit): MemoryFlowPlannedWorkUnit { return { @@ -528,6 +529,7 @@ export class IngestBundleRunner { let sourceContextReport: { capped?: boolean; warnings?: string[] } | undefined; let parseArtifacts: unknown; let postProcessorOutcome: IngestReportPostProcessorOutcome | undefined; + let wikiSlRefRepairResult: WikiSlRefRepairResult | null = null; let reconcileNotes: string[] = []; let triageResult: PageTriageRunResult | null = null; if (overrideReport) { @@ -662,6 +664,7 @@ export class IngestBundleRunner { touchedSlSources: session.touchedSlSources, actions: sessionActions, allowedRawPaths: new Set(wu.rawFiles), + allowedConnectionNames: new Set(slConnectionIds), semanticLayerService: scopedSemanticLayerService, wikiService: scopedWikiService, configService: sessionWorktree.config, @@ -898,6 +901,7 @@ export class IngestBundleRunner { touchedSlSources: reconcileSession.touchedSlSources, actions: reconcileActions, allowedRawPaths: reconciliationAllowedRawPaths, + allowedConnectionNames: new Set(slConnectionIds), semanticLayerService: rcScopedSl, wikiService: rcScopedWiki, configService: sessionWorktree.config, @@ -1138,6 +1142,19 @@ export class IngestBundleRunner { } } + const repairConnectionIds = [ + ...new Set([ + ...slConnectionIds, + ...(postProcessorOutcome?.touchedSources ?? []).map((source) => source.connectionId), + ]), + ].sort(); + wikiSlRefRepairResult = await repairWikiSlRefs({ + wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), + semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), + configService: sessionWorktree.config, + connectionIds: repairConnectionIds, + }); + // Stage 6 — squash commit const stage6 = ctx?.startPhase(0.04); await stage6?.updateProgress(0.0, 'Saving changes'); @@ -1354,6 +1371,8 @@ export class IngestBundleRunner { provenanceRows: reportProvenanceRows, toolTranscripts: reportToolTranscripts, postProcessor: postProcessorOutcome, + wikiSlRefRepairs: wikiSlRefRepairResult.repairs, + wikiSlRefRepairWarnings: wikiSlRefRepairResult.warnings, ...(reportMemoryFlow ? { memoryFlow: reportMemoryFlow } : {}), context: contextReport ? { diff --git a/packages/context/src/ingest/ingest-prompts.test.ts b/packages/context/src/ingest/ingest-prompts.test.ts index 43985ee9..8adcbfef 100644 --- a/packages/context/src/ingest/ingest-prompts.test.ts +++ b/packages/context/src/ingest/ingest-prompts.test.ts @@ -29,6 +29,18 @@ describe('ingest prompt assets', () => { expect(prompt).not.toMatch(forbiddenProductPattern()); }); + it('uses shipped warehouse verification tools in the WorkUnit prompt', async () => { + const prompt = await readFile( + new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), + 'utf-8', + ); + + expect(prompt).toContain('discover_data'); + expect(prompt).toContain('entity_details'); + expect(prompt).not.toContain('wiki_sl_search'); + expect(prompt).not.toContain('sl_describe_table'); + }); + it('does not route historic-SQL through page-triage prompt examples', async () => { const prompt = await readFile(new URL('../../prompts/skills/page_triage_classifier.md', import.meta.url), 'utf-8'); diff --git a/packages/context/src/ingest/ingest-runtime-assets.test.ts b/packages/context/src/ingest/ingest-runtime-assets.test.ts index 2fafd69b..4b75fcdf 100644 --- a/packages/context/src/ingest/ingest-runtime-assets.test.ts +++ b/packages/context/src/ingest/ingest-runtime-assets.test.ts @@ -91,4 +91,14 @@ describe('ingest runtime assets', () => { expect(body).toContain('cross-table'); expect(body).not.toMatch(forbiddenProductPattern()); }); + + it('packages identifier verification prompt assets', async () => { + const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); + expect(shared).toContain('## Identifier Verification Protocol'); + expect(shared).toContain('discover_data'); + expect(shared).toContain('entity_details'); + expect(shared).toContain('sql_execution'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); + }); }); diff --git a/packages/context/src/ingest/local-adapters.test.ts b/packages/context/src/ingest/local-adapters.test.ts index 141827be..ad3b23f4 100644 --- a/packages/context/src/ingest/local-adapters.test.ts +++ b/packages/context/src/ingest/local-adapters.test.ts @@ -498,6 +498,60 @@ describe('local ingest adapters', () => { await expect(adapter?.listTargetConnectionIds?.('/tmp/staged-dbt')).resolves.toEqual(['warehouse']); }); + it('passes primary warehouse connection ids to the local Notion adapter', async () => { + const adapters = createDefaultLocalIngestAdapters( + projectWithConnections({ + notion: { + driver: 'notion', + auth_token: 'secret', + crawl_mode: 'selected_roots', + root_page_ids: ['page-1'], + }, + warehouse: { + driver: 'postgres', + url: 'postgresql://readonly@db.example.test/analytics', + }, + docs: { + driver: 'dbt', + source_dir: './dbt', + }, + } as never), + ); + + const notion = adapters.find((adapter) => adapter.source === 'notion'); + + await expect(notion?.listTargetConnectionIds?.('/tmp/staged-notion')).resolves.toEqual(['warehouse']); + }); + + it('passes primary warehouse connection ids to local LookML and MetricFlow adapters', async () => { + const adapters = createDefaultLocalIngestAdapters( + projectWithConnections({ + warehouse: { + driver: 'postgres', + url: 'postgresql://readonly@db.example.test/analytics', + }, + lookml_docs: { + driver: 'lookml', + lookml: { + repoUrl: 'https://github.com/acme/lookml.git', + }, + }, + metrics_repo: { + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/metrics.git', + }, + }, + } as never), + ); + + const lookml = adapters.find((adapter) => adapter.source === 'lookml'); + const metricflow = adapters.find((adapter) => adapter.source === 'metricflow'); + + await expect(lookml?.listTargetConnectionIds?.('/tmp/staged-lookml')).resolves.toEqual(['warehouse']); + await expect(metricflow?.listTargetConnectionIds?.('/tmp/staged-metricflow')).resolves.toEqual(['warehouse']); + }); + it('resolves MetricFlow auth_token_ref without writing literal tokens to config', async () => { const project = projectWithConnections({ metricflow_main: { diff --git a/packages/context/src/ingest/local-adapters.ts b/packages/context/src/ingest/local-adapters.ts index 59daf6d1..533bd526 100644 --- a/packages/context/src/ingest/local-adapters.ts +++ b/packages/context/src/ingest/local-adapters.ts @@ -88,7 +88,10 @@ export function createDefaultLocalIngestAdapters( ...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}), }), }), - new LookmlSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }), + new LookmlSourceAdapter({ + homeDir: join(project.projectDir, '.ktx/cache'), + targetConnectionIds: primaryWarehouseConnectionIds(project), + }), new DbtSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache'), targetConnectionIds: primaryWarehouseConnectionIds(project), @@ -106,8 +109,12 @@ export function createDefaultLocalIngestAdapters( }, }, }), - new MetricflowSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }), + new MetricflowSourceAdapter({ + homeDir: join(project.projectDir, '.ktx/cache'), + targetConnectionIds: primaryWarehouseConnectionIds(project), + }), new NotionSourceAdapter({ + targetConnectionIds: primaryWarehouseConnectionIds(project), ...(options.logger ? { logger: options.logger } : {}), }), ]; diff --git a/packages/context/src/ingest/local-bundle-ingest.test.ts b/packages/context/src/ingest/local-bundle-ingest.test.ts index f631e6ed..fe781b33 100644 --- a/packages/context/src/ingest/local-bundle-ingest.test.ts +++ b/packages/context/src/ingest/local-bundle-ingest.test.ts @@ -27,6 +27,18 @@ class LookerSlWritingAgentRunner extends AgentRunnerService { params.telemetryTags?.operationName === 'ingest-bundle-wu' && params.telemetryTags?.unitKey === 'looker-explore-ecommerce-orders' ) { + const ledger = params.toolSet.record_verification_ledger; + if (!ledger?.execute) { + throw new Error('record_verification_ledger tool was not available to the Looker WorkUnit'); + } + await ledger.execute( + { + summary: 'Test fixture verified Looker explore target identifiers before writing SL.', + verifiedIdentifiers: ['prod-warehouse', 'public.orders'], + unverifiedIdentifiers: [], + }, + { toolCallId: 'looker-verification-ledger', messages: [] }, + ); const slWrite = params.toolSet.sl_write_source; if (!slWrite?.execute) { throw new Error('sl_write_source tool was not available to the Looker WorkUnit'); @@ -63,6 +75,18 @@ class LookerSlWritingAgentRunner extends AgentRunnerService { class WikiWritingAgentRunner extends AgentRunnerService { override runLoop = vi.fn(async (params: any) => { if (params.telemetryTags?.operationName === 'ingest-bundle-wu') { + const ledger = params.toolSet.record_verification_ledger; + if (!ledger?.execute) { + throw new Error('record_verification_ledger tool was not available to the WorkUnit'); + } + await ledger.execute( + { + summary: 'Test fixture writes wiki-only context with no warehouse identifiers.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 'wiki-verification-ledger', messages: [] }, + ); const wikiWrite = params.toolSet.wiki_write; if (!wikiWrite?.execute) { throw new Error('wiki_write tool was not available to the WorkUnit'); @@ -91,6 +115,18 @@ class WikiWritingAgentRunner extends AgentRunnerService { class WikiWritingWithRawPathAgentRunner extends AgentRunnerService { override runLoop = vi.fn(async (params: any) => { if (params.telemetryTags?.operationName === 'ingest-bundle-wu') { + const ledger = params.toolSet.record_verification_ledger; + if (!ledger?.execute) { + throw new Error('record_verification_ledger tool was not available to the WorkUnit'); + } + await ledger.execute( + { + summary: 'Test fixture writes wiki-only context with explicit raw provenance and no warehouse identifiers.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 'wiki-raw-path-verification-ledger', messages: [] }, + ); const wikiWrite = params.toolSet.wiki_write; if (!wikiWrite?.execute) { throw new Error('wiki_write tool was not available to the WorkUnit'); diff --git a/packages/context/src/ingest/local-bundle-runtime.test.ts b/packages/context/src/ingest/local-bundle-runtime.test.ts index af40d44f..e9be5a14 100644 --- a/packages/context/src/ingest/local-bundle-runtime.test.ts +++ b/packages/context/src/ingest/local-bundle-runtime.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { AgentRunnerService } from '../agent/index.js'; import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; @@ -12,6 +12,7 @@ type RuntimeWithConnectionDeps = { connections: { listEnabledConnections(ids: string[]): Promise>; getConnectionById(connectionId: string): Promise<{ id: string; name: string; connectionType: string } | null>; + executeQuery(connectionId: string, sql: string): Promise; }; }; }; @@ -113,6 +114,37 @@ describe('createLocalBundleIngestRuntime', () => { ]); }); + it('passes project connection config to local ingest query executors', async () => { + const agentRunner = new AgentRunnerService({ llmProvider: { getModel: () => ({}) as never } as any }); + const queryExecutor = { + execute: vi.fn(async () => ({ + headers: ['answer'], + rows: [[1]], + totalRows: 1, + command: 'SELECT', + rowCount: 1, + })), + }; + + const runtime = createLocalBundleIngestRuntime({ + project, + adapters: [new FakeSourceAdapter()], + agentRunner, + queryExecutor, + }); + const connections = (runtime.runner as unknown as RuntimeWithConnectionDeps).deps.connections; + + await expect(connections.executeQuery('warehouse', 'select 1')).resolves.toMatchObject({ + headers: ['answer'], + }); + expect(queryExecutor.execute).toHaveBeenCalledWith({ + connectionId: 'warehouse', + projectDir: project.projectDir, + connection: project.config.connections.warehouse, + sql: 'select 1', + }); + }); + it('accepts a debug LLM request file when constructing the default agent runner', async () => { await writeFile( join(project.projectDir, 'ktx.yaml'), diff --git a/packages/context/src/ingest/local-bundle-runtime.ts b/packages/context/src/ingest/local-bundle-runtime.ts index b453cae6..9eeda894 100644 --- a/packages/context/src/ingest/local-bundle-runtime.ts +++ b/packages/context/src/ingest/local-bundle-runtime.ts @@ -6,7 +6,7 @@ import type { Tool } from 'ai'; import YAML from 'yaml'; import type { AgentRunnerService } from '../agent/index.js'; import { AgentRunnerService as DefaultAgentRunnerService } from '../agent/index.js'; -import { localConnectionInfoFromConfig } from '../connections/index.js'; +import { localConnectionInfoFromConfig, type KtxSqlQueryExecutorPort } from '../connections/index.js'; import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; import { noopLogger, SessionWorktreeService } from '../core/index.js'; import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; @@ -56,6 +56,7 @@ import { buildKnowledgeSearchText, type KnowledgeEventPort, type KnowledgeIndexPort, + type KnowledgeIndexPageListing, KnowledgeWikiService, searchLocalKnowledgePages, SqliteKnowledgeIndex, @@ -77,6 +78,7 @@ import { ContextEvidenceIndexService, SqliteContextEvidenceStore } from './conte import { DiffSetService } from './diff-set.service.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; import { PageTriageService } from './page-triage/index.js'; +import { createWarehouseVerificationTools } from './tools/warehouse-verification/index.js'; import type { IngestBundleRunnerDeps, IngestCommitMessagePort, @@ -103,7 +105,7 @@ export interface CreateLocalBundleIngestRuntimeOptions { llmDebugRequestFile?: string; memoryModel?: string; semanticLayerCompute?: KtxSemanticLayerComputePort; - queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise }; + queryExecutor?: KtxSqlQueryExecutorPort; jobIdFactory?: () => string; logger?: KtxLogger; } @@ -169,9 +171,7 @@ class LocalAuthorResolver implements GitAuthorResolverPort { class LocalConnectionCatalog implements SlConnectionCatalogPort { constructor( private readonly project: KtxLocalProject, - private readonly queryExecutor?: { - execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise; - }, + private readonly queryExecutor?: KtxSqlQueryExecutorPort, ) {} async listEnabledConnections(ids: string[]): Promise { @@ -192,7 +192,12 @@ class LocalConnectionCatalog implements SlConnectionCatalogPort { if (!this.queryExecutor) { throw new Error('Local ingest has no query executor configured'); } - return this.queryExecutor.execute({ connectionId, sql }); + return this.queryExecutor.execute({ + connectionId, + projectDir: this.project.projectDir, + connection: this.project.config.connections[connectionId], + sql, + }); } } @@ -347,15 +352,19 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { async listPagesForUser( userId: string, - ): Promise> { - const pages: Array<{ page_key: string; summary: string; scope: string; scope_id: string | null }> = []; + ): Promise { + const pages: KnowledgeIndexPageListing[] = []; for (const scope of [ { scope: 'GLOBAL', scopeId: null, dir: 'knowledge/global' }, { scope: 'USER', scopeId: userId, dir: `knowledge/user/${userId}` }, ]) { const listed = await this.project.fileStore.listFiles(scope.dir, true); for (const file of listed.files.filter((entry) => entry.endsWith('.md'))) { - const pageKey = file.replace(/\.md$/, ''); + const parsedPath = parseKnowledgeIndexPath(file.startsWith('global/') || file.startsWith('user/') ? file : `${scope.dir.replace('knowledge/', '')}/${file}`); + if (!parsedPath || parsedPath.scope !== scope.scope) { + continue; + } + const pageKey = parsedPath.pageKey; const raw = await this.project.fileStore.readFile(`${scope.dir}/${file}`); const parsed = parseWiki(raw.content); pages.push({ @@ -363,6 +372,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { summary: parsed.summary, scope: scope.scope, scope_id: scope.scopeId, + tags: parseWikiTags(raw.content), }); } } @@ -432,13 +442,6 @@ function parseKnowledgeIndexPath(file: string): { scope: 'GLOBAL' | 'USER'; page const pageKey = segments[1].replace(/\.md$/, ''); return /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(pageKey) ? { scope: 'GLOBAL', pageKey } : null; } - if (segments.length >= 3 && segments[0] === 'global' && segments[1] === 'historic-sql') { - const historicPath = segments.slice(2).join('/').replace(/\.md$/, ''); - if (historicPath.split('/').every((segment) => /^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(segment))) { - return { scope: 'GLOBAL', pageKey: `historic-sql/${historicPath}` }; - } - return null; - } if (segments.length === 3 && segments[0] === 'user') { const pageKey = segments[2].replace(/\.md$/, ''); return /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(pageKey) ? { scope: 'USER', pageKey } : null; @@ -486,38 +489,47 @@ class LocalIngestToolsetFactory implements IngestToolsetFactoryPort { slSearchService: deps.slSearchService, authorResolver: deps.authorResolver, }; + const wikiSearchTool = new WikiSearchTool({ + search: async (input) => { + const results = await searchLocalKnowledgePages(deps.project, { + userId: input.userId, + query: input.query, + limit: input.limit, + embeddingService: deps.embedding, + }); + return { + results: results.slice(0, input.limit).map((result) => ({ + key: result.key, + path: result.path, + summary: result.summary, + score: result.score, + matchReasons: result.matchReasons, + lanes: result.lanes, + })), + totalFound: results.length, + }; + }, + }); + const slDiscoverTool = new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 }); + const warehouseVerificationTools = createWarehouseVerificationTools({ + connections: deps.connections, + fallbackFileStore: deps.project.fileStore, + wikiSearchTool, + slDiscoverTool, + }); this.baseTools = [ new WikiReadTool(deps.wikiService, deps.knowledgeIndex), - new WikiSearchTool({ - search: async (input) => { - const results = await searchLocalKnowledgePages(deps.project, { - userId: input.userId, - query: input.query, - limit: input.limit, - embeddingService: deps.embedding, - }); - return { - results: results.slice(0, input.limit).map((result) => ({ - key: result.key, - path: result.path, - summary: result.summary, - score: result.score, - matchReasons: result.matchReasons, - lanes: result.lanes, - })), - totalFound: results.length, - }; - }, - }), - new WikiListTagsTool(deps.wikiService, deps.knowledgeIndex), + wikiSearchTool, + new WikiListTagsTool(deps.knowledgeIndex), new WikiWriteTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), new WikiRemoveTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), - new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 }), + slDiscoverTool, new SlEditSourceTool(slDeps), new SlReadSourceTool(slDeps), new SlWriteSourceTool(slDeps), new SlValidateTool(slDeps), new SlRollbackTool(deps.slSourcesRepository, deps.connections, 0), + ...warehouseVerificationTools, ]; this.contextTools = [ new ContextEvidenceSearchTool(deps.contextStore, deps.embedding), diff --git a/packages/context/src/ingest/local-ingest.ts b/packages/context/src/ingest/local-ingest.ts index 2ebb3e7b..6056f6ed 100644 --- a/packages/context/src/ingest/local-ingest.ts +++ b/packages/context/src/ingest/local-ingest.ts @@ -3,11 +3,11 @@ import { cp, mkdir, rm } from 'node:fs/promises'; import { isAbsolute, resolve } from 'node:path'; import type { KtxLlmProvider } from '@ktx/llm'; import type { AgentRunnerService } from '../agent/index.js'; +import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; import type { KtxLogger } from '../core/index.js'; import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; import type { KtxLocalProject } from '../project/index.js'; import { ktxLocalStateDbPath } from '../project/index.js'; -import type { KtxQueryResult } from '../sl/index.js'; import { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js'; import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js'; import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } from './local-adapters.js'; @@ -33,7 +33,7 @@ export interface RunLocalIngestOptions { llmDebugRequestFile?: string; memoryModel?: string; semanticLayerCompute?: KtxSemanticLayerComputePort; - queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise }; + queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; } @@ -171,7 +171,7 @@ async function runScheduledPullJob(options: { llmProvider?: KtxLlmProvider; memoryModel?: string; semanticLayerCompute?: KtxSemanticLayerComputePort; - queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise }; + queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; }): Promise { const runtime = createLocalBundleIngestRuntime(options); diff --git a/packages/context/src/ingest/memory-flow/summary.ts b/packages/context/src/ingest/memory-flow/summary.ts index a23d7146..b76bde90 100644 --- a/packages/context/src/ingest/memory-flow/summary.ts +++ b/packages/context/src/ingest/memory-flow/summary.ts @@ -53,7 +53,6 @@ function fixSuggestions(input: MemoryFlowReplayInput): string[] { export function formatMemoryFlowFinalSummary(input: MemoryFlowReplayInput): string { const sources = eventsOf(input.events, 'source_acquired'); - const source = sources.at(-1); const totalFiles = sources.reduce((sum, s) => sum + s.fileCount, 0); const saved = latest(input.events, 'saved'); const provenance = latest(input.events, 'provenance_recorded'); diff --git a/packages/context/src/ingest/reports.ts b/packages/context/src/ingest/reports.ts index 2c3020b4..cda4d7c1 100644 --- a/packages/context/src/ingest/reports.ts +++ b/packages/context/src/ingest/reports.ts @@ -9,6 +9,7 @@ import type { StageIndex, UnmappedFallbackRecord, } from './stages/stage-index.types.js'; +import type { WikiSlRefRepair } from './wiki-sl-ref-repair.js'; import type { IngestDiffSummary, SourceFetchReport, UnresolvedCardInfo } from './types.js'; export interface IngestReportWorkUnit { @@ -70,6 +71,8 @@ export interface IngestReportBody { provenanceRows: IngestReportProvenanceDetail[]; toolTranscripts: IngestReportToolTranscriptSummary[]; postProcessor?: IngestReportPostProcessorOutcome; + wikiSlRefRepairs?: WikiSlRefRepair[]; + wikiSlRefRepairWarnings?: string[]; memoryFlow?: MemoryFlowReplayInput; } diff --git a/packages/context/src/ingest/stages/build-reconcile-context.test.ts b/packages/context/src/ingest/stages/build-reconcile-context.test.ts index 7db0bb23..9ac95356 100644 --- a/packages/context/src/ingest/stages/build-reconcile-context.test.ts +++ b/packages/context/src/ingest/stages/build-reconcile-context.test.ts @@ -107,6 +107,7 @@ describe('buildReconcileToolSet', () => { 'eviction_list', 'load_skill', 'read_raw_span', + 'record_verification_ledger', 'sl_write_source', 'stage_diff', 'stage_list', @@ -114,4 +115,54 @@ describe('buildReconcileToolSet', () => { ].sort(), ); }); + + it('requires the verification ledger before reconciliation write tools run', async () => { + const slWrite = vi.fn().mockResolvedValue({ markdown: 'written', structured: { success: true } }); + const toolSet = buildReconcileToolSet({ + loadSkillTool: { load_skill: { description: 'load', inputSchema: {} as any, execute: vi.fn() } } as any, + stageListTool: { stage_list: { description: 'stage list', inputSchema: {} as any, execute: vi.fn() } } as any, + stageDiffTool: { stage_diff: { description: 'stage diff', inputSchema: {} as any, execute: vi.fn() } } as any, + evictionListTool: { + eviction_list: { description: 'eviction list', inputSchema: {} as any, execute: vi.fn() }, + } as any, + emitConflictResolutionTool: { + emit_conflict_resolution: { description: 'conflict', inputSchema: {} as any, execute: vi.fn() }, + } as any, + emitEvictionDecisionTool: { + emit_eviction_decision: { description: 'eviction', inputSchema: {} as any, execute: vi.fn() }, + } as any, + emitArtifactResolutionTool: { + emit_artifact_resolution: { description: 'resolution', inputSchema: {} as any, execute: vi.fn() }, + } as any, + emitUnmappedFallbackTool: { + emit_unmapped_fallback: { description: 'fallback', inputSchema: {} as any, execute: vi.fn() }, + } as any, + readRawSpanTool: { read_raw_span: { description: 'raw span', inputSchema: {} as any, execute: vi.fn() } } as any, + toolsetTools: { sl_write_source: { description: 'sl write', inputSchema: {} as any, execute: slWrite } as any }, + }); + + const correction = await toolSet.sl_write_source.execute?.( + { connectionId: 'warehouse', sourceName: 'accounts' }, + { toolCallId: 't1' } as any, + ); + + expect(slWrite).not.toHaveBeenCalled(); + expect(correction).toMatchObject({ structured: { success: false, reason: 'verification_ledger_required' } }); + + await toolSet.record_verification_ledger.execute?.( + { + summary: 'Verified warehouse.accounts with entity_details.', + verifiedIdentifiers: ['warehouse.accounts'], + unverifiedIdentifiers: [], + }, + { toolCallId: 't2' } as any, + ); + const written = await toolSet.sl_write_source.execute?.( + { connectionId: 'warehouse', sourceName: 'accounts' }, + { toolCallId: 't3' } as any, + ); + + expect(slWrite).toHaveBeenCalledTimes(1); + expect(written).toMatchObject({ structured: { success: true } }); + }); }); diff --git a/packages/context/src/ingest/stages/build-reconcile-context.ts b/packages/context/src/ingest/stages/build-reconcile-context.ts index 30ff6341..9533acbd 100644 --- a/packages/context/src/ingest/stages/build-reconcile-context.ts +++ b/packages/context/src/ingest/stages/build-reconcile-context.ts @@ -1,5 +1,10 @@ import type { Tool, ToolSet } from 'ai'; import { buildCanonicalPinsPromptBlock, type CanonicalPin } from '../canonical-pins.js'; +import { + createVerificationLedgerState, + VERIFICATION_LEDGER_PROMPT, + withVerificationLedger, +} from '../tools/verification-ledger.tool.js'; import type { EvictionUnit } from '../types.js'; import type { StageIndex } from './stage-index.types.js'; @@ -12,6 +17,7 @@ export function buildReconcileSystemPrompt(params: { }): string { return [ params.baseFraming.trimEnd(), + VERIFICATION_LEDGER_PROMPT, params.skillsPrompt.trimEnd(), buildCanonicalPinsPromptBlock(params.canonicalPins), `\n\nsyncId: ${params.syncId}\nsource: ${params.sourceKey}\n`, @@ -188,16 +194,20 @@ export interface ReconcileToolSetInput { } export function buildReconcileToolSet(input: ReconcileToolSetInput): ToolSet { - return { - ...input.toolsetTools, - ...input.loadSkillTool, - ...input.stageListTool, - ...input.stageDiffTool, - ...input.evictionListTool, - ...input.emitConflictResolutionTool, - ...input.emitEvictionDecisionTool, - ...input.emitArtifactResolutionTool, - ...input.emitUnmappedFallbackTool, - ...input.readRawSpanTool, - }; + const state = createVerificationLedgerState(); + return withVerificationLedger( + { + ...input.toolsetTools, + ...input.loadSkillTool, + ...input.stageListTool, + ...input.stageDiffTool, + ...input.evictionListTool, + ...input.emitConflictResolutionTool, + ...input.emitEvictionDecisionTool, + ...input.emitArtifactResolutionTool, + ...input.emitUnmappedFallbackTool, + ...input.readRawSpanTool, + }, + state, + ); } diff --git a/packages/context/src/ingest/stages/build-wu-context.test.ts b/packages/context/src/ingest/stages/build-wu-context.test.ts index 13a3ff8f..db17154e 100644 --- a/packages/context/src/ingest/stages/build-wu-context.test.ts +++ b/packages/context/src/ingest/stages/build-wu-context.test.ts @@ -68,12 +68,45 @@ describe('buildWuToolSet', () => { 'load_skill', 'read_raw_file', 'read_raw_span', + 'record_verification_ledger', 'sl_write_source', 'wiki_search', ].sort(), ); }); + it('requires the verification ledger before write-capable tools run', async () => { + const wikiWrite = vi.fn().mockResolvedValue({ markdown: 'written', structured: { success: true } }); + const toolSet = buildWuToolSet({ + stagedDir: '/tmp/staged', + wu: { unitKey: 'u1', rawFiles: ['a.yml'], peerFileIndex: [], dependencyPaths: [] }, + loadSkillTool: { load_skill: { description: 'load', inputSchema: {} as any, execute: vi.fn() } } as any, + emitUnmappedFallbackTool: { + emit_unmapped_fallback: { description: 'fallback', inputSchema: {} as any, execute: vi.fn() }, + } as any, + toolsetTools: { wiki_write: { description: 'write', inputSchema: {} as any, execute: wikiWrite } as any }, + }); + + const correction = await toolSet.wiki_write.execute?.({ key: 'customer-rules' }, { toolCallId: 't1' } as any); + + expect(wikiWrite).not.toHaveBeenCalled(); + expect(correction).toMatchObject({ structured: { success: false, reason: 'verification_ledger_required' } }); + expect(String((correction as any).markdown)).toContain('record_verification_ledger'); + + await toolSet.record_verification_ledger.execute?.( + { + summary: 'No warehouse identifiers will be emitted in this wiki write.', + verifiedIdentifiers: [], + unverifiedIdentifiers: [], + }, + { toolCallId: 't2' } as any, + ); + const written = await toolSet.wiki_write.execute?.({ key: 'customer-rules' }, { toolCallId: 't3' } as any); + + expect(wikiWrite).toHaveBeenCalledTimes(1); + expect(written).toMatchObject({ structured: { success: true } }); + }); + it('includes looker_query_to_sl only for Looker WorkUnits', () => { const toolSet = buildWuToolSet({ sourceKey: 'looker', @@ -93,6 +126,7 @@ describe('buildWuToolSet', () => { 'looker_query_to_sl', 'read_raw_file', 'read_raw_span', + 'record_verification_ledger', 'sl_write_source', 'wiki_search', ].sort(), diff --git a/packages/context/src/ingest/stages/build-wu-context.ts b/packages/context/src/ingest/stages/build-wu-context.ts index 6ba26fd7..bfa1bd9c 100644 --- a/packages/context/src/ingest/stages/build-wu-context.ts +++ b/packages/context/src/ingest/stages/build-wu-context.ts @@ -4,6 +4,11 @@ import { createLookerQueryToSlTool } from '../adapters/looker/tools/looker-query import type { IngestProvenanceRow } from '../ports.js'; import { createReadRawFileTool } from '../tools/read-raw-file.tool.js'; import { createReadRawSpanTool } from '../tools/read-raw-span.tool.js'; +import { + createVerificationLedgerState, + VERIFICATION_LEDGER_PROMPT, + withVerificationLedger, +} from '../tools/verification-ledger.tool.js'; import type { WorkUnit } from '../types.js'; const PEER_FILE_INDEX_PROMPT_LIMIT = 100; @@ -24,6 +29,7 @@ export function buildWuSystemPrompt(params: { }): string { const parts = [ params.baseFraming.trimEnd(), + VERIFICATION_LEDGER_PROMPT, params.skillsPrompt.trimEnd(), buildCanonicalPinsPromptBlock(params.canonicalPins ?? []), `\n\nsyncId: ${params.syncId}\nsource: ${params.sourceKey}\n`, @@ -100,15 +106,19 @@ function withoutWriteSlTools(toolset: ToolSet, wu: WorkUnit): ToolSet { export function buildWuToolSet(input: BuildWuToolSetInput): ToolSet { const allowedPaths = new Set([...input.wu.rawFiles, ...input.wu.dependencyPaths]); const lookerTools: ToolSet = input.sourceKey === 'looker' ? { looker_query_to_sl: createLookerQueryToSlTool() } : {}; - return withoutWriteSlTools( - { - ...input.toolsetTools, - ...lookerTools, - ...input.loadSkillTool, - ...input.emitUnmappedFallbackTool, - read_raw_file: createReadRawFileTool({ stagedDir: input.stagedDir, allowedPaths }), - read_raw_span: createReadRawSpanTool({ stagedDir: input.stagedDir, allowedPaths }), - }, - input.wu, + const state = createVerificationLedgerState(); + return withVerificationLedger( + withoutWriteSlTools( + { + ...input.toolsetTools, + ...lookerTools, + ...input.loadSkillTool, + ...input.emitUnmappedFallbackTool, + read_raw_file: createReadRawFileTool({ stagedDir: input.stagedDir, allowedPaths }), + read_raw_span: createReadRawSpanTool({ stagedDir: input.stagedDir, allowedPaths }), + }, + input.wu, + ), + state, ); } diff --git a/packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts b/packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts index aaba3509..33a8610e 100644 --- a/packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts +++ b/packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts @@ -64,7 +64,7 @@ export function createEmitUnmappedFallbackTool(deps: EmitUnmappedFallbackDeps) { tableRef: z .string() .optional() - .describe('The fully-qualified table or source reference that triggered the fallback (e.g. "orbit_analytics.customer"). Used to generate canonical detail text.'), + .describe('The fully-qualified table or source reference that triggered the fallback (e.g. ".
"). Used to generate canonical detail text.'), clarification: z .string() .optional() diff --git a/packages/context/src/ingest/tools/tool-transcript-summary.test.ts b/packages/context/src/ingest/tools/tool-transcript-summary.test.ts index bc836e97..9e110789 100644 --- a/packages/context/src/ingest/tools/tool-transcript-summary.test.ts +++ b/packages/context/src/ingest/tools/tool-transcript-summary.test.ts @@ -36,6 +36,28 @@ describe('tool transcript summaries', () => { expect(summary.fatalErrorCount).toBe(0); }); + it('treats a suggested flat wiki key retry as recovery for an invalid nested key', () => { + const summary = createMutableToolTranscriptSummary('wu-1', '/tmp/wu-1.jsonl'); + + recordToolTranscriptEntry( + summary, + entry({ + input: { key: 'historic-sql/top-accounts-by-contract-arr' }, + output: { structured: { success: false, key: 'historic-sql/top-accounts-by-contract-arr' } }, + }), + ); + recordToolTranscriptEntry( + summary, + entry({ + input: { key: 'historic-sql-top-accounts-by-contract-arr' }, + output: { structured: { success: true, key: 'historic-sql-top-accounts-by-contract-arr' } }, + }), + ); + + expect(summary.errorCount).toBe(1); + expect(summary.fatalErrorCount).toBe(0); + }); + it('counts unrecovered wiki_remove structured failures as fatal transcript errors', () => { const summary = createMutableToolTranscriptSummary('reconcile', '/tmp/reconcile.jsonl'); diff --git a/packages/context/src/ingest/tools/tool-transcript-summary.ts b/packages/context/src/ingest/tools/tool-transcript-summary.ts index de7ee668..4af450f0 100644 --- a/packages/context/src/ingest/tools/tool-transcript-summary.ts +++ b/packages/context/src/ingest/tools/tool-transcript-summary.ts @@ -1,4 +1,5 @@ import type { ToolCallLogEntry } from './tool-call-logger.js'; +import { isFlatWikiKey, suggestFlatWikiKey } from '../../wiki/keys.js'; export interface MutableToolTranscriptSummary { unitKey: string; @@ -112,7 +113,10 @@ function structuredSuccess(output: unknown): boolean | null { function wikiTargetKey(entry: ToolCallLogEntry): string | null { const key = stringField(recordField(entry.output, 'structured'), 'key') ?? stringField(entry.input, 'key'); - return key ? `wiki:${key}` : null; + if (!key) { + return null; + } + return `wiki:${isFlatWikiKey(key) ? key : suggestFlatWikiKey(key)}`; } function slTargetKey(entry: ToolCallLogEntry): string | null { diff --git a/packages/context/src/ingest/tools/verification-ledger.tool.ts b/packages/context/src/ingest/tools/verification-ledger.tool.ts new file mode 100644 index 00000000..ac880607 --- /dev/null +++ b/packages/context/src/ingest/tools/verification-ledger.tool.ts @@ -0,0 +1,97 @@ +import { tool, type ToolExecuteFunction, type ToolExecutionOptions, type ToolSet } from 'ai'; +import { z } from 'zod'; + +const verificationLedgerInputSchema = z.object({ + summary: z.string().min(1).max(2000), + verifiedIdentifiers: z.array(z.string().min(1)).max(100).default([]), + unverifiedIdentifiers: z.array(z.string().min(1)).max(100).default([]), + notes: z.string().max(2000).optional(), +}); + +interface VerificationLedgerEntry { + summary: string; + verifiedIdentifiers: string[]; + unverifiedIdentifiers: string[]; + notes?: string; +} + +export interface VerificationLedgerState { + entries: VerificationLedgerEntry[]; +} + +const WRITE_TOOL_NAMES = new Set([ + 'wiki_write', + 'wiki_remove', + 'sl_write_source', + 'sl_edit_source', + 'emit_unmapped_fallback', +]); + +export const VERIFICATION_LEDGER_PROMPT = ` +Before any write-capable tool call (wiki_write, wiki_remove, sl_write_source, sl_edit_source, emit_unmapped_fallback), call record_verification_ledger. +The ledger is a model-authored checkpoint, not a deterministic parser gate. Summarize the verification protocol from the loaded skill, list identifiers verified with discover_data/entity_details/sql_execution, and list anything intentionally left unverified. If the write contains no warehouse identifiers, say that explicitly. +If a write tool returns verification_ledger_required, complete the ledger and retry the write. +`; + +export function createVerificationLedgerState(): VerificationLedgerState { + return { entries: [] }; +} + +export function withVerificationLedger(tools: ToolSet, state: VerificationLedgerState): ToolSet { + const wrapped: ToolSet = {}; + for (const [name, original] of Object.entries(tools)) { + if (!WRITE_TOOL_NAMES.has(name) || typeof original.execute !== 'function') { + wrapped[name] = original; + continue; + } + const originalExecute = original.execute; + const guardedExecute: ToolExecuteFunction = async ( + input: unknown, + opts: ToolExecutionOptions, + ) => { + if (state.entries.length === 0) { + return verificationRequiredOutput(name); + } + return (originalExecute as ToolExecuteFunction)(input, opts); + }; + wrapped[name] = { ...original, execute: guardedExecute }; + } + wrapped.record_verification_ledger = createRecordVerificationLedgerTool(state); + return wrapped; +} + +function createRecordVerificationLedgerTool(state: VerificationLedgerState) { + return tool({ + description: + 'Record the pre-write verification ledger required by loaded ingest skills. Call this before wiki/SL/fallback writes to state what was verified, which tool calls support it, and what remains intentionally unverified.', + inputSchema: verificationLedgerInputSchema, + execute: async (input) => { + const entry = verificationLedgerInputSchema.parse(input); + state.entries.push(entry); + return { + markdown: + `Verification ledger recorded. Summary: ${entry.summary}\n` + + `Verified identifiers: ${entry.verifiedIdentifiers.length ? entry.verifiedIdentifiers.join(', ') : '(none)'}\n` + + `Unverified identifiers: ${ + entry.unverifiedIdentifiers.length ? entry.unverifiedIdentifiers.join(', ') : '(none)' + }`, + structured: { success: true, entry }, + }; + }, + }); +} + +function verificationRequiredOutput(toolName: string) { + return { + markdown: + `Pre-write verification required before calling ${toolName}. ` + + 'Call record_verification_ledger first. In the ledger, summarize the loaded skill protocol you followed, ' + + 'list identifiers verified via discover_data/entity_details/sql_execution, and list any identifiers intentionally left unverified. ' + + 'If the write contains no warehouse identifiers, say that explicitly in the ledger summary.', + structured: { + success: false, + reason: 'verification_ledger_required', + toolName, + }, + }; +} diff --git a/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts b/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts new file mode 100644 index 00000000..979873a4 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import { DiscoverDataTool } from './discover-data.tool.js'; +import type { WarehouseCatalogService } from './warehouse-catalog.service.js'; + +describe('DiscoverDataTool', () => { + const wikiSearchTool = { call: vi.fn() } as unknown as BaseTool & { call: ReturnType }; + const slDiscoverTool = { call: vi.fn() } as unknown as BaseTool & { call: ReturnType }; + const catalog = { searchByName: vi.fn() } as unknown as WarehouseCatalogService & { + searchByName: ReturnType; + }; + const context: ToolContext = { + sourceId: 'ingest', + messageId: 'm1', + userId: 'system', + session: { allowedConnectionNames: new Set(['warehouse']) } as any, + }; + const tool = new DiscoverDataTool({ + wikiSearchTool, + slDiscoverTool, + catalogFactory: () => catalog, + }); + + beforeEach(() => { + wikiSearchTool.call.mockReset(); + slDiscoverTool.call.mockReset(); + catalog.searchByName.mockReset(); + wikiSearchTool.call.mockResolvedValue({ + markdown: '- orders wiki', + structured: { totalFound: 1, results: [{ key: 'orders' }] }, + }); + slDiscoverTool.call.mockResolvedValue({ + markdown: '- orders source', + structured: { totalSources: 1, sources: [{ sourceName: 'orders' }] }, + }); + catalog.searchByName.mockResolvedValue([ + { + kind: 'table', + connectionName: 'warehouse', + ref: { catalog: null, db: 'public', name: 'orders' }, + display: 'public.orders', + matchedOn: 'name', + }, + ]); + }); + + it('groups wiki, semantic layer, and raw schema hits with routing hints', async () => { + const result = await tool.call({ query: 'orders', connectionName: 'warehouse', limit: 5 }, context); + + expect(result.markdown).toContain('## Wiki Pages'); + expect(result.markdown).toContain('use `wiki_read(blockKey)` for full content'); + expect(result.markdown).toContain('## Semantic Layer Sources'); + expect(result.markdown).toContain('use `sl_read_source(sourceName)` for the YAML'); + expect(result.markdown).toContain('## Raw Warehouse Schema'); + expect(result.markdown).toContain('use `entity_details({connectionName, targets: [{display}]})`'); + expect(result.structured.raw?.hits).toHaveLength(1); + }); + + it('includes connectionName on raw schema hits so entity_details can follow up', async () => { + const multiConnectionContext: ToolContext = { + ...context, + session: { allowedConnectionNames: new Set(['warehouse', 'analytics']) } as any, + }; + catalog.searchByName.mockImplementation(async (connectionName: string, query: string) => [ + { + kind: 'table', + connectionName, + ref: { catalog: null, db: 'public', name: `${connectionName}_${query}` }, + display: `public.${connectionName}_${query}`, + matchedOn: 'name', + }, + ]); + + const result = await tool.call({ query: 'orders', limit: 10 }, multiConnectionContext); + + expect(catalog.searchByName).toHaveBeenCalledWith('analytics', 'orders', 10); + expect(catalog.searchByName).toHaveBeenCalledWith('warehouse', 'orders', 10); + expect(result.markdown).toContain('connectionName=analytics'); + expect(result.markdown).toContain('connectionName=warehouse'); + expect(result.markdown).toContain( + 'entity_details({connectionName: "analytics", targets: [{display: "public.analytics_orders"}]})', + ); + expect(result.structured.raw?.hits.map((hit) => hit.connectionName)).toEqual(['analytics', 'warehouse']); + }); + + it('refuses explicit out-of-scope connection names', async () => { + const result = await tool.call({ query: 'orders', connectionName: 'billing' }, context); + + expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.'); + expect(result.structured).toEqual({ wiki: null, sl: null, raw: null }); + expect(wikiSearchTool.call).not.toHaveBeenCalled(); + expect(slDiscoverTool.call).not.toHaveBeenCalled(); + expect(catalog.searchByName).not.toHaveBeenCalled(); + }); + + it('delegates sourceName inspect mode to sl_discover only', async () => { + slDiscoverTool.call.mockResolvedValueOnce({ + markdown: 'source detail', + structured: { sourceName: 'orders' }, + }); + + const result = await tool.call({ sourceName: 'orders', connectionName: 'warehouse' }, context); + + expect(slDiscoverTool.call).toHaveBeenCalledWith({ sourceName: 'orders', connectionId: 'warehouse' }, context); + expect(wikiSearchTool.call).not.toHaveBeenCalled(); + expect(catalog.searchByName).not.toHaveBeenCalled(); + expect(result.markdown).toContain('source detail'); + }); + + it('returns the empty-state message when all sections are empty', async () => { + wikiSearchTool.call.mockResolvedValueOnce({ markdown: '', structured: { totalFound: 0, results: [] } }); + slDiscoverTool.call.mockResolvedValueOnce({ markdown: '', structured: { totalSources: 0, sources: [] } }); + catalog.searchByName.mockResolvedValueOnce([]); + + const result = await tool.call({ query: 'customer source', connectionName: 'warehouse' }, context); + + expect(result.markdown).toContain('No matches for "customer source" across wiki, semantic layer, or raw warehouse schema.'); + }); +}); diff --git a/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts b/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts new file mode 100644 index 00000000..667d8f83 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts @@ -0,0 +1,142 @@ +import { z } from 'zod'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { WarehouseCatalogService, type RawSchemaHit } from './warehouse-catalog.service.js'; + +const discoverDataInputSchema = z.object({ + query: z.string().optional(), + connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(), + limit: z.number().int().positive().max(50).optional().default(10), + sourceName: z.string().optional(), +}); + +type DiscoverDataInput = z.input; + +export interface DiscoverDataStructured { + wiki: unknown | null; + sl: unknown | null; + raw: { hits: RawSchemaHit[] } | null; +} + +interface DiscoverDataDeps { + wikiSearchTool: BaseTool; + slDiscoverTool: BaseTool; + catalogFactory: (context: ToolContext) => WarehouseCatalogService; +} + +function totalFound(structured: unknown): number { + return typeof structured === 'object' && + structured !== null && + 'totalFound' in structured && + typeof structured.totalFound === 'number' + ? structured.totalFound + : 0; +} + +function totalSources(structured: unknown): number { + return typeof structured === 'object' && + structured !== null && + 'totalSources' in structured && + typeof structured.totalSources === 'number' + ? structured.totalSources + : 0; +} + +function allowedConnectionNames(context: ToolContext): ReadonlySet | null { + return context.session?.allowedConnectionNames ?? null; +} + +export class DiscoverDataTool extends BaseTool { + readonly name = 'discover_data'; + + constructor(private readonly deps: DiscoverDataDeps) { + super(); + } + + get description(): string { + return 'Discover existing wiki pages, semantic layer sources, and raw warehouse schema hits before writing ingest output.'; + } + + get inputSchema() { + return discoverDataInputSchema; + } + + async call(input: DiscoverDataInput, context: ToolContext): Promise> { + const allowed = allowedConnectionNames(context); + if (input.connectionName && allowed && !allowed.has(input.connectionName)) { + return { + markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, + structured: { wiki: null, sl: null, raw: null }, + }; + } + + if (input.sourceName) { + const sl = await this.deps.slDiscoverTool.call( + { sourceName: input.sourceName, connectionId: input.connectionName }, + context, + ); + return { markdown: sl.markdown, structured: { wiki: null, sl: sl.structured, raw: null } }; + } + + const query = input.query?.trim() || ''; + const limit = input.limit ?? 10; + const parts: string[] = []; + let wiki: unknown | null = null; + let sl: unknown | null = null; + let raw: DiscoverDataStructured['raw'] = null; + + if (query) { + const wikiResult = await this.deps.wikiSearchTool.call({ query, limit }, context); + if (totalFound(wikiResult.structured) > 0) { + parts.push('## Wiki Pages', '> use `wiki_read(blockKey)` for full content', wikiResult.markdown, ''); + wiki = wikiResult.structured; + } + } + + const slResult = await this.deps.slDiscoverTool.call( + { query: query || undefined, connectionId: input.connectionName }, + context, + ); + if (totalSources(slResult.structured) > 0) { + parts.push( + '## Semantic Layer Sources', + '> use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details', + slResult.markdown, + '', + ); + sl = slResult.structured; + } + + const catalog = this.deps.catalogFactory(context); + const connections = input.connectionName ? [input.connectionName] : [...(allowed ?? [])].sort(); + const rawHits: RawSchemaHit[] = []; + for (const connectionName of connections) { + rawHits.push(...(await catalog.searchByName(connectionName, query, limit))); + } + if (rawHits.length > 0) { + parts.push( + '## Raw Warehouse Schema', + '> use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values', + ); + parts.push( + rawHits + .slice(0, limit) + .map( + (hit) => + `- ${hit.kind}: ${hit.display} [connectionName=${hit.connectionName}] (matched on ${hit.matchedOn}) - ` + + `follow up with \`entity_details({connectionName: "${hit.connectionName}", targets: [{display: "${hit.display}"}]})\``, + ) + .join('\n'), + ); + raw = { hits: rawHits.slice(0, limit) }; + } + + if (parts.length === 0) { + return { + markdown: `No matches for "${query}" across wiki, semantic layer, or raw warehouse schema. Try broader terms; this concept may not exist yet.`, + structured: { wiki, sl, raw }, + }; + } + + return { markdown: parts.join('\n'), structured: { wiki, sl, raw } }; + } +} diff --git a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts b/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts new file mode 100644 index 00000000..9188bc68 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts @@ -0,0 +1,192 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { initKtxProject, type KtxLocalProject } from '../../../project/index.js'; +import type { ToolContext } from '../../../tools/index.js'; +import { EntityDetailsTool } from './entity-details.tool.js'; +import { WarehouseCatalogService } from './warehouse-catalog.service.js'; + +describe('EntityDetailsTool', () => { + let tempDir: string; + let project: KtxLocalProject; + let tool: EntityDetailsTool; + let context: ToolContext; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-entity-details-')); + project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + await seedLiveDatabaseScan(); + tool = new EntityDetailsTool(() => new WarehouseCatalogService({ fileStore: project.fileStore })); + context = { + sourceId: 'ingest', + messageId: 'm1', + userId: 'system', + session: { + allowedConnectionNames: new Set(['warehouse']), + } as any, + }; + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + async function seedLiveDatabaseScan(connectionName = 'warehouse', syncId = 'sync-1') { + const root = `raw-sources/${connectionName}/live-database/${syncId}`; + await project.fileStore.writeFile( + `${root}/connection.json`, + JSON.stringify({ connectionId: connectionName, driver: 'postgres', extractedAt: '2026-05-12T00:00:00.000Z' }, null, 2), + 'ktx', + 'ktx@example.com', + 'seed connection', + ); + await project.fileStore.writeFile( + `${root}/tables/orders.json`, + JSON.stringify( + { + catalog: null, + db: 'public', + name: 'orders', + kind: 'table', + comment: 'Customer orders', + estimatedRows: 12, + columns: [ + { + name: 'id', + nativeType: 'integer', + normalizedType: 'integer', + dimensionType: 'number', + nullable: false, + primaryKey: true, + comment: 'Order id', + }, + { + name: 'status', + nativeType: 'text', + normalizedType: 'text', + dimensionType: 'string', + nullable: false, + primaryKey: false, + comment: 'Order status', + }, + ], + foreignKeys: [], + }, + null, + 2, + ), + 'ktx', + 'ktx@example.com', + 'seed orders', + ); + await project.fileStore.writeFile( + `${root}/enrichment/relationship-profile.json`, + JSON.stringify( + { + connectionId: connectionName, + driver: 'postgres', + tables: [{ table: { catalog: null, db: 'public', name: 'orders' }, rowCount: 12 }], + columns: { + 'orders.status': { + table: { catalog: null, db: 'public', name: 'orders' }, + column: 'status', + rowCount: 12, + nullCount: 0, + distinctCount: 2, + nullRate: 0, + sampleValues: ['paid', 'refunded'], + }, + }, + }, + null, + 2, + ), + 'ktx', + 'ktx@example.com', + 'seed profile', + ); + } + + it('returns scoped table detail for a display target', async () => { + const result = await tool.call({ connectionName: 'warehouse', targets: [{ display: 'public.orders' }] }, context); + + expect(result.markdown).toContain('### public.orders'); + expect(result.markdown).toContain('- status (text, nullable=false)'); + expect(result.markdown).toContain('sample: ["paid","refunded"]'); + expect(result.structured.scanAvailable).toBe(true); + expect(result.structured.resolved).toHaveLength(1); + }); + + it('resolves display targets that include a column name', async () => { + const result = await tool.call( + { connectionName: 'warehouse', targets: [{ display: 'public.orders.status' }] }, + context, + ); + + expect(result.markdown).toContain('### public.orders'); + expect(result.markdown).toContain('- status (text, nullable=false)'); + expect(result.markdown).not.toContain('- id (integer'); + expect(result.structured.resolved).toHaveLength(1); + expect(result.structured.resolved[0]?.columns.map((column) => column.name)).toEqual(['status']); + }); + + it('reports missing explicit columns instead of returning an empty column list', async () => { + const result = await tool.call( + { connectionName: 'warehouse', targets: [{ display: 'public.orders.plan_tier' }] }, + context, + ); + + expect(result.markdown).toContain('Column not found in scan: public.orders.plan_tier'); + expect(result.markdown).toContain('Available columns: id, status'); + expect(result.structured.resolved).toHaveLength(0); + expect(result.structured.missing).toHaveLength(1); + }); + + it('reports missing structured table targets in model-visible markdown', async () => { + const result = await tool.call( + { + connectionName: 'warehouse', + targets: [{ catalog: null, db: 'public', name: 'orderz' }], + }, + context, + ); + + expect(result.markdown).toContain('Not found in scan: public.orderz'); + expect(result.markdown).toContain('Closest matches: orders'); + expect(result.structured.resolved).toHaveLength(0); + expect(result.structured.missing).toHaveLength(1); + }); + + it('reports missing structured column targets in model-visible markdown', async () => { + const result = await tool.call( + { + connectionName: 'warehouse', + targets: [{ catalog: null, db: 'public', name: 'orders', column: 'plan_tier' }], + }, + context, + ); + + expect(result.markdown).toContain('Column not found in scan: public.orders.plan_tier'); + expect(result.markdown).toContain('Available columns: id, status'); + expect(result.structured.resolved).toHaveLength(0); + expect(result.structured.missing).toHaveLength(1); + }); + + it('returns a no-scan state distinct from not found', async () => { + const result = await tool.call( + { connectionName: 'empty', targets: [{ display: 'public.orders' }] }, + { ...context, session: { ...context.session!, allowedConnectionNames: new Set(['empty']) } }, + ); + + expect(result.markdown).toContain('No live-database scan available for connection "empty"; run `ktx scan` first.'); + expect(result.structured.scanAvailable).toBe(false); + }); + + it('refuses out-of-scope connections', async () => { + const result = await tool.call({ connectionName: 'billing', targets: [{ display: 'public.orders' }] }, context); + + expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.'); + expect(result.structured.scanAvailable).toBe(false); + }); +}); diff --git a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts b/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts new file mode 100644 index 00000000..27cf55a0 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts @@ -0,0 +1,170 @@ +import { z } from 'zod'; +import type { KtxTableRef } from '../../../scan/types.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { WarehouseCatalogService, type TableDetail } from './warehouse-catalog.service.js'; + +const targetSchema = z.union([ + z.object({ display: z.string().min(1) }), + z.object({ + catalog: z.string().nullable(), + db: z.string().nullable(), + name: z.string().min(1), + column: z.string().optional(), + }), +]); + +const entityDetailsInputSchema = z.object({ + connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + targets: z.array(targetSchema).min(1).max(50), +}); + +type EntityDetailsInput = z.infer; +type EntityDetailsTarget = EntityDetailsInput['targets'][number]; + +export interface EntityDetailsStructured { + resolved: TableDetail[]; + missing: Array<{ target: unknown; candidates: KtxTableRef[] }>; + scanAvailable: boolean; +} + +function allowedConnectionNames(context: ToolContext): ReadonlySet | null { + return context.session?.allowedConnectionNames ?? null; +} + +function targetLabel(target: EntityDetailsTarget): string { + if ('display' in target) { + return target.display; + } + return [target.catalog, target.db, target.name, target.column].filter((part): part is string => !!part).join('.'); +} + +function appendMissingTargetMarkdown(parts: string[], target: EntityDetailsTarget, candidates: KtxTableRef[]): void { + parts.push(`Not found in scan: ${targetLabel(target)}`); + if (candidates.length > 0) { + parts.push(`Closest matches: ${candidates.map((candidate) => candidate.name).join(', ')}`); + } +} + +async function resolveTarget( + catalog: WarehouseCatalogService, + connectionName: string, + target: EntityDetailsTarget, +): Promise<{ resolved: (KtxTableRef & { column?: string }) | null; candidates: KtxTableRef[] }> { + if ('display' in target) { + return catalog.resolveDisplayTarget(connectionName, target.display); + } + + const candidateResolution = await catalog.resolveDisplayTarget(connectionName, targetLabel(target)); + return { + resolved: { + catalog: target.catalog, + db: target.db, + name: target.name, + column: target.column, + }, + candidates: candidateResolution.candidates, + }; +} + +function sampleText(values: string[]): string { + return values.length > 0 ? ` - sample: ${JSON.stringify(values.slice(0, 10))}` : ''; +} + +function appendTableMarkdown(parts: string[], detail: TableDetail, columnName?: string): void { + const columns = columnName ? detail.columns.filter((column) => column.name === columnName) : detail.columns; + parts.push(`### ${detail.display}`); + parts.push(`Type: ${detail.kind} | Native columns: ${detail.columns.length}`); + if (detail.description || detail.comment) { + parts.push(`Description: ${detail.description ?? detail.comment}`); + } + parts.push('', 'Columns:'); + for (const column of columns) { + const pk = column.primaryKey ? ', PK' : ''; + parts.push(`- ${column.name} (${column.nativeType}, nullable=${column.nullable}${pk})${sampleText(column.sampleValues)}`); + } + parts.push(''); +} + +function findColumn(detail: TableDetail, columnName: string): TableDetail['columns'][number] | null { + const normalized = columnName.toLowerCase(); + return detail.columns.find((column) => column.name.toLowerCase() === normalized) ?? null; +} + +export class EntityDetailsTool extends BaseTool { + readonly name = 'entity_details'; + + constructor(private readonly catalogFactory: (context: ToolContext) => WarehouseCatalogService) { + super(); + } + + get description(): string { + return 'Verify warehouse tables and columns from the latest live-database scan before writing them into wiki or semantic-layer output.'; + } + + get inputSchema() { + return entityDetailsInputSchema; + } + + async call(input: EntityDetailsInput, context: ToolContext): Promise> { + const allowed = allowedConnectionNames(context); + if (allowed && !allowed.has(input.connectionName)) { + return { + markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, + structured: { resolved: [], missing: [], scanAvailable: false }, + }; + } + + const catalog = this.catalogFactory(context); + const scanAvailable = await catalog.hasScan(input.connectionName); + if (!scanAvailable) { + return { + markdown: `No live-database scan available for connection "${input.connectionName}"; run \`ktx scan\` first.`, + structured: { resolved: [], missing: [], scanAvailable: false }, + }; + } + + const parts: string[] = []; + const resolved: TableDetail[] = []; + const missing: EntityDetailsStructured['missing'] = []; + + for (const target of input.targets) { + const resolution = await resolveTarget(catalog, input.connectionName, target); + if (!resolution.resolved) { + missing.push({ target, candidates: resolution.candidates }); + appendMissingTargetMarkdown(parts, target, resolution.candidates); + continue; + } + const detail = await catalog.getTable({ connectionName: input.connectionName, ...resolution.resolved }); + if (!detail) { + missing.push({ target, candidates: resolution.candidates }); + appendMissingTargetMarkdown(parts, target, resolution.candidates); + continue; + } + const requestedColumn = resolution.resolved.column; + if (requestedColumn) { + const column = findColumn(detail, requestedColumn); + if (!column) { + missing.push({ + target, + candidates: [{ catalog: detail.catalog, db: detail.db, name: detail.name }], + }); + parts.push(`Column not found in scan: ${detail.display}.${requestedColumn}`); + parts.push(`Available columns: ${detail.columns.map((candidate) => candidate.name).join(', ')}`); + continue; + } + const scopedDetail = { ...detail, columns: [column] }; + resolved.push(scopedDetail); + appendTableMarkdown(parts, scopedDetail, column.name); + continue; + } + + resolved.push(detail); + appendTableMarkdown(parts, detail); + } + + return { + markdown: parts.join('\n').trim(), + structured: { resolved, missing, scanAvailable: true }, + }; + } +} diff --git a/packages/context/src/ingest/tools/warehouse-verification/index.ts b/packages/context/src/ingest/tools/warehouse-verification/index.ts new file mode 100644 index 00000000..e6ac2c1c --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/index.ts @@ -0,0 +1,28 @@ +import type { KtxFileStorePort } from '../../../core/index.js'; +import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import { DiscoverDataTool } from './discover-data.tool.js'; +import { EntityDetailsTool } from './entity-details.tool.js'; +import { SqlExecutionTool } from './sql-execution.tool.js'; +import { WarehouseCatalogService } from './warehouse-catalog.service.js'; + +export function createWarehouseVerificationTools(deps: { + connections: SlConnectionCatalogPort; + fallbackFileStore: KtxFileStorePort; + wikiSearchTool: BaseTool; + slDiscoverTool: BaseTool; +}): BaseTool[] { + const catalogFactory = (context: ToolContext) => + new WarehouseCatalogService({ + fileStore: context.session?.configService ?? deps.fallbackFileStore, + }); + return [ + new EntityDetailsTool(catalogFactory), + new SqlExecutionTool(deps.connections), + new DiscoverDataTool({ + wikiSearchTool: deps.wikiSearchTool, + slDiscoverTool: deps.slDiscoverTool, + catalogFactory, + }), + ]; +} diff --git a/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts b/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts new file mode 100644 index 00000000..1cc63cac --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import type { ToolContext } from '../../../tools/index.js'; +import { SqlExecutionTool } from './sql-execution.tool.js'; + +describe('SqlExecutionTool', () => { + const connections = { + executeQuery: vi.fn(), + } as unknown as SlConnectionCatalogPort & { executeQuery: ReturnType }; + const tool = new SqlExecutionTool(connections); + const context: ToolContext = { + sourceId: 'ingest', + messageId: 'm1', + userId: 'system', + session: { allowedConnectionNames: new Set(['warehouse']) } as any, + }; + + it('wraps read-only SQL with a capped row limit', async () => { + connections.executeQuery.mockResolvedValue({ headers: ['status'], rows: [['paid']], totalRows: 1 }); + + const result = await tool.call( + { connectionName: 'warehouse', sql: 'select status from public.orders', rowLimit: 5 }, + context, + ); + + expect(connections.executeQuery).toHaveBeenCalledWith( + 'warehouse', + 'select * from (select status from public.orders) as ktx_query_result limit 5', + ); + expect(result.markdown).toContain('| status |'); + expect(result.structured.wrappedSql).toContain('limit 5'); + }); + + it.each(['insert into x values (1)', 'drop table x', 'vacuum'])('rejects mutating SQL: %s', async (sql) => { + connections.executeQuery.mockClear(); + + const result = await tool.call({ connectionName: 'warehouse', sql }, context); + + expect(result.markdown).toContain('Only read-only SELECT/WITH queries can be executed locally.'); + expect(connections.executeQuery).not.toHaveBeenCalled(); + }); + + it('surfaces connector errors verbatim', async () => { + connections.executeQuery.mockRejectedValue(new Error('relation "orbit_analytics.customer" does not exist')); + + const result = await tool.call( + { connectionName: 'warehouse', sql: 'select 1 from orbit_analytics.customer', rowLimit: 1 }, + context, + ); + + expect(result.markdown).toContain('relation "orbit_analytics.customer" does not exist'); + expect(result.structured.error).toContain('relation "orbit_analytics.customer" does not exist'); + }); +}); diff --git a/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts b/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts new file mode 100644 index 00000000..03375938 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts @@ -0,0 +1,102 @@ +import { z } from 'zod'; +import { assertReadOnlySql, limitSqlForExecution } from '../../../connections/index.js'; +import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; + +const sqlExecutionInputSchema = z.object({ + connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + sql: z.string().min(1), + rowLimit: z.number().int().positive().max(1000).optional().default(100), +}); + +type SqlExecutionInput = z.input; + +export interface SqlExecutionStructured { + headers: string[]; + rows: unknown[][]; + rowCount: number; + truncated: boolean; + sql: string; + wrappedSql: string; + error?: string; +} + +function markdownTable(headers: string[], rows: unknown[][], totalRows: number): string { + if (headers.length === 0) { + return rows.length === 0 ? 'Query returned no rows.' : JSON.stringify(rows.slice(0, 20)); + } + const visible = rows.slice(0, 20); + const lines = [ + `| ${headers.join(' | ')} |`, + `| ${headers.map(() => '---').join(' | ')} |`, + ...visible.map((row) => `| ${row.map((value) => String(value ?? '')).join(' | ')} |`), + ]; + if (totalRows > visible.length) { + lines.push(`... +${totalRows - visible.length} more rows`); + } + return lines.join('\n'); +} + +export class SqlExecutionTool extends BaseTool { + readonly name = 'sql_execution'; + + constructor(private readonly connections: SlConnectionCatalogPort) { + super(); + } + + get description(): string { + return 'Run a single read-only SELECT or WITH probe against an allowed warehouse connection and return a capped markdown table or the warehouse error.'; + } + + get inputSchema() { + return sqlExecutionInputSchema; + } + + async call(input: SqlExecutionInput, context: ToolContext): Promise> { + const allowed = context.session?.allowedConnectionNames; + if (allowed && !allowed.has(input.connectionName)) { + return { + markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, + structured: { + headers: [], + rows: [], + rowCount: 0, + truncated: false, + sql: input.sql, + wrappedSql: '', + error: 'connection_not_allowed', + }, + }; + } + + let sql: string; + let wrappedSql: string; + try { + sql = assertReadOnlySql(input.sql); + wrappedSql = limitSqlForExecution(sql, input.rowLimit); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + markdown: message, + structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql: input.sql, wrappedSql: '', error: message }, + }; + } + + try { + const result = await this.connections.executeQuery(input.connectionName, wrappedSql); + const headers = result.headers ?? []; + const rows = result.rows ?? []; + const rowCount = result.totalRows ?? rows.length; + return { + markdown: markdownTable(headers, rows, rowCount), + structured: { headers, rows, rowCount, truncated: rowCount > rows.length, sql, wrappedSql }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + markdown: `SQL execution failed: ${message}`, + structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql, wrappedSql, error: message }, + }; + } + } +} diff --git a/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts new file mode 100644 index 00000000..c2ab1f36 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts @@ -0,0 +1,196 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { initKtxProject, type KtxLocalProject } from '../../../project/index.js'; +import { WarehouseCatalogService } from './warehouse-catalog.service.js'; + +describe('WarehouseCatalogService', () => { + let tempDir: string; + let project: KtxLocalProject; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-warehouse-catalog-')); + project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + async function seedLiveDatabaseScan(connectionName = 'warehouse', syncId = 'sync-2', driver = 'postgres') { + const root = `raw-sources/${connectionName}/live-database/${syncId}`; + const tableRef = { + catalog: driver === 'bigquery' ? 'analytics' : null, + db: driver === 'sqlite' ? null : 'public', + name: 'orders', + }; + await project.fileStore.writeFile( + `${root}/connection.json`, + JSON.stringify({ connectionId: connectionName, driver, extractedAt: '2026-05-12T00:00:00.000Z' }, null, 2), + 'ktx', + 'ktx@example.com', + 'seed connection', + ); + await project.fileStore.writeFile( + `${root}/tables/orders.json`, + JSON.stringify( + { + catalog: tableRef.catalog, + db: tableRef.db, + name: tableRef.name, + kind: 'table', + comment: 'Customer orders', + estimatedRows: 12, + columns: [ + { + name: 'id', + nativeType: 'integer', + normalizedType: 'integer', + dimensionType: 'number', + nullable: false, + primaryKey: true, + comment: 'Order id', + }, + { + name: 'status', + nativeType: 'text', + normalizedType: 'text', + dimensionType: 'string', + nullable: false, + primaryKey: false, + comment: 'Order status', + }, + ], + foreignKeys: [], + }, + null, + 2, + ), + 'ktx', + 'ktx@example.com', + 'seed orders', + ); + await project.fileStore.writeFile( + `${root}/enrichment/relationship-profile.json`, + JSON.stringify( + { + connectionId: connectionName, + driver, + sqlAvailable: true, + queryCount: 3, + tables: [{ table: { catalog: tableRef.catalog, db: tableRef.db, name: tableRef.name }, rowCount: 12 }], + columns: { + 'orders.status': { + table: { catalog: tableRef.catalog, db: tableRef.db, name: tableRef.name }, + column: 'status', + nativeType: 'text', + normalizedType: 'text', + rowCount: 12, + nullCount: 0, + distinctCount: 2, + uniquenessRatio: 0.1667, + nullRate: 0, + sampleValues: ['paid', 'refunded'], + minTextLength: 4, + maxTextLength: 8, + }, + }, + warnings: [], + }, + null, + 2, + ), + 'ktx', + 'ktx@example.com', + 'seed profile', + ); + } + + it('finds the latest sync and merges table schema with relationship profile values', async () => { + await seedLiveDatabaseScan('warehouse', 'sync-1'); + await seedLiveDatabaseScan('warehouse', 'sync-2'); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.getLatestSyncId('warehouse')).resolves.toBe('sync-2'); + const detail = await catalog.getTable({ connectionName: 'warehouse', catalog: null, db: 'public', name: 'orders' }); + + expect(detail).toMatchObject({ + connectionName: 'warehouse', + display: 'public.orders', + rowCount: 12, + columns: [ + { name: 'id', nativeType: 'integer', primaryKey: true }, + { name: 'status', nativeType: 'text', sampleValues: ['paid', 'refunded'], distinctCount: 2 }, + ], + }); + }); + + it('returns scanAvailable=false when no live-database scan exists', async () => { + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + await expect(catalog.getTable({ connectionName: 'missing', catalog: null, db: 'public', name: 'orders' })).resolves.toBeNull(); + await expect(catalog.hasScan('missing')).resolves.toBe(false); + }); + + it('resolves postgres display strings and returns closest candidates for missing tables', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplay('warehouse', 'public.orders')).resolves.toMatchObject({ + resolved: { catalog: null, db: 'public', name: 'orders' }, + candidates: [], + dialect: 'postgres', + }); + await expect(catalog.resolveDisplay('warehouse', 'public.orderz')).resolves.toMatchObject({ + resolved: null, + candidates: [{ name: 'orders' }], + }); + }); + + it('treats two-part BigQuery identifiers as ambiguous instead of guessing', async () => { + await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplay('warehouse', 'public.orders')).resolves.toMatchObject({ + resolved: null, + dialect: 'bigquery', + }); + }); + + it('resolves postgres column display strings without treating the column as a table', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplayTarget('warehouse', 'public.orders.status')).resolves.toMatchObject({ + resolved: { catalog: null, db: 'public', name: 'orders', column: 'status' }, + candidates: [], + dialect: 'postgres', + }); + }); + + it('resolves BigQuery column display strings with four parts', async () => { + await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplayTarget('warehouse', 'analytics.public.orders.status')).resolves.toMatchObject({ + resolved: { catalog: 'analytics', db: 'public', name: 'orders', column: 'status' }, + candidates: [], + dialect: 'bigquery', + }); + }); + + it('searches table names, column names, comments, and descriptions', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.searchByName('warehouse', 'status', 10)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'column', + ref: expect.objectContaining({ db: 'public', name: 'orders', column: 'status' }), + matchedOn: 'name', + }), + ]), + ); + }); +}); diff --git a/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts new file mode 100644 index 00000000..a1edf807 --- /dev/null +++ b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts @@ -0,0 +1,452 @@ +import { getDialectForDriver } from '../../../connections/index.js'; +import type { KtxFileStorePort } from '../../../core/index.js'; +import type { + KtxConnectionDriver, + KtxSchemaColumn, + KtxSchemaForeignKey, + KtxSchemaTable, + KtxTableRef, +} from '../../../scan/types.js'; + +type CatalogDriver = KtxConnectionDriver | 'sqlite3'; + +export interface WarehouseCatalogServiceDeps { + fileStore: KtxFileStorePort; +} + +interface WarehouseColumnDetail extends KtxSchemaColumn { + descriptions: Record; + rowCount: number | null; + nullCount: number | null; + distinctCount: number | null; + nullRate: number | null; + sampleValues: string[]; +} + +export interface TableDetail { + connectionName: string; + catalog: string | null; + db: string | null; + name: string; + display: string; + kind: string; + comment: string | null; + description: string | null; + rowCount: number | null; + columns: WarehouseColumnDetail[]; + foreignKeys: KtxSchemaForeignKey[]; +} + +export type RawSchemaHit = + | { + kind: 'table'; + connectionName: string; + ref: KtxTableRef; + display: string; + matchedOn: 'name' | 'db' | 'comment' | 'description'; + } + | { + kind: 'column'; + connectionName: string; + ref: KtxTableRef & { column: string }; + display: string; + matchedOn: 'name' | 'comment' | 'description'; + }; + +export interface DisplayTargetResolution { + resolved: (KtxTableRef & { column?: string }) | null; + candidates: KtxTableRef[]; + dialect: string; +} + +interface ConnectionArtifact { + driver?: CatalogDriver; +} + +interface RelationshipProfileColumn { + table?: KtxTableRef; + column?: string; + rowCount?: number; + nullCount?: number; + distinctCount?: number; + nullRate?: number; + sampleValues?: unknown[]; +} + +interface RelationshipProfileArtifact { + driver?: CatalogDriver; + tables?: Array<{ table?: KtxTableRef; rowCount?: number }>; + columns?: Record; +} + +interface ConnectionCatalog { + connectionName: string; + syncId: string; + driver: CatalogDriver; + tables: KtxSchemaTable[]; + profile: RelationshipProfileArtifact | null; +} + +type TableWithDescriptions = KtxSchemaTable & { + description?: string | null; + descriptions?: Record; + columns: Array }>; +}; + +function normalize(value: string | null | undefined): string { + return (value ?? '').toLowerCase(); +} + +function refsEqual(left: KtxTableRef, right: KtxTableRef): boolean { + return ( + normalize(left.catalog) === normalize(right.catalog) && + normalize(left.db) === normalize(right.db) && + normalize(left.name) === normalize(right.name) + ); +} + +function refKey(ref: KtxTableRef): string { + return [ref.catalog, ref.db, ref.name].map((part) => normalize(part)).join('.'); +} + +function columnKey(ref: KtxTableRef, column: string): string { + return `${refKey(ref)}.${normalize(column)}`; +} + +function readJson(content: string): T { + return JSON.parse(content) as T; +} + +function cleanIdentifierPart(part: string): string { + return part.trim().replace(/^["'`\[]|["'`\]]$/g, ''); +} + +function splitDisplay(display: string): string[] { + return display + .trim() + .split('.') + .map(cleanIdentifierPart) + .filter(Boolean); +} + +function formatDisplay(driver: CatalogDriver, table: KtxTableRef): string { + if (driver === 'sqlite' || driver === 'sqlite3') { + return table.name; + } + return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); +} + +function parseDisplay(driver: CatalogDriver, display: string): KtxTableRef | null { + const parts = splitDisplay(display); + if (driver === 'sqlite' || driver === 'sqlite3') { + return parts.length === 1 ? { catalog: null, db: null, name: parts[0]! } : null; + } + if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { + if (parts.length !== 3) { + return null; + } + return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; + } + if (parts.length === 2) { + return { catalog: null, db: parts[0]!, name: parts[1]! }; + } + if (parts.length === 3) { + return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; + } + return parts.length === 1 ? { catalog: null, db: null, name: parts[0]! } : null; +} + +function expectedDisplayPartCount(driver: CatalogDriver): number { + if (driver === 'sqlite' || driver === 'sqlite3') { + return 1; + } + if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { + return 3; + } + return 2; +} + +function parseColumnDisplay(driver: CatalogDriver, display: string): (KtxTableRef & { column: string }) | null { + const parts = splitDisplay(display); + const tablePartCount = expectedDisplayPartCount(driver); + if (parts.length !== tablePartCount + 1) { + return null; + } + const column = parts.at(-1); + if (!column) { + return null; + } + const table = parseDisplay(driver, parts.slice(0, -1).join('.')); + return table ? { ...table, column } : null; +} + +function bestCandidates(tables: KtxSchemaTable[], display: string, limit = 5): KtxTableRef[] { + const needle = normalize(splitDisplay(display).at(-1) ?? display); + return tables + .map((table) => { + const name = normalize(table.name); + let score = 0; + if (name === needle) { + score = 100; + } else if (name.includes(needle) || needle.includes(name)) { + score = 80; + } else { + const samePrefix = [...name].filter((char, index) => needle[index] === char).length; + score = samePrefix / Math.max(name.length, needle.length, 1); + } + return { table, score }; + }) + .filter((entry) => entry.score > 0) + .sort((left, right) => right.score - left.score || left.table.name.localeCompare(right.table.name)) + .slice(0, limit) + .map(({ table }) => ({ catalog: table.catalog, db: table.db, name: table.name })); +} + +function firstDescription(descriptions: Record | undefined): string | null { + return Object.values(descriptions ?? {}).find((value) => value.trim().length > 0) ?? null; +} + +function matchedOnTable(table: TableWithDescriptions, query: string): RawSchemaHit['matchedOn'] | null { + const q = normalize(query); + if (!q) { + return null; + } + if (normalize(table.name).includes(q)) { + return 'name'; + } + if (normalize(table.db).includes(q)) { + return 'db'; + } + if (normalize(table.comment).includes(q)) { + return 'comment'; + } + if (normalize(firstDescription(table.descriptions) ?? table.description).includes(q)) { + return 'description'; + } + return null; +} + +function matchedOnColumn( + column: KtxSchemaColumn & { description?: string | null; descriptions?: Record }, + query: string, +): 'name' | 'comment' | 'description' | null { + const q = normalize(query); + if (!q) { + return null; + } + if (normalize(column.name).includes(q)) { + return 'name'; + } + if (normalize(column.comment).includes(q)) { + return 'comment'; + } + if (normalize(firstDescription(column.descriptions) ?? column.description).includes(q)) { + return 'description'; + } + return null; +} + +export class WarehouseCatalogService { + private readonly catalogs = new Map>(); + + constructor(private readonly deps: WarehouseCatalogServiceDeps) {} + + async hasScan(connectionName: string): Promise { + return (await this.loadCatalog(connectionName)) !== null; + } + + async getLatestSyncId(connectionName: string): Promise { + return (await this.loadCatalog(connectionName))?.syncId ?? null; + } + + async listTables(connectionName: string): Promise { + const catalog = await this.loadCatalog(connectionName); + return catalog?.tables.map((table) => ({ catalog: table.catalog, db: table.db, name: table.name })) ?? []; + } + + async getTable(ref: { connectionName: string } & KtxTableRef): Promise { + const catalog = await this.loadCatalog(ref.connectionName); + if (!catalog) { + return null; + } + const table = catalog.tables.find((candidate) => refsEqual(candidate, ref)) as TableWithDescriptions | undefined; + if (!table) { + return null; + } + const profileTables = catalog.profile?.tables ?? []; + const profileTable = profileTables.find((candidate) => candidate.table && refsEqual(candidate.table, table)); + const profileColumns = catalog.profile?.columns ?? {}; + + return { + connectionName: ref.connectionName, + catalog: table.catalog, + db: table.db, + name: table.name, + display: formatDisplay(catalog.driver, table), + kind: table.kind, + comment: table.comment, + description: table.description ?? firstDescription(table.descriptions), + rowCount: profileTable?.rowCount ?? table.estimatedRows ?? null, + columns: table.columns.map((rawColumn) => { + const column = rawColumn as KtxSchemaColumn & { + description?: string | null; + descriptions?: Record; + }; + const profileColumn = + profileColumns[columnKey(table, column.name)] ?? + Object.entries(profileColumns).find( + ([key, value]) => + normalize(key) === `${normalize(table.name)}.${normalize(column.name)}` || + (value.table && refsEqual(value.table, table) && normalize(value.column) === normalize(column.name)), + )?.[1]; + return { + ...column, + descriptions: column.descriptions ?? {}, + rowCount: profileColumn?.rowCount ?? null, + nullCount: profileColumn?.nullCount ?? null, + distinctCount: profileColumn?.distinctCount ?? null, + nullRate: profileColumn?.nullRate ?? null, + sampleValues: (profileColumn?.sampleValues ?? []).map((value) => String(value)), + }; + }), + foreignKeys: table.foreignKeys, + }; + } + + async resolveDisplay( + connectionName: string, + display: string, + ): Promise<{ + resolved: KtxTableRef | null; + candidates: KtxTableRef[]; + dialect: string; + }> { + const catalog = await this.loadCatalog(connectionName); + if (!catalog) { + return { resolved: null, candidates: [], dialect: 'unknown' }; + } + const dialect = getDialectForDriver(catalog.driver).type; + const parsed = parseDisplay(catalog.driver, display); + if (!parsed) { + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + } + const table = catalog.tables.find((candidate) => refsEqual(candidate, parsed)); + if (!table) { + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + } + return { resolved: { catalog: table.catalog, db: table.db, name: table.name }, candidates: [], dialect }; + } + + async resolveDisplayTarget(connectionName: string, display: string): Promise { + const catalog = await this.loadCatalog(connectionName); + if (!catalog) { + return { resolved: null, candidates: [], dialect: 'unknown' }; + } + + const dialect = getDialectForDriver(catalog.driver).type; + const tableResolution = await this.resolveDisplay(connectionName, display); + if (tableResolution.resolved) { + return tableResolution; + } + + const parsedColumn = parseColumnDisplay(catalog.driver, display); + if (!parsedColumn) { + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + } + + const table = catalog.tables.find((candidate) => refsEqual(candidate, parsedColumn)); + if (!table) { + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + } + + return { + resolved: { + catalog: table.catalog, + db: table.db, + name: table.name, + column: parsedColumn.column, + }, + candidates: [], + dialect, + }; + } + + async searchByName(connectionName: string, query: string, limit: number): Promise { + const catalog = await this.loadCatalog(connectionName); + if (!catalog) { + return []; + } + const hits: RawSchemaHit[] = []; + for (const table of catalog.tables as TableWithDescriptions[]) { + const tableMatch = matchedOnTable(table, query); + if (tableMatch) { + hits.push({ + kind: 'table', + connectionName, + ref: { catalog: table.catalog, db: table.db, name: table.name }, + display: formatDisplay(catalog.driver, table), + matchedOn: tableMatch, + }); + } + for (const column of table.columns) { + const columnMatch = matchedOnColumn(column, query); + if (!columnMatch) { + continue; + } + hits.push({ + kind: 'column', + connectionName, + ref: { catalog: table.catalog, db: table.db, name: table.name, column: column.name }, + display: `${formatDisplay(catalog.driver, table)}.${column.name}`, + matchedOn: columnMatch, + }); + } + } + return hits.slice(0, Math.max(0, limit)); + } + + private loadCatalog(connectionName: string): Promise { + const existing = this.catalogs.get(connectionName); + if (existing) { + return existing; + } + const pending = this.readCatalog(connectionName); + this.catalogs.set(connectionName, pending); + return pending; + } + + private async readCatalog(connectionName: string): Promise { + const root = `raw-sources/${connectionName}/live-database`; + const listed = await this.deps.fileStore.listFiles(root); + const connectionFiles = listed.files.filter((file) => file.endsWith('/connection.json')).sort(); + const latestConnectionPath = connectionFiles.at(-1); + if (!latestConnectionPath) { + return null; + } + const latestRoot = latestConnectionPath.slice(0, -'/connection.json'.length); + const syncId = latestRoot.split('/').at(-1) ?? ''; + const connection = readJson((await this.deps.fileStore.readFile(latestConnectionPath)).content); + const tablesListing = await this.deps.fileStore.listFiles(`${latestRoot}/tables`); + const tables: KtxSchemaTable[] = []; + for (const tablePath of tablesListing.files.filter((file) => file.endsWith('.json')).sort()) { + tables.push(readJson((await this.deps.fileStore.readFile(tablePath)).content)); + } + + let profile: RelationshipProfileArtifact | null = null; + try { + profile = readJson( + (await this.deps.fileStore.readFile(`${latestRoot}/enrichment/relationship-profile.json`)).content, + ); + } catch { + profile = null; + } + + return { + connectionName, + syncId, + driver: connection.driver ?? profile?.driver ?? 'postgres', + tables, + profile, + }; + } +} diff --git a/packages/context/src/ingest/wiki-sl-ref-repair.test.ts b/packages/context/src/ingest/wiki-sl-ref-repair.test.ts new file mode 100644 index 00000000..958386c7 --- /dev/null +++ b/packages/context/src/ingest/wiki-sl-ref-repair.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from 'vitest'; +import { repairWikiSlRefs } from './wiki-sl-ref-repair.js'; + +describe('repairWikiSlRefs', () => { + it('removes missing measure refs while keeping source, measure, segment, and manifest-backed refs', async () => { + type TestPage = { pageKey: string; frontmatter: Record; content: string }; + const pages = new Map([ + [ + 'GLOBAL:accounts-at-risk', + { + pageKey: 'accounts-at-risk', + frontmatter: { + summary: 'Accounts at risk', + usage_mode: 'auto', + sl_refs: [ + 'mart_customer_health', + 'mart_customer_health.high_risk_account_count', + 'mart_customer_health.medium_risk_account_count', + 'mart_customer_health.high_risk', + 'int_procurement_qualifying_actions', + ], + }, + content: 'Risk context.', + }, + ], + ]); + const wikiService = { + readPage: vi.fn(async (scope: string, _scopeId: string | null, key: string) => pages.get(`${scope}:${key}`)), + writePage: vi.fn( + async ( + scope: string, + _scopeId: string | null, + key: string, + frontmatter: Record, + content: string, + ) => { + pages.set(`${scope}:${key}`, { pageKey: key, frontmatter, content }); + }, + ), + }; + const configService = { + listFiles: vi.fn(async () => ({ + files: ['global/accounts-at-risk.md', 'global/historic-sql/nested-legacy.md'], + })), + }; + const semanticLayerService = { + loadAllSources: vi.fn(async () => [ + { + name: 'mart_customer_health', + grain: [], + columns: [], + joins: [], + measures: [{ name: 'high_risk_account_count', expr: 'count(*)' }], + segments: [{ name: 'high_risk', expr: "risk_level = 'high'" }], + }, + { + name: 'int_procurement_qualifying_actions', + grain: [], + columns: [], + joins: [], + measures: [], + }, + ]), + }; + + const result = await repairWikiSlRefs({ + wikiService: wikiService as never, + semanticLayerService: semanticLayerService as never, + configService: configService as never, + connectionIds: ['warehouse'], + }); + + expect(result.repairs).toEqual([ + { + pageKey: 'accounts-at-risk', + scope: 'GLOBAL', + scopeId: null, + removedRefs: ['mart_customer_health.medium_risk_account_count'], + }, + ]); + expect(wikiService.writePage).toHaveBeenCalledWith( + 'GLOBAL', + null, + 'accounts-at-risk', + expect.objectContaining({ + sl_refs: [ + 'mart_customer_health', + 'mart_customer_health.high_risk_account_count', + 'mart_customer_health.high_risk', + 'int_procurement_qualifying_actions', + ], + }), + 'Risk context.', + 'System User', + 'system@example.com', + 'Repair semantic-layer refs: accounts-at-risk', + ); + }); +}); diff --git a/packages/context/src/ingest/wiki-sl-ref-repair.ts b/packages/context/src/ingest/wiki-sl-ref-repair.ts new file mode 100644 index 00000000..7d3d48f3 --- /dev/null +++ b/packages/context/src/ingest/wiki-sl-ref-repair.ts @@ -0,0 +1,140 @@ +import type { KtxFileStorePort } from '../core/index.js'; +import type { SemanticLayerService, SemanticLayerSource } from '../sl/index.js'; +import { isFlatWikiKey } from '../wiki/keys.js'; +import type { KnowledgeWikiService, WikiFrontmatter } from '../wiki/index.js'; + +const SYSTEM_AUTHOR = 'System User'; +const SYSTEM_EMAIL = 'system@example.com'; + +export interface WikiSlRefRepair { + pageKey: string; + scope: 'GLOBAL' | 'USER'; + scopeId: string | null; + removedRefs: string[]; +} + +export interface WikiSlRefRepairResult { + repairs: WikiSlRefRepair[]; + warnings: string[]; +} + +interface WikiPath { + scope: 'GLOBAL' | 'USER'; + scopeId: string | null; + pageKey: string; +} + +function parseKnowledgeFilePath(path: string): WikiPath | null { + if (!path.endsWith('.md')) { + return null; + } + const segments = path.split('/'); + if (segments.length === 2 && segments[0] === 'global') { + const pageKey = segments[1].replace(/\.md$/, ''); + return isFlatWikiKey(pageKey) ? { scope: 'GLOBAL', scopeId: null, pageKey } : null; + } + if (segments.length === 3 && segments[0] === 'user') { + const pageKey = segments[2].replace(/\.md$/, ''); + return isFlatWikiKey(pageKey) ? { scope: 'USER', scopeId: segments[1], pageKey } : null; + } + return null; +} + +function entityRefsForSource(source: SemanticLayerSource): string[] { + return [ + source.name, + ...(source.measures ?? []).map((measure) => `${source.name}.${measure.name}`), + ...(source.segments ?? []).map((segment) => `${source.name}.${segment.name}`), + ]; +} + +async function loadVisibleSlRefs( + semanticLayerService: SemanticLayerService, + connectionIds: string[], +): Promise<{ refs: Set; warnings: string[] }> { + const refs = new Set(); + const warnings: string[] = []; + for (const connectionId of connectionIds) { + try { + for (const source of await semanticLayerService.loadAllSources(connectionId)) { + for (const ref of entityRefsForSource(source)) { + refs.add(ref); + } + } + } catch (error) { + warnings.push( + `Skipped wiki sl_refs repair for connection ${connectionId}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + return { refs, warnings }; +} + +function uniqueStringArray(value: string[] | undefined): string[] { + return [...new Set((value ?? []).filter((entry) => typeof entry === 'string' && entry.trim().length > 0))]; +} + +export async function repairWikiSlRefs(input: { + wikiService: KnowledgeWikiService; + semanticLayerService: SemanticLayerService; + configService: KtxFileStorePort; + connectionIds: string[]; +}): Promise { + const { refs: validRefs, warnings } = await loadVisibleSlRefs(input.semanticLayerService, input.connectionIds); + const listFiles = + typeof input.configService.listFiles === 'function' + ? input.configService.listFiles.bind(input.configService) + : null; + if (!listFiles) { + return { + repairs: [], + warnings: [...warnings, 'Skipped wiki sl_refs repair: config service cannot list wiki files.'], + }; + } + const listed = await listFiles('knowledge', true); + const repairs: WikiSlRefRepair[] = []; + + for (const file of listed.files.sort()) { + const parsedPath = parseKnowledgeFilePath(file); + if (!parsedPath) { + continue; + } + const page = await input.wikiService.readPage(parsedPath.scope, parsedPath.scopeId, parsedPath.pageKey); + const refs = uniqueStringArray(page?.frontmatter.sl_refs); + if (!page || refs.length === 0) { + continue; + } + const keptRefs = refs.filter((ref) => validRefs.has(ref)); + const removedRefs = refs.filter((ref) => !validRefs.has(ref)); + if (removedRefs.length === 0) { + continue; + } + + const frontmatter: WikiFrontmatter = { + ...page.frontmatter, + sl_refs: keptRefs, + }; + await input.wikiService.writePage( + parsedPath.scope, + parsedPath.scopeId, + parsedPath.pageKey, + frontmatter, + page.content, + SYSTEM_AUTHOR, + SYSTEM_EMAIL, + `Repair semantic-layer refs: ${parsedPath.pageKey}`, + ); + repairs.push({ ...parsedPath, removedRefs }); + } + + return { + repairs, + warnings: [ + ...warnings, + ...repairs.map( + (repair) => + `Removed invalid sl_refs from ${repair.pageKey}: ${repair.removedRefs.join(', ')}`, + ), + ], + }; +} diff --git a/packages/context/src/memory/local-memory.ts b/packages/context/src/memory/local-memory.ts index 25ddb2c1..af65b54e 100644 --- a/packages/context/src/memory/local-memory.ts +++ b/packages/context/src/memory/local-memory.ts @@ -36,6 +36,7 @@ import { BaseTool, type GitAuthorResolverPort, type ToolContext } from '../tools import { type KnowledgeEventPort, type KnowledgeIndexPort, + type KnowledgeIndexPageListing, KnowledgeWikiService, searchLocalKnowledgePages, WikiListTagsTool, @@ -219,7 +220,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { } async listPagesForUser(userId: string) { - const pages: Array<{ id?: string; page_key: string; summary: string; scope: string; scope_id: string | null }> = []; + const pages: KnowledgeIndexPageListing[] = []; for (const scope of [ { scope: 'GLOBAL', scopeId: null, dir: 'knowledge/global' }, { scope: 'USER', scopeId: userId, dir: `knowledge/user/${userId}` }, @@ -234,6 +235,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { summary: parsed.summary, scope: scope.scope, scope_id: scope.scopeId, + tags: parseWikiTags(raw.content), }); } } @@ -433,7 +435,7 @@ class LocalMemoryToolsetFactory implements MemoryToolsetFactoryPort { }; }, }), - new WikiListTagsTool(deps.wikiService, deps.knowledgeIndex), + new WikiListTagsTool(deps.knowledgeIndex), new WikiWriteTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), new WikiRemoveTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), ]; @@ -468,6 +470,17 @@ function parseWiki(raw: string): { summary: string; content: string } { }; } +function parseWikiTags(raw: string): string[] { + const match = raw.match(/^---\n([\s\S]*?)\n---\n?/); + if (!match) { + return []; + } + const frontmatter = (YAML.parse(match[1]) ?? {}) as Record; + return Array.isArray(frontmatter.tags) + ? frontmatter.tags.filter((tag): tag is string => typeof tag === 'string') + : []; +} + function scoreText(text: string, query: string): number { const normalized = query.toLowerCase().trim(); if (!normalized) { diff --git a/packages/context/src/memory/memory-runtime-assets.test.ts b/packages/context/src/memory/memory-runtime-assets.test.ts index ef056d18..bd18e524 100644 --- a/packages/context/src/memory/memory-runtime-assets.test.ts +++ b/packages/context/src/memory/memory-runtime-assets.test.ts @@ -23,11 +23,42 @@ const expectedAdapterSkillHeadings: Record = { metabase_ingest: '# Metabase to KTX Semantic Layer', metricflow_ingest: '# MetricFlow to KTX Semantic Layer', }; +const verificationWriterSkills = [ + 'notion_synthesize', + 'dbt_ingest', + 'lookml_ingest', + 'looker_ingest', + 'metabase_ingest', + 'metricflow_ingest', + 'live_database_ingest', + 'historic_sql_table_digest', + 'historic_sql_patterns', + 'knowledge_capture', + 'sl_capture', +] as const; function forbiddenProductPattern() { return new RegExp([['Kae', 'lio'].join(''), ['kae', 'lio'].join(''), ['KAE', 'LIO_'].join('')].join('|')); } +function sqlExecutionCallBlocks(body: string): string[] { + const blocks: string[] = []; + const marker = 'sql_execution({'; + let offset = 0; + + while (offset < body.length) { + const start = body.indexOf(marker, offset); + if (start === -1) { + break; + } + const end = body.indexOf('})', start + marker.length); + blocks.push(body.slice(start, end === -1 ? start + marker.length : end + 2)); + offset = start + marker.length; + } + + return blocks; +} + describe('memory runtime assets', () => { it('packages every memory-agent base prompt referenced by promptNameFor()', async () => { const prompts = new PromptService({ promptsDir, partials: [] }); @@ -117,4 +148,50 @@ describe('memory runtime assets', () => { expect(body).toContain('Do not call `sl_write_source` or `sl_edit_source`'); expect(body).toContain('LookML writes target the run connection directly'); }); + + it('ships identifier verification protocol in every synthesis writer skill', async () => { + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + expect(body).toContain('## Identifier Verification Protocol'); + expect(body).toMatch(/discover_data|entity_details/); + } + }); + + it('does not ship stale warehouse verification tool names or fictional identifiers', async () => { + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + expect(body).not.toContain('orbit_analytics.customer'); + expect(body).not.toContain('wiki_sl_search'); + expect(body).not.toContain('sl_describe_table'); + } + }); + + it('ships only the KTX connectionName sql_execution call shape in writer guidance', async () => { + const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); + const bodies = [{ name: '_shared/identifier-verification.md', body: shared }]; + + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); + expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); + + for (const skillName of verificationWriterSkills) { + const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); + bodies.push({ name: `${skillName}/SKILL.md`, body }); + expect(body).toContain('sql_execution({connectionName'); + expect(body).not.toContain('sql_execution({ sql'); + expect(body).not.toContain('session shape'); + expect(body).not.toContain('connection is already pinned by the ingest session'); + } + + for (const { name, body } of bodies) { + const calls = sqlExecutionCallBlocks(body); + expect(calls.length, `${name} should contain sql_execution guidance`).toBeGreaterThan(0); + expect( + calls.filter((call) => !call.includes('connectionName')), + `${name} has sql_execution calls without connectionName`, + ).toEqual([]); + expect(body, `${name} has a connectionless multiline sql_execution call`).not.toMatch( + /sql_execution\(\{\s*sql\s*:/, + ); + } + }); }); diff --git a/packages/context/src/scan/enrichment-types.ts b/packages/context/src/scan/enrichment-types.ts index 2b186976..645951c3 100644 --- a/packages/context/src/scan/enrichment-types.ts +++ b/packages/context/src/scan/enrichment-types.ts @@ -68,15 +68,6 @@ export interface KtxDescriptionUpdate { columnDescriptions?: Record; } -const PREFERRED_METADATA_FIELD_NAMES = [ - 'tags', - 'constraints', - 'enum_values', - 'freshness', - 'tests', - 'lineage', -] as const; - export interface KtxMetadataUpdate { connectionId: string; table: KtxTableRef; diff --git a/packages/context/src/search/pglite-runtime-boundary.test.ts b/packages/context/src/search/pglite-runtime-boundary.test.ts index ce2f5b7a..2db3209b 100644 --- a/packages/context/src/search/pglite-runtime-boundary.test.ts +++ b/packages/context/src/search/pglite-runtime-boundary.test.ts @@ -46,7 +46,8 @@ describe('PGlite hybrid search runtime boundary', () => { } const productionRoutingFiles = [ - 'packages/cli/src/agent.ts', + 'packages/cli/src/sl.ts', + 'packages/cli/src/knowledge.ts', 'packages/context/src/mcp/local-project-ports.ts', 'packages/context/src/wiki/local-knowledge.ts', 'packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.ts', diff --git a/packages/context/src/sl/tools/sl-warehouse-validation.ts b/packages/context/src/sl/tools/sl-warehouse-validation.ts index f9c5e4fd..a200dad9 100644 --- a/packages/context/src/sl/tools/sl-warehouse-validation.ts +++ b/packages/context/src/sl/tools/sl-warehouse-validation.ts @@ -90,7 +90,7 @@ export async function validateSingleSource( `writing it as-is drops the manifest's columns and joins. ` + `Remove "sql:", "table:", "grain:", "columns:", and "joins:" and keep only ` + `"name:" plus "measures:"/"segments:"/"description:" to write an overlay ` + - `that inherits the manifest schema. Call sl_describe_table to see it first.`, + `that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`, ); return { errors, warnings }; } diff --git a/packages/context/src/tools/tool-session.ts b/packages/context/src/tools/tool-session.ts index d9156258..023a8c8e 100644 --- a/packages/context/src/tools/tool-session.ts +++ b/packages/context/src/tools/tool-session.ts @@ -47,6 +47,7 @@ export interface ToolSession { touchedSlSources: TouchedSlSourceSet; actions: MemoryAction[]; allowedRawPaths?: ReadonlySet; + allowedConnectionNames?: ReadonlySet; semanticLayerService: SemanticLayerService; wikiService: KnowledgeWikiService; configService: KtxFileStorePort; diff --git a/packages/context/src/wiki/index.ts b/packages/context/src/wiki/index.ts index 6eae10f0..17d37399 100644 --- a/packages/context/src/wiki/index.ts +++ b/packages/context/src/wiki/index.ts @@ -12,6 +12,7 @@ export type { KnowledgeEventPort, KnowledgeGitDiffPort, KnowledgeIndexPort, + KnowledgeIndexPageListing, UpsertPageParams, WikiFileStorePort, } from './ports.js'; diff --git a/packages/context/src/wiki/knowledge-wiki.service.test.ts b/packages/context/src/wiki/knowledge-wiki.service.test.ts index ecbf954a..40056edc 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.test.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.test.ts @@ -113,13 +113,13 @@ describe('KnowledgeWikiService.syncFromCommit', () => { expect(call.deletes).toEqual([{ scope: 'GLOBAL', scopeId: null, pageKey: 'gone-page' }]); }); - it('indexes historic-SQL nested pages but skips other nested wiki paths from commit sync', async () => { + it('indexes only flat wiki pages and skips nested paths from commit sync', async () => { const { service, pagesRepository, gitService, logger } = makeService(); gitService.diffNameStatus.mockResolvedValue([ { status: 'A', path: 'knowledge/global/revenue-policy.md' }, + { status: 'A', path: 'knowledge/global/historic-sql-order-lifecycle.md' }, { status: 'A', path: 'knowledge/global/historic-sql/order-lifecycle.md' }, - { status: 'A', path: 'knowledge/global/historic-sql/_archived/retired-pattern.md' }, { status: 'A', path: 'knowledge/global/orbit/company-overview.md' }, ]); gitService.getFileAtCommit.mockImplementation((path: string) => { @@ -138,26 +138,25 @@ describe('KnowledgeWikiService.syncFromCommit', () => { await service.syncFromCommit('sha-before', 'sha-after', 'run-uuid'); expect(gitService.getFileAtCommit).not.toHaveBeenCalledWith('knowledge/global/orbit/company-overview.md', 'sha-after'); + expect(gitService.getFileAtCommit).not.toHaveBeenCalledWith('knowledge/global/historic-sql/order-lifecycle.md', 'sha-after'); expect(logger.warn).toHaveBeenCalledWith( '[knowledge.sync] skipping unparseable path: knowledge/global/orbit/company-overview.md', ); + expect(logger.warn).toHaveBeenCalledWith( + '[knowledge.sync] skipping unparseable path: knowledge/global/historic-sql/order-lifecycle.md', + ); const call = pagesRepository.applyDiffTransactional.mock.calls[0][0]; expect(call.upserts).toEqual( expect.arrayContaining([ expect.objectContaining({ scope: 'GLOBAL', pageKey: 'revenue-policy', summary: 'revenue' }), expect.objectContaining({ scope: 'GLOBAL', - pageKey: 'historic-sql/order-lifecycle', + pageKey: 'historic-sql-order-lifecycle', summary: 'order lifecycle', }), - expect.objectContaining({ - scope: 'GLOBAL', - pageKey: 'historic-sql/_archived/retired-pattern', - summary: 'retired', - }), ]), ); - expect(call.upserts).toHaveLength(3); + expect(call.upserts).toHaveLength(2); }); it('is a no-op when the diff between shas has no knowledge changes', async () => { diff --git a/packages/context/src/wiki/knowledge-wiki.service.ts b/packages/context/src/wiki/knowledge-wiki.service.ts index 2ca32f79..fb152e83 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.ts @@ -11,10 +11,6 @@ const WIKI_PREFIX = 'knowledge'; export type { WikiFrontmatter }; -function isHistoricSqlPathSegment(segment: string): boolean { - return /^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(segment); -} - export class KnowledgeWikiService { private isWorktreeScoped = false; @@ -422,7 +418,6 @@ export class KnowledgeWikiService { * Parse a `knowledge//...` file path into its scope and page key. * `knowledge/global/foo.md` → { scope: 'GLOBAL', scopeId: null, pageKey: 'foo' } * `knowledge/user//bar.md` → { scope: 'USER', scopeId: '', pageKey: 'bar' } - * `knowledge/global/historic-sql/foo.md` → { scope: 'GLOBAL', scopeId: null, pageKey: 'historic-sql/foo' } */ function parseKnowledgePath(path: string): { scope: string; scopeId: string | null; pageKey: string } | null { if (!path.endsWith('.md')) { @@ -437,13 +432,6 @@ function parseKnowledgePath(path: string): { scope: string; scopeId: string | nu const pageKey = rest[1].replace(/\.md$/, ''); return isFlatWikiKey(pageKey) ? { scope: 'GLOBAL', scopeId: null, pageKey } : null; } - if (rest.length >= 3 && rest[0] === 'global' && rest[1] === 'historic-sql') { - const historicPath = rest.slice(2).join('/').replace(/\.md$/, ''); - if (historicPath.split('/').every(isHistoricSqlPathSegment)) { - return { scope: 'GLOBAL', scopeId: null, pageKey: `historic-sql/${historicPath}` }; - } - return null; - } if (rest.length === 3 && rest[0] === 'user') { const pageKey = rest[2].replace(/\.md$/, ''); return isFlatWikiKey(pageKey) ? { scope: 'USER', scopeId: rest[1], pageKey } : null; diff --git a/packages/context/src/wiki/local-knowledge.test.ts b/packages/context/src/wiki/local-knowledge.test.ts index 78da841f..5ad66eb1 100644 --- a/packages/context/src/wiki/local-knowledge.test.ts +++ b/packages/context/src/wiki/local-knowledge.test.ts @@ -244,4 +244,30 @@ describe('local knowledge helpers', () => { }), ).rejects.toThrow('Invalid wiki key "orbit/company-overview". Wiki keys must be flat; use "orbit-company-overview".'); }); + + it('ignores nested historic-SQL legacy paths when listing local knowledge pages', async () => { + await writeLocalKnowledgePage(project, { + key: 'historic-sql-paid-orders', + scope: 'GLOBAL', + summary: 'Flat historic SQL page', + content: 'Flat page body.', + tags: ['historic-sql'], + }); + await project.fileStore.writeFile( + 'knowledge/global/historic-sql/paid-orders.md', + '---\nsummary: Nested historic SQL page\nusage_mode: auto\n---\n\nNested body\n', + 'Test', + 'test@example.com', + 'Write nested legacy page', + ); + + await expect(listLocalKnowledgePages(project, { userId: 'local' })).resolves.toEqual([ + { + key: 'historic-sql-paid-orders', + path: 'knowledge/global/historic-sql-paid-orders.md', + scope: 'GLOBAL', + summary: 'Flat historic SQL page', + }, + ]); + }); }); diff --git a/packages/context/src/wiki/local-knowledge.ts b/packages/context/src/wiki/local-knowledge.ts index 007b006e..5d1314a8 100644 --- a/packages/context/src/wiki/local-knowledge.ts +++ b/packages/context/src/wiki/local-knowledge.ts @@ -80,26 +80,12 @@ function knowledgePath(scope: LocalKnowledgeScope, userId: string | undefined, k return `knowledge/user/${assertSafePathToken('user id', userId ?? 'local')}/${safeKey}.md`; } -function isHistoricSqlPathSegment(segment: string): boolean { - return /^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(segment); -} - function keyFromKnowledgePath(path: string, scope: LocalKnowledgeScope, userId: string): string | null { const prefix = scope === 'GLOBAL' ? 'knowledge/global/' : `knowledge/user/${assertSafePathToken('user id', userId)}/`; const key = path.slice(prefix.length).replace(/\.md$/, ''); if (isFlatWikiKey(key)) { return key; } - if ( - scope === 'GLOBAL' && - key.startsWith('historic-sql/') && - key - .slice('historic-sql/'.length) - .split('/') - .every(isHistoricSqlPathSegment) - ) { - return key; - } return null; } diff --git a/packages/context/src/wiki/ports.ts b/packages/context/src/wiki/ports.ts index 7fa48c29..075a6c20 100644 --- a/packages/context/src/wiki/ports.ts +++ b/packages/context/src/wiki/ports.ts @@ -13,6 +13,15 @@ export interface UpsertPageParams { sourceRunId?: string | null; } +export interface KnowledgeIndexPageListing { + id?: string; + page_key: string; + summary: string; + scope: string; + scope_id: string | null; + tags: string[]; +} + export interface KnowledgeIndexPort { upsertPage(params: UpsertPageParams): Promise; applyDiffTransactional(params: { @@ -32,9 +41,7 @@ export interface KnowledgeIndexPort { scopeId: string | null, pageKey: string, ): Promise<{ id?: string; page_key: string } | null | undefined>; - listPagesForUser( - userId: string, - ): Promise>; + listPagesForUser(userId: string): Promise; getUserPageCount(userId: string): Promise; incrementUsageCount(pageIds: string[]): Promise; searchRRF( diff --git a/packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts b/packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts index a47b5912..e4b5b7f3 100644 --- a/packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts +++ b/packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts @@ -8,22 +8,11 @@ describe('WikiListTagsTool', () => { it("returns distinct sorted tags across the user's visible pages", async () => { const pagesRepository = { listPagesForUser: vi.fn().mockResolvedValue([ - { scope: 'GLOBAL', scope_id: null, page_key: 'k1' }, - { scope: 'USER', scope_id: 'u', page_key: 'k2' }, + { scope: 'GLOBAL', scope_id: null, page_key: 'k1', tags: ['metrics', 'finance'] }, + { scope: 'USER', scope_id: 'u', page_key: 'k2', tags: ['metrics'] }, ]), }; - const wikiService = { - readPage: vi.fn().mockImplementation((_scope, _scopeId, key) => { - if (key === 'k1') { - return Promise.resolve({ frontmatter: { tags: ['metrics', 'finance'] }, content: '' }); - } - if (key === 'k2') { - return Promise.resolve({ frontmatter: { tags: ['metrics'] }, content: '' }); - } - return Promise.resolve(null); - }), - }; - const tool = new WikiListTagsTool(wikiService as any, pagesRepository as any); + const tool = new WikiListTagsTool(pagesRepository as any); const result = await tool.call({}, baseContext); expect(result.markdown).toContain('finance'); @@ -31,10 +20,23 @@ describe('WikiListTagsTool', () => { expect(result.structured.tags).toEqual(['finance', 'metrics']); }); + it('lists tags from historic-SQL indexed pages with flat wiki keys', async () => { + const pagesRepository = { + listPagesForUser: vi.fn().mockResolvedValue([ + { scope: 'GLOBAL', scope_id: null, page_key: 'company-overview', tags: ['notion'] }, + { scope: 'GLOBAL', scope_id: null, page_key: 'historic-sql-revenue-pattern', tags: ['historic-sql', 'pattern'] }, + ]), + }; + const tool = new WikiListTagsTool(pagesRepository as any); + + const result = await tool.call({}, baseContext); + + expect(result.structured.tags).toEqual(['historic-sql', 'notion', 'pattern']); + }); + it('returns a friendly message when no pages have tags', async () => { const pagesRepository = { listPagesForUser: vi.fn().mockResolvedValue([]) }; - const wikiService = { readPage: vi.fn() }; - const tool = new WikiListTagsTool(wikiService as any, pagesRepository as any); + const tool = new WikiListTagsTool(pagesRepository as any); const result = await tool.call({}, baseContext); expect(result.markdown).toMatch(/no tags/i); diff --git a/packages/context/src/wiki/tools/wiki-list-tags.tool.ts b/packages/context/src/wiki/tools/wiki-list-tags.tool.ts index cd3c5aac..3a31ee41 100644 --- a/packages/context/src/wiki/tools/wiki-list-tags.tool.ts +++ b/packages/context/src/wiki/tools/wiki-list-tags.tool.ts @@ -1,7 +1,5 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; -type BlockScope = 'GLOBAL' | 'USER'; -import { KnowledgeWikiService } from '../index.js'; import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; const wikiListTagsInputSchema = z.object({}); @@ -11,10 +9,7 @@ type WikiListTagsInput = z.infer; export class WikiListTagsTool extends BaseTool { readonly name = 'wiki_list_tags'; - constructor( - private readonly wikiService: KnowledgeWikiService, - private readonly pagesRepository: KnowledgeIndexPort, - ) { + constructor(private readonly pagesRepository: KnowledgeIndexPort) { super(); } @@ -33,10 +28,7 @@ Call before writing a new page so you can reuse existing tags consistently inste const pages = await this.pagesRepository.listPagesForUser(context.userId); const set = new Set(); for (const p of pages) { - const scope = p.scope as BlockScope; - const scopeId = scope === 'USER' ? p.scope_id : null; - const page = await this.wikiService.readPage(scope, scopeId, p.page_key); - for (const t of page?.frontmatter.tags ?? []) { + for (const t of p.tags) { set.add(t); } } diff --git a/packages/context/src/wiki/tools/wiki-write.tool.ts b/packages/context/src/wiki/tools/wiki-write.tool.ts index edd34f8f..70668950 100644 --- a/packages/context/src/wiki/tools/wiki-write.tool.ts +++ b/packages/context/src/wiki/tools/wiki-write.tool.ts @@ -150,6 +150,7 @@ export class WikiWriteTool extends BaseTool { Create or update a knowledge page. Provide content for create/rewrite, or replacements for targeted edits. For existing pages, you may provide only frontmatter fields such as summary, tags, refs, or sl_refs to update metadata while preserving content. tags/refs/sl_refs use REPLACE semantics: omit to keep existing on update, [] to clear, [values] to set. +Keys must be flat file names, not directory paths. Use tags/source frontmatter for grouping. `; } diff --git a/packages/context/src/wiki/types.ts b/packages/context/src/wiki/types.ts index 317b17ab..bff57aa5 100644 --- a/packages/context/src/wiki/types.ts +++ b/packages/context/src/wiki/types.ts @@ -25,6 +25,7 @@ export interface WikiFrontmatter { usage?: HistoricSqlWikiUsageFrontmatter; fingerprints?: string[]; stale_since?: string; + archived_since?: string; } export interface WikiPage { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0f7ff8bd..6f9f19b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,15 +12,24 @@ importers: .: devDependencies: + '@biomejs/biome': + specifier: ^2.4.15 + version: 2.4.15 '@types/node': specifier: ^24.3.0 version: 24.12.2 + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + knip: + specifier: ^6.12.2 + version: 6.12.2(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) typescript: specifier: ^6.0.3 version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + yaml: + specifier: ^2.9.0 + version: 2.9.0 docs-site: dependencies: @@ -741,6 +750,63 @@ packages: resolution: {integrity: sha512-SriLPKezypIsiZ+TtlFfE46uuBIap2HeaQVS78e1P7rz5OSbq0rsd52WE1mC5f7vAeLiXqv7I7oRhL3WFZEw3Q==} engines: {node: '>=18.0.0'} + '@biomejs/biome@2.4.15': + resolution: {integrity: sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==} + engines: {node: '>=14.21.3'} + hasBin: true + + '@biomejs/cli-darwin-arm64@2.4.15': + resolution: {integrity: sha512-rF3PPqLq1yoST79zaQbDjVJwsuIeci/O+9bgNmC5QpgOqz6aqYuzA4abyAGx+mgyiDXn4A049xAN8gijbuR1Qg==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [darwin] + + '@biomejs/cli-darwin-x64@2.4.15': + resolution: {integrity: sha512-/5KHXYMfSJs1fNXiX30xFtI8JcCFV6zaVVLxOa0M2sfqBKHkpQhRTv94yxQWxeTY2lzo2OuTlNvPC+hDQt2wcQ==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [darwin] + + '@biomejs/cli-linux-arm64-musl@2.4.15': + resolution: {integrity: sha512-ZPcxznxm0pogHBLZhYntyR3sR+MrZjqJIKEr7ZqVen0Rl+P/4upVmfYXjftizi9RoqZntg33fv/1fbdhbYXpEQ==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@biomejs/cli-linux-arm64@2.4.15': + resolution: {integrity: sha512-owaAMZD/T4LrD0ELNCk0Km3qrRHuM0X6EAyVE1FSqGY0rbLoiDLrO4Us2tllm6cAeB2Ioa9C2C08NZPdr8+0Ug==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@biomejs/cli-linux-x64-musl@2.4.15': + resolution: {integrity: sha512-CNq/9W38SYSH023lfcQ4KKU8K0YX8T//FZUhcgtMMRABDojx5XsMV7jlweAvGSl389wJQB29Qo6Zb/a+jdvt+w==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@biomejs/cli-linux-x64@2.4.15': + resolution: {integrity: sha512-0jj7THz12GbUOLmMibktK6DZjqz2zV64KFxyBtcFTKPiiOIY0a7vns1elpO1dERvxpsZ5ik0oFfz0oGwFde1+g==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@biomejs/cli-win32-arm64@2.4.15': + resolution: {integrity: sha512-ouhkYdlhp/1GghEJPdWwD/Vi3gQ1nFxuSpMolWsbq3Lsq3QUR4jl6UdhhscdCugKU5vOEuMiJhvKj66O0OCq+w==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [win32] + + '@biomejs/cli-win32-x64@2.4.15': + resolution: {integrity: sha512-zBrGq5mx5wwpnow4+2BxUvleDM+GNd4sLbPaMapsSLQLD0NGRCquqPBTgN+7XkUteHvj7M+BstuI8tmnV7+HgQ==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [win32] + '@clack/core@1.3.1': resolution: {integrity: sha512-fT1qHVGAag4IEkrupZ6lRRbNCs1vS9P01KB/sG8zKgvUztbYtFBtQpjSITNwooDZ83tpsPzP0mRNs1/KVszCRA==} engines: {node: '>= 20.12.0'} @@ -1317,9 +1383,247 @@ packages: resolution: {integrity: sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA==} engines: {node: '>= 20.0.0'} + '@oxc-parser/binding-android-arm-eabi@0.128.0': + resolution: {integrity: sha512-aca6ZvzmCBUGOANQRiRQRZuRKYI3ENhcit6GisnknOOmcezfQc7xJ4dxlPU7MV7mOvrC7RNR1u3LAD7xyaiCxA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@oxc-parser/binding-android-arm64@0.128.0': + resolution: {integrity: sha512-BbeDmuohoJ7Rz/it5wnkj69i/OsCPS3Z51nLEzwO/Y6YshtC4JU+15oNwhY8v4LRKRYclRc7ggOikwrsJ/eOEQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@oxc-parser/binding-darwin-arm64@0.128.0': + resolution: {integrity: sha512-tRUHPt80417QmvNpoSslJT1VY8NUbWdrWR+L14Zn+RbOTcaqB8E6PYE/ZGN8jjWBzqporiA/H4MfO50ew/NCNA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@oxc-parser/binding-darwin-x64@0.128.0': + resolution: {integrity: sha512-rWI2Hb1Nt3U/vKsjyNvZzDC8i/l144U20DKjhzaTmwIhIiSRGeroPWWiImwypmKLqrw8GuIixbWJkpGWLbkzrQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@oxc-parser/binding-freebsd-x64@0.128.0': + resolution: {integrity: sha512-hhpdVMaNCLgQxjgNPeeFzSeJMmZPc5lKfv0NGSI3egZq9EdnEGqeC8JsYsQjK7PoQgbvZ17xlj0SO5ziH5Obkg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@oxc-parser/binding-linux-arm-gnueabihf@0.128.0': + resolution: {integrity: sha512-093zNw0zZ/e/obML+rhlSdmnzR0mVZluPcAkxunEc5E3F0yBVsFn24Y1ILfsEte11Ud041qn/gp2OJ1jxNqUng==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm-musleabihf@0.128.0': + resolution: {integrity: sha512-fq7DmKmfC+dvD97IXrgbph6Jzwe0EDu+PYMofmzZ6fv5X1k9vtaqLpDGMuICO9MmUnyKAQmVl+wIv2RNy4Dz8g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm64-gnu@0.128.0': + resolution: {integrity: sha512-Xvm48jJah8TlIrURIjNOP/gNiGe6aKvCB+r06VliflFo8Kq7VOLE8PxtgShJzZIqubrgdMdYfvuPPozn7F6MbQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-arm64-musl@0.128.0': + resolution: {integrity: sha512-M7iwBGmYJTx+pKOYFjI0buop4gJvlmcVzFGaXPt21DKpQkbQZG1f63Yg7LloIYT/t9yLxCw0Lhfx/RFlAlMSjA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-ppc64-gnu@0.128.0': + resolution: {integrity: sha512-21LGNIZb1Pcfk5/EGsqabrxv4yqQOWis1407JJrClS7XpFCrbvr74YAB1V+m54cYbwvO6UWwQqS4WecxiyfCRg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-gnu@0.128.0': + resolution: {integrity: sha512-gyHjOTFpg9bTTYjxPmQirvufb89+VdZwVfcMtAUyPr6F5H8ZswvCQshK4qOW+Q+2Xyb33hduRgY/eFHJQjU/vQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-musl@0.128.0': + resolution: {integrity: sha512-X6Q2oKUrP5GyDd2xniuEBLk6aFQCZ97W2+aVXGgJXdjx5t4/oFuA9ri0wLOUrBIX+qdSuK581snMBio4z910eA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-s390x-gnu@0.128.0': + resolution: {integrity: sha512-BdzTmqxfxoYkpgokoLaSnOX6T+R3/goL42klre2tnG+kHbG2TXS0VN+P5BPofH1axdKOHy5ei4ENZrjmCOt2lA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-gnu@0.128.0': + resolution: {integrity: sha512-OO1nW2Q7sSYYvJZpDHdvyFSdRaVcQqRijZSSmWVMqFxPYy8cEF45zJ9fcdIYuzIT3jYq6YRhEFm/VMWNWhE22Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-musl@0.128.0': + resolution: {integrity: sha512-4NehAe404MRdoZVS9DW8C5XbJwbXIc/KfVlYdpi5vE4081zc9Y0YzKVqyOYj/Puye7/Do+ohaONBFWlEHYl9hw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-openharmony-arm64@0.128.0': + resolution: {integrity: sha512-kVbqgW9xLL8bh8oc7aYOJilRKXE5G33+tE0jan+duo/9OriaFRpijcCwT2waWs2oqYROYq0GlE7/p3ywoshVeg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@oxc-parser/binding-wasm32-wasi@0.128.0': + resolution: {integrity: sha512-L38ojghJYHmgiz6fJd7jwLB/ESDBpB02NdFxh+smqVM6P2anCEvHn0jhaSrt5eVNR1Ak8+moOeftUlofeyvniA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@oxc-parser/binding-win32-arm64-msvc@0.128.0': + resolution: {integrity: sha512-xgvO35GyHBtjlQ5AEpaYr7Rll1rvY7zqIhT6ty8E3ezBW2J1SFLjIDEvI/tcgDg6oaseDAqVcM+jU1HuCekgZw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@oxc-parser/binding-win32-ia32-msvc@0.128.0': + resolution: {integrity: sha512-OY+3eM2SN72prHKRB22mPz8o5A/7dJ+f5DFLBVvggyZhEaNDAH9IB+ElMjmOkOIwf5MDCUAowCK7pAncNxzpBA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ia32] + os: [win32] + + '@oxc-parser/binding-win32-x64-msvc@0.128.0': + resolution: {integrity: sha512-NE9ny+cPUCCObXa0IKLfj0tCdPd7pe/dz9ZpkxpUOymB3miNeMPybdlYYTBSGJUalMWeBM85/4JcCErCNTqOXw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + '@oxc-project/types@0.127.0': resolution: {integrity: sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==} + '@oxc-project/types@0.128.0': + resolution: {integrity: sha512-huv1Y/LzBJkBVHt3OlC7u0zHBW9qXf1FdD7sGmc1rXc2P1mTwHssYv7jyGx5KAACSCH+9B3Bhn6Z9luHRvf7pQ==} + + '@oxc-resolver/binding-android-arm-eabi@11.19.1': + resolution: {integrity: sha512-aUs47y+xyXHUKlbhqHUjBABjvycq6YSD7bpxSW7vplUmdzAlJ93yXY6ZR0c1o1x5A/QKbENCvs3+NlY8IpIVzg==} + cpu: [arm] + os: [android] + + '@oxc-resolver/binding-android-arm64@11.19.1': + resolution: {integrity: sha512-oolbkRX+m7Pq2LNjr/kKgYeC7bRDMVTWPgxBGMjSpZi/+UskVo4jsMU3MLheZV55jL6c3rNelPl4oD60ggYmqA==} + cpu: [arm64] + os: [android] + + '@oxc-resolver/binding-darwin-arm64@11.19.1': + resolution: {integrity: sha512-nUC6d2i3R5B12sUW4O646qD5cnMXf2oBGPLIIeaRfU9doJRORAbE2SGv4eW6rMqhD+G7nf2Y8TTJTLiiO3Q/dQ==} + cpu: [arm64] + os: [darwin] + + '@oxc-resolver/binding-darwin-x64@11.19.1': + resolution: {integrity: sha512-cV50vE5+uAgNcFa3QY1JOeKDSkM/9ReIcc/9wn4TavhW/itkDGrXhw9jaKnkQnGbjJ198Yh5nbX/Gr2mr4Z5jQ==} + cpu: [x64] + os: [darwin] + + '@oxc-resolver/binding-freebsd-x64@11.19.1': + resolution: {integrity: sha512-xZOQiYGFxtk48PBKff+Zwoym7ScPAIVp4c14lfLxizO2LTTTJe5sx9vQNGrBymrf/vatSPNMD4FgsaaRigPkqw==} + cpu: [x64] + os: [freebsd] + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.19.1': + resolution: {integrity: sha512-lXZYWAC6kaGe/ky2su94e9jN9t6M0/6c+GrSlCqL//XO1cxi5lpAhnJYdyrKfm0ZEr/c7RNyAx3P7FSBcBd5+A==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm-musleabihf@11.19.1': + resolution: {integrity: sha512-veG1kKsuK5+t2IsO9q0DErYVSw2azvCVvWHnfTOS73WE0STdLLB7Q1bB9WR+yHPQM76ASkFyRbogWo1GR1+WbQ==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-gnu@11.19.1': + resolution: {integrity: sha512-heV2+jmXyYnUrpUXSPugqWDRpnsQcDm2AX4wzTuvgdlZfoNYO0O3W2AVpJYaDn9AG4JdM6Kxom8+foE7/BcSig==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-arm64-musl@11.19.1': + resolution: {integrity: sha512-jvo2Pjs1c9KPxMuMPIeQsgu0mOJF9rEb3y3TdpsrqwxRM+AN6/nDDwv45n5ZrUnQMsdBy5gIabioMKnQfWo9ew==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@oxc-resolver/binding-linux-ppc64-gnu@11.19.1': + resolution: {integrity: sha512-vLmdNxWCdN7Uo5suays6A/+ywBby2PWBBPXctWPg5V0+eVuzsJxgAn6MMB4mPlshskYbppjpN2Zg83ArHze9gQ==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-riscv64-gnu@11.19.1': + resolution: {integrity: sha512-/b+WgR+VTSBxzgOhDO7TlMXC1ufPIMR6Vj1zN+/x+MnyXGW7prTLzU9eW85Aj7Th7CCEG9ArCbTeqxCzFWdg2w==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-riscv64-musl@11.19.1': + resolution: {integrity: sha512-YlRdeWb9j42p29ROh+h4eg/OQ3dTJlpHSa+84pUM9+p6i3djtPz1q55yLJhgW9XfDch7FN1pQ/Vd6YP+xfRIuw==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@oxc-resolver/binding-linux-s390x-gnu@11.19.1': + resolution: {integrity: sha512-EDpafVOQWF8/MJynsjOGFThcqhRHy417sRyLfQmeiamJ8qVhSKAn2Dn2VVKUGCjVB9C46VGjhNo7nOPUi1x6uA==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-x64-gnu@11.19.1': + resolution: {integrity: sha512-NxjZe+rqWhr+RT8/Ik+5ptA3oz7tUw361Wa5RWQXKnfqwSSHdHyrw6IdcTfYuml9dM856AlKWZIUXDmA9kkiBQ==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-x64-musl@11.19.1': + resolution: {integrity: sha512-cM/hQwsO3ReJg5kR+SpI69DMfvNCp+A/eVR4b4YClE5bVZwz8rh2Nh05InhwI5HR/9cArbEkzMjcKgTHS6UaNw==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@oxc-resolver/binding-openharmony-arm64@11.19.1': + resolution: {integrity: sha512-QF080IowFB0+9Rh6RcD19bdgh49BpQHUW5TajG1qvWHvmrQznTZZjYlgE2ltLXyKY+qs4F/v5xuX1XS7Is+3qA==} + cpu: [arm64] + os: [openharmony] + + '@oxc-resolver/binding-wasm32-wasi@11.19.1': + resolution: {integrity: sha512-w8UCKhX826cP/ZLokXDS6+milN8y4X7zidsAttEdWlVoamTNf6lhBJldaWr3ukTDiye7s4HRcuPEPOXNC432Vg==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@oxc-resolver/binding-win32-arm64-msvc@11.19.1': + resolution: {integrity: sha512-nJ4AsUVZrVKwnU/QRdzPCCrO0TrabBqgJ8pJhXITdZGYOV28TIYystV1VFLbQ7DtAcaBHpocT5/ZJnF78YJPtQ==} + cpu: [arm64] + os: [win32] + + '@oxc-resolver/binding-win32-ia32-msvc@11.19.1': + resolution: {integrity: sha512-EW+ND5q2Tl+a3pH81l1QbfgbF3HmqgwLfDfVithRFheac8OTcnbXt/JxqD2GbDkb7xYEqy1zNaVFRr3oeG8npA==} + cpu: [ia32] + os: [win32] + + '@oxc-resolver/binding-win32-x64-msvc@11.19.1': + resolution: {integrity: sha512-6hIU3RQu45B+VNTY4Ru8ppFwjVS/S5qwYyGhBotmjxfEKk41I2DlGtRfGJndZ5+6lneE2pwloqunlOyZuX/XAw==} + cpu: [x64] + os: [win32] + '@radix-ui/number@1.1.1': resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} @@ -2799,6 +3103,9 @@ packages: resolution: {integrity: sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==} engines: {node: '>= 4.9.1'} + fd-package-json@2.0.0: + resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -2838,6 +3145,11 @@ packages: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} + formatly@0.3.0: + resolution: {integrity: sha512-9XNj/o4wrRFyhSMJOvsuyMwy8aUfBaZ1VrqHVfohyXf0Sw0e+yfKG+xZaY3arGCOMdwFsqObtzVOc1gU9KiT9w==} + engines: {node: '>=18.3.0'} + hasBin: true + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -3019,6 +3331,9 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-tsconfig@4.14.0: + resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -3269,6 +3584,11 @@ packages: jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + knip@6.12.2: + resolution: {integrity: sha512-RcZpT1sVziKZgDk1F0hAcp+bq71VJAF8vg1Y9ZLXc1+UXQaMm1rjiUqpJQTIj+lqwmiBQT19/u7ikgazs23cvA==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} @@ -3766,6 +4086,13 @@ packages: zod: optional: true + oxc-parser@0.128.0: + resolution: {integrity: sha512-XkOw3eiIxAgQ19WRew/Bq9wc5Ga/guaWIzDBzq80z1PyuDNGvWBpPby9k6YGwV8A8uMw+Nlq3xqlzuDYmUFYUw==} + engines: {node: ^20.19.0 || >=22.12.0} + + oxc-resolver@11.19.1: + resolution: {integrity: sha512-qE/CIg/spwrTBFt5aKmwe3ifeDdLfA2NESN30E42X/lII5ClF8V7Wt6WIJhcGZjp0/Q+nQ+9vgxGk//xZNX2hg==} + p-limit@7.3.0: resolution: {integrity: sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==} engines: {node: '>=20'} @@ -4028,6 +4355,9 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + restore-cursor@4.0.0: resolution: {integrity: sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -4138,6 +4468,10 @@ packages: resolution: {integrity: sha512-SO/3iYL5S3W57LLEniscOGPZgOqZUPCx6d3dB+52B80yJ0XstzsC/eV8gnA4tM3MHDrKz+OCFSLNjswdSC+/bA==} engines: {node: '>=22'} + smol-toml@1.6.1: + resolution: {integrity: sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==} + engines: {node: '>= 18'} + snowflake-sdk@2.4.1: resolution: {integrity: sha512-JIdqz9ed2FzkU8oEstf06hTJRoX9+PRRG9LJT1vfGTXN3A52kGxhGoWzmK0GtFTUnxTMxMoMYgD5QdoQbckyag==} engines: {node: '>=18'} @@ -4211,6 +4545,10 @@ packages: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} + strip-json-comments@5.0.3: + resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} + engines: {node: '>=14.16'} + strnum@2.2.3: resolution: {integrity: sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==} @@ -4336,6 +4674,10 @@ packages: engines: {node: '>=0.8.0'} hasBin: true + unbash@3.0.0: + resolution: {integrity: sha512-FeFPZ/WFT0mbRCuydiZzpPFlrYN8ZUpphQKoq4EeElVIYjYyGzPMxQR/simUwCOJIyVhpFk4RbtyO7RuMpMnHA==} + engines: {node: '>=14'} + undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} @@ -4487,6 +4829,10 @@ packages: jsdom: optional: true + walk-up-path@4.0.0: + resolution: {integrity: sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==} + engines: {node: 20 || >=22} + web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} @@ -5546,6 +5892,41 @@ snapshots: transitivePeerDependencies: - supports-color + '@biomejs/biome@2.4.15': + optionalDependencies: + '@biomejs/cli-darwin-arm64': 2.4.15 + '@biomejs/cli-darwin-x64': 2.4.15 + '@biomejs/cli-linux-arm64': 2.4.15 + '@biomejs/cli-linux-arm64-musl': 2.4.15 + '@biomejs/cli-linux-x64': 2.4.15 + '@biomejs/cli-linux-x64-musl': 2.4.15 + '@biomejs/cli-win32-arm64': 2.4.15 + '@biomejs/cli-win32-x64': 2.4.15 + + '@biomejs/cli-darwin-arm64@2.4.15': + optional: true + + '@biomejs/cli-darwin-x64@2.4.15': + optional: true + + '@biomejs/cli-linux-arm64-musl@2.4.15': + optional: true + + '@biomejs/cli-linux-arm64@2.4.15': + optional: true + + '@biomejs/cli-linux-x64-musl@2.4.15': + optional: true + + '@biomejs/cli-linux-x64@2.4.15': + optional: true + + '@biomejs/cli-win32-arm64@2.4.15': + optional: true + + '@biomejs/cli-win32-x64@2.4.15': + optional: true + '@clack/core@1.3.1': dependencies: fast-wrap-ansi: 0.2.0 @@ -6130,8 +6511,139 @@ snapshots: '@orama/orama@3.1.18': {} + '@oxc-parser/binding-android-arm-eabi@0.128.0': + optional: true + + '@oxc-parser/binding-android-arm64@0.128.0': + optional: true + + '@oxc-parser/binding-darwin-arm64@0.128.0': + optional: true + + '@oxc-parser/binding-darwin-x64@0.128.0': + optional: true + + '@oxc-parser/binding-freebsd-x64@0.128.0': + optional: true + + '@oxc-parser/binding-linux-arm-gnueabihf@0.128.0': + optional: true + + '@oxc-parser/binding-linux-arm-musleabihf@0.128.0': + optional: true + + '@oxc-parser/binding-linux-arm64-gnu@0.128.0': + optional: true + + '@oxc-parser/binding-linux-arm64-musl@0.128.0': + optional: true + + '@oxc-parser/binding-linux-ppc64-gnu@0.128.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-gnu@0.128.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-musl@0.128.0': + optional: true + + '@oxc-parser/binding-linux-s390x-gnu@0.128.0': + optional: true + + '@oxc-parser/binding-linux-x64-gnu@0.128.0': + optional: true + + '@oxc-parser/binding-linux-x64-musl@0.128.0': + optional: true + + '@oxc-parser/binding-openharmony-arm64@0.128.0': + optional: true + + '@oxc-parser/binding-wasm32-wasi@0.128.0': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + optional: true + + '@oxc-parser/binding-win32-arm64-msvc@0.128.0': + optional: true + + '@oxc-parser/binding-win32-ia32-msvc@0.128.0': + optional: true + + '@oxc-parser/binding-win32-x64-msvc@0.128.0': + optional: true + '@oxc-project/types@0.127.0': {} + '@oxc-project/types@0.128.0': {} + + '@oxc-resolver/binding-android-arm-eabi@11.19.1': + optional: true + + '@oxc-resolver/binding-android-arm64@11.19.1': + optional: true + + '@oxc-resolver/binding-darwin-arm64@11.19.1': + optional: true + + '@oxc-resolver/binding-darwin-x64@11.19.1': + optional: true + + '@oxc-resolver/binding-freebsd-x64@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-arm-musleabihf@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-arm64-gnu@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-arm64-musl@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-ppc64-gnu@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-riscv64-gnu@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-riscv64-musl@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-s390x-gnu@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-x64-gnu@11.19.1': + optional: true + + '@oxc-resolver/binding-linux-x64-musl@11.19.1': + optional: true + + '@oxc-resolver/binding-openharmony-arm64@11.19.1': + optional: true + + '@oxc-resolver/binding-wasm32-wasi@11.19.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': + dependencies: + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + optional: true + + '@oxc-resolver/binding-win32-arm64-msvc@11.19.1': + optional: true + + '@oxc-resolver/binding-win32-ia32-msvc@11.19.1': + optional: true + + '@oxc-resolver/binding-win32-x64-msvc@11.19.1': + optional: true + '@radix-ui/number@1.1.1': {} '@radix-ui/primitive@1.1.3': {} @@ -7691,6 +8203,10 @@ snapshots: fastest-levenshtein@1.0.16: {} + fd-package-json@2.0.0: + dependencies: + walk-up-path: 4.0.0 + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -7727,6 +8243,10 @@ snapshots: hasown: 2.0.3 mime-types: 2.1.35 + formatly@0.3.0: + dependencies: + fd-package-json: 2.0.0 + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -7895,6 +8415,10 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + get-tsconfig@4.14.0: + dependencies: + resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} github-slugger@2.0.0: {} @@ -8234,6 +8758,26 @@ snapshots: jwa: 2.0.1 safe-buffer: 5.2.1 + knip@6.12.2(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0): + dependencies: + fdir: 6.5.0(picomatch@4.0.4) + formatly: 0.3.0 + get-tsconfig: 4.14.0 + jiti: 2.7.0 + minimist: 1.2.8 + oxc-parser: 0.128.0 + oxc-resolver: 11.19.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + picomatch: 4.0.4 + smol-toml: 1.6.1 + strip-json-comments: 5.0.3 + tinyglobby: 0.2.16 + unbash: 3.0.0 + yaml: 2.9.0 + zod: 4.4.3 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + kuler@2.0.0: {} lightningcss-android-arm64@1.32.0: @@ -8963,6 +9507,57 @@ snapshots: optionalDependencies: zod: 4.4.3 + oxc-parser@0.128.0: + dependencies: + '@oxc-project/types': 0.128.0 + optionalDependencies: + '@oxc-parser/binding-android-arm-eabi': 0.128.0 + '@oxc-parser/binding-android-arm64': 0.128.0 + '@oxc-parser/binding-darwin-arm64': 0.128.0 + '@oxc-parser/binding-darwin-x64': 0.128.0 + '@oxc-parser/binding-freebsd-x64': 0.128.0 + '@oxc-parser/binding-linux-arm-gnueabihf': 0.128.0 + '@oxc-parser/binding-linux-arm-musleabihf': 0.128.0 + '@oxc-parser/binding-linux-arm64-gnu': 0.128.0 + '@oxc-parser/binding-linux-arm64-musl': 0.128.0 + '@oxc-parser/binding-linux-ppc64-gnu': 0.128.0 + '@oxc-parser/binding-linux-riscv64-gnu': 0.128.0 + '@oxc-parser/binding-linux-riscv64-musl': 0.128.0 + '@oxc-parser/binding-linux-s390x-gnu': 0.128.0 + '@oxc-parser/binding-linux-x64-gnu': 0.128.0 + '@oxc-parser/binding-linux-x64-musl': 0.128.0 + '@oxc-parser/binding-openharmony-arm64': 0.128.0 + '@oxc-parser/binding-wasm32-wasi': 0.128.0 + '@oxc-parser/binding-win32-arm64-msvc': 0.128.0 + '@oxc-parser/binding-win32-ia32-msvc': 0.128.0 + '@oxc-parser/binding-win32-x64-msvc': 0.128.0 + + oxc-resolver@11.19.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0): + optionalDependencies: + '@oxc-resolver/binding-android-arm-eabi': 11.19.1 + '@oxc-resolver/binding-android-arm64': 11.19.1 + '@oxc-resolver/binding-darwin-arm64': 11.19.1 + '@oxc-resolver/binding-darwin-x64': 11.19.1 + '@oxc-resolver/binding-freebsd-x64': 11.19.1 + '@oxc-resolver/binding-linux-arm-gnueabihf': 11.19.1 + '@oxc-resolver/binding-linux-arm-musleabihf': 11.19.1 + '@oxc-resolver/binding-linux-arm64-gnu': 11.19.1 + '@oxc-resolver/binding-linux-arm64-musl': 11.19.1 + '@oxc-resolver/binding-linux-ppc64-gnu': 11.19.1 + '@oxc-resolver/binding-linux-riscv64-gnu': 11.19.1 + '@oxc-resolver/binding-linux-riscv64-musl': 11.19.1 + '@oxc-resolver/binding-linux-s390x-gnu': 11.19.1 + '@oxc-resolver/binding-linux-x64-gnu': 11.19.1 + '@oxc-resolver/binding-linux-x64-musl': 11.19.1 + '@oxc-resolver/binding-openharmony-arm64': 11.19.1 + '@oxc-resolver/binding-wasm32-wasi': 11.19.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + '@oxc-resolver/binding-win32-arm64-msvc': 11.19.1 + '@oxc-resolver/binding-win32-ia32-msvc': 11.19.1 + '@oxc-resolver/binding-win32-x64-msvc': 11.19.1 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + p-limit@7.3.0: dependencies: yocto-queue: 1.2.2 @@ -9273,6 +9868,8 @@ snapshots: require-from-string@2.0.2: {} + resolve-pkg-maps@1.0.0: {} + restore-cursor@4.0.0: dependencies: onetime: 5.1.2 @@ -9467,6 +10064,8 @@ snapshots: ansi-styles: 6.2.3 is-fullwidth-code-point: 5.1.0 + smol-toml@1.6.1: {} + snowflake-sdk@2.4.1(asn1.js@5.4.1): dependencies: '@aws-crypto/sha256-js': 5.2.0 @@ -9558,6 +10157,8 @@ snapshots: strip-json-comments@2.0.1: {} + strip-json-comments@5.0.3: {} + strnum@2.2.3: {} stubs@3.0.0: {} @@ -9689,6 +10290,8 @@ snapshots: uglify-js@3.19.3: optional: true + unbash@3.0.0: {} + undici-types@7.16.0: {} unified@11.0.5: @@ -9811,6 +10414,8 @@ snapshots: transitivePeerDependencies: - msw + walk-up-path@4.0.0: {} + web-namespaces@2.0.1: {} web-streams-polyfill@3.3.3: {} diff --git a/python/ktx-sl/semantic_layer/generator.py b/python/ktx-sl/semantic_layer/generator.py index 4e1ec891..a5979299 100644 --- a/python/ktx-sl/semantic_layer/generator.py +++ b/python/ktx-sl/semantic_layer/generator.py @@ -687,6 +687,12 @@ class SqlGenerator: if isinstance(node, exp.AggFunc): if isinstance(node, exp.Count): count_arg = node.this + if isinstance(count_arg, exp.Star): + node.set( + "this", + _make_case(exp.Literal.number(1)), + ) + return node if ( isinstance(count_arg, exp.Distinct) and count_arg.expressions diff --git a/python/ktx-sl/tests/test_corner_case_regressions.py b/python/ktx-sl/tests/test_corner_case_regressions.py index cb99d446..92eeb2a5 100644 --- a/python/ktx-sl/tests/test_corner_case_regressions.py +++ b/python/ktx-sl/tests/test_corner_case_regressions.py @@ -243,6 +243,37 @@ def test_filtered_count_distinct_keeps_distinct_inside_count(): assert_valid_sql(result.sql) +def test_filtered_count_star_uses_case_one_not_case_star(): + engine = make_engine( + { + "accounts": { + "name": "accounts", + "table": "public.accounts", + "grain": ["id"], + "columns": [ + {"name": "id", "type": "number"}, + {"name": "risk_level", "type": "string"}, + ], + "measures": [ + { + "name": "high_risk_account_count", + "expr": "count(*)", + "filter": "risk_level = 'high'", + } + ], + } + } + ) + + result = engine.query( + {"measures": ["accounts.high_risk_account_count"], "dimensions": []} + ) + + assert "THEN *" not in result.sql + assert "COUNT(CASE WHEN accounts.risk_level = 'high' THEN 1 END)" in result.sql + assert_valid_sql(result.sql) + + def test_predefined_measure_via_alias_uses_real_table_and_alias_qualification(): engine = make_engine(_alias_measure_sources()) result = engine.query( diff --git a/scripts/build-public-npm-package.mjs b/scripts/build-public-npm-package.mjs index 1a9ef8bc..9df91c69 100644 --- a/scripts/build-public-npm-package.mjs +++ b/scripts/build-public-npm-package.mjs @@ -151,10 +151,6 @@ export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUB } function bundledWorkspacePackageJson(packageJson) { - const dependencies = Object.fromEntries( - Object.entries(packageJson.dependencies ?? {}).filter(([name]) => !isWorkspacePackageName(name)), - ); - return { name: packageJson.name, version: packageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, @@ -164,7 +160,6 @@ function bundledWorkspacePackageJson(packageJson) { types: packageJson.types, exports: packageJson.exports, files: packageJson.files, - dependencies: sortedObject(Object.entries(dependencies)), license: packageJson.license ?? 'Apache-2.0', }; } diff --git a/scripts/build-public-npm-package.test.mjs b/scripts/build-public-npm-package.test.mjs index d4461049..c68c2aed 100644 --- a/scripts/build-public-npm-package.test.mjs +++ b/scripts/build-public-npm-package.test.mjs @@ -250,7 +250,7 @@ describe('createPublicNpmPackageTree', () => { await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'package.json'), 'utf8'), ); assert.equal(bundledContextJson.private, true); - assert.deepEqual(bundledContextJson.dependencies, { yaml: '^2.8.2' }); + assert.equal(bundledContextJson.dependencies, undefined); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 5378b8ce..793566ed 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -22,10 +22,6 @@ function publicPackagePattern(text) { return new RegExp(text.replaceAll('{package}', escapeRegExp(publicNpmPackageName()))); } -function runtimeWheelPackagePattern(text) { - return new RegExp(text.replaceAll('{package}', escapeRegExp(runtimeWheelPackageName()))); -} - describe('standalone example docs', () => { it('documents the local warehouse example from the examples index', async () => { const examples = await readText('examples/README.md'); @@ -156,14 +152,12 @@ describe('standalone example docs', () => { const servingAgents = await readText('docs-site/content/docs/guides/serving-agents.mdx'); for (const command of [ - 'ktx agent tools --json', - 'ktx agent context --json', - 'ktx agent sl list --json', - 'ktx agent sl read orders --json', - 'ktx agent sl query --json', - 'ktx agent wiki search "revenue recognition" --json', - 'ktx agent wiki read order-status-definitions --json', - 'ktx agent sql execute --json', + 'ktx status --json', + 'ktx sl list --json', + 'ktx sl read orders --json', + 'ktx sl query --json', + 'ktx wiki search "revenue recognition" --json', + 'ktx wiki read order-status-definitions --json', ]) { assert.match(servingAgents, new RegExp(command.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); } @@ -192,8 +186,7 @@ describe('standalone example docs', () => { assert.match(packageArtifacts, /requires `uv` on `PATH`/); assert.match(packageArtifacts, /ktx dev runtime status/); assert.match(packageArtifacts, /ktx dev runtime status/); - assert.match(packageArtifacts, /ktx dev runtime prune --dry-run/); - assert.match(packageArtifacts, /ktx dev runtime prune --yes/); + assert.doesNotMatch(packageArtifacts, /ktx dev runtime prune/); assert.match( packageArtifacts, new RegExp( @@ -226,8 +219,7 @@ describe('standalone example docs', () => { assert.match(readme, /requires `uv` on `PATH`/); assert.match(readme, /ktx dev runtime status/); assert.match(readme, /ktx dev runtime status/); - assert.match(readme, /ktx dev runtime prune --dry-run/); - assert.match(readme, /ktx dev runtime prune --yes/); + assert.doesNotMatch(readme, /ktx dev runtime prune/); assert.doesNotMatch(readme, /@ktx\/context/); assert.doesNotMatch(readme, /@ktx\/cli/); assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); diff --git a/scripts/local-embeddings-runtime-smoke.mjs b/scripts/local-embeddings-runtime-smoke.mjs index 064cd070..8b0243a0 100644 --- a/scripts/local-embeddings-runtime-smoke.mjs +++ b/scripts/local-embeddings-runtime-smoke.mjs @@ -205,6 +205,17 @@ function parseJsonStdout(label, result) { } } +function parseJsonStdoutWithExitCode(label, result, expectedCode) { + if (result.code !== expectedCode) { + throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } + try { + return JSON.parse(result.stdout); + } catch (error) { + throw new Error(`${label} did not write JSON stdout: ${error.message}\nstdout:\n${result.stdout}`); + } +} + function requireOutput(label, result, pattern) { if (!pattern.test(result.stdout)) { throw new Error(`${label} stdout did not match ${pattern}\nstdout:\n${result.stdout}`); @@ -283,13 +294,14 @@ export async function runLocalEmbeddingsRuntimeSmoke(options = {}) { requireSuccess(commands[0].label, version); requireOutput(commands[0].label, version, expectedPublicKtxVersionPattern()); - const missingStatus = parseJsonStdout( + const missingStatus = parseJsonStdoutWithExitCode( commands[1].label, await run(commands[1].command, commands[1].args, { cwd: installDir, env: smokeEnv, timeoutMs: commands[1].timeoutMs, }), + 1, ); if (missingStatus.kind !== 'missing') { throw new Error(`Expected missing runtime before install, got ${JSON.stringify(missingStatus)}`); diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index d99509d9..ab8d7adf 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -459,9 +459,6 @@ export function npmSmokePackageJson(layout) { dependencies: { '@kaelio/ktx': `file:${layout.cliTarball}`, }, - devDependencies: { - 'better-sqlite3': '^12.6.2', - }, }; } @@ -485,11 +482,11 @@ if (typeof cli.runKtxCli !== 'function') { export function npmRuntimeSmokeSource() { return ` import assert from 'node:assert/strict'; -import Database from 'better-sqlite3'; import { execFile } from 'node:child_process'; import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { promisify } from 'node:util'; const execFileAsync = promisify(execFile); @@ -548,10 +545,13 @@ function parseJsonResult(label, result) { return JSON.parse(result.stdout); } -function parseJsonFailure(label, result) { - assert.equal(result.code, 1, label + ' should fail with exit code 1'); - assert.equal(result.stdout, '', label + ' should not write stdout when failing'); - return JSON.parse(result.stderr); +function parseJsonResultWithExitCode(label, result, expectedCode) { + assert.equal( + result.code, + expectedCode, + label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr, + ); + return JSON.parse(result.stdout); } function requireIncludes(values, expected, label) { @@ -566,7 +566,7 @@ function getRunId(stdout) { } async function writeSqliteWarehouse(projectDir) { - const database = new Database(join(projectDir, 'warehouse.db')); + const database = new DatabaseSync(join(projectDir, 'warehouse.db')); try { database.exec(\` DROP TABLE IF EXISTS orders; @@ -594,38 +594,15 @@ try { requireSuccess('ktx public package version', version); requireOutput('ktx public package version', version, /@kaelio\\/ktx 0\\.1\\.0/); - const runtimeStatusBefore = parseJsonResult( + const runtimeStatusBefore = parseJsonResultWithExitCode( 'ktx dev runtime status missing', await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']), + 1, ); assert.equal(runtimeStatusBefore.kind, 'missing'); assert.equal(runtimeStatusBefore.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); process.stdout.write('ktx managed runtime starts missing in isolated root\\n'); - const missingProjectDir = join(root, 'missing-project'); - await mkdir(missingProjectDir, { recursive: true }); - const missingProjectSearch = await run('pnpm', [ - 'exec', - 'ktx', - 'agent', - 'sl', - 'list', - '--json', - '--query', - 'revenue', - '--project-dir', - missingProjectDir, - ]); - const missingProjectError = parseJsonFailure('ktx agent sl list missing project', missingProjectSearch); - assert.equal(missingProjectError.error.code, 'agent_sl_search_missing_project'); - assert.deepEqual(missingProjectError.error.nextSteps, [ - 'ktx setup --project-dir ' + missingProjectDir, - 'ktx status --project-dir ' + missingProjectDir, - 'ktx ingest run --connection-id --adapter ', - 'ktx agent sl list --json --query "revenue" --project-dir ' + missingProjectDir, - ]); - process.stdout.write('ktx agent sl list missing project guidance verified\\n'); - const init = await run('pnpm', [ 'exec', 'ktx', @@ -661,28 +638,6 @@ try { '--skip-agents', ]); requireProjectStderr('ktx setup empty project', emptyInit, emptyProjectDir); - const emptySearch = await run('pnpm', [ - 'exec', - 'ktx', - 'agent', - 'sl', - 'list', - '--json', - '--query', - 'revenue', - '--project-dir', - emptyProjectDir, - ]); - const emptySearchError = parseJsonFailure('ktx agent sl list no connections', emptySearch); - assert.equal(emptySearchError.error.code, 'agent_sl_search_no_connections'); - assert.deepEqual(emptySearchError.error.nextSteps, [ - 'ktx setup --project-dir ' + emptyProjectDir, - 'ktx status --project-dir ' + emptyProjectDir, - 'ktx ingest run --connection-id --adapter ', - 'ktx agent sl list --json --query "revenue" --project-dir ' + emptyProjectDir, - ]); - process.stdout.write('ktx agent sl list no connections guidance verified\\n'); - await writeFile( join(projectDir, 'ktx.yaml'), [ @@ -727,10 +682,9 @@ try { 'utf-8', ); - const agentWikiSearch = await run('pnpm', [ + const wikiSearch = await run('pnpm', [ 'exec', 'ktx', - 'agent', 'wiki', 'search', 'revenue', @@ -740,40 +694,17 @@ try { '--project-dir', projectDir, ]); - const agentWikiSearchJson = parseJsonResult('ktx agent wiki search', agentWikiSearch); - assert.equal(agentWikiSearchJson.totalFound, 1); - assert.equal(agentWikiSearchJson.results[0].key, 'revenue'); - assert.equal(agentWikiSearchJson.results[0].path, 'knowledge/global/revenue.md'); - assert.equal(typeof agentWikiSearchJson.results[0].score, 'number'); - requireIncludes(agentWikiSearchJson.results[0].matchReasons, 'lexical', 'agent wiki search match reasons'); - process.stdout.write('ktx agent wiki search hybrid metadata verified\\n'); + const wikiSearchJson = parseJsonResult('ktx wiki search', wikiSearch); + assert.equal(wikiSearchJson.kind, 'list'); + assert.equal(wikiSearchJson.data.items.length, 1); + assert.equal(wikiSearchJson.data.items[0].key, 'revenue'); + assert.equal(wikiSearchJson.data.items[0].path, 'knowledge/global/revenue.md'); + assert.equal(typeof wikiSearchJson.data.items[0].score, 'number'); + requireIncludes(wikiSearchJson.data.items[0].matchReasons, 'lexical', 'wiki search match reasons'); + process.stdout.write('ktx wiki search hybrid metadata verified\\n'); await access(join(projectDir, '.ktx', 'db.sqlite')); process.stdout.write('SQLite knowledge index: ' + join(projectDir, '.ktx', 'db.sqlite') + '\\n'); - const noSourceSearch = await run('pnpm', [ - 'exec', - 'ktx', - 'agent', - 'sl', - 'list', - '--json', - '--connection-id', - 'warehouse', - '--query', - 'revenue', - '--project-dir', - projectDir, - ]); - const noSourceSearchError = parseJsonFailure('ktx agent sl list no indexed sources', noSourceSearch); - assert.equal(noSourceSearchError.error.code, 'agent_sl_search_no_indexed_sources'); - assert.deepEqual(noSourceSearchError.error.nextSteps, [ - 'ktx setup --project-dir ' + projectDir, - 'ktx status --project-dir ' + projectDir, - 'ktx ingest run --connection-id --adapter ', - 'ktx agent sl list --json --query "revenue" --project-dir ' + projectDir, - ]); - process.stdout.write('ktx agent sl list no indexed sources guidance verified\\n'); - const slYaml = [ 'name: orders', 'table: orders', @@ -794,10 +725,9 @@ try { await mkdir(join(projectDir, 'semantic-layer', 'warehouse'), { recursive: true }); await writeFile(join(projectDir, 'semantic-layer', 'warehouse', 'orders.yaml'), slYaml, 'utf-8'); - const agentSlSearch = await run('pnpm', [ + const slSearch = await run('pnpm', [ 'exec', 'ktx', - 'agent', 'sl', 'list', '--json', @@ -808,13 +738,14 @@ try { '--project-dir', projectDir, ]); - const agentSlSearchJson = parseJsonResult('ktx agent sl list', agentSlSearch); - assert.equal(agentSlSearchJson.totalSources, 1); - assert.equal(agentSlSearchJson.sources[0].connectionId, 'warehouse'); - assert.equal(agentSlSearchJson.sources[0].name, 'orders'); - assert.equal(typeof agentSlSearchJson.sources[0].score, 'number'); - requireIncludes(agentSlSearchJson.sources[0].matchReasons, 'lexical', 'agent sl search match reasons'); - process.stdout.write('ktx agent sl list hybrid metadata verified\\n'); + const slSearchJson = parseJsonResult('ktx sl list', slSearch); + assert.equal(slSearchJson.kind, 'list'); + assert.equal(slSearchJson.data.items.length, 1); + assert.equal(slSearchJson.data.items[0].connectionId, 'warehouse'); + assert.equal(slSearchJson.data.items[0].name, 'orders'); + assert.equal(typeof slSearchJson.data.items[0].score, 'number'); + requireIncludes(slSearchJson.data.items[0].matchReasons, 'lexical', 'sl search match reasons'); + process.stdout.write('ktx sl list hybrid metadata verified\\n'); const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', '--connection-id', @@ -889,27 +820,6 @@ try { requireOutput('ktx dev runtime stop', runtimeStop, /Stopped KTX Python daemon/); process.stdout.write('ktx dev runtime daemon lifecycle verified\\n'); - const staleRuntimeDir = join(process.env.KTX_RUNTIME_ROOT, '0.0.0'); - await mkdir(staleRuntimeDir, { recursive: true }); - - const runtimePruneDryRun = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'prune', '--dry-run']); - requireSuccess('ktx dev runtime prune dry run', runtimePruneDryRun); - requireOutput('ktx dev runtime prune dry run', runtimePruneDryRun, /Stale KTX Python runtimes/); - requireOutput('ktx dev runtime prune dry run', runtimePruneDryRun, /0\\.0\\.0/); - await access(staleRuntimeDir); - - const runtimePruneNeedsConfirmation = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'prune']); - assert.equal(runtimePruneNeedsConfirmation.code, 1, 'ktx dev runtime prune needs confirmation'); - assert.equal(runtimePruneNeedsConfirmation.stdout, '', 'ktx dev runtime prune needs confirmation wrote stdout'); - assert.match(runtimePruneNeedsConfirmation.stderr, /Refusing to prune without --yes/); - - const runtimePruneConfirmed = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'prune', '--yes']); - requireSuccess('ktx dev runtime prune confirmed', runtimePruneConfirmed); - requireOutput('ktx dev runtime prune confirmed', runtimePruneConfirmed, /Removed stale KTX Python runtimes/); - requireOutput('ktx dev runtime prune confirmed', runtimePruneConfirmed, /0\\.0\\.0/); - await assert.rejects(() => access(staleRuntimeDir)); - process.stdout.write('ktx dev runtime prune verified\\n'); - const structuralScan = await run('pnpm', ['exec', 'ktx', 'scan', 'warehouse', '--project-dir', projectDir, diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index a8527c98..9fe5a2c1 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -27,8 +27,6 @@ import { writeArtifactManifest, } from './package-artifacts.mjs'; -const STALE_METABASE_UNSUPPORTED = ['Standalone Metabase scheduled fetch', 'is intentionally unsupported'].join(' '); - async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } @@ -420,9 +418,7 @@ describe('verification snippets', () => { assert.deepEqual(packageJson.dependencies, { '@kaelio/ktx': `file:${layout.cliTarball}`, }); - assert.deepEqual(packageJson.devDependencies, { - 'better-sqlite3': '^12.6.2', - }); + assert.equal(packageJson.devDependencies, undefined); assert.equal( npmSmokePnpmWorkspaceYaml(), ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'), @@ -461,11 +457,11 @@ describe('verification snippets', () => { assert.doesNotMatch(source, /startSemanticDaemon/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'setup'/); assert.match(source, /knowledge', 'global', 'revenue\.md'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'wiki',\s*'search'/); + assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'wiki',\s*'search'/); assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'agent',\s*'sl',\s*'list'/); + assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'list'/); assert.match(source, /orders\.order_count/); - assert.match(source, /sqlite3/); + assert.match(source, /node:sqlite/); assert.match(source, /driver: sqlite/); assert.match(source, /path: warehouse\.db/); assert.match(source, /live-database/); @@ -473,7 +469,7 @@ describe('verification snippets', () => { assert.match(source, /"mode": "compile_only"/); assert.match(source, /"mode": "executed"/); assert.match(source, /ktx sl query sqlite execute/); - assert.match(source, /import Database from 'better-sqlite3'/); + assert.match(source, /import \{ DatabaseSync \} from 'node:sqlite'/); assert.doesNotMatch(source, /run\('python'/); assert.match(source, /KTX_RUNTIME_ROOT/); assert.match(source, /managed-runtime/); @@ -490,13 +486,8 @@ describe('verification snippets', () => { assert.match(source, /ktx dev runtime start reuse/); assert.match(source, /Using existing KTX Python daemon/); assert.match(source, /ktx dev runtime stop/); - assert.match(source, /ktx dev runtime prune dry run/); - assert.match(source, /0\.0\.0/); - assert.match(source, /ktx dev runtime prune needs confirmation/); - assert.match(source, /Refusing to prune without --yes/); - assert.match(source, /ktx dev runtime prune confirmed/); - assert.match(source, /Removed stale KTX Python runtimes/); - assert.match(source, /assert\.rejects\(\(\) => access\(staleRuntimeDir\)\)/); + assert.doesNotMatch(source, /ktx dev runtime prune/); + assert.doesNotMatch(source, /staleRuntimeDir/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'scan',\s*'warehouse'/); assert.match(source, /'--mode',\s*'enriched'/); assert.doesNotMatch(source, /'--enrich'/); diff --git a/scripts/relationship-orbit-verification.mjs b/scripts/relationship-orbit-verification.mjs index aa6309dd..81f81187 100644 --- a/scripts/relationship-orbit-verification.mjs +++ b/scripts/relationship-orbit-verification.mjs @@ -9,7 +9,6 @@ import { runWorkspaceKtx } from './run-ktx.mjs'; const scriptDir = dirname(fileURLToPath(import.meta.url)); const ktxRootDir = resolve(scriptDir, '..'); -const repoRootDir = resolve(ktxRootDir, '..'); const defaultProjectDir = resolve(ktxRootDir, 'examples/orbit-relationship-verification'); const defaultReportPath = resolve( ktxRootDir,