diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de9d056d..fbfe41d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,9 +191,8 @@ jobs: coverage: name: Coverage runs-on: ubuntu-latest - permissions: - contents: read - id-token: write + env: + CODECOV_TOKEN_CONFIGURED: ${{ secrets.CODECOV_TOKEN != '' }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -232,25 +231,39 @@ jobs: run: pnpm run test:coverage:ts - name: Upload TypeScript coverage + if: env.CODECOV_TOKEN_CONFIGURED == 'true' uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: - use_oidc: true + token: ${{ secrets.CODECOV_TOKEN }} + slug: Kaelio/ktx files: ./packages/cli/coverage/lcov.info,./packages/connector-bigquery/coverage/lcov.info,./packages/connector-clickhouse/coverage/lcov.info,./packages/connector-mysql/coverage/lcov.info,./packages/connector-postgres/coverage/lcov.info,./packages/connector-snowflake/coverage/lcov.info,./packages/connector-sqlite/coverage/lcov.info,./packages/connector-sqlserver/coverage/lcov.info,./packages/context/coverage/lcov.info,./packages/llm/coverage/lcov.info flags: typescript name: typescript - fail_ci_if_error: false + disable_search: true + fail_ci_if_error: true + + - name: Warn when Codecov token is missing for TypeScript + if: env.CODECOV_TOKEN_CONFIGURED != 'true' + run: echo "::warning::CODECOV_TOKEN is not configured; skipping TypeScript coverage upload" - name: Generate Python coverage run: pnpm run test:coverage:py - name: Upload Python coverage + if: env.CODECOV_TOKEN_CONFIGURED == 'true' uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: - use_oidc: true + token: ${{ secrets.CODECOV_TOKEN }} + slug: Kaelio/ktx files: ./coverage/python.xml flags: python name: python - fail_ci_if_error: false + disable_search: true + fail_ci_if_error: true + + - name: Warn when Codecov token is missing for Python + if: env.CODECOV_TOKEN_CONFIGURED != 'true' + run: echo "::warning::CODECOV_TOKEN is not configured; skipping Python coverage upload" artifact-checks: name: Artifact checks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95fdf9d4..d00a8fdd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,10 +7,10 @@ on: description: "Release kind: rc publishes to next, stable publishes to latest" required: true type: choice - default: "rc" + default: "stable" options: - - rc - stable + - rc force_release: description: "Force a patch release even if semantic-release finds no releasable commits" required: false @@ -20,7 +20,7 @@ on: description: "Create the release and publish @kaelio/ktx to npm instead of running a dry-run" required: true type: boolean - default: false + default: true permissions: contents: write @@ -90,6 +90,26 @@ jobs: env: KTX_PRERELEASE_BRANCH: next + - name: Prepare npm package root for release verification + run: | + set -euo pipefail + + mkdir -p dist/public-npm-package + node --input-type=module <<'EOF' + import { writeFile } from 'node:fs/promises'; + + const packageJson = { + name: '@kaelio/ktx', + version: '0.0.0', + private: false + }; + + await writeFile( + 'dist/public-npm-package/package.json', + `${JSON.stringify(packageJson, null, 2)}\n` + ); + EOF + - name: Dry-run semantic release if: ${{ !inputs.publish_live }} run: | diff --git a/.gitignore b/.gitignore index d1098953..0ad46067 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ yarn-error.log* .agents .claude .superpowers +docs/superpowers # Editors and OS files .idea/ diff --git a/AGENTS.md b/AGENTS.md index 15b9c8aa..a5447c9d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,8 +72,9 @@ KTX is a pnpm + uv workspace. - Python daemon: `python/ktx-daemon` - Examples and fixtures: `examples/` - Workspace scripts: `scripts/` -- Local agent skills are private overlays. Do not commit `.agents/` or - `.claude/` to this public repository. +- Local agent skills and internal planning docs are private overlays. Do not + commit `.agents/`, `.claude/`, or `docs/superpowers/` to this public + repository. Some package names still contain `ktx` during the split. Do not mass-rename symbols, package names, paths, or docs to `ktx` unless the task asks for that diff --git a/README.md b/README.md index e058b828..d3d55d03 100644 --- a/README.md +++ b/README.md @@ -6,35 +6,68 @@ The context layer for analytics agents -

by Kaelio

-

npm version - Codecov + Codecov + Tests + Documentation License - GitHub stars

--- -KTX turns warehouse metadata, semantic definitions, and business knowledge into -reviewable project files that agents can use to plan, query, and update -analytics work. +KTX is a self-improving context layer that teaches agents how to query your +warehouse accurately - from approved metric definitions, joinable columns, and +business knowledge it builds and maintains for you. -Use KTX when you want agents to: +Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and +SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, and Notion. -- Generate SQL from approved measures and joins -- Repair semantic definitions through reviewable diffs -- Explain metric provenance with warehouse evidence -- Work alongside dbt, MetricFlow, LookML, Looker, Metabase, and Notion +## Why KTX -Supports PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and -SQLite. +General-purpose agents struggle on data tasks. They re-explore your warehouse +on every question, invent their own metric logic, and return numbers that +don't match approved definitions. + +Traditional semantic layers don't fix this. They demand constant manual +upkeep and don't absorb the rest of your company's knowledge. + +KTX does both, automatically: + +- **Learns from company knowledge.** Ingests wiki content, organizes it, + removes duplicates, and flags contradictions for human review. +- **Maps the data stack.** Samples tables, captures metadata and usage + patterns, detects joinable columns, and annotates sources so agents write + better queries. +- **Builds a semantic layer.** Combines raw tables and high-level metrics + through a join graph that automatically resolves chasm and fan traps, so + agents fetch metrics declaratively instead of rewriting canonical SQL each + time. +- **Serves agents at execution.** Exposes CLI and MCP tools with combined + full-text and semantic search across wiki and semantic-layer entities. + +Agents can run raw SQL when they need it, or compose semantic-layer queries +when they want approved metrics with reliable joins. + +

+ KTX ingestion flow from source systems through validation to wiki and semantic-layer outputs +

+ +## Agent Setup + +Ask an agent such as Claude Code, Codex, Cursor, or OpenCode to install and +configure KTX from your project directory: + +```text +Follow instructions from +https://docs.kaelio.com/ktx/docs/agents-setup.md +to install and configure ktx +``` ## Quick Start ```bash -pnpm add --global @kaelio/ktx +npm install -g @kaelio/ktx ktx setup ktx status ``` @@ -49,8 +82,8 @@ KTX project: /home/user/analytics Project ready: yes LLM ready: yes (claude-sonnet-4-6) Embeddings ready: yes (text-embedding-3-small) -Databases configured: yes (postgres-warehouse) -Context sources configured: yes (dbt-main) +Databases configured: yes (warehouse) +Context sources configured: yes (dbt_main) KTX context built: yes Agent integration ready: yes (codex:project) ``` @@ -65,7 +98,7 @@ Agent integration ready: yes (codex:project) | `ktx connection test ` | Test one connection | | `ktx ingest ` | Build context for one connection | | `ktx ingest --all` | Build context for every configured connection | -| `ktx ingest text ` | Capture free-form notes into memory | +| `ktx ingest text --connection-id ` | Capture free-form notes into memory | | `ktx sl list` | List semantic-layer sources | | `ktx sl search "revenue"` | Search semantic-layer sources | | `ktx sl validate --connection-id ` | Validate a semantic source | @@ -93,14 +126,17 @@ Commit `ktx.yaml`, `semantic-layer/`, and `wiki/`. Keep `.ktx/` local. ## Agent Usage -Setup can install KTX instructions for Claude Code, Codex, Cursor, OpenCode, -and universal `.agents` clients: +Install KTX integration for Claude Code, Claude Desktop, Codex, Cursor, +OpenCode, and generic `.agents` clients: ```bash -ktx setup --agents --target codex +ktx setup --agents ``` -Agent-facing workflows typically start with: +Pass `--target ` to install or repair one specific integration. + +A typical agent workflow combines wiki and semantic-layer search before +querying: ```bash ktx sl search "revenue" --json @@ -108,31 +144,14 @@ ktx wiki search "refund policy" --json ktx sl query --connection-id warehouse --measure orders.revenue --format sql ``` -During agent setup, choose **MCP tools + analytics skill** for client agents. -Choose **MCP tools + analytics skill + admin CLI skill** only when a developer -or operator agent also needs pinned `ktx` admin commands. +During setup, choose **Ask data questions with KTX MCP** for client agents. +Choose **Ask data questions + manage KTX with CLI commands** when an operator +agent also needs pinned `ktx` admin commands. -The analytics skill teaches client agents the MCP workflow: discover data, -prefer semantic-layer measures, inspect entity details before raw SQL, and -capture durable learnings. Admin CLI skills call `ktx` commands directly -through a skill file installed in your agent's config: - -```bash -ktx sl query --measure orders.revenue --dimension orders.status --format sql -ktx wiki search "revenue definition" -ktx sl validate orders -``` - -Supported client agents: Claude Code, Claude Desktop, Codex, Cursor, OpenCode, -and clients that can use the printed MCP endpoint or `.agents` admin skills. -Claude Desktop setup registers a local `ktx mcp stdio` server in Claude -Desktop's config and generates `.ktx/agents/claude/ktx-plugin.zip` with the -analytics skill. - -The release artifact manifest contains the public npm tarball and the bundled -`kaelio-ktx` runtime wheel. The `python/ktx-sl` and `python/ktx-daemon` -directories remain source packages for development, not public release -artifacts. +After setup, KTX prints **Required before using agents** with the exact +commands to run. If the output includes `ktx mcp start --project-dir ...`, run +it before opening your agent. Claude Desktop uses its own launcher and prints +separate skill upload steps under `.ktx/agents/claude/`. ## Workspace packages @@ -189,7 +208,7 @@ uv run pytest -q ## Docs - [Quickstart](docs-site/content/docs/getting-started/quickstart.mdx) -- [CLI Reference](docs-site/content/docs/cli-reference/index.mdx) +- [CLI Reference](docs-site/content/docs/cli-reference/ktx.mdx) - [Building Context](docs-site/content/docs/guides/building-context.mdx) - [Contributing](docs-site/content/docs/community/contributing.mdx) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..0ff83e2f --- /dev/null +++ b/codecov.yml @@ -0,0 +1,109 @@ +codecov: + branch: main + require_ci_to_pass: true + notify: + after_n_builds: 2 + +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + target: auto + threshold: 1% + if_ci_failed: error + typescript: + target: auto + threshold: 1% + flags: + - typescript + if_ci_failed: error + python: + target: auto + threshold: 1% + flags: + - python + if_ci_failed: error + patch: + default: + target: 75% + threshold: 5% + if_ci_failed: error + informational: true + +comment: + layout: "header, diff, flags, components, files" + behavior: default + require_changes: false + require_base: false + require_head: true + +flags: + typescript: + paths: + - packages/ + carryforward: false + python: + paths: + - python/ + carryforward: false + +component_management: + individual_components: + - component_id: pkg_cli + name: CLI + paths: + - packages/cli/src/** + - component_id: pkg_context + name: Context engine + paths: + - packages/context/src/** + - component_id: pkg_llm + name: LLM + paths: + - packages/llm/src/** + - component_id: connector_bigquery + name: BigQuery connector + paths: + - packages/connector-bigquery/src/** + - component_id: connector_clickhouse + name: ClickHouse connector + paths: + - packages/connector-clickhouse/src/** + - component_id: connector_mysql + name: MySQL connector + paths: + - packages/connector-mysql/src/** + - component_id: connector_postgres + name: Postgres connector + paths: + - packages/connector-postgres/src/** + - component_id: connector_snowflake + name: Snowflake connector + paths: + - packages/connector-snowflake/src/** + - component_id: connector_sqlite + name: SQLite connector + paths: + - packages/connector-sqlite/src/** + - component_id: connector_sqlserver + name: SQL Server connector + paths: + - packages/connector-sqlserver/src/** + - component_id: py_semantic_layer + name: Python semantic layer + paths: + - python/ktx-sl/semantic_layer/** + - component_id: py_daemon + name: Python daemon + paths: + - python/ktx-daemon/src/ktx_daemon/** + +ignore: + - docs-site/** + - examples/** + - packages/*/coverage/** + - packages/*/dist/** + - python/ktx-sl/demos/** diff --git a/docs-site/app/api/search/route.ts b/docs-site/app/api/search/route.ts new file mode 100644 index 00000000..d86bfc5b --- /dev/null +++ b/docs-site/app/api/search/route.ts @@ -0,0 +1,4 @@ +import { source } from "@/lib/source"; +import { createFromSource } from "fumadocs-core/search/server"; + +export const { GET } = createFromSource(source); diff --git a/docs-site/app/docs/[[...slug]]/page.tsx b/docs-site/app/docs/[[...slug]]/page.tsx index dd5d944c..e0677c9e 100644 --- a/docs-site/app/docs/[[...slug]]/page.tsx +++ b/docs-site/app/docs/[[...slug]]/page.tsx @@ -51,10 +51,7 @@ export default async function Page(props: { <>
{page.data.title} - +
{page.data.description} diff --git a/docs-site/app/global.css b/docs-site/app/global.css index bc4ed8a4..d0f7ac21 100644 --- a/docs-site/app/global.css +++ b/docs-site/app/global.css @@ -69,7 +69,11 @@ --color-fd-muted-foreground: #7a8d96; } -html, body { +/* Keep html overflow at the default `visible` so body's overflow + propagates to the viewport (per CSS Overflow spec). That lets + `react-remove-scroll-bar` lock viewport scroll via body alone while + leaving the sticky sidebar placeholder anchored to the viewport. */ +body { overflow-x: clip; } @@ -161,6 +165,17 @@ pre { line-height: 1.7 !important; } +/* Disable monospace ligatures so `--flag` keeps a visible space and double + dashes don't fuse into an em-dash glyph. */ +code, +pre, +pre code, +.ktx-code, +.ktx-code code { + font-variant-ligatures: none !important; + font-feature-settings: "liga" 0, "calt" 0 !important; +} + .dark pre { background: transparent !important; } @@ -206,6 +221,72 @@ pre { padding-inline: 0 !important; } +.ktx-code .ktx-token-key { + color: #0f766e; +} + +.ktx-code .ktx-token-keyword { + color: #0e7490; + font-weight: 650; +} + +.ktx-code .ktx-token-function { + color: #7c3aed; + font-weight: 650; +} + +.ktx-code .ktx-token-flag { + color: #0369a1; +} + +.ktx-code .ktx-token-string { + color: #b45309; +} + +.ktx-code .ktx-token-number, +.ktx-code .ktx-token-constant { + color: #be123c; +} + +.ktx-code .ktx-token-comment { + color: #64748b; + font-style: italic; +} + +.ktx-code .ktx-token-punctuation { + color: #64748b; +} + +.dark .ktx-code .ktx-token-key { + color: #5eead4; +} + +.dark .ktx-code .ktx-token-keyword { + color: #67e8f9; +} + +.dark .ktx-code .ktx-token-function { + color: #c4b5fd; +} + +.dark .ktx-code .ktx-token-flag { + color: #7dd3fc; +} + +.dark .ktx-code .ktx-token-string { + color: #fbbf24; +} + +.dark .ktx-code .ktx-token-number, +.dark .ktx-code .ktx-token-constant { + color: #fb7185; +} + +.dark .ktx-code .ktx-token-comment, +.dark .ktx-code .ktx-token-punctuation { + color: #94a3b8; +} + /* Neutralize the outer figure styling that our wrapper now owns */ figure:has(> .ktx-code), figure[data-rehype-pretty-code-figure]:has(.ktx-code) { @@ -216,57 +297,10 @@ figure[data-rehype-pretty-code-figure]:has(.ktx-code) { margin: 0; } -/* ── Mode A: Terminal ─────────────────────── */ -.ktx-code-terminal { - background: #0c1417; - border: 1px solid rgba(255, 255, 255, 0.08); - color: #c8c3bc; - box-shadow: - 0 1px 2px rgba(0, 0, 0, 0.1), - 0 12px 32px -16px rgba(0, 0, 0, 0.3); -} - -.ktx-code-terminal:hover { - border-color: rgba(34, 211, 238, 0.2); - box-shadow: - 0 1px 2px rgba(0, 0, 0, 0.1), - 0 14px 32px -12px rgba(34, 211, 238, 0.18); -} - -.ktx-code-terminal-head { - display: flex; - align-items: center; - gap: 6px; - padding: 10px 12px; - border-bottom: 1px solid rgba(255, 255, 255, 0.06); - background: linear-gradient(180deg, rgba(255, 255, 255, 0.03), transparent); -} - -.ktx-tl-dot { - width: 11px; - height: 11px; - border-radius: 999px; - flex-shrink: 0; -} - -.ktx-code-terminal-label { - margin-left: 8px; - font-size: 11px; - font-weight: 500; - letter-spacing: 0.02em; - color: rgba(255, 255, 255, 0.4); -} - -.ktx-code-body-terminal { - background: transparent !important; - color: #c8c3bc !important; -} - /* ── Mode D: Output preview (wizard prompts, status output) ── */ .ktx-code-output { background: var(--color-fd-muted); border: 1px solid var(--color-fd-border); - border-left: 3px solid color-mix(in oklch, var(--color-fd-primary) 50%, var(--color-fd-border)); position: relative; box-shadow: 0 1px 2px rgba(27, 27, 24, 0.02); } @@ -274,17 +308,14 @@ figure[data-rehype-pretty-code-figure]:has(.ktx-code) { .dark .ktx-code-output { background: #111a1e; border-color: rgba(255, 255, 255, 0.05); - border-left-color: rgba(34, 211, 238, 0.25); } .ktx-code-output:hover { border-color: color-mix(in oklch, var(--color-fd-primary) 25%, var(--color-fd-border)); - border-left-color: var(--color-fd-primary); } .dark .ktx-code-output:hover { border-color: rgba(255, 255, 255, 0.08); - border-left-color: rgba(34, 211, 238, 0.45); } .ktx-code-output-label { @@ -304,8 +335,8 @@ figure[data-rehype-pretty-code-figure]:has(.ktx-code) { .ktx-code-output-copy { position: absolute !important; - top: 6px !important; - right: 6px !important; + top: 7px !important; + right: 8px !important; opacity: 0; transform: translateY(-4px); transition: opacity 0.2s var(--ktx-ease), transform 0.2s var(--ktx-ease); @@ -362,55 +393,32 @@ figure[data-rehype-pretty-code-figure]:has(.ktx-code) { display: flex; align-items: center; gap: 8px; - padding: 8px 10px 8px 14px; + padding: 5px 8px 5px 12px; border-bottom: 1px solid var(--color-fd-border); - background: linear-gradient(180deg, var(--color-fd-muted), transparent); + background: rgba(0, 0, 0, 0.025); } .dark .ktx-code-tab-head { border-bottom-color: rgba(255, 255, 255, 0.05); - background: linear-gradient(180deg, rgba(255, 255, 255, 0.02), transparent); + background: rgba(255, 255, 255, 0.02); } -.ktx-file-glyph { - display: inline-block; - width: 8px; - height: 8px; - border-radius: 999px; - background: var(--color-fd-muted-foreground); - flex-shrink: 0; -} -.ktx-file-glyph[data-lang="yaml"], -.ktx-file-glyph[data-lang="yml"] { background: #fbbf24; } -.ktx-file-glyph[data-lang="ts"], -.ktx-file-glyph[data-lang="tsx"], -.ktx-file-glyph[data-lang="typescript"] { background: #3b82f6; } -.ktx-file-glyph[data-lang="js"], -.ktx-file-glyph[data-lang="jsx"], -.ktx-file-glyph[data-lang="javascript"] { background: #facc15; } -.ktx-file-glyph[data-lang="json"] { background: #84cc16; } -.ktx-file-glyph[data-lang="md"], -.ktx-file-glyph[data-lang="mdx"] { background: #a3a3a3; } -.ktx-file-glyph[data-lang="sql"] { background: #f97316; } -.ktx-file-glyph[data-lang="py"], -.ktx-file-glyph[data-lang="python"] { background: #22d3ee; } - .ktx-code-tab-filename { font-family: var(--font-mono), ui-monospace, monospace; - font-size: 12.5px; - color: var(--color-fd-foreground); + font-size: 11.5px; + color: #6b7280; } .ktx-lang-pill { - margin-left: 4px; - padding: 1px 6px; - font-size: 10px; - font-weight: 600; + margin-right: 4px; + padding: 0 7px; + font-size: 9px; + font-weight: 500; text-transform: uppercase; - letter-spacing: 0.04em; - color: var(--color-fd-muted-foreground); + letter-spacing: 0.06em; + color: #9ca3af; border: 1px solid var(--color-fd-border); - border-radius: 4px; + border-radius: 3px; background: var(--color-fd-card); font-family: var(--font-display), var(--font-sans), sans-serif; } @@ -445,30 +453,10 @@ figure[data-rehype-pretty-code-figure]:has(.ktx-code) { border-color: rgba(34, 211, 238, 0.2); } -.ktx-code-minimal-lang { - position: absolute; - top: 8px; - left: 14px; - font-size: 10px; - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.05em; - color: var(--color-fd-muted-foreground); - font-family: var(--font-display), var(--font-sans), sans-serif; - opacity: 0; - transition: opacity 0.2s var(--ktx-ease); - pointer-events: none; - z-index: 1; -} - -.ktx-code-minimal:hover .ktx-code-minimal-lang { - opacity: 0.5; -} - .ktx-code-minimal-copy { position: absolute !important; - top: 6px !important; - right: 6px !important; + top: 7px !important; + right: 8px !important; opacity: 0; transform: translateY(-4px); transition: opacity 0.2s var(--ktx-ease), transform 0.2s var(--ktx-ease); @@ -778,8 +766,8 @@ body::after { mix-blend-mode: overlay; } -/* Make sure content stays above background */ -body > * { +/* Make sure page content stays above the decorative background. */ +.ktx-site-shell { position: relative; z-index: 2; } @@ -1058,8 +1046,7 @@ body > * { .pill-badge .pill-dot { animation: none; } .card-lift { transition: none; } .ktx-code, - .ktx-code-minimal-copy, - .ktx-code-minimal-lang { + .ktx-code-minimal-copy { transition: none; } #nd-sidebar div[data-state]:not([class]) > button[data-state] svg { diff --git a/docs-site/app/layout.tsx b/docs-site/app/layout.tsx index 48e12a3f..230fd232 100644 --- a/docs-site/app/layout.tsx +++ b/docs-site/app/layout.tsx @@ -28,8 +28,8 @@ export const metadata: Metadata = { description: "Open-source context infrastructure that makes agentic analytics reliable.", icons: { - icon: "/brand/ktx-mascot.svg", - shortcut: "/brand/ktx-mascot.svg", + icon: "/ktx/brand/ktx-mascot.svg", + shortcut: "/ktx/brand/ktx-mascot.svg", }, }; @@ -41,7 +41,9 @@ export default function RootLayout({ children }: { children: ReactNode }) { suppressHydrationWarning > - {children} + +
{children}
+ ); diff --git a/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts b/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts index 1372d556..87dcbd42 100644 --- a/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts +++ b/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts @@ -3,6 +3,11 @@ import { getLlmDocsPages, getPageMarkdown, } from "@/lib/llm-docs"; +import { + agentSetupSlug, + isAgentSetupSlug, + readAgentSetupMarkdown, +} from "@/lib/agent-setup-markdown"; export const dynamic = "force-static"; @@ -11,6 +16,14 @@ export async function GET( props: { params: Promise<{ slug?: string[] }> }, ) { const params = await props.params; + if (isAgentSetupSlug(params.slug)) { + return new Response(await readAgentSetupMarkdown(), { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); + } + const page = getLlmDocsPage(params.slug); if (!page) { return new Response("Documentation page not found.\n", { @@ -29,5 +42,8 @@ export async function GET( } export function generateStaticParams() { - return getLlmDocsPages().map((page) => ({ slug: page.slug })); + return [ + ...getLlmDocsPages().map((page) => ({ slug: page.slug })), + { slug: [...agentSetupSlug] }, + ]; } diff --git a/docs-site/components/code-block.tsx b/docs-site/components/code-block.tsx index 7d6a22af..80f29bdc 100644 --- a/docs-site/components/code-block.tsx +++ b/docs-site/components/code-block.tsx @@ -1,5 +1,3 @@ -"use client"; - import { type ComponentPropsWithoutRef, type ReactNode, @@ -13,8 +11,57 @@ type Props = ComponentPropsWithoutRef<"pre"> & { "data-language"?: string; }; -const TERMINAL_LANGS = new Set(["bash", "sh", "shell", "zsh"]); +const OUTPUT_LANGS = new Set(["text", "plain", "plaintext", "console", "output"]); const WIZARD_GLYPHS = /^\s*[◆◇◯◐○●]/; +const JSON_TOKEN_PATTERN = + /"(?:\\.|[^"\\])*"|-?\b\d+(?:\.\d+)?\b|\b(?:true|false|null)\b|[{}[\],:]/g; +const SQL_TOKEN_PATTERN = + /--[^\n]*|'(?:''|[^'])*'|\b\d+(?:\.\d+)?\b|\b(?:select|from|join|left|right|inner|outer|on|where|group|by|order|limit|as|sum|avg|min|max|count|coalesce|date_trunc|case|when|then|else|end|and|or|is|not|null|false|true|with|having|over|partition|insert|update|delete|create|alter|drop|table|view)\b|[(),.;=*<>+-]/gi; +const CODE_LIKE_TOKEN_PATTERN = + /\/\/[^\n]*|\/\*[\s\S]*?\*\/|#(?![{\w-]+:)[^\n]*|`(?:\\.|[^`\\])*`|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|-?\b\d+(?:\.\d+)?\b|\b(?:const|let|var|function|return|import|export|from|type|interface|extends|async|await|if|else|for|while|switch|case|break|continue|try|catch|throw|new|class|public|private|protected|readonly|true|false|null|undefined|pnpm|uv|ktx|node|npx|curl|git)\b|--?[\w-]+|[{}[\](),.;:=*<>|&+-]/g; +const SQL_FUNCTIONS = new Set([ + "sum", + "avg", + "min", + "max", + "count", + "coalesce", + "date_trunc", +]); +const CODE_KEYWORDS = new Set([ + "const", + "let", + "var", + "function", + "return", + "import", + "export", + "from", + "type", + "interface", + "extends", + "async", + "await", + "if", + "else", + "for", + "while", + "switch", + "case", + "break", + "continue", + "try", + "catch", + "throw", + "new", + "class", + "public", + "private", + "protected", + "readonly", +]); +const COMMAND_KEYWORDS = new Set(["pnpm", "uv", "ktx", "node", "npx", "curl", "git"]); +const CODE_CONSTANTS = new Set(["true", "false", "null", "undefined"]); function extractText(node: ReactNode): string { if (typeof node === "string") return node; @@ -27,6 +74,33 @@ function extractText(node: ReactNode): string { return ""; } +function findLanguageInNode(node: ReactNode): string | null { + if (!isValidElement(node)) return null; + const props = (node as ReactElement<{ + className?: string; + "data-language"?: string; + children?: ReactNode; + }>).props; + + const dataLang = props["data-language"]; + if (typeof dataLang === "string" && dataLang) return dataLang; + + const className = typeof props.className === "string" ? props.className : ""; + const m = className.match(/language-([\w-]+)/); + if (m) return m[1]; + + const children = props.children; + if (Array.isArray(children)) { + for (const child of children) { + const found = findLanguageInNode(child); + if (found) return found; + } + } else if (children) { + return findLanguageInNode(children); + } + return null; +} + function detectLanguage(props: Props, children: ReactNode): string | null { const dataLang = props["data-language"]; if (typeof dataLang === "string" && dataLang) return dataLang; @@ -35,54 +109,288 @@ function detectLanguage(props: Props, children: ReactNode): string | null { const m = className.match(/language-([\w-]+)/); if (m) return m[1]; - if (isValidElement(children)) { - const childProps = (children as ReactElement<{ className?: string }>).props; - const childClass = typeof childProps.className === "string" ? childProps.className : ""; - const cm = childClass.match(/language-([\w-]+)/); - if (cm) return cm[1]; + return findLanguageInNode(children); +} + +function stripOneLeadingBlankLine(text: string) { + return text.startsWith("\n") ? text.slice(1) : text; +} + +function extractCodeHeader(language: string | null, code: string) { + const normalized = normalizeLanguage(language); + const firstLineEnd = code.indexOf("\n"); + const firstLine = firstLineEnd === -1 ? code : code.slice(0, firstLineEnd); + const rest = firstLineEnd === -1 ? "" : code.slice(firstLineEnd + 1); + const commentPrefix = + normalized === "sql" + ? "--" + : normalized === "javascript" || + normalized === "js" || + normalized === "jsx" || + normalized === "typescript" || + normalized === "ts" || + normalized === "tsx" + ? "//" + : "#"; + + if (!firstLine.trimStart().startsWith(commentPrefix)) { + return { header: null, code }; } - return null; + const candidate = firstLine + .trim() + .slice(commentPrefix.length) + .trim(); + const looksLikePath = + candidate.includes("/") && + /\.[A-Za-z0-9]+(?:["'`)]*)?$/.test(candidate); + + if (!looksLikePath) return { header: null, code }; + + return { + header: candidate, + code: stripOneLeadingBlankLine(rest), + }; +} + +function normalizeLanguage(language: string | null) { + return language?.toLowerCase() ?? ""; +} + +function pushMatchedToken( + parts: ReactNode[], + token: string, + className: string, + key: string, +) { + parts.push( + + {token} + , + ); +} + +function highlightJson(code: string) { + const parts: ReactNode[] = []; + let lastIndex = 0; + let tokenIndex = 0; + + for (const match of code.matchAll(JSON_TOKEN_PATTERN)) { + const token = match[0]; + const index = match.index ?? 0; + if (index > lastIndex) parts.push(code.slice(lastIndex, index)); + + const nextText = code.slice(index + token.length); + const className = token.startsWith('"') + ? /^\s*:/.test(nextText) + ? "ktx-token-key" + : "ktx-token-string" + : /^-?\d/.test(token) + ? "ktx-token-number" + : /^(true|false|null)$/.test(token) + ? "ktx-token-constant" + : "ktx-token-punctuation"; + + pushMatchedToken(parts, token, className, `json-${tokenIndex}`); + lastIndex = index + token.length; + tokenIndex += 1; + } + + if (lastIndex < code.length) parts.push(code.slice(lastIndex)); + return parts; +} + +function highlightYaml(code: string) { + const parts: ReactNode[] = []; + const lines = code.split(/(\n)/); + let tokenIndex = 0; + + for (const line of lines) { + if (line === "\n") { + parts.push(line); + continue; + } + + const commentIndex = line.search(/\s#/); + const fullLineComment = line.trimStart().startsWith("#"); + const contentEnd = + fullLineComment || commentIndex === -1 ? line.length : commentIndex + 1; + const content = fullLineComment ? "" : line.slice(0, contentEnd); + const comment = fullLineComment ? line : line.slice(contentEnd); + const keyMatch = content.match(/^(\s*(?:-\s*)?)([A-Za-z_][\w.-]*)(\s*:)/); + + if (keyMatch) { + parts.push(keyMatch[1]); + pushMatchedToken(parts, keyMatch[2], "ktx-token-key", `yaml-key-${tokenIndex}`); + pushMatchedToken( + parts, + keyMatch[3], + "ktx-token-punctuation", + `yaml-colon-${tokenIndex}`, + ); + const rest = content.slice(keyMatch[0].length); + if (rest) parts.push(...highlightInlineValue(rest, `yaml-${tokenIndex}`)); + } else if (content) { + parts.push(...highlightInlineValue(content, `yaml-${tokenIndex}`)); + } + + if (comment) { + pushMatchedToken(parts, comment, "ktx-token-comment", `yaml-comment-${tokenIndex}`); + } + tokenIndex += 1; + } + + return parts; +} + +function highlightInlineValue(value: string, keyPrefix: string) { + const parts: ReactNode[] = []; + let lastIndex = 0; + let tokenIndex = 0; + const pattern = /'(?:''|[^'])*'|"(?:\\.|[^"\\])*"|-?\b\d+(?:\.\d+)?\b|\b(?:true|false|null)\b|[()[\]{},:=!<>+-]/g; + + for (const match of value.matchAll(pattern)) { + const token = match[0]; + const index = match.index ?? 0; + if (index > lastIndex) parts.push(value.slice(lastIndex, index)); + + const className = + token.startsWith("'") || token.startsWith('"') + ? "ktx-token-string" + : /^-?\d/.test(token) + ? "ktx-token-number" + : /^(true|false|null)$/.test(token) + ? "ktx-token-constant" + : "ktx-token-punctuation"; + + pushMatchedToken(parts, token, className, `${keyPrefix}-value-${tokenIndex}`); + lastIndex = index + token.length; + tokenIndex += 1; + } + + if (lastIndex < value.length) parts.push(value.slice(lastIndex)); + return parts; +} + +function highlightSql(code: string) { + const parts: ReactNode[] = []; + let lastIndex = 0; + let tokenIndex = 0; + + for (const match of code.matchAll(SQL_TOKEN_PATTERN)) { + const token = match[0]; + const index = match.index ?? 0; + if (index > lastIndex) parts.push(code.slice(lastIndex, index)); + + const lowerToken = token.toLowerCase(); + const className = token.startsWith("--") + ? "ktx-token-comment" + : token.startsWith("'") + ? "ktx-token-string" + : /^\d/.test(token) + ? "ktx-token-number" + : SQL_FUNCTIONS.has(lowerToken) + ? "ktx-token-function" + : /^[a-z_]+$/i.test(token) + ? "ktx-token-keyword" + : "ktx-token-punctuation"; + + pushMatchedToken(parts, token, className, `sql-${tokenIndex}`); + lastIndex = index + token.length; + tokenIndex += 1; + } + + if (lastIndex < code.length) parts.push(code.slice(lastIndex)); + return parts; +} + +function highlightCodeLike(code: string) { + const parts: ReactNode[] = []; + let lastIndex = 0; + let tokenIndex = 0; + + for (const match of code.matchAll(CODE_LIKE_TOKEN_PATTERN)) { + const token = match[0]; + const index = match.index ?? 0; + if (index > lastIndex) parts.push(code.slice(lastIndex, index)); + + const lowerToken = token.toLowerCase(); + const className = + token.startsWith("//") || token.startsWith("/*") || token.startsWith("#") + ? "ktx-token-comment" + : token.startsWith("'") || token.startsWith('"') || token.startsWith("`") + ? "ktx-token-string" + : /^-?\d/.test(token) + ? "ktx-token-number" + : CODE_CONSTANTS.has(lowerToken) + ? "ktx-token-constant" + : CODE_KEYWORDS.has(lowerToken) + ? "ktx-token-keyword" + : COMMAND_KEYWORDS.has(lowerToken) + ? "ktx-token-function" + : token.startsWith("-") + ? "ktx-token-flag" + : "ktx-token-punctuation"; + + pushMatchedToken(parts, token, className, `code-${tokenIndex}`); + lastIndex = index + token.length; + tokenIndex += 1; + } + + if (lastIndex < code.length) parts.push(code.slice(lastIndex)); + return parts; +} + +function highlightCode(language: string | null, code: string) { + const normalized = normalizeLanguage(language); + if (normalized === "json" || normalized === "jsonc") return highlightJson(code); + if (normalized === "yaml" || normalized === "yml") return highlightYaml(code); + if (normalized === "sql") return highlightSql(code); + if ( + [ + "bash", + "sh", + "shell", + "zsh", + "javascript", + "js", + "jsx", + "typescript", + "ts", + "tsx", + "python", + "py", + ].includes(normalized) + ) { + return highlightCodeLike(code); + } + return code; } export function CodeBlock(props: Props) { const { children, title, className: _ignored, ...rest } = props; const language = detectLanguage(props, children); - const codeText = extractText(children); + const rawCodeText = extractText(children); + const extractedHeader = extractCodeHeader(language, rawCodeText); + const codeText = extractedHeader.code; + const headerTitle = + typeof title === "string" && title.length > 0 + ? title + : extractedHeader.header; + const highlightedCode = highlightCode(language, codeText); - const isTerminal = language !== null && TERMINAL_LANGS.has(language); - const isOutput = !isTerminal && WIZARD_GLYPHS.test(codeText); - const hasTitle = typeof title === "string" && title.length > 0; - - // Mode A - Terminal (commands the user types) - if (isTerminal) { - return ( -
-
- - - - - {hasTitle ? title : "zsh"} - - -
- 
-          {children}
-
-
- ); - } + const hasHeader = typeof headerTitle === "string" && headerTitle.length > 0; + const isOutput = + !hasHeader && + (WIZARD_GLYPHS.test(rawCodeText) || + (language !== null && OUTPUT_LANGS.has(language))); // Mode D - Output preview (wizard prompts, terminal output) if (isOutput) { return (
output - + 
           {children}
@@ -90,18 +398,17 @@ export function CodeBlock(props: Props) { ); } - // Mode B - VS Code tab (filename present) - if (hasTitle) { + // Mode B - Header (filename present) + if (hasHeader) { return (
- - {title} {language && {language}} + {headerTitle} 
-          {children}
+          {highlightedCode}
); @@ -110,10 +417,9 @@ export function CodeBlock(props: Props) { // Mode C - Minimal default return (
- {language && {language}} 
-        {children}
+        {highlightedCode}
); diff --git a/docs-site/components/copy-button.tsx b/docs-site/components/copy-button.tsx index 876f5de0..c0dd1f31 100644 --- a/docs-site/components/copy-button.tsx +++ b/docs-site/components/copy-button.tsx @@ -25,12 +25,12 @@ export function CopyButton({ text, className = "" }: Props) { type="button" onClick={onClick} aria-label={copied ? "Copied" : "Copy code"} - className={`inline-flex items-center justify-center w-7 h-7 rounded-md transition-all hover:bg-white/5 ${className}`} + className={`inline-flex items-center justify-center w-9 h-9 rounded-md transition-all hover:bg-fd-muted ${className}`} > {copied ? ( ) : ( { + try { + await navigator.clipboard.writeText(stripFrontmatter(mdxSource)); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch { + // Clipboard denied - fail silently + } + }; + return (
- - - View MD - - + {copied ? "Copied" : "Copy as Markdown"} +
); } - -function CopyMarkdownButton({ markdownUrl }: { markdownUrl: string }) { - const [state, setState] = useState("idle"); - - const onClick = async () => { - try { - const response = await fetch(markdownUrl, { - headers: { Accept: "text/markdown" }, - }); - - if (!response.ok) { - throw new Error(`Failed to fetch ${markdownUrl}`); - } - - await navigator.clipboard.writeText(await response.text()); - flash(setState, "copied"); - } catch { - flash(setState, "error"); - } - }; - - return ( - - ); -} - -function CopyTextButton({ label, text }: { label: string; text: string }) { - const [state, setState] = useState("idle"); - - const onClick = async () => { - try { - await navigator.clipboard.writeText(text); - flash(setState, "copied"); - } catch { - flash(setState, "error"); - } - }; - - return ( - - ); -} - -function ActionButton({ - label, - onClick, - state, -}: { - label: string; - onClick: () => void; - state: CopyState; -}) { - return ( - - ); -} - -function labelForState(state: CopyState, label: string) { - if (state === "copied") return "Copied"; - if (state === "error") return "Copy failed"; - return label; -} - -function flash( - setState: (state: CopyState) => void, - state: Exclude, -) { - setState(state); - window.setTimeout(() => setState("idle"), 1500); -} diff --git a/docs-site/components/product-mechanics.tsx b/docs-site/components/product-mechanics.tsx index 1c10e9aa..45baaa31 100644 --- a/docs-site/components/product-mechanics.tsx +++ b/docs-site/components/product-mechanics.tsx @@ -1,115 +1,502 @@ -import type { ReactNode } from "react"; +"use client"; -type SourceInput = { - name: string; - sources: string[]; - detail: string; - signal: string; +import { + Background, + BackgroundVariant, + type Edge, + type EdgeProps, + getSmoothStepPath, + Handle, + MarkerType, + type Node, + type NodeProps, + Position, + ReactFlow, +} from "@xyflow/react"; +import "@xyflow/react/dist/style.css"; +import { useEffect, useMemo, useState } from "react"; + +type SourceNodeData = { accent: string; + body: string; + items: string[]; + title: string; }; -const sourceInputs = [ - { - name: "Database structure", - sources: [ - "Postgres", - "Snowflake", - "BigQuery", - "and many others", - ], - detail: "tables, columns, types, constraints, row counts", - signal: "grounds definitions in live database structure", - accent: "border-fd-primary", - }, - { - name: "BI and usage evidence", - sources: ["Metabase", "Looker"], - detail: "historic SQL, questions, dashboards, usage patterns", - signal: "extracts joins, filters, grain, and trusted examples", - accent: "border-orange-500", - }, - { - name: "Semantic modeling", - sources: ["dbt", "MetricFlow", "LookML"], - detail: "models, metrics, dimensions, explores, joins", - signal: "maps existing modeling logic into semantic entities", - accent: "border-amber-500", - }, - { - name: "Company documentation", - sources: ["Notion"], - detail: "Notion pages, policies, caveats", - signal: "links business language back to semantic references", - accent: "border-slate-500 dark:border-cyan-200", - }, -] satisfies SourceInput[]; +type StageNodeData = { + body: string; + index: number; + title: string; +}; -const ingestSteps = [ +type OutputNodeData = { + accent: string; + body: string; + path: string; + tags: string[]; + title: string; +}; + +type SourceNode = Node; +type StageNode = Node; +type OutputNode = Node; +type FlowNode = SourceNode | StageNode | OutputNode; + +const SOURCE_W = 210; +const SOURCE_H = 200; +const STAGE_W = 280; +const STAGE_H = 120; +const OUTPUT_W = 340; +const OUTPUT_H = 232; + +const ROW_SOURCES_Y = 80; +const ROW_STAGE_START_Y = 360; +const STAGE_GAP = 30; +const ROW_OUTPUTS_Y = 1000; + +const STAGE_CENTER_X = 460; +const STAGE_X = STAGE_CENTER_X - STAGE_W / 2; + +const SOURCE_GAP_X = 24; +const SOURCES_TOTAL = SOURCE_W * 4 + SOURCE_GAP_X * 3; +const SOURCES_START_X = STAGE_CENTER_X - SOURCES_TOTAL / 2; + +const OUTPUT_GAP_X = 180; +const OUTPUTS_TOTAL = OUTPUT_W * 2 + OUTPUT_GAP_X; +const OUTPUTS_START_X = STAGE_CENTER_X - OUTPUTS_TOTAL / 2; + +const EDGE_STROKE = "#94a3b8"; + +const sourceData: SourceNodeData[] = [ { - title: "extract evidence", - body: "Pull structured facts from schemas, SQL, BI metadata, and docs.", + title: "Databases", + body: "Schemas, columns, keys, row counts, and query history.", + items: ["PostgreSQL", "Snowflake", "BigQuery", "SQLite"], + accent: "#3b82f6", }, { - title: "reconcile entities", - body: "Merge names, measures, joins, and caveats into one project model.", + title: "BI tools", + body: "Dashboards, questions, explores, usage, and trusted examples.", + items: ["Metabase", "Looker"], + accent: "#f97316", }, { - title: "validate references", - body: "Check semantic fields and joins against database context before agents use them.", + title: "Modeling code", + body: "Existing metrics, dimensions, models, joins, and entities.", + items: ["dbt", "LookML", "MetricFlow"], + accent: "#f59e0b", + }, + { + title: "Docs and notes", + body: "Policies, caveats, team definitions, and analyst context.", + items: ["Notion", "Any text"], + accent: "#10b981", }, ]; -const artifacts = [ +const stageData: Omit[] = [ { - path: "semantic-layer/*.yaml", - title: "Typed query model", - body: "sources, grain, joins, dimensions, measures, filters, segments", + title: "Source adapters", + body: "Read each configured system in its native shape.", }, { + title: "Context builder", + body: "Turn source evidence into proposed context updates.", + }, + { + title: "Reconciliation", + body: "Merge new evidence with the context that already exists.", + }, + { + title: "Validation", + body: "Check references and semantics before agents rely on them.", + }, +]; + +const outputData: OutputNodeData[] = [ + { + title: "Wiki", path: "wiki/*.md", - title: "Business context", - body: "rules and caveats with sl_refs back to semantic-layer entities", + tags: ["free-form", "auto-maintained"], + body: "Definitions, caveats, policies, analyst notes, and business language that agents can search.", + accent: "#10b981", }, { - path: "raw-sources/", - title: "Evidence trail", - body: "scan artifacts, extracted metadata, relationship evidence", - }, - { - path: ".ktx/", - title: "Local indexes", - body: "embeddings and search indexes, not the source of truth", + title: "Semantic layer", + path: "semantic-layer/*.yaml", + tags: ["structured", "executable", "auto-maintained"], + body: "Metrics, joins, tables, dimensions, filters, and segments that KTX can validate and compile into SQL.", + accent: "#3b82f6", }, ]; -const runtimeSteps = [ - { - title: "Search wiki", - body: "Find business rules, caveats, synonyms, and sl_refs.", - }, - { - title: "Resolve semantic refs", - body: "Map measure and dimension names to approved entities.", - }, - { - title: "Validate fields", - body: "Check source, columns, joins, grain, filters, and segments.", - }, - { - title: "Build query plan", - body: "Create a semantic query plan before SQL is generated.", - }, - { - title: "Compile dialect SQL", - body: "Generate warehouse-shaped SQL instead of copying examples.", - }, - { - title: "Execute with bounds", - body: "Optionally run with bounded rows and return provenance.", - }, +const nodes: FlowNode[] = [ + ...sourceData.map((source, index) => ({ + id: `source-${index}`, + type: "source", + position: { + x: SOURCES_START_X + index * (SOURCE_W + SOURCE_GAP_X), + y: ROW_SOURCES_Y, + }, + data: source, + draggable: false, + selectable: false, + })), + ...stageData.map((stage, index) => ({ + id: `stage-${index}`, + type: "stage", + position: { + x: STAGE_X, + y: ROW_STAGE_START_Y + index * (STAGE_H + STAGE_GAP), + }, + data: { ...stage, index: index + 1 }, + draggable: false, + selectable: false, + })), + ...outputData.map((output, index) => ({ + id: `output-${index}`, + type: "output", + position: { + x: OUTPUTS_START_X + index * (OUTPUT_W + OUTPUT_GAP_X), + y: ROW_OUTPUTS_Y, + }, + data: output, + draggable: false, + selectable: false, + })), ]; +const REF_EDGE_STROKE = "#64748b"; + +const flowEdges = [ + ...sourceData.map((_, index) => ({ + id: `e-source-${index}-stage-0`, + source: `source-${index}`, + target: "stage-0", + })), + ...stageData.slice(0, -1).map((_, index) => ({ + id: `e-stage-${index}-stage-${index + 1}`, + source: `stage-${index}`, + target: `stage-${index + 1}`, + })), + ...outputData.map((_, index) => ({ + id: `e-stage-3-output-${index}`, + source: "stage-3", + target: `output-${index}`, + })), +].map((edge) => ({ + ...edge, + type: "smoothstep" as const, + style: { stroke: EDGE_STROKE, strokeWidth: 1.5 }, + markerEnd: { + type: MarkerType.ArrowClosed, + color: EDGE_STROKE, + width: 16, + height: 16, + }, +})); + +const refsEdge = { + id: "e-output-refs", + source: "output-0", + sourceHandle: "right", + target: "output-1", + targetHandle: "left", + type: "straight" as const, + label: "references", + labelBgPadding: [6, 3] as [number, number], + labelBgBorderRadius: 4, + labelStyle: { + fontSize: 13, + fontWeight: 500, + fill: "var(--color-fd-muted-foreground)", + }, + labelBgStyle: { + fill: "var(--color-fd-background)", + stroke: "var(--color-fd-border)", + strokeWidth: 1, + }, + style: { + stroke: REF_EDGE_STROKE, + strokeWidth: 1.25, + strokeDasharray: "4 4", + }, + markerStart: { + type: MarkerType.ArrowClosed, + color: REF_EDGE_STROKE, + width: 14, + height: 14, + }, + markerEnd: { + type: MarkerType.ArrowClosed, + color: REF_EDGE_STROKE, + width: 14, + height: 14, + }, +}; + + +function SourceNodeView({ data }: NodeProps) { + return ( +
+ +

+ {data.title} +

+

+ {data.body} +

+
+ {data.items.map((item) => ( + + {item} + + ))} +
+ +
+ ); +} + +function StageNodeView({ data }: NodeProps) { + return ( +
+ + + {data.index} + +
+

+ {data.title} +

+

+ {data.body} +

+
+ +
+ ); +} + +function OutputNodeView({ data }: NodeProps) { + return ( +
+ + + +

+ {data.path} +

+

+ {data.title} +

+
+ {data.tags.map((tag) => ( + + {tag} + + ))} +
+

+ {data.body} +

+
+ ); +} + +const PARTICLES_PER_SOURCE = 2; +const PARTICLE_SPEED_PX_PER_SEC = 130; +const PARTICLE_MIN_DURATION_SEC = 4; + +const stageTopY = (i: number) => ROW_STAGE_START_Y + i * (STAGE_H + STAGE_GAP); +const stageBottomY = (i: number) => stageTopY(i) + STAGE_H; + +function buildParticlePath( + sourceIndex: number, + outputIndex: number, +): { d: string; length: number } { + const sourceCenterX = + SOURCES_START_X + sourceIndex * (SOURCE_W + SOURCE_GAP_X) + SOURCE_W / 2; + const sourceBottomYVal = ROW_SOURCES_Y + SOURCE_H; + const outputCenterX = + OUTPUTS_START_X + outputIndex * (OUTPUT_W + OUTPUT_GAP_X) + OUTPUT_W / 2; + + const legs: Array<[number, number, number, number]> = [ + [sourceCenterX, sourceBottomYVal, STAGE_CENTER_X, stageTopY(0)], + [STAGE_CENTER_X, stageBottomY(0), STAGE_CENTER_X, stageTopY(1)], + [STAGE_CENTER_X, stageBottomY(1), STAGE_CENTER_X, stageTopY(2)], + [STAGE_CENTER_X, stageBottomY(2), STAGE_CENTER_X, stageTopY(3)], + [STAGE_CENTER_X, stageBottomY(3), outputCenterX, ROW_OUTPUTS_Y], + ]; + + const segments = legs.map(([sx, sy, tx, ty]) => { + const [segment] = getSmoothStepPath({ + sourceX: sx, + sourceY: sy, + sourcePosition: Position.Bottom, + targetX: tx, + targetY: ty, + targetPosition: Position.Top, + }); + return segment; + }); + + let d = segments[0]; + for (let i = 1; i < segments.length; i += 1) { + d += ` ${segments[i].replace(/^M/, "L")}`; + } + + const length = legs.reduce( + (sum, [sx, sy, tx, ty]) => sum + Math.abs(tx - sx) + Math.abs(ty - sy), + 0, + ); + + return { d, length }; +} + +type ParticleEdgeData = { + d: string; + duration: number; + beginOffset: number; + color: string; +}; + +type ParticleEdge = Edge; + +function ParticleEdgeView({ id, data }: EdgeProps) { + if (!data) return null; + const pathId = `mechanics-particle-path-${id}`; + return ( + <> + + + + + + + + + + + ); +} + +const nodeTypes = { + source: SourceNodeView, + stage: StageNodeView, + output: OutputNodeView, +}; + +const edgeTypes = { + particle: ParticleEdgeView, +}; + +const staticEdges = [...flowEdges, refsEdge]; + +type ParticleSpec = { + id: string; + sourceIndex: number; + outputIndex: number; +}; + +function makeRandomParticles(perSource: number): ParticleSpec[] { + const specs: ParticleSpec[] = []; + for (let sourceIndex = 0; sourceIndex < sourceData.length; sourceIndex += 1) { + for (let n = 0; n < perSource; n += 1) { + specs.push({ + id: `particle-${sourceIndex}-${n}`, + sourceIndex, + outputIndex: Math.floor(Math.random() * outputData.length), + }); + } + } + return specs; +} + +function specToEdge(spec: ParticleSpec): { + id: string; + source: string; + target: string; + type: "particle"; + data: ParticleEdgeData; +} { + const { d, length } = buildParticlePath(spec.sourceIndex, spec.outputIndex); + const duration = Math.max( + PARTICLE_MIN_DURATION_SEC, + length / PARTICLE_SPEED_PX_PER_SEC, + ); + return { + id: spec.id, + source: `source-${spec.sourceIndex}`, + target: `output-${spec.outputIndex}`, + type: "particle", + data: { + d, + duration, + beginOffset: Math.random() * duration, + color: sourceData[spec.sourceIndex].accent, + }, + }; +} + export function ProductMechanics() { + const [particles, setParticles] = useState([]); + + useEffect(() => { + setParticles(makeRandomParticles(PARTICLES_PER_SOURCE)); + }, []); + + const edges = useMemo( + () => [...staticEdges, ...particles.map(specToEdge)], + [particles], + ); + return (
- How KTX works + How ingestion works

- KTX reads source evidence, writes local context files, and gives - agents semantic search, validation, SQL, and provenance. + KTX ingests source evidence, reconciles it with your existing project, + and produces durable context that agents can search, review, and + execute.

-
- - -
+
+
+

+ Ingestion flow +

+

+ From scattered source systems to agent-ready context +

+

+ The inputs can be structured systems or loose team knowledge. The + outputs are the two files agents need: a readable wiki and an + executable semantic layer. +

+
+ +
+ + + +
+
+ ); } - -function IngestionDiagram() { - return ( -
- - -
-
- Inputs KTX reads -
- {sourceInputs.map((source) => ( -
-
-

- {source.name} -

-

- {source.detail} -

-

- {source.signal} -

-
- -
- ))} -
-
- -
- KTX transforms evidence -
-
-

- KTX builds the model -

-
    - {ingestSteps.map((step, index) => ( - - ))} -
-
- -
-

- Outputs KTX writes -

-
- {artifacts.map((artifact) => ( - - ))} -
-
-
-
-
-
- ); -} - -function RuntimeDiagram() { - return ( -
- - -
-
- Agent sends - -
connection: warehouse
-
measure: orders.total_revenue
-
dimension: customers.segment
-
filter: orders.created_date >= '2024-01-01'
- -

- This is the API surface agents should use: compact semantic intent, - not hand-written warehouse SQL. -

-
- -
- KTX planning and execution -
    - {runtimeSteps.map((step, index) => ( - - ))} -
-
-
- -
-
- Semantic query plan -
-

- source:{" "} - orders joined to customers as many_to_one -

-

- measure:{" "} - total_revenue = sum(amount) with refund filter -

-

- grain: segment - group-by with date predicate -

-

- result: dialect - SQL, bounded rows, and provenance -

-
-
- -
- KTX returns - -
select
-
customers.segment,
-
sum(orders.amount) as total_revenue
-
from analytics.orders
-
join analytics.customers
-
on orders.customer_id = customers.id
-
where orders.status != 'refunded'
-
and orders.created_date >= '2024-01-01'
-
group by 1
- -

- The output can be SQL-only or executed results with provenance, so - the agent can show where the answer came from. -

-
-
-
- ); -} - -function DiagramHeader({ - body, - eyebrow, - id, - title, -}: { - body: string; - eyebrow: string; - id: string; - title: string; -}) { - return ( -
-

- {eyebrow} -

-

- {title} -

-

- {body} -

-
- ); -} - -function Artifact({ - body, - path, - title, -}: { - body: string; - path: string; - title: string; -}) { - return ( -
-

- {path} -

-

{title}

-

- {body} -

-
- ); -} - -function PipelineStep({ - body, - dark = false, - index, - title, -}: { - body: string; - dark?: boolean; - index: number; - title: string; -}) { - return ( -
  • - - {index} - - - - {title} - - - {body} - - -
    • - ); -} - -function ColumnLabel({ children }: { children: ReactNode }) { - return ( -

    - {children} -

    - ); -} - -function SourceList({ - sources, -}: { - sources: string[]; -}) { - return ( -
    -

    - Sources -

    -
    - {sources.map((source) => - source === "and many others" ? ( - - {source} - - ) : ( - - {source} - - ), - )} -
    -
    - ); -} - -function CodeBox({ children }: { children: ReactNode }) { - return ( -
    -
    {children}
    -
    - ); -} diff --git a/docs-site/components/semantic-layer-flow.tsx b/docs-site/components/semantic-layer-flow.tsx new file mode 100644 index 00000000..770516ad --- /dev/null +++ b/docs-site/components/semantic-layer-flow.tsx @@ -0,0 +1,1258 @@ +"use client"; + +import { useCallback, useState } from "react"; +import { + Background, + BackgroundVariant, + Controls, + Handle, + MarkerType, + type Node, + type NodeProps, + type OnInit, + Position, + ReactFlow, +} from "@xyflow/react"; +import "@xyflow/react/dist/style.css"; + +type LaneVariant = "manual" | "ktx"; + +type AgentNodeData = { + variant: "single"; + title: string; + subtitle: string; +}; + +type IssueNote = { + id: number; + label: string; +}; + +type ManualSqlNodeData = { + variant: "manual"; + badge: string; + title: string; + caption: string; + code: string; + notes: IssueNote[]; + lineIssues: Record; +}; + +type SlQueryNodeData = { + variant: "slQuery"; + badge: string; + title: string; + caption: string; + code: string; +}; + +type EngineNodeData = { + variant: "engine"; + badge: string; + title: string; + stages: Array<{ index: number; title: string; detail: string }>; +}; + +type CompiledSqlNodeData = { + variant: "compiled"; + badge: string; + title: string; + caption: string; + code: string; + notes: string[]; +}; + +type WarehouseNodeData = { + variant: "warehouse"; + title: string; + drivers: string[]; +}; + +type AgentNode = Node; +type ManualSqlNode = Node; +type SlQueryNode = Node; +type EngineNode = Node; +type CompiledSqlNode = Node; +type WarehouseNode = Node; + +type FlowNode = + | AgentNode + | ManualSqlNode + | SlQueryNode + | EngineNode + | CompiledSqlNode + | WarehouseNode; + +const CANVAS_W = 1120; + +const AGENT_W = 380; +const AGENT_H = 104; +const AGENT_X = (CANVAS_W - AGENT_W) / 2; +const AGENT_Y = 16; + +const LANE_W = 488; +const LEFT_LANE_X = 32; +const RIGHT_LANE_X = CANVAS_W - LEFT_LANE_X - LANE_W; + +const LANE_TOP_Y = 248; + +const SL_QUERY_H = 510; +const ENGINE_H = 380; +const COMPILED_H = 1380; +const RIGHT_GAP = 24; + +const RIGHT_LANE_TOTAL = SL_QUERY_H + RIGHT_GAP + ENGINE_H + RIGHT_GAP + COMPILED_H; +const MANUAL_SQL_H = 840; +const LANES_BOTTOM_Y = + LANE_TOP_Y + Math.max(MANUAL_SQL_H, RIGHT_LANE_TOTAL); + +const SL_QUERY_Y = LANE_TOP_Y; +const ENGINE_Y = SL_QUERY_Y + SL_QUERY_H + RIGHT_GAP; +const COMPILED_Y = ENGINE_Y + ENGINE_H + RIGHT_GAP; + +const WAREHOUSE_W = 304; +const WAREHOUSE_H = 92; +const WAREHOUSE_X = (CANVAS_W - WAREHOUSE_W) / 2; +const WAREHOUSE_Y = LANES_BOTTOM_Y + 56; + +const MANUAL_STROKE = "#94a3b8"; +const KTX_STROKE = "#0891b2"; +const FIT_VIEW_OPTIONS = { padding: 0.05 }; + +const agent: AgentNode = { + id: "agent", + type: "agent", + position: { x: AGENT_X, y: AGENT_Y }, + data: { + variant: "single", + title: "Analytics agent", + subtitle: + "Asks: monthly net revenue and open tickets per segment, high-value orders only, no test customers", + }, + draggable: false, + selectable: false, +}; + +const manualSql: ManualSqlNode = { + id: "manual-sql", + type: "manualSql", + position: { x: LEFT_LANE_X, y: LANE_TOP_Y }, + data: { + variant: "manual", + badge: "Without KTX", + title: "Agent writes the SQL", + caption: + "Stitches four tables, mixes grains, and ships numbers that won't match the dashboard.", + code: `-- agent stitches four tables, mixes facts, +-- and ships numbers that won't match the dashboard + +SELECT + c.segment, + DATE_TRUNC('month', o.created_at) AS month, + SUM(o.amount) - SUM(r.amount) AS net_revenue, + COUNT(t.id) AS open_tickets +FROM customers c +LEFT JOIN orders o + ON o.customer_id = c.id +LEFT JOIN refunds r + ON r.order_id = o.id +LEFT JOIN tickets t + ON t.customer_id = c.id +WHERE + c.is_test = false + AND o.amount >= 100 + AND t.status = 'open' -- turns LEFT JOIN into INNER +GROUP BY + c.segment, + DATE_TRUNC('month', o.created_at) +ORDER BY + month, + c.segment +LIMIT 1000; + +-- chasm trap: orders rows multiply by tickets and refunds +-- net_revenue and open_tickets are both inflated +-- DATE_TRUNC syntax breaks on BigQuery`, + notes: [ + { id: 1, label: "Re-stitches a 4-way join on every question" }, + { id: 2, label: "Reinvents net_revenue and the high-value rule" }, + { id: 3, label: "Hides a chasm trap across three facts" }, + { id: 4, label: "Filters a LEFT JOIN target in WHERE" }, + { id: 5, label: "Hardcodes one warehouse's date functions" }, + ], + lineIssues: { + 5: [5], + 6: [2, 3], + 7: [3], + 8: [1], + 9: [1], + 10: [1], + 11: [1], + 12: [1], + 13: [1], + 14: [1], + 17: [2], + 18: [4], + 21: [5], + 27: [3], + 28: [3], + 29: [5], + }, + }, + draggable: false, + selectable: false, +}; + +const slQuery: SlQueryNode = { + id: "sl-query", + type: "slQuery", + position: { x: RIGHT_LANE_X, y: SL_QUERY_Y }, + data: { + variant: "slQuery", + badge: "With KTX", + title: "Agent sends a Semantic Query", + caption: + "Names the measures, dimensions, segments, and filters it wants. No SQL, no joins.", + code: `{ + "measures": [ + "orders.revenue", + "refunds.amount", + "tickets.open_count", + { + "name": "net_revenue", + "expr": "orders.revenue - refunds.amount" + } + ], + "dimensions": [ + "customers.segment", + { "field": "orders.created_at", "granularity": "month" } + ], + "segments": ["orders.high_value"], + "filters": ["customers.is_test = false"], + "limit": 1000 +}`, + }, + draggable: false, + selectable: false, +}; + +const engine: EngineNode = { + id: "engine", + type: "engine", + position: { x: RIGHT_LANE_X, y: ENGINE_Y }, + data: { + variant: "engine", + badge: "Semantic-layer engine", + title: "Plans the query against the reviewed graph", + stages: [ + { + index: 1, + title: "Resolve refs", + detail: "qualify columns, look up measure formulas", + }, + { + index: 2, + title: "Build join tree", + detail: "Dijkstra over typed edges from an anchor source", + }, + { + index: 3, + title: "Detect fan-out", + detail: "group measures by source, flag chasm traps", + }, + { + index: 4, + title: "Localize aggregation", + detail: "pre-aggregate each fact as its own CTE", + }, + { + index: 5, + title: "Transpile dialect", + detail: "emit Postgres-shaped SQL, then target dialect", + }, + ], + }, + draggable: false, + selectable: false, +}; + +const compiledSql: CompiledSqlNode = { + id: "compiled-sql", + type: "compiledSql", + position: { x: RIGHT_LANE_X, y: COMPILED_Y }, + data: { + variant: "compiled", + badge: "Generated SQL", + title: "KTX returns dialect-correct SQL", + caption: + "Pre-aggregates each fact at its own grain, then joins back on the shared dimension.", + code: `WITH orders_agg AS ( + SELECT + customer_id, + DATE_TRUNC('month', created_at) AS month, + SUM(amount) AS revenue + FROM public.orders + WHERE amount >= 100 + GROUP BY + customer_id, + DATE_TRUNC('month', created_at) +), +refunds_agg AS ( + SELECT + o.customer_id, + DATE_TRUNC('month', o.created_at) AS month, + SUM(r.amount) AS refund_amount + FROM public.refunds r + JOIN public.orders o + ON o.id = r.order_id + WHERE o.amount >= 100 + GROUP BY + o.customer_id, + DATE_TRUNC('month', o.created_at) +), +tickets_agg AS ( + SELECT + customer_id, + DATE_TRUNC('month', opened_at) AS month, + COUNT(*) AS open_count + FROM public.tickets + WHERE status = 'open' + GROUP BY + customer_id, + DATE_TRUNC('month', opened_at) +) +SELECT + c.segment, + o.month, + SUM(o.revenue - COALESCE(r.refund_amount, 0)) AS net_revenue, + SUM(o.revenue) AS revenue, + SUM(r.refund_amount) AS refund_amount, + SUM(COALESCE(t.open_count, 0)) AS open_tickets +FROM public.customers c +JOIN orders_agg o + ON o.customer_id = c.id +LEFT JOIN refunds_agg r + ON r.customer_id = c.id + AND r.month = o.month +LEFT JOIN tickets_agg t + ON t.customer_id = c.id + AND t.month = o.month +WHERE c.is_test = false +GROUP BY + c.segment, + o.month +ORDER BY + o.month, + c.segment +LIMIT 1000;`, + notes: [ + "Walks the reviewed join graph automatically", + "Uses the canonical net_revenue formula", + "Pre-aggregates each fact to avoid the chasm trap", + "Keeps LEFT JOIN filters on the dimension source", + "Transpiles DATE_TRUNC to the target dialect", + ], + }, + draggable: false, + selectable: false, +}; + +const warehouse: WarehouseNode = { + id: "warehouse", + type: "warehouse", + position: { x: WAREHOUSE_X, y: WAREHOUSE_Y }, + data: { + variant: "warehouse", + title: "Warehouse", + drivers: ["PostgreSQL", "Snowflake", "BigQuery"], + }, + draggable: false, + selectable: false, +}; + +const nodes: FlowNode[] = [ + agent, + manualSql, + slQuery, + engine, + compiledSql, + warehouse, +]; + +const arrowMarker = (color: string) => ({ + type: MarkerType.ArrowClosed, + color, + width: 16, + height: 16, +}); + +const edges = [ + { + id: "agent-manual", + source: "agent", + target: "manual-sql", + type: "default" as const, + label: "writes raw SQL", + labelBgPadding: [6, 3] as [number, number], + labelBgBorderRadius: 4, + labelStyle: { + fontSize: 12, + fontWeight: 500, + fill: "var(--color-fd-muted-foreground)", + }, + labelBgStyle: { + fill: "var(--color-fd-background)", + stroke: "var(--color-fd-border)", + strokeWidth: 1, + }, + style: { + stroke: MANUAL_STROKE, + strokeWidth: 1.5, + strokeDasharray: "5 4", + }, + markerEnd: arrowMarker(MANUAL_STROKE), + }, + { + id: "manual-warehouse", + source: "manual-sql", + target: "warehouse", + targetHandle: "warehouse-manual", + type: "default" as const, + style: { + stroke: MANUAL_STROKE, + strokeWidth: 1.5, + strokeDasharray: "5 4", + }, + markerEnd: arrowMarker(MANUAL_STROKE), + }, + { + id: "agent-slquery", + source: "agent", + target: "sl-query", + type: "default" as const, + label: "sends Semantic Query", + labelBgPadding: [6, 3] as [number, number], + labelBgBorderRadius: 4, + labelStyle: { + fontSize: 12, + fontWeight: 600, + fill: KTX_STROKE, + }, + labelBgStyle: { + fill: "var(--color-fd-background)", + stroke: "var(--color-fd-border)", + strokeWidth: 1, + }, + style: { stroke: KTX_STROKE, strokeWidth: 1.75 }, + markerEnd: arrowMarker(KTX_STROKE), + }, + { + id: "slquery-engine", + source: "sl-query", + target: "engine", + type: "straight" as const, + style: { stroke: KTX_STROKE, strokeWidth: 1.75 }, + markerEnd: arrowMarker(KTX_STROKE), + }, + { + id: "engine-compiled", + source: "engine", + target: "compiled-sql", + type: "straight" as const, + style: { stroke: KTX_STROKE, strokeWidth: 1.75 }, + markerEnd: arrowMarker(KTX_STROKE), + }, + { + id: "compiled-warehouse", + source: "compiled-sql", + target: "warehouse", + targetHandle: "warehouse-compiled", + type: "straight" as const, + style: { stroke: KTX_STROKE, strokeWidth: 1.75 }, + markerEnd: arrowMarker(KTX_STROKE), + }, +]; + +type FlowEdge = (typeof edges)[number]; + +function AgentNodeView({ data }: NodeProps) { + return ( +
    + +
    + +
    +
    +

    + {data.title} +

    +

    + {data.subtitle} +

    +
    +
    + ); +} + +function LaneBadge({ + variant, + children, +}: { + variant: LaneVariant; + children: React.ReactNode; +}) { + const cls = + variant === "manual" + ? "border-slate-300 bg-slate-100 text-slate-700 dark:border-slate-600/60 dark:bg-slate-700/40 dark:text-slate-200" + : "border-cyan-300/70 bg-cyan-50 text-cyan-800 dark:border-cyan-400/40 dark:bg-cyan-400/15 dark:text-cyan-100"; + return ( + + + {children} + + ); +} + +const JSON_TOKEN_PATTERN = + /"(?:\\.|[^"\\])*"|-?\b\d+(?:\.\d+)?\b|\b(?:true|false|null)\b|[{}[\],:]/g; +const SQL_TOKEN_PATTERN = + /--[^\n]*|'(?:''|[^'])*'|\b\d+(?:\.\d+)?\b|\b(?:select|from|join|left|right|inner|outer|on|where|group|by|order|limit|as|sum|count|coalesce|date_trunc|case|when|then|else|end|and|or|is|not|null|false|true|with|having|over|partition)\b|[(),.;=*<>+-]/gi; +const SQL_FUNCTIONS = new Set(["sum", "count", "coalesce", "date_trunc"]); + +function highlightJson(code: string) { + const parts = []; + let lastIndex = 0; + let tokenIndex = 0; + + for (const match of code.matchAll(JSON_TOKEN_PATTERN)) { + const token = match[0]; + const index = match.index ?? 0; + if (index > lastIndex) parts.push(code.slice(lastIndex, index)); + + const nextText = code.slice(index + token.length); + const className = token.startsWith('"') + ? /^\s*:/.test(nextText) + ? "syntax-json-key" + : "syntax-string" + : /^-?\d/.test(token) + ? "syntax-number" + : /^(true|false|null)$/.test(token) + ? "syntax-constant" + : "syntax-punctuation"; + + parts.push( + + {token} + , + ); + lastIndex = index + token.length; + tokenIndex += 1; + } + + if (lastIndex < code.length) parts.push(code.slice(lastIndex)); + return parts; +} + +function highlightSql(code: string) { + const parts = []; + let lastIndex = 0; + let tokenIndex = 0; + + for (const match of code.matchAll(SQL_TOKEN_PATTERN)) { + const token = match[0]; + const index = match.index ?? 0; + if (index > lastIndex) parts.push(code.slice(lastIndex, index)); + + const lowerToken = token.toLowerCase(); + const className = token.startsWith("--") + ? "syntax-comment" + : token.startsWith("'") + ? "syntax-string" + : /^\d/.test(token) + ? "syntax-number" + : SQL_FUNCTIONS.has(lowerToken) + ? "syntax-function" + : /^[a-z_]+$/i.test(token) + ? "syntax-keyword" + : "syntax-punctuation"; + + parts.push( + + {token} + , + ); + lastIndex = index + token.length; + tokenIndex += 1; + } + + if (lastIndex < code.length) parts.push(code.slice(lastIndex)); + return parts; +} + +function highlightCode(language: string, code: string) { + if (language === "json") return highlightJson(code); + if (language === "sql") return highlightSql(code); + return code; +} + +function CodeBlock({ + language, + code, + tone, +}: { + language: string; + code: string; + tone: "manual" | "slQuery" | "compiled"; +}) { + const toneClass = + tone === "manual" + ? "text-slate-600 dark:text-slate-300" + : tone === "slQuery" + ? "text-fd-primary" + : "text-fd-primary/90"; + const highlightedCode = highlightCode(language, code); + + return ( +
    +
    + + {language} + + + {tone === "compiled" ? "ktx-compiled" : "agent-authored"} + +
    + 
    +        {highlightedCode}
+
    +
    + ); +} + +function AnnotatedSqlBlock({ + code, + lineIssues, + activeIssue, + onIssueEnter, + onIssueLeave, +}: { + code: string; + lineIssues: Record; + activeIssue: number | null; + onIssueEnter: (n: number) => void; + onIssueLeave: () => void; +}) { + const lines = code.split("\n"); + + return ( +
    +
    + + sql + + + agent-authored + +
    + 
    +        {lines.map((line, idx) => {
+          const issues = lineIssues[idx] ?? [];
+          const hasIssue = issues.length > 0;
+          const dim =
+            activeIssue !== null && !issues.includes(activeIssue);
+          const active =
+            activeIssue !== null && issues.includes(activeIssue);
+          const classes = [
+            "sl-sql-line",
+            hasIssue ? "is-issue" : "",
+            active ? "is-active" : "",
+            dim ? "is-dim" : "",
+          ]
+            .filter(Boolean)
+            .join(" ");
+          return (
+            
    
+              
+                {issues.map((n) => (
+                  
+                ))}
+              
+              
+                {line.length ? highlightSql(line) : " "}
+              
+            
    
+          );
+        })}
+
    +
    + ); +} + +function ManualSqlNodeView({ data }: NodeProps) { + const [activeIssue, setActiveIssue] = useState(null); + const clearActive = useCallback(() => setActiveIssue(null), []); + + return ( +
    + +
    +
    + {data.badge} +

    + {data.title} +

    +

    + {data.caption} +

    +
    +
    +
    + +
    +
      + {data.notes.map((note) => { + const dim = activeIssue !== null && activeIssue !== note.id; + const active = activeIssue === note.id; + return ( +
    • setActiveIssue(note.id)} + onMouseLeave={clearActive} + onFocus={() => setActiveIssue(note.id)} + onBlur={clearActive} + tabIndex={0} + > + + {note.label} +
      • + ); + })} +
    + +
    + ); +} + +function SlQueryNodeView({ data }: NodeProps) { + return ( +
    + +
    +
    + {data.badge} +

    + {data.title} +

    +

    + {data.caption} +

    +
    +
    +
    + +
    + +
    + ); +} + +function EngineNodeView({ data }: NodeProps) { + return ( +
    +
    + ); +} + +function CompiledSqlNodeView({ data }: NodeProps) { + return ( +
    + +
    +
    + {data.badge} +

    + {data.title} +

    +

    + {data.caption} +

    +
    +
    +
    + +
    +
      + {data.notes.map((note) => ( +
    • +
      • + ))} +
    + +
    + ); +} + +function WarehouseNodeView({ data }: NodeProps) { + return ( +
    + + +
    + +
    +
    +

    + {data.title} +

    +

    + {data.drivers.join(" • ")} +

    +
    +
    + ); +} + +const nodeTypes = { + agent: AgentNodeView, + manualSql: ManualSqlNodeView, + slQuery: SlQueryNodeView, + engine: EngineNodeView, + compiledSql: CompiledSqlNodeView, + warehouse: WarehouseNodeView, +}; + +export function SemanticLayerFlow() { + const [minZoom, setMinZoom] = useState(0.2); + const handleFlowInit = useCallback>((instance) => { + requestAnimationFrame(() => { + void instance.fitView(FIT_VIEW_OPTIONS).then(() => { + setMinZoom(instance.getZoom()); + }); + }); + }, []); + + return ( +
    +
    +
    + + Imperative vs declarative + + +

    + Same answer, two contracts +

    +

    + On the left, the agent works imperatively: chooses tables, writes + joins, picks the grain, and remembers each warehouse's dialect. On + the right, the agent only declares what it wants. KTX handles + every how. +

    +
    + +
    +
    + Pan / zoom +
    + + + + +
    +
    + +
    + ); +} diff --git a/docs-site/content/agents-setup.md b/docs-site/content/agents-setup.md new file mode 100644 index 00000000..c7709639 --- /dev/null +++ b/docs-site/content/agents-setup.md @@ -0,0 +1,201 @@ +# Goal + +Set up KTX from scratch end-to-end as a fully autonomous, agent-driven replacement for the interactive `ktx setup` wizard. Detect the environment, install missing prerequisites, ask the user only for information you genuinely need (which connections to add, credentials), write a valid configuration, verify it works, and run a fast schema ingest. Keep the user updated throughout. + +# Operating principles + +- **Be autonomous.** Detect, decide, and act. Only ask the user when you need information that only they can provide: project location, which databases/sources to connect, credentials, and similar choices. +- **Stream short status updates.** Before each major phase ("Checking prerequisites…", "Installing uv…", "Configuring warehouse connection…", "Running fast ingest…") print a one-line update. Not chatty — just enough that the user can see what's happening. +- **Verify against docs, never guess.** CLI flags, config keys, and command names must come from the docs or from `ktx --help`. If something looks wrong or missing, say so explicitly. +- **Print every command you run and its exit code.** Terse, not silent. +- **Fail loudly with cause + fix.** When a command fails: capture the exact error, identify the cause, change something, retry. Never retry an unchanged command. Exceptions for *known soft-failures* are listed in Phase 4 — handle those without retrying. +- **No LLM-based ingestion in this flow.** Only `--fast` ingest (schema-only). The user can run `--deep` later. +- **Platform-agnostic.** Detect the host OS first and pick the right install commands / path syntax. Anything path- or shell-specific must branch on OS. + +# Authoritative docs + +KTX docs are served at `https://docs.kaelio.com/ktx/`. **Start by fetching `https://docs.kaelio.com/ktx/llms.txt`** to discover the docs map. Scan it for a "troubleshooting" entry — if one exists, read it **before** running install/setup so you can apply known fixes preemptively rather than after failing. If no troubleshooting page is listed (current state of the docs), proceed. Then fetch any other `.md` pages you need (setup, ingest, status, connection types). **Never invent CLI flags or config keys** — verify against the docs or `ktx --help` / `ktx --help`. + +> **Note on the `ktx status` JSON example in the docs.** The docs page for `ktx status` shows an example shaped like `{"title": "...", "checks": [...]}`. That example is outdated. The real CLI output uses a top-level `verdict` field plus a `connections[]` array — see Phase 5 for the canonical success criteria. Trust the shape in this prompt over the docs example. + +# Workflow + +## Phase 1 — Detect environment + +Determine the host OS (e.g. via `uname -s`, `process.platform`, or `$env:OS`). Use the right install commands per OS for the rest of this flow. + +| Tool | macOS / Linux | Windows (PowerShell) | +|------|---------------|----------------------| +| `uv` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` then re-source shell env | `irm https://astral.sh/uv/install.ps1 \| iex` | +| Node.js | use system / fnm / nvm — **do not** auto-install | use system / nvm-windows — **do not** auto-install | +| KTX CLI | `npm install -g …` (see Phase 2) | `npm install -g …` (see Phase 2) | + +If Node.js is missing, **stop and ask the user** to install it (https://nodejs.org/). Do not attempt to auto-install Node. + +## Phase 2 — Verify and install prerequisites + +Check each tool in order; install only if missing. + +1. **Node.js** — run `node --version`. Require >= 22. If missing or older, stop and instruct the user. +2. **`uv`** — run `uv --version`. If missing, run the OS-appropriate install command, then re-source the shell environment (`export PATH="$HOME/.local/bin:$PATH"` on Linux/macOS) so `uv` is on `PATH`. +3. **KTX CLI** — + - Install ktx with `npm install -g @kaelio/ktx` + - Verify with `ktx --version`. + +Print one status line per tool ("✓ uv 0.11.15 found", "Installing uv…", "✓ ktx 0.x.y installed"). + +## Phase 3 — Gather user choices + +Ask the user (grouped if your harness supports it; otherwise sequentially): + +1. **Project directory.** Default: current working directory. Confirm before continuing. +2. **LLM provider.** Default: `claude-code` with model `sonnet` (the user is already inside Claude Code; no extra API key needed). Offer `anthropic` (paste API key, stored as `env:` or `file:` ref) and `vertex` (GCP project + location) as alternatives. Skip if defaults are accepted. +3. **Embeddings backend.** Default: `sentence-transformers` (local, no API key, managed Python runtime). Offer `openai` only if the user has a key. +4. **Database connections.** Ask how many to add, then loop. For each, collect: + - Connection name (e.g. `warehouse`, `analytics`). + - Driver: one of `sqlite`, `postgres`, `mysql`, `sqlserver`, `bigquery`, `snowflake`. + - Connection URL/DSN (or service-account file for BigQuery). Accept `env:VAR_NAME` or `file:/abs/path` to avoid pasting raw secrets. + - **Heads-up for the user**: even if they paste a literal URL, KTX will silently relocate it into `/.ktx/secrets/-url` and rewrite `ktx.yaml` to `url: file:…` — this is correct, secure behavior and not a bug. + - Schemas / datasets to include (postgres / sqlserver / snowflake / bigquery only). + - Optional `enabled_tables` allowlist if the user wants to scope ingest to specific tables. +5. **BI / metadata sources** (dbt, Metabase, Looker, LookML, MetricFlow, Notion). Default: none. Ask only if the user mentions them. + +## Phase 4 — Configure the project + +Drive the existing wizard non-interactively (verify exact flag names with `ktx setup --help` and the docs — the automation flags are hidden from help but accepted): + +``` +ktx setup \ + --project-dir \ + --no-input --yes \ + --llm-backend --llm-model \ + [--anthropic-api-key-env ANTHROPIC_API_KEY | --anthropic-api-key-file ] \ + [--vertex-project

    --vertex-location ] \ + --embedding-backend \ + [--embedding-api-key-env OPENAI_API_KEY] \ + --skip-sources \ + --database --database-connection-id --database-url \ + [--database-schema …] +``` + +Notes on the flags above: +- **Project creation is automatic with `--no-input --yes`.** When + `ktx.yaml` exists, setup resumes it. When it doesn't exist, setup creates it + at `--project-dir`. +- **`--database-connection-id` is dual-purpose.** With `--database` or + `--database-url`, it names the new connection. Without those flags, it + selects an existing connection id. +- **Configure one new database connection per setup command.** If the user + wants multiple new connections, run setup again for each connection. +- **You don't need `--skip-agents` in this flow.** The agent integration step + is opt-in: setup leaves it alone unless you pass `--agents --target + `. +- **`--skip-sources`** is correct and is the documented way to leave BI/metadata sources unconfigured. + +### Known soft-failure: `ktx setup` exits 1 after a successful fast build + +When you select a configuration that only does fast (schema-only) ingest, `ktx setup`'s final readiness verification fails with: + +``` +KTX context build did not pass agent-readiness verification. + : deep database context has not completed. +``` + +This is **expected** and **does not mean setup failed**. Treat the exit code as a soft-failure **only if all of the following hold**: + +- The build log shows the fast ingest reached `[100%] Scan completed` for every configured connection. +- `ktx connection test ` (run next) exits 0 for every connection. +- `ktx status --json --no-input` reports `verdict: "ready"`. + +If those three conditions hold, proceed to Phase 5 without retrying setup, and **do not** switch to `--deep` to "fix" the readiness gate — deep ingest is explicitly out of scope. Mention this in the final report under "Docs / CLI gaps" so the user is aware. + +If any of those three conditions do not hold, this is a real failure — capture the error, fetch the relevant docs page, fix the cause, retry. + +After `ktx setup` writes `ktx.yaml`, edit it directly for anything flags don't cover: +- Per-connection `enabled_tables` allowlist (snake_case, under `connections..enabled_tables`). +- Any advanced settings the user requested. + +Use a YAML-aware editor (e.g. `uv run python -c "import yaml; …"`) — do not hand-edit blindly. + +## Phase 5 — Verify + +`ktx setup` already runs a fast schema ingest of every database connection it configures, so you do not need to re-ingest by default. For each configured connection: + +``` +ktx connection test # must exit 0 +``` + +Only re-run ingest if setup's build log did **not** reach 100% for that connection: + +``` +ktx ingest --fast --no-input +``` + +**Mutex warning on `ktx ingest`**: passing both `--yes` and `--no-input` fails with `Choose only one runtime install mode: --yes or --no-input`. Setup already installed the managed Python runtime, so pass **only `--no-input`** to `ktx ingest`. (`--yes` is only needed when an ingest invocation has to install the runtime itself, which is not the case here.) + +Then run the global health check: + +``` +ktx status --json --no-input +``` + +Success requires (canonical shape — supersedes the example in the docs): +- `verdict: "ready"` at the top of the JSON. +- Every `connections[].status === "ok"`. +- `ktx connection test ` exited 0 for every connection. + +Do **not** run `--deep` ingest in this flow — that requires LLM time and is out of scope. + +### Optional: directly probe the embeddings daemon + +If the user asks for stronger verification that `sentence-transformers` is actually serving (not just that setup said "ok"), do all of: + +1. `ktx dev runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`. +2. `pgrep -fa ktx-daemon` → expect a process running `ktx-daemon serve-http`. +3. `curl -sS http://127.0.0.1:/health` → expect HTTP 200 with `{"status":"healthy",…}`. +4. `curl -sS -X POST http://127.0.0.1:/embeddings/compute -H 'content-type: application/json' -d '{"text":"hello"}'` → expect `{"embedding": [...384 floats...]}`. + +Discover the port from setup's log line `Started KTX local embeddings daemon: http://127.0.0.1:` or from the daemon's OpenAPI at `GET /openapi.json`. Note: the routes are `/health` and `/embeddings/compute` — not `/healthz` or `/embeddings`. + +## Phase 6 — Final report + +Print a structured report: + +``` +KTX SETUP COMPLETE + +Project: +LLM: / +Embeddings: / +Runtime: managed Python ✓ (if sentence-transformers daemon was started) + +Connections: + - () status=ok schemas=[…] tables= + - … + +Sources: +Verdict: ready +``` + +Then **Next steps** (copy-pasteable): +1. Enrich with AI descriptions and embeddings: `ktx ingest --deep` (several minutes per connection). +2. Add more connections later by rerunning this setup or via `ktx setup --database … --database-connection-id …`. +3. Configure BI sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) — see `ktx setup --help` for `--source …` flags. +4. Install agent integration: `ktx setup --agents --target ` (with optional `--global` for `claude-code`/`codex`). +5. Connect the agent / MCP: see docs at `https://docs.kaelio.com/ktx/`. + +Under **Docs / CLI gaps to flag** include any of these that applied during your run: +- `ktx setup` exits non-zero after a successful fast build (deep-readiness gate); status reports ready. +- `ktx ingest` rejects `--yes` and `--no-input` together; docs don't note the conflict. +- `ktx status --json` real shape (`verdict`, `connections[]`) doesn't match the example in the docs page. +- The pasted DB URL was moved to `.ktx/secrets/-url` automatically. + +End with a single line: `RESULT: PASS` or `RESULT: FAIL — `. + +# Operating rules (recap) + +- Print every command you run and its exit code. Status updates may be terse, but never silent. +- On failure: capture the error, fetch the relevant docs page, fix the cause, retry. Never retry an unchanged command. +- Known soft-failures (listed in Phase 4 and Phase 5) are not real failures — handle them as documented; do not retry or escalate. +- If you find a docs/CLI gap ("docs say X but CLI does Y"), call it out in the final report. +- Never commit credentials — KTX accepts `env:` and `file:` references; prefer those. KTX will also auto-relocate literal URLs into `.ktx/secrets/`, but that does not protect anyone who pasted the URL into chat history. diff --git a/docs-site/content/docs/ai-resources/agent-quickstart.mdx b/docs-site/content/docs/ai-resources/agent-quickstart.mdx index 6fd6e5ac..0d49ab00 100644 --- a/docs-site/content/docs/ai-resources/agent-quickstart.mdx +++ b/docs-site/content/docs/ai-resources/agent-quickstart.mdx @@ -5,6 +5,10 @@ description: A task-first route for coding agents that need to understand KTX do This page is for coding assistants reading or citing the KTX docs. It is intentionally limited to documentation lookup, docs navigation, and safe command discovery. +For Markdown endpoints, use [Markdown Access](/docs/ai-resources/markdown-access). +For reusable task prompts, use [Prompt Recipes](/docs/ai-resources/prompt-recipes). +To install KTX into an agent client, use [Agent Clients](/docs/integrations/agent-clients). + ## First read Agents should start with the smallest source that answers the task: diff --git a/docs-site/content/docs/ai-resources/index.mdx b/docs-site/content/docs/ai-resources/index.mdx deleted file mode 100644 index 37bae650..00000000 --- a/docs-site/content/docs/ai-resources/index.mdx +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: AI Resources -description: Machine-readable docs, retrieval paths, and prompt recipes for coding assistants using KTX documentation. ---- - -Use this section when a coding assistant, IDE agent, or automation system needs -to read, cite, or update KTX documentation. These resources are optimized for -retrieval: agents can fetch small Markdown pages, use the full corpus only when -needed, and copy prompts that point them at current setup and CLI behavior. - -> **Documentation index** -> -> Start with [`/llms.txt`](/llms.txt) to discover the available docs. Use -> [`/llms-full.txt`](/llms-full.txt) when the assistant needs the complete docs -> corpus in one Markdown response. - -## What agents can do - -| Need | Recommended path | -|------|------------------| -| Find the right setup or CLI page | Fetch [`/llms.txt`](/llms.txt), then read the smallest matching `.md` page | -| Answer a setup question | Read [Agent Quickstart](/docs/ai-resources/agent-quickstart), then [Quickstart](/docs/getting-started/quickstart) or [ktx setup](/docs/cli-reference/ktx-setup) | -| Quote a command or flag | Read the matching [CLI Reference](/docs/cli-reference) page as Markdown | -| Update docs in this repo | Use [Agent Instructions](/docs/ai-resources/agent-instructions) and verify generated Markdown routes after editing | -| Reuse a prompt | Copy from [Prompt Recipes](/docs/ai-resources/prompt-recipes) | - -## Section map - -| Goal | Use this page | -|------|---------------| -| Give an assistant a task-first route through the docs | [Agent Quickstart](/docs/ai-resources/agent-quickstart) | -| Fetch docs as Markdown instead of rendered HTML | [Markdown Access](/docs/ai-resources/markdown-access) | -| Add lightweight KTX docs guidance to a system prompt | [Agent Instructions](/docs/ai-resources/agent-instructions) | -| Copy prompts for setup, command lookup, and docs editing | [Prompt Recipes](/docs/ai-resources/prompt-recipes) | - -## Available resources - -| Resource | What it gives agents | -|----------|----------------------| -| [`/llms.txt`](/llms.txt) | Curated index of high-value KTX docs and Markdown endpoints | -| [`/llms-full.txt`](/llms-full.txt) | Complete docs corpus in one plain-text Markdown response | -| `/docs/.md` | Per-page Markdown for any docs page | -| Page-level actions | Copy Markdown, view Markdown, or copy MDX from rendered docs pages | -| Prompt recipes | Reusable prompts for docs lookup, setup help, command discovery, and docs editing | - -## Agent usage notes - -When an assistant is unsure where to begin, use this retrieval order: - -1. Read [`/llms.txt`](/llms.txt). -2. Fetch one or two specific Markdown pages for the task. -3. Use [Agent Quickstart](/docs/ai-resources/agent-quickstart) to choose the - next command, guide, or CLI reference page. -4. Use [`/llms-full.txt`](/llms-full.txt) only when the answer requires broad - context across setup, integrations, concepts, and CLI reference. -5. Use page-level copy actions when the user wants exact generated Markdown or - source MDX. - -## Boundaries - -AI Resources explain how agents consume the docs. To install KTX into an -agent client, use [Agent Clients](/docs/integrations/agent-clients). To set up a -project, use [Quickstart](/docs/getting-started/quickstart) or -[`ktx setup`](/docs/cli-reference/ktx-setup). diff --git a/docs-site/content/docs/ai-resources/meta.json b/docs-site/content/docs/ai-resources/meta.json index ff555283..a925bf37 100644 --- a/docs-site/content/docs/ai-resources/meta.json +++ b/docs-site/content/docs/ai-resources/meta.json @@ -2,7 +2,6 @@ "title": "AI Resources", "defaultOpen": true, "pages": [ - "index", "agent-quickstart", "markdown-access", "agent-instructions", diff --git a/docs-site/content/docs/ai-resources/prompt-recipes.mdx b/docs-site/content/docs/ai-resources/prompt-recipes.mdx index 35f8a4ae..6498fc1d 100644 --- a/docs-site/content/docs/ai-resources/prompt-recipes.mdx +++ b/docs-site/content/docs/ai-resources/prompt-recipes.mdx @@ -14,7 +14,7 @@ Read https://docs.kaelio.com/ktx/llms.txt first. Then fetch only the KTX Markdow ## Set up a project ```text -Set up KTX in this repository. Start by reading /docs/ai-resources/agent-quickstart.md and /docs/getting-started/quickstart.md. Use pnpm, not npm. After setup, run ktx status and summarize which steps are complete, which files changed, and what still needs credentials or user input. +Set up KTX in this repository. Start by reading /docs/ai-resources/agent-quickstart.md and /docs/getting-started/quickstart.md. Install the published CLI with npm; use pnpm only when working from a KTX source checkout. After setup, run ktx status and summarize which steps are complete, which files changed, and what still needs credentials or user input. ``` ## Find a command diff --git a/docs-site/content/docs/cli-reference/ktx-dev.mdx b/docs-site/content/docs/cli-reference/ktx-dev.mdx index 50d4ea77..efa3d74b 100644 --- a/docs-site/content/docs/cli-reference/ktx-dev.mdx +++ b/docs-site/content/docs/cli-reference/ktx-dev.mdx @@ -36,7 +36,7 @@ directory. Use it from any directory to generate editor or agent schema files. | Flag | Description | Default | |------|-------------|---------| -| `--output ` | Write the schema to a file instead of stdout | — | +| `--output ` | Write the schema to a file instead of stdout | - | ## `dev runtime` Subcommands diff --git a/docs-site/content/docs/cli-reference/ktx-ingest.mdx b/docs-site/content/docs/cli-reference/ktx-ingest.mdx index 9d94cd88..49485d10 100644 --- a/docs-site/content/docs/cli-reference/ktx-ingest.mdx +++ b/docs-site/content/docs/cli-reference/ktx-ingest.mdx @@ -33,7 +33,7 @@ connections when you use `--all`. | `--plain` | Print plain text output | `true` | | `--json` | Print JSON output | `false` | | `--yes` | Install required managed runtime features without prompting | `false` | -| `--no-input` | Disable interactive terminal input | — | +| `--no-input` | Disable interactive terminal input | - | `--fast` and `--deep` are mutually exclusive. Depth flags apply only to database connections. Query-history flags apply only to database connections @@ -60,7 +60,7 @@ read one item from stdin. | Flag | Description | Default | |------|-------------|---------| | `--text ` | Text content to ingest; repeat for a batch | `[]` | -| `--connection-id ` | Optional KTX connection id for semantic-layer capture | — | +| `--connection-id ` | Optional KTX connection id for semantic-layer capture | - | | `--user-id ` | Memory user id for capture attribution | `local-cli` | | `--json` | Print JSON output | `false` | | `--fail-fast` | Stop after the first failed text item | `false` | diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index 562b5f28..6dbe54c2 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -26,10 +26,10 @@ below. | Flag | Description | Default | |------|-------------|---------| -| `--agents` | Install agent integration only | `false` | -| `--target ` | Agent target: `claude-code`, `codex`, `cursor`, `opencode`, or `universal` | - | +| `--agents` | Install agent configuration and rules only | `false` | +| `--target ` | Agent target: `claude-code`, `claude-desktop`, `codex`, `cursor`, `opencode`, or `universal` | - | | `--global` | Install agent integration into the global target scope for `claude-code` or `codex` | `false` | -| `--yes` | Accept safe defaults in non-interactive setup | `false` | +| `--yes` | Accept project creation and runtime install defaults where setup asks for confirmation | `false` | | `--no-input` | Disable interactive terminal input | - | Use the global `--project-dir ` option when setup should target a @@ -40,12 +40,12 @@ specific directory. These flags are useful for repeatable setup in examples, tests, CI fixtures, and scripted project creation. They are not shown in `ktx setup --help`. -### Project Mode +### Project Creation -| Flag | Description | Default | -|------|-------------|---------| -| `--new` | Create a new KTX project before setup | `false` | -| `--existing` | Use an existing KTX project | `false` | +Setup resumes an existing `ktx.yaml` when one is present. When no project +exists, interactive setup prompts for where to create it. In scripts, pass +`--project-dir

    --no-input --yes` to create the target directory without +prompts. ### LLM Provider @@ -56,7 +56,6 @@ scripted project creation. They are not shown in `ktx setup --help`. | `--llm-model ` | LLM model ID or backend model alias to validate and save | | `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | | `--anthropic-api-key-file ` | File containing the Anthropic API key | -| `--anthropic-model ` | Legacy alias for `--llm-model` | | `--vertex-project ` | Vertex AI project ID, `env:NAME`, or `file:/path` reference | | `--vertex-location ` | Vertex AI location, `env:NAME`, or `file:/path` reference | | `--skip-llm` | Leave LLM setup incomplete | @@ -82,15 +81,19 @@ embedding credential source. ### Runtime Setup prepares the managed Python runtime when your selected configuration -needs it. The runtime step runs after database and source setup and before the -initial context build. +needs it. In the full setup flow, the runtime step runs after database and +source setup and before the initial context build. -KTX prepares the `core` runtime feature when agent integration, query-history -ingest, Looker source ingest, or daemon-backed context build paths need it. KTX -prepares the `local-embeddings` runtime feature when you choose managed local -`sentence-transformers` embeddings. Existing external daemon URLs, such as -`KTX_DAEMON_URL` or `KTX_SQL_ANALYSIS_URL`, satisfy the matching dependency and -skip managed runtime installation for that dependency. +KTX prepares the `core` runtime feature when query-history ingest, Looker +source ingest, database introspection fallback, or daemon-backed context build +paths need it. KTX prepares the `local-embeddings` runtime feature when you +choose managed local `sentence-transformers` embeddings. Existing external +daemon URLs, such as `KTX_DAEMON_URL` or `KTX_SQL_ANALYSIS_URL`, satisfy the +matching dependency and skip managed runtime installation for that dependency. + +`ktx setup --agents` doesn't prepare runtime features or build context. It only +installs agent configuration and rules. Start MCP with `ktx mcp start` before +using HTTP-based agents; MCP startup prepares the runtime it needs. Interactive setup prompts before installing runtime features. Use `--yes` to install them without prompting. Use `--no-input` to fail fast when required @@ -100,9 +103,8 @@ runtime features are missing. | Flag | Description | |------|-------------| -| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` | -| `--database-connection-id ` | Existing selected connection id; repeatable | -| `--new-database-connection-id ` | Connection id for one new database connection | +| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `sqlserver`, `bigquery`, `snowflake` | +| `--database-connection-id ` | Existing selected connection id; repeatable. With `--database` or `--database-url`, connection id for the new connection. | | `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite path | | `--database-schema ` | Database schema or dataset to include; repeatable | | `--skip-databases` | Leave database setup incomplete | @@ -173,10 +175,11 @@ ktx setup \ ktx setup \ --project-dir ./analytics \ --no-input \ + --yes \ --skip-llm \ --skip-embeddings \ --database postgres \ - --new-database-connection-id warehouse \ + --database-connection-id warehouse \ --database-url env:DATABASE_URL \ --database-schema public @@ -184,7 +187,7 @@ ktx setup \ ktx setup \ --project-dir ./analytics \ --database postgres \ - --new-database-connection-id warehouse \ + --database-connection-id warehouse \ --database-url env:DATABASE_URL \ --enable-query-history \ --query-history-min-executions 5 @@ -232,4 +235,5 @@ Use `ktx status` for repeatable readiness checks after setup exits. | `--enable-query-history` is rejected | The selected database driver does not support query history | Use Postgres, BigQuery, or Snowflake, or rerun without query-history flags | | Source setup rejects location flags | Both `--source-path` and `--source-git-url` were supplied | Choose the local path or the Git URL, not both | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | +| Agent setup cannot prompt for a target | Non-TTY `ktx setup --agents` needs a target | Run `ktx setup --agents --target ` or rerun in a TTY | | Global agent install is rejected | `--global` was used with a target other than `claude-code` or `codex` | Omit `--global`, or choose `--target claude-code` or `--target codex` | diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index b0282a38..f395a170 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -20,7 +20,7 @@ ktx sl [options] | `list` | List semantic-layer sources | | `search ` | Search semantic-layer sources | | `validate ` | Validate a semantic-layer source against the database schema | -| `query` | Compile or execute a semantic-layer query | +| `query` | Compile or execute a Semantic Query | ## Options @@ -52,7 +52,7 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| | `--connection-id ` | KTX connection id | - | -| `--query-file ` | JSON semantic-layer query file | - | +| `--query-file ` | JSON Semantic Query file | - | | `--measure ` | Measure to query; repeatable (at least one required) | - | | `--dimension ` | Dimension to include; repeatable | - | | `--filter ` | Filter expression; repeatable | - | @@ -67,7 +67,7 @@ ktx sl [options] | `--max-rows ` | Maximum rows to return when executing | - | `sl query` requires at least one `--measure` unless `--query-file` is set. -`--query-file` should point to a JSON semantic-layer query object. +`--query-file` should point to a JSON Semantic Query object. ## Examples diff --git a/docs-site/content/docs/cli-reference/index.mdx b/docs-site/content/docs/cli-reference/ktx.mdx similarity index 78% rename from docs-site/content/docs/cli-reference/index.mdx rename to docs-site/content/docs/cli-reference/ktx.mdx index 2b681cba..937d2529 100644 --- a/docs-site/content/docs/cli-reference/index.mdx +++ b/docs-site/content/docs/cli-reference/ktx.mdx @@ -1,13 +1,29 @@ --- -title: "Overview" -description: "Command map and shared options for the KTX CLI." +title: "ktx" +description: "Root command map, global options, and project resolution for the KTX CLI." --- The `ktx` CLI sets up local projects, builds agent-ready context, checks connections, queries semantic-layer sources, searches wiki pages, runs the MCP server, and manages the bundled Python runtime. -## Command Map +## Command signature + +```bash +ktx [global-options] +``` + +When you run bare `ktx` in an interactive terminal outside any KTX project, the +CLI starts the same guided setup flow as `ktx setup`. Inside an existing +project, use command-specific help: + +```bash +ktx --help +ktx setup --help +ktx ingest --help +``` + +## Command map ```text ktx @@ -45,7 +61,7 @@ ktx The public context-build entrypoint is `ktx ingest [connectionId]` or `ktx ingest --all`. -## Global Options +## Global options | Flag | Description | |------|-------------| @@ -54,14 +70,14 @@ The public context-build entrypoint is `ktx ingest [connectionId]` or | `-v`, `--version` | Show the CLI package name and version. | | `-h`, `--help` | Show help for the current command. | -## Project Resolution +## Project resolution Most commands are project-aware. Pass `--project-dir ` when scripting or when you are outside the project directory. If you omit it, KTX checks `KTX_PROJECT_DIR`, then walks upward for the nearest `ktx.yaml`, then falls back to the current directory. -## Common Workflows +## Common workflows ```bash # Start or resume setup diff --git a/docs-site/content/docs/cli-reference/meta.json b/docs-site/content/docs/cli-reference/meta.json index 6757e5bc..6385c7d4 100644 --- a/docs-site/content/docs/cli-reference/meta.json +++ b/docs-site/content/docs/cli-reference/meta.json @@ -2,7 +2,7 @@ "title": "CLI Reference", "defaultOpen": true, "pages": [ - "index", + "ktx", "ktx-setup", "ktx-connection", "ktx-ingest", diff --git a/docs-site/content/docs/community/contributing.mdx b/docs-site/content/docs/community/contributing.mdx index 791d865c..63b08cd6 100644 --- a/docs-site/content/docs/community/contributing.mdx +++ b/docs-site/content/docs/community/contributing.mdx @@ -91,7 +91,6 @@ packages/ connector-postgres/ # PostgreSQL connector connector-snowflake/ # Snowflake connector connector-bigquery/ # BigQuery connector - connector-clickhouse/ # ClickHouse connector connector-mysql/ # MySQL connector connector-sqlserver/ # SQL Server connector connector-sqlite/ # SQLite connector diff --git a/docs-site/content/docs/concepts/semantic-layer-internals.mdx b/docs-site/content/docs/concepts/semantic-layer-internals.mdx index aec0ccfa..a011d1cb 100644 --- a/docs-site/content/docs/concepts/semantic-layer-internals.mdx +++ b/docs-site/content/docs/concepts/semantic-layer-internals.mdx @@ -1,141 +1,115 @@ --- -title: Context-Aware SQL -description: How KTX turns reviewed context, grain, and relationship evidence into safe SQL for agents. +title: Semantic Querying +description: How KTX compiles a short Semantic Query into safe, dialect-correct SQL using a reviewed join graph. --- -## Why query planning needs context +import { SemanticLayerFlow } from "@/components/semantic-layer-flow"; -Agents can generate SQL from schema alone, but safe analytics SQL needs more -than table names. KTX uses reviewed context to understand grain, joins, measures, -filters, and where aggregation must happen. +KTX's semantic layer is a compiler that turns intent into SQL. The agent +declares _what_ it wants — measures, dimensions, filters — in a small +Semantic Query. KTX figures out the _how_: which tables to join, what +grain to aggregate at, how to keep fan-out from inflating measures, and +what dialect the warehouse speaks. -Read this page as four mechanics: +This page covers four mechanics: -- context files feed the semantic engine; -- evidence becomes a join graph with grain and relationship metadata; -- review keeps the graph current; -- query planning avoids fan-out and ambiguous joins. +- The Semantic Query contract agents send to the compiler. +- The planner steps that turn a Semantic Query into SQL. +- The join graph that backs those steps, and how it's built. +- The fan-out failure mode the compiler is designed to prevent. -## Where the semantic layer fits +## Imperative SQL vs declarative Semantic Querying -This planner is one subsystem inside KTX's broader context layer. It uses source -YAML, wiki context, scan evidence, and provenance to make context actionable for -SQL generation. +Writing analytics SQL is imperative work. Every question forces the +agent to hold two things in mind at once: _what_ it wants — a measure, a +slice, a filter — and _how_ to compute it: which tables to join, which +key links them, what grain to aggregate at, how to keep one fact from +inflating another, and what dialect the warehouse speaks. Plumbing on +top of intent, every query. -
    -
    -
    -

    - {"Context inputs"} -

    -
    -
    -

    semantic-layer/

    -

    - {"source YAML, measures, joins, grain"} -

    -
    -
    -

    wiki/

    -

    - {"business rules, definitions, caveats"} -

    -
    -
    -

    raw-sources/

    -

    - {"schema scans, keys, imported metadata"} -

    -
    -
    -

    provenance

    -

    - {"ingest decisions and review history"} -

    -
    -
    -
    +KTX's semantic layer separates those concerns: - +- **You and KTX maintain the how.** Sources, joins, grain, measures, and + segments live in reviewable YAML — the analytical contract the team + agrees on, version-controlled. +- **The agent declares the what.** It sends a Semantic Query and trusts + the compiler to produce safe SQL. -
    -
    -

    - {"Semantic layer engine"} -

    -
    -
    -

    Join graph

    -

    - {"sources as nodes, joins as typed edges"} -

    -
    -
    -

    Grain

    -

    - {"row identity before aggregation"} -

    -
    -
    -

    Measures

    -

    - {"verified formulas and filters"} -

    -
    -
    -

    Relationships

    -

    - {"many_to_one, one_to_many, one_to_one"} -

    -
    -
    -
    - {"Safe query planning before SQL is generated."} -
    -
    +The agent stops reasoning about plumbing. It states intent. KTX turns +that into SQL the warehouse can run. - + -
    -

    - {"Agent workflows"} -

    -
    -
    - {"Search sources and wiki pages"} -
    -
    - {"Compile trusted SQL"} -
    -
    - {"Explain metrics and provenance"} -
    -
    - {"Patch files and validate review"} -
    -
    -
    -
    -
    +## The Semantic Query contract -## Join graph +A Semantic Query is the JSON payload the agent sends. Every field is optional +except `measures`, and column references are fully qualified +(`source.column`) so the compiler never has to guess where a name came +from. -A semantic source is a node. A join is a typed edge. KTX uses the graph to -choose valid paths and detect row-multiplying joins before SQL is generated. +Notice what's _not_ in the payload: no `FROM`, no `JOIN`, no `GROUP BY`, +no `WITH`. The agent states what it wants. KTX picks the join path, the +grain, the SQL shape, and the dialect. + +| Field | Purpose | +|-------|---------| +| `measures` | Names of pre-defined measures, or inline expressions like `sum(orders.amount)` | +| `dimensions` | Columns to group by, optionally with a `granularity` for time fields | +| `filters` | Row-level predicates, classified into `WHERE` or `HAVING` at planning time | +| `segments` | Named filter sets defined on a source, applied as additional predicates | +| `order_by` | Sort fields with optional direction | +| `limit` | Row cap on the result | + +A typical agent call looks like this: + +```json +{ + "measures": ["orders.revenue", "tickets.ticket_count"], + "dimensions": ["customers.segment"], + "filters": ["orders.created_at >= '2025-01-01'"], + "limit": 1000 +} +``` + +That payload is enough for KTX to plan and compile. The agent never +authors a join, a CTE, or a dialect-specific cast. + +## What the planner does + +The planner is a deterministic pipeline. Each Semantic Query runs through the +same ordered steps before any SQL is emitted. + +1. **Resolve refs.** Qualify bare column names, look up pre-defined + measure expressions, and classify each measure as raw or derived. +2. **Pick an anchor and build the join tree.** Choose the largest measure + source as the root, then run a shortest-path search across the typed + join graph to reach every required source. +3. **Detect fan-out.** Group measures by their owning source. If more + than one group exists, the planner marks the query as a chasm trap + and switches to aggregate-locality compilation. +4. **Classify filters.** Split predicates into row-level (`WHERE`) and + aggregate-level (`HAVING`) based on whether they reference a measure. +5. **Generate SQL.** Emit Postgres-shaped SQL with the right shape: + single-source aggregation when the query is safe, per-source CTEs + when fan-out is present. +6. **Transpile to the target dialect.** Run the result through `sqlglot` + so the warehouse receives syntax it understands. + +The output is the SQL string, the resolved plan, and any warnings +surfaced during planning. + +## The join graph + +A semantic source is a node. A declared join is a typed edge. The graph +is bidirectional: every forward edge has a reverse with the relationship +inverted, so the planner can traverse from any anchor. | Relationship | Planning impact | |--------------|-----------------| -| `many_to_one` | Usually safe for adding dimensions | -| `one_to_many` | Can multiply measures and trigger fan-out handling | -| `one_to_one` | Usually safe when keys are correct | -| Equal-cost paths | Ambiguous unless aliases or explicit joins disambiguate | +| `many_to_one` | Safe direction for adding dimensions | +| `one_to_many` | Multiplies measures and triggers fan-out handling | +| `one_to_one` | Safe in either direction when keys match | +| Equal-cost paths | Treated as ambiguous; aliases or explicit joins resolve them |
    -

    customers

    -

    grain: customer_id

    +

    {"customers"}

    +

    {"grain: customer_id"}

    -

    orders

    -

    grain: order_id

    +

    {"orders"}

    +

    {"grain: order_id"}

    -

    order_items

    -

    grain: order_id, line_id

    +

    {"order_items"}

    +

    {"grain: order_id, line_id"}

    -
    orders -> customers: many_to_one
    -
    orders -> order_items: one_to_many
    +
    {"orders -> customers: many_to_one"}
    +
    {"orders -> order_items: one_to_many"}
    {"Example: "} - {"refunds joins to orders. Used carefully, it explains net revenue. Joined naively, it can duplicate order-level measures."} + {"refunds joins to orders. Used carefully, it explains net revenue. Joined naively, it duplicates order-level measures."}
    -The graph is bidirectional for planning. If `orders -> customers` is -`many_to_one`, the reverse path is `one_to_many`. +Edges and grain come from your YAML. The compiler treats them as fact, +not a guess. + +```yaml +# semantic-layer/warehouse/orders.yaml +name: orders +table: public.orders +grain: [order_id] +joins: + - to: customers + on: customer_id = customers.id + relationship: many_to_one + - to: order_items + on: id = order_items.order_id + relationship: one_to_many +measures: + - name: revenue + expr: sum(case when status != 'refunded' then amount end) +``` ## Building and maintaining the graph -KTX starts from evidence, writes reviewable source YAML, and treats the merged -diff as the accepted graph. +KTX builds the graph from evidence and accepted edits, not from runtime +inference. Each input contributes a different kind of authority. | Evidence | What it contributes | |----------|---------------------| | Declared primary keys | Initial row grain | | Declared foreign keys | Formal join candidates | -| Inferred relationships | Edges when warehouses lack constraints | +| Inferred relationships | Edges when the warehouse lacks constraints | | dbt, MetricFlow, and LookML imports | Existing metrics, dimensions, explores, and joins | -| Query history | Real join and filter patterns | +| Query history | Real join and filter patterns from analyst SQL | | Analyst review | Final authority before context is merged |
    -## Modeling problems +## Fan-out and aggregate locality -Fan-out is the classic failure mode: an order-level measure joins to line-item -rows before aggregation, so one order becomes many rows. +Fan-out is the classic analytics failure mode. Two fact tables join to a +shared dimension. A naive query joins them all together first, so each +row from one fact is multiplied by the matching rows from the other. +Measures duplicate, numbers go wrong, and the agent doesn't notice. -| Problem | What happens | How KTX handles it | -|---------|--------------|--------------------| -| Order measure joins to `order_items` | `orders.revenue` repeats once per item | Detect `one_to_many` and pre-aggregate | -| Two fact sources share `customers` | Measures multiply across the shared dimension | Treat as a chasm trap and plan each fact locally | -| Filter crosses `one_to_many` | Filtering changes measure grain | Reject or localize the filter | -| Equal-cost paths connect sources | Join choice is ambiguous | Prefer safer paths or require aliases | - -## Execution planning - -The planner resolves sources, chooses a join tree, checks relationship paths, -and picks a simple or aggregate-locality SQL shape. +KTX's planner detects the shape by grouping measures by their owning +source. If more than one source contributes raw measures, the generator +switches to aggregate locality: each fact is pre-aggregated at its own +grain inside a CTE, and the CTEs are joined back to the dimension at the +end. | Naive SQL shape | Semantic-layer SQL shape | |-----------------|--------------------------| -| Join facts and dimensions first, then aggregate | Aggregate each fact source at its own grain, then join results | -| Put every filter in one outer `WHERE` clause | Keep measure filters with the measure source when locality is needed | -| Trust the shortest textual join path | Prefer safe relationship paths and reject disconnected sources | -| Let dimension grain differ across facts | Raise when asymmetric dimensions would fan out another measure | +| Join facts and dimensions first, then aggregate | Aggregate each fact at its own grain, then join | +| Put every filter in one outer `WHERE` clause | Keep measure filters with the measure source | +| Trust the shortest textual join path | Prefer typed safe paths, reject disconnected sources | +| Let dimension grain differ across facts | Raise when an asymmetric dimension would fan out another measure | -
    -
    -

    - {"Fan-out handling"} -

    -

    - {"The same question planned before and after KTX preserves the measure grain."} -

    -
    -
    -
    -
    -

    - {"Unsafe shape"} -

    -

    - {"Join first, aggregate later"} -

    -
    - 
    -{`orders
-  -> join order_items
-  -> join customers
+The result is the same analyst answer, computed with the join shape an
+analyst would have written by hand.
 
-group by
-  customer_segment
+## Where the context comes from
 
-measure
-  sum(orders.amount)`}
-
    -
    - {"Order-level revenue is exposed to line-item fan-out before aggregation."} -
    -
    -
    -
    -

    - {"KTX shape"} -

    -

    - {"Aggregate locally, then join"} -

    -
    - 
    -{`orders_agg as (
-  select customer_id, sum(amount) revenue
-  from orders
-  group by customer_id
-)
-select customers.segment, sum(revenue)
-from orders_agg
-join customers`}
-
    -
    - {"The measure is pre-aggregated at order grain before dimensions are joined."} -
    -
    -
    -
    +The planner is only as good as the YAML it reads. KTX builds and +maintains that YAML for you. -The result is structured planning: validated sources, typed relationships, -graph search, fan-out detection, aggregate locality, and dialect transpilation. +- `raw-sources//` holds scan evidence from your warehouse: + schemas, columns, keys, samples, and observed usage patterns. +- `wiki/` holds business language, definitions, and caveats. The + planner doesn't read wiki at compile time, but the agent does, so + measure names and dimensions stay anchored to terms the team uses. +- `semantic-layer//` holds the structured sources, joins, + grain, measures, and segments the planner actually compiles against. + +Every accepted edit flows back into the next ingest, so the graph stays +current as the warehouse changes. ## Agent usage notes -Use this page when an agent needs to explain how KTX turns reviewed semantic -context into SQL, why relationship metadata matters, or why a query was rejected -as unsafe. +Point an agent at this page when it needs to explain why KTX asks for +grain, why a query was rejected as unsafe, or why the compiled SQL looks +different from what the agent first proposed. | Agent task | Relevant section | Next page | |------------|------------------|-----------| -| Explain why KTX asks for `grain` and relationship types | Join graph | [Writing Context](/docs/guides/writing-context) | -| Diagnose duplicated measures after a join | Modeling problems | [ktx sl](/docs/cli-reference/ktx-sl) | -| Explain safe SQL generation | Execution planning | [ktx sl](/docs/cli-reference/ktx-sl) | -| Describe how semantic context stays current | Building and maintaining the graph | [Context as Code](/docs/concepts/context-as-code) | +| Explain the Semantic Query shape | The Semantic Query contract | [ktx sl](/docs/cli-reference/ktx-sl) | +| Describe what the planner does between query and SQL | What the planner does | [ktx sl](/docs/cli-reference/ktx-sl) | +| Explain why KTX asks for grain and relationship types | The join graph | [Writing context](/docs/guides/writing-context) | +| Diagnose duplicated measures after a join | Fan-out and aggregate locality | [ktx sl](/docs/cli-reference/ktx-sl) | +| Describe how semantic context stays current | Building and maintaining the graph | [Context as code](/docs/concepts/context-as-code) | diff --git a/docs-site/content/docs/concepts/the-context-layer.mdx b/docs-site/content/docs/concepts/the-context-layer.mdx index 9a8130d0..c56327c5 100644 --- a/docs-site/content/docs/concepts/the-context-layer.mdx +++ b/docs-site/content/docs/concepts/the-context-layer.mdx @@ -74,7 +74,7 @@ measures: ``` For join graphs, fan-out handling, and execution mechanics, read -[Context-Aware SQL](/docs/concepts/semantic-layer-internals). +[Semantic Querying](/docs/concepts/semantic-layer-internals). ## Wiki pages diff --git a/docs-site/content/docs/getting-started/introduction.mdx b/docs-site/content/docs/getting-started/introduction.mdx index 72891d99..d0ee126d 100644 --- a/docs-site/content/docs/getting-started/introduction.mdx +++ b/docs-site/content/docs/getting-started/introduction.mdx @@ -1,6 +1,6 @@ --- title: Introduction -description: What KTX is, how it works, and where to start. +description: KTX is an open-source, self-improving context layer for data agents. --- import { ProductMechanics } from "@/components/product-mechanics"; @@ -23,54 +23,75 @@ import { ProductMechanics } from "@/components/product-mechanics"; Make analytics context usable by agents

    - {'KTX turns warehouse metadata, semantic definitions, BI usage, and team knowledge into local files and runtime tools that database agents can trust.'} + {'KTX is an open-source context layer for database agents. It turns warehouse metadata, BI models, query history, docs, and approved metric definitions into reviewable files agents can search and execute.'}

    -## Why KTX +## Why KTX helps -- Schemas show columns, not business rules. -- Agents need trusted metrics, joins, filters, caveats, and provenance. -- KTX captures that context before agents write SQL, docs, or semantic edits. +KTX gives agents a shared context workspace before they write SQL, answer a +question, or update analytics definitions. -## What KTX creates +- **Context as code.** KTX writes wiki pages and semantic-layer definitions as + git-based files you can review, diff, and merge. +- **Self-improving ingest.** KTX reads warehouses, BI tools, modeling code, + query history, and notes, then reconciles new evidence with accepted context. +- **Executable semantics.** Agents can use approved measures, joins, filters, + dimensions, and segments instead of rebuilding canonical SQL from scratch. +- **Agent-native access.** CLI and MCP tools let agents search context, compile + semantic queries, run read-only SQL, and propose updates. -| Path | What it gives agents | -|------|----------------------| -| `semantic-layer/` | Measures, dimensions, joins, grain, filters, segments | -| `wiki/` | Business definitions, caveats, policies, analyst notes | -| `raw-sources/` | Extracted metadata, scan output, relationship evidence | -| `.ktx/` | Local indexes, embeddings, setup state, runtime data | +KTX complements existing semantic layers by pairing metric definitions with the +surrounding business knowledge, caveats, provenance, and review workflow agents +need for data work. + +## How KTX works + +KTX has two connected sides: it builds and maintains the context layer, then +serves that context to agents at runtime. + +| Side | What KTX does | +|------|---------------| +| **Ingest and auto-maintain knowledge** | Reads your data stack and company knowledge, reconciles new evidence with accepted context, and keeps changes to `semantic-layer/` plus `wiki/` as version-controlled diffs automatically. | +| **Serve agents at runtime** | Helps agents find the right wiki pages and semantic-layer entities, then compile or execute semantic queries through CLI and MCP tools. | ## Use it for -- **Generate SQL** from approved measures, dimensions, joins, and filters -- **Explain provenance** with wiki context and warehouse evidence -- **Repair context** through reviewable YAML and Markdown diffs -- **Work alongside** dbt, LookML, MetricFlow, Looker, Metabase, and warehouses +Use KTX when agents need more than raw database access. Agents can search wiki +context, find semantic-layer entities, compile trusted semantic queries, run +read-only SQL, and use the same tools through MCP. -Databases: SQLite, PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL -Server. +- Generate SQL from approved metrics, joins, filters, and dimensions. +- Explain metric provenance with wiki context and source evidence. +- Repair context through reviewable YAML and Markdown diffs. +- Work alongside dbt, MetricFlow, LookML, Looker, Metabase, Notion, and + supported databases. ## Start here +Choose the route that matches what you want to do next. The quickstart is the +best first step for users; contributor setup lives in the community docs. + - Set up KTX and build your first context in under 10 minutes. + Install KTX, run setup, build context, and connect an agent. - - Hands-on workflows for scanning, ingesting, writing, and serving. + + Understand why agents need more than schema access and raw SQL. + + + Refresh context from databases, BI tools, query history, and documents. Edit semantic-layer YAML and wiki Markdown safely. - + Complete flag and subcommand reference for every KTX command. - + Machine-readable docs and agent-facing setup notes. diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 0118522c..d0566ec9 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -1,135 +1,137 @@ --- title: Quickstart -description: Set up KTX, build local context, and connect your coding agent. +description: Install KTX, run setup, and connect your coding agent. --- -This guide gets a local analytics project ready for KTX. You will install the -CLI, run the setup wizard, connect a database, build context, and install agent -rules that teach your coding assistant which KTX commands to run. +import { CopyButton } from "@/components/copy-button"; -If you are a coding assistant choosing a docs route, start with the -[Agent Quickstart](/docs/ai-resources/agent-quickstart). This page is the -human setup walkthrough. +This guide takes a local analytics project from empty to agent-ready. You'll +install the CLI, run one guided setup command, and hand the context to a +coding assistant. -## What setup does +If you're a coding assistant choosing a docs route, start with the +[Agent Quickstart](/docs/ai-resources/agent-quickstart) instead. -`ktx setup` is the main project workflow. It can create or resume `ktx.yaml`, -configure model and embedding providers, add database connections, add optional -context sources, build the first context artifacts, and install agent -integration. +
    +
    + Need a warehouse to play with? +
    +
    + Try KTX against a real data stack - Postgres, dbt, Metabase, and Notion + pre-loaded with the Orbit demo corpus. The page lists demo credentials + you can paste straight into `ktx setup`. +
    + + Get demo credentials at kaelio.com/start → + +
    -When you run bare `ktx` in an interactive terminal outside a KTX project, the -CLI opens the same setup experience. Inside an existing project, `ktx setup` -resumes incomplete work or opens a menu for changing setup, connecting an -agent, checking status, or exploring a demo project. +
    +
    + Run setup from an agent +
    +
    + You can ask an agent such as Claude Code, Codex, Cursor, or OpenCode to + install and configure KTX for you. The{' '} + + agent setup Markdown prompt + {' '} + tells the agent how to check prerequisites, ask only for credentials or + connection choices, run ktx setup, verify connections, and + report the result. +
    +
    + Use a prompt like this from the project you want to configure: +
    +
    +
    + + Prompt + + +
    +
    +
    Follow instructions from
    +
    https://docs.kaelio.com/ktx/docs/agents-setup.md
    +
    to install and configure ktx
    +
    +
    +
    ## Install the CLI -Install the published `@kaelio/ktx` package: +Install the published package globally: ```bash npm install -g @kaelio/ktx ``` -Then run setup from the analytics project directory: +KTX is open source. If you'd like to hack on it or run from a local checkout, +the source lives at [github.com/kaelio/ktx](https://github.com/kaelio/ktx) - +see [Contributing](/docs/community/contributing) to get set up. + +## Run setup + +From your project directory, run: ```bash ktx setup ``` -The local checkout workflow is only for KTX contributors. See -[Contributing](/docs/community/contributing) for that path. +The wizard walks you through everything KTX needs in one pass: -## Step 1: Choose the project +1. **Project** - creates or resumes `ktx.yaml` in the current directory. +2. **LLM** - picks a Claude backend. The default uses your local Claude Code + session, so no API key is required. You can also use an Anthropic API key + or Vertex AI. +3. **Embeddings** - picks an embeddings backend. Choose OpenAI for hosted + embeddings or `sentence-transformers` to run locally without an API key. +4. **Database** - adds at least one primary connection. Supported drivers: + SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, and Snowflake. +5. **Context sources** - optionally adds dbt, MetricFlow, LookML, Looker, + Metabase, or Notion. You can skip and add them later. +6. **Build** - runs the first ingest so semantic-layer sources and wiki pages + are ready for agents. +7. **Agent integration** - installs project-local rules for Claude Code, + Codex, Cursor, OpenCode, or universal `.agents`. -In an interactive terminal, setup can create a new KTX project or resume the -nearest existing project. The main project file is `ktx.yaml`. - -For scripted setup, pass the project directory explicitly: - -```bash -ktx setup --project-dir ./analytics -``` - -If setup exits early, rerun `ktx setup` in the same directory. KTX keeps local -setup progress under `.ktx/setup/` and resumes from the remaining work. - -## Step 2: Configure the LLM - -KTX uses a Claude model for ingest agents that turn schemas, SQL, BI metadata, -and documents into semantic-layer sources and wiki context. - -Setup supports three LLM provider paths: - -| Provider | Use when | Credential model | -|----------|----------|------------------| -| Claude subscription (Pro/Max) | You want KTX to use your local Claude Code session | Claude Code local authentication | -| Anthropic API key | You have an Anthropic API key | `ANTHROPIC_API_KEY` or a local `file:` secret | -| Google Vertex AI for Anthropic Claude | Your organization runs Claude through Google Cloud | Application Default Credentials plus Vertex project and location | - -For Anthropic API, setup can read the key from the environment or save a pasted -key to `.ktx/secrets/anthropic-api-key`. `ktx.yaml` stores an `env:` or `file:` -reference, not the raw key. - -For Vertex AI, setup uses Google Application Default Credentials. It can read -your active `gcloud` project, list visible projects, or accept explicit -`--vertex-project` and `--vertex-location` values. - -To use your local Claude Code session instead of an API key, set: - -```yaml -llm: - provider: - backend: claude-code - models: - default: sonnet - triage: haiku - candidateExtraction: sonnet - curator: sonnet - reconcile: sonnet - repair: sonnet -``` - -`claude-code` uses the Claude Code authentication already configured on your -machine. It doesn't use `ANTHROPIC_API_KEY`, Vertex credentials, AI Gateway -tokens, or Bedrock credentials. In non-interactive setup, pass -`--llm-model opus`, `--llm-model sonnet`, `--llm-model haiku`, or a full Claude -model ID to select the Claude Code model. - -Setup checks the selected model before saving. Anthropic API setup fetches live -Claude model choices when possible and falls back to bundled defaults if model -discovery is unavailable. - -## Step 3: Configure embeddings - -KTX uses embeddings for semantic search over semantic-layer sources, wiki -context, schema metadata, and relationship evidence. - -| Backend | Default model | Notes | -|---------|---------------|-------| -| OpenAI | `text-embedding-3-small` | Recommended for hosted embeddings. Requires an OpenAI API key. | -| Local sentence-transformers | `all-MiniLM-L6-v2` | Runs through the KTX-managed Python runtime. No hosted embedding key is required. | - -OpenAI setup reads `OPENAI_API_KEY` or saves a local secret file. Local -sentence-transformers setup can install and start the managed runtime during -setup. To prepare that runtime before setup, run: +If you choose local `sentence-transformers` embeddings, KTX uses the managed +Python runtime. To prepare it before setup, run: ```bash ktx dev runtime install --feature local-embeddings --yes ktx dev runtime start --feature local-embeddings ``` -## Step 4: Add a database - -KTX needs at least one primary database connection before it can build database -context. The wizard supports SQLite, PostgreSQL, MySQL, ClickHouse, SQL Server, -BigQuery, and Snowflake. - -You can usually enter connection fields interactively or provide a URL. Secret -URLs can be stored as local files under `.ktx/secrets/` or referenced with -`env:NAME` in `ktx.yaml`. - -After saving a connection, setup tests it and builds fast schema context: +During the database step, setup tests the saved connection and builds initial +schema context: ```text Testing warehouse @@ -137,114 +139,24 @@ Testing warehouse Building schema context for warehouse Running fast database ingest - -Database ready - warehouse - PostgreSQL - schema context complete ``` -PostgreSQL, BigQuery, and Snowflake can also enable query-history ingest. Query -history helps KTX learn common query patterns, joins, service-account filters, -and warehouse-specific usage. BigQuery and Snowflake support a lookback window; -Postgres reads the current `pg_stat_statements` aggregate data instead. +If setup exits early, rerun `ktx setup` in the same directory. KTX keeps +progress under `.ktx/setup/` and resumes from the remaining work. -## Step 5: Add context sources +> **Note:** Running bare `ktx` in an interactive terminal outside a KTX +> project opens the same wizard. Inside a project, it opens a menu for +> resuming setup, connecting an agent, checking status, or exploring a +> pre-built demo project. -Context sources are optional, but they make the first context layer much richer. -Setup can add: +## Verify -| Source | Typical input | What KTX learns | -|--------|---------------|-----------------| -| dbt | Local project or Git repo | Models, columns, tests, descriptions, tags | -| MetricFlow | Local project or Git repo | Semantic models, metrics, dimensions, entities | -| LookML | Local files or Git repo | Views, explores, dimensions, measures, joins | -| Looker | API URL and credentials | Explores, looks, dashboards, model metadata | -| Metabase | API URL and key | Questions, dashboards, BI database mappings | -| Notion | Integration token and crawl settings | Business docs and knowledge pages | - -Setup maps BI and source metadata back to your primary warehouse connection so -generated context points at the right tables. - -You can skip this step and add sources later by rerunning `ktx setup`. - -## Step 6: Build context - -The context build turns configured databases and sources into local artifacts -agents can read. It runs database ingest first, then source ingest and memory -updates. - -Fast database ingest records deterministic schema grounding. Deep ingest adds -AI-enriched descriptions, embeddings, relationship evidence, and query-history -context when configured. - -When the build finishes, setup verifies that agent-ready context exists: - -```text -KTX context is ready for agents. - -Databases: - warehouse: deep context complete - -Context sources: - dbt_main: memory update complete - -Verification: - Agent context: ready - Semantic search: ready -``` - -If a foreground build is interrupted, rerun `ktx setup` or build the same target -with `ktx ingest `. - -## Step 7: Install agent integration - -The final setup step installs project-local rules for your coding assistant. -Supported targets are Claude Code, Codex, Cursor, OpenCode, and universal -`.agents`. - -You can also run this step later: - -```bash -ktx setup --agents --target codex -``` - -Claude Code and Codex also support global installs: - -```bash -ktx setup --agents --target codex --global -``` - -Agent rules are CLI-based. They point agents at the KTX CLI path that created -the file, so agents do not need a separate `ktx` binary in `PATH`. If the CLI -path changes after reinstalling or moving a checkout, rerun `ktx setup --agents`. - -## Generated files - -KTX writes plain files so people and agents can inspect changes in git. - -| Path | Purpose | -|------|---------| -| `ktx.yaml` | Project configuration for LLMs, embeddings, connections, context sources, and query-history settings | -| `.ktx/secrets/*` | Local secret files referenced from `ktx.yaml`; do not commit these | -| `.ktx/setup/*` | Local setup and context-build state | -| `.ktx/agents/install-manifest.json` | Manifest used to manage installed agent files | -| `semantic-layer//*.yaml` | Semantic source definitions used for SQL generation | -| `wiki/global/*.md` | Shared business context and metric definitions | -| `wiki/user//*.md` | User-scoped notes and local context | -| `.claude/skills/ktx/SKILL.md` | Claude Code project skill | -| `.agents/skills/ktx/SKILL.md` | Codex or universal project skill | -| `.cursor/rules/ktx.mdc` | Cursor project rule | -| `.opencode/commands/ktx.md` | OpenCode project command | - -## Verify setup - -Run: +When setup finishes, check readiness: ```bash ktx status ``` -Example output: - ```text KTX project: /home/user/analytics Project ready: yes @@ -256,24 +168,59 @@ KTX context built: yes Agent integration ready: yes (codex:project) ``` -Use JSON when an agent or script needs a structured readiness check: +For a structured check inside scripts, use `ktx status --json`. -```bash -ktx status --json +When setup builds deep context, its final context check looks like: + +```text +KTX context is ready for agents. + +Databases: + warehouse: deep context complete + +Context sources: + dbt_main: memory update complete ``` -## Scripted setup example +## Connect a coding agent -Use non-interactive setup when creating repeatable fixtures or automation: +The setup wizard installs project-local agent rules in the last step. To +install or change targets later: + +```bash +ktx setup --agents +``` + +Claude Code and Codex also support global installs with `--global`. Agent +rules point at the KTX CLI path that created them, so agents don't need a +separate `ktx` binary on `PATH`. If the CLI path changes, rerun +`ktx setup --agents`. + +## What setup writes + +KTX writes plain files so people and agents can review changes in git. + +| Path | Purpose | +|------|---------| +| `ktx.yaml` | Project configuration | +| `.ktx/secrets/*` | Local secret files referenced from `ktx.yaml` - do not commit | +| `semantic-layer//*.yaml` | Semantic sources for SQL generation | +| `wiki/global/*.md` | Shared business context and metric definitions | +| `.claude/skills/ktx/`, `.agents/skills/ktx/`, `.cursor/rules/ktx.mdc`, `.opencode/commands/ktx.md` | Installed agent rules | + +## Scripted setup + +For repeatable fixtures and automation, skip prompts with flags: ```bash ktx setup \ --project-dir ./analytics \ --no-input \ + --yes \ --skip-llm \ --skip-embeddings \ --database postgres \ - --new-database-connection-id warehouse \ + --database-connection-id warehouse \ --database-url env:DATABASE_URL \ --database-schema public ``` @@ -287,23 +234,21 @@ ktx ingest warehouse --fast See [ktx setup](/docs/cli-reference/ktx-setup) for the full automation flag surface. -## Common errors +## Common issues -| Symptom | Likely cause | Recovery | -|---------|--------------|----------| -| `ktx: command not found` | The global package is not installed or your shell cannot find it | Reinstall `@kaelio/ktx` and open a new shell | -| Setup resumes the wrong project | `KTX_PROJECT_DIR` or the nearest `ktx.yaml` points somewhere else | Pass `--project-dir ` | -| Anthropic health check fails | API key, model id, or access is invalid | Fix `ANTHROPIC_API_KEY` or rerun setup with a different key or model | -| Vertex AI health check fails | Vertex API, Claude access, project, location, or IAM permissions are missing | Check the project, location, Application Default Credentials, and Vertex AI permissions | -| OpenAI embeddings fail | `OPENAI_API_KEY` is missing or invalid | Export the key or choose local sentence-transformers embeddings | -| Local embeddings fail | Managed Python runtime cannot install or start | Run `ktx dev runtime status`, then install the local embeddings runtime | -| Database test fails | Credentials, network access, database, warehouse, or schema is wrong | Test the same values with the database's native client, then rerun setup | -| Context is not built | Setup saved configuration but skipped or interrupted the build | Run `ktx setup` or `ktx ingest --all` | -| Agent integration is incomplete | Setup skipped the agents step or installed a different target | Run `ktx setup --agents --target ` | +| Symptom | Fix | +|---------|-----| +| `ktx: command not found` | Reinstall `@kaelio/ktx` and open a new shell | +| Setup resumes the wrong project | Pass `--project-dir ` | +| LLM or embeddings health check fails | Rerun setup and pick a different credential, model, or backend | +| Database test fails | Verify the same connection with the database's native client, then rerun setup | +| Agent integration is incomplete | Run `ktx setup --agents --target ` | ## Next steps -- Build and refresh context with [Building Context](/docs/guides/building-context). -- Edit semantic sources and wiki pages with [Writing Context](/docs/guides/writing-context). +- Refresh context with [Building Context](/docs/guides/building-context). +- Edit semantic sources and wiki pages with + [Writing Context](/docs/guides/writing-context). - Connect more tools with [Agent Clients](/docs/integrations/agent-clients). -- Read [The Context Layer](/docs/concepts/the-context-layer) to understand the architecture. +- Read [The Context Layer](/docs/concepts/the-context-layer) to understand + the architecture. diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx index 584e9003..39edaa85 100644 --- a/docs-site/content/docs/guides/building-context.mdx +++ b/docs-site/content/docs/guides/building-context.mdx @@ -3,12 +3,9 @@ title: Building Context description: Build and refresh KTX context from databases, source tools, query history, and text. --- -Building context turns configured connections into local semantic-layer sources -and wiki pages. Agents use those files to understand your schema, business -definitions, metric logic, joins, and known caveats before they write SQL. - -Use this guide after `ktx setup` has created `ktx.yaml` and at least one -database or context-source connection. +Build context after `ktx setup` creates `ktx.yaml` and at least one database or +context-source connection. KTX writes local semantic-layer sources and wiki +pages for agents to use before writing SQL. ## The build loop @@ -22,15 +19,12 @@ Most projects use this loop: 5. Validate and query representative sources before handing the context to an agent. -`ktx ingest --all` runs database connections first, then context-source -connections. That order lets dbt, BI, Notion, and text ingest attach context to -known warehouse tables. +`ktx ingest --all` runs databases first, then context-source connections, so +external metadata can attach to known warehouse tables. ## Database ingest -Database ingest connects to a configured warehouse and records local schema -context. It gives agents table, column, type, constraint, and row-count -grounding without requiring them to inspect the database directly. +Database ingest records table, column, type, constraint, and row-count context. ```bash # Build one configured database connection @@ -55,20 +49,16 @@ ktx ingest warehouse --deep ktx ingest --all --deep ``` -Deep ingest needs LLM and embedding readiness. If those providers are not -configured, run `ktx setup` or use `--fast`. +Deep ingest needs LLM and embedding readiness. Otherwise run `ktx setup` or use +`--fast`. -When you use `claude-code`, KTX still controls the tool surface for ingest and -memory capture. Claude Code built-in tools, discovered MCP servers, plugins, -skills, agents, and slash commands are not invokable by KTX agent loops unless -they are exact KTX MCP tools for the current run. +With `claude-code`, KTX agent loops can invoke only the KTX MCP tools for the +current run. ## Query history -PostgreSQL, BigQuery, and Snowflake can add query-history context. This helps -KTX learn common joins, filters, service-account patterns, redaction rules, and -usage-heavy query templates. BigQuery and Snowflake support a lookback window; -Postgres reads the current `pg_stat_statements` aggregate data instead. +PostgreSQL, BigQuery, and Snowflake can add query-history context: common joins, +filters, service-account patterns, redaction rules, and high-usage templates. Enable it during setup, store it under `connections..context.queryHistory`, or request it for one run: @@ -84,19 +74,13 @@ for one run. ## Relationship evidence -Many databases do not declare all foreign keys. KTX can score relationship -candidates using signals such as name similarity, type compatibility, value -overlap, embedding similarity, uniqueness, null rate, and structural priors. - -The public CLI does not expose separate relationship review subcommands. -Relationship evidence is built as part of deep database ingest when the -connector and readiness checks support it. +KTX scores relationship candidates during supported deep database ingest. The +public CLI does not expose separate relationship review subcommands. ## Context-source ingest -Context-source connections pull business metadata from tools your team already -uses. The current public `ktx ingest` command is connection-centric: pass one -configured connection id, or pass `--all`. +Context-source connections pull metadata from dbt, BI tools, Notion, and other +configured systems. Pass one connection id or `--all`. ```bash # Build one source connection @@ -117,14 +101,13 @@ Supported source types: | `metabase` | Metabase API | Questions, dashboards, table metadata, and mappings | | `notion` | Notion API | Wiki pages and business knowledge | -Source ingest extracts metadata, reconciles it with existing local context, and -writes semantic-layer YAML plus wiki Markdown. It merges rather than blindly -overwriting local edits. +Source ingest writes semantic-layer YAML and wiki Markdown, merging with local +edits. ## Text ingest -Use `ktx ingest text` for notes, Markdown files, runbooks, Slack exports, or -other free-form knowledge that should become searchable KTX memory. +Use `ktx ingest text` for notes, Markdown, runbooks, Slack exports, or other +searchable memory. ```bash # Capture a Markdown file @@ -146,14 +129,12 @@ Useful flags: | `--json` | Print structured output | | `--fail-fast` | Stop after the first failed text item | -Text ingest is a good fit for small, high-signal documents. For system-specific -connectors such as Notion, dbt, or Metabase, prefer configured source ingest so -KTX can preserve source metadata. +Use text ingest for small, high-signal documents. Prefer configured source +ingest for Notion, dbt, Metabase, and similar systems. ## Output and artifacts -Every ingest run prints a summary. Use `--json` when an agent or script needs a -structured plan and per-target results. +Every ingest run prints a summary. Use `--json` for scripts and agents. ```bash ktx ingest --all --json @@ -168,9 +149,7 @@ Typical generated files: | `wiki/user//*.md` | Text and memory ingest | User-scoped context | | `.ktx/setup/context-build.json` | Setup context build | Resume and readiness state for setup | -Ingest sessions also record transcripts with tool calls, LLM responses, and -write decisions. Inspect them when you need to debug why a source or wiki page -was written a certain way. +Ingest transcripts include tool calls, LLM responses, and write decisions. ## Example: first full refresh diff --git a/docs-site/content/docs/guides/llm-configuration.mdx b/docs-site/content/docs/guides/llm-configuration.mdx index 054d0b58..7e86cf96 100644 --- a/docs-site/content/docs/guides/llm-configuration.mdx +++ b/docs-site/content/docs/guides/llm-configuration.mdx @@ -3,8 +3,8 @@ title: LLM configuration description: Configure KTX LLM providers, model roles, and prompt caching. --- -KTX uses the top-level `llm` block in `ktx.yaml` for text generation, -structured extraction, and ingest or memory agent loops. +Configure text generation, structured extraction, and ingest or memory loops in +the top-level `llm` block. ## Backends @@ -15,9 +15,7 @@ Set `llm.provider.backend` to one of these values: - `vertex`: Use Vertex AI Anthropic models through Google Cloud credentials. - `gateway`: Use AI Gateway-compatible Anthropic model ids. - `claude-code`: Use your local Claude Code session through the Claude Agent - SDK. KTX removes provider-routing environment variables from Claude Code - child processes, so this backend doesn't silently fall back to - `ANTHROPIC_API_KEY`, Vertex, Gateway, or Bedrock credentials. + SDK. KTX strips provider-routing environment variables from child processes. ## Claude Code @@ -36,26 +34,20 @@ llm: repair: sonnet ``` -During setup, choose the Claude Code backend interactively or pass the model in -automation: +During setup, choose the backend interactively or pass the model in automation: ```bash ktx setup --llm-backend claude-code --llm-model opus --no-input ``` -For Claude Code, `sonnet`, `opus`, and `haiku` map to the current KTX defaults. -You can also pass a full Claude model ID, such as `claude-opus-4-7`. +For Claude Code, `sonnet`, `opus`, and `haiku` map to KTX defaults. Full Claude +model IDs are also accepted. -`claude-code` keeps KTX tool boundaries intact. KTX exposes only the MCP tools -needed for the current KTX agent loop, disables Claude Code built-in tools, -keeps plugins empty, and denies every non-KTX tool request through -`canUseTool`. The Claude Agent SDK may still report host-discovered slash -commands, skills, and subagent names in init metadata; that metadata is not an -execution grant for KTX agent loops. +`claude-code` exposes only KTX MCP tools for the current agent loop. SDK init +metadata may still list host slash commands, skills, and subagents; KTX does not +grant execution access to them. ## Prompt caching -`llm.promptCaching` has partial parity on `claude-code`. KTX doesn't pass -Anthropic cache-control markers to the Claude Agent SDK. Status and doctor warn -when you configure prompt-cache TTL, tool, or history fields that the Claude -Agent SDK backend ignores. +`llm.promptCaching` has partial parity on `claude-code`. Status and doctor warn +when the Claude Agent SDK backend ignores configured cache fields. diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 192b1c7f..8710d3ba 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -3,9 +3,8 @@ title: Serving Agents description: Expose KTX context to Claude Code, Codex, Cursor, OpenCode, and custom agents. --- -KTX serves agents through the public CLI and project-local instruction files. -Agents do not need a separate server. They read the generated rules, call KTX -commands, inspect local context files, and use JSON output when they need +KTX serves agents through the CLI and project-local instruction files. Agents +read generated rules, call KTX commands, inspect context files, and use JSON for structured results. ## Recommended setup @@ -39,14 +38,13 @@ ktx setup --agents --target claude-code --global ktx setup --agents --target codex --global ``` -KTX records installed files in `.ktx/agents/install-manifest.json`. Rerun -`ktx setup --agents` after moving a checkout or reinstalling the CLI so the -generated instructions point at the current CLI path. +Installed files are recorded in `.ktx/agents/install-manifest.json`. Rerun +`ktx setup --agents` after moving a checkout or reinstalling the CLI. ## Agent command set -All supported agent clients use the same command surface. Use `--project-dir` -when the agent is running outside the KTX project directory. +All supported clients use the same command surface. Use `--project-dir` outside +the KTX project directory. ### Readiness @@ -54,9 +52,8 @@ when the agent is running outside the KTX project directory. ktx status --json ``` -Agents should run this before relying on context. It reports project, LLM, -embedding, database, context-source, context-build, and agent-integration -readiness. +Run this before relying on context. It reports project, provider, connection, +context-build, and agent-integration readiness. ### Semantic layer discovery @@ -66,8 +63,8 @@ ktx sl list --connection-id warehouse --json ktx sl search "revenue" --json --limit 10 ``` -Agents use these commands to discover source names, connection ids, measures, -dimensions, and likely files to inspect. +Use these commands to find source names, connection ids, measures, dimensions, +and files to inspect. ### Semantic-layer validation and queries @@ -106,9 +103,8 @@ ktx wiki list --json ktx wiki search "revenue recognition" --json --limit 10 ``` -Agents should search wiki context when a question depends on business -definitions, metric caveats, process rules, or terms that are not obvious from -schema names. +Search wiki context for business definitions, metric caveats, process rules, and +non-obvious terms. ### Context refresh @@ -120,8 +116,7 @@ ktx ingest --all ktx ingest text docs/revenue-notes.md --connection-id warehouse ``` -Use `--deep` only when LLM and embedding setup is ready and the user expects an -AI-enriched refresh. +Use `--deep` only when LLM and embedding setup is ready. ## Good agent behavior @@ -135,14 +130,12 @@ Agents should: - Validate edited semantic sources with `ktx sl validate`. - Keep generated context changes reviewable in git. -Agents should not assume a background server, ORPC route, frontend app, or -external migration system exists. KTX is a local context layer with a CLI and -plain project files. +KTX is a local context layer with a CLI and plain project files. Do not assume a +background server, ORPC route, frontend app, or external migration system. ## Manual setup -Manual setup is useful for custom agents that can read project-local -instructions but are not yet a named target. +Use manual setup for custom agents that can read project-local instructions. 1. Install the universal target: diff --git a/docs-site/content/docs/guides/writing-context.mdx b/docs-site/content/docs/guides/writing-context.mdx index b68960bf..2b9824c8 100644 --- a/docs-site/content/docs/guides/writing-context.mdx +++ b/docs-site/content/docs/guides/writing-context.mdx @@ -3,12 +3,8 @@ title: Writing Context description: Edit semantic sources and wiki pages so agents use your business logic. --- -KTX context is meant to be edited. Ingest gives you a grounded first draft, then -you refine source YAML and wiki Markdown until agents can answer data questions -with the same definitions your team uses. - -Use this guide when you are adding measures, fixing joins, documenting business -rules, or reviewing context changes made by an agent. +Ingest creates the first draft. Edit source YAML and wiki Markdown when you need +sharper metrics, joins, or business rules. ## Editing workflow @@ -45,10 +41,8 @@ Use this order for most context changes: ## Semantic sources -Semantic sources are YAML files that describe queryable entities. A source is -usually a table, but it can also point at a custom SQL expression. Sources -define the vocabulary agents use for measures, dimensions, segments, joins, and -grain-aware query planning. +Semantic sources are YAML files for queryable tables or custom SQL. They define +agent-facing measures, dimensions, segments, joins, and grain. Source files live at: @@ -198,8 +192,8 @@ joins: ## Measures -Good measures have precise names, SQL expressions at the correct grain, and -descriptions that say what is included and excluded. +Good measures have precise names, correct-grain SQL, and descriptions that name +key inclusions and exclusions. ```yaml measures: @@ -209,14 +203,13 @@ measures: description: Completed order revenue after refunds, excluding cancelled orders. ``` -Prefer one canonical measure plus wiki synonyms over several nearly identical -measures. If your team uses multiple definitions, document the distinction in a -wiki page and link it with `sl_refs`. +Prefer one canonical measure plus wiki synonyms. Put competing definitions in a +linked wiki page. ## Joins and grain -`grain` and `relationship` prevent agents from producing double-counted SQL. -State the row grain even when it seems obvious. +`grain` and `relationship` prevent double-counted SQL. State the row grain even +when it seems obvious. ```yaml grain: @@ -228,8 +221,7 @@ joins: ``` Use `many_to_one` for dimensions such as customer, account, product, or plan. -Use `one_to_many` only when the target can fan out the source rows, such as -orders to order items. +Use `one_to_many` only when the target can fan out rows. ## Validate and query @@ -239,8 +231,7 @@ Validation checks source YAML against the live database schema: ktx sl validate orders --connection-id warehouse ``` -It catches missing columns, invalid join targets, and table-reference problems -before an agent relies on the source. +It catches missing columns, invalid joins, and table-reference problems. Compile a query to inspect generated SQL: @@ -268,9 +259,8 @@ ktx sl query \ ## Wiki pages -Wiki pages capture business context that does not belong in a single source -file: metric policies, dashboard caveats, company vocabulary, data freshness, -known issues, and source-of-truth notes. +Wiki pages hold context that does not belong in one source file: policies, +caveats, vocabulary, freshness, known issues, and source-of-truth notes. Wiki files live under: @@ -280,8 +270,7 @@ wiki/ user// ``` -Use global pages for shared business rules. Use user-scoped pages for local -notes, personal conventions, or context that should not be shared broadly. +Use global pages for shared rules and user-scoped pages for local notes. ### Wiki page example @@ -338,8 +327,7 @@ ktx sl search "revenue" --json ktx wiki search "revenue recognition" --json --limit 10 ``` -Check that definitions are specific, hidden columns stay hidden, joins have -explicit relationships, and measures compile into the expected SQL. +Check definitions, hidden columns, join relationships, and generated SQL. ## Common errors diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 1934c978..4a670315 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -7,32 +7,32 @@ KTX exposes context to end-user agents through MCP tools. The CLI remains the admin surface for setup, ingest, status, daemon lifecycle, and debugging. Run `ktx setup` and select your client agent targets, or configure manually -using the snippets below. Choose **MCP tools + analytics skill** for client -agents. Choose **MCP tools + analytics skill + admin CLI skill** only when a -developer or operator agent also needs pinned `ktx` admin commands. +using the snippets below. Choose **Ask data questions with KTX MCP** for client +agents. Choose **Ask data questions + manage KTX with CLI commands** only when +a developer or operator agent also needs pinned `ktx` admin commands. ## Install with setup -Start the MCP server before connecting an end-user agent: - -```bash -ktx mcp start -``` - -Then install client integration: +Install client integration first: ```bash ktx setup --agents ``` +Then start the MCP server before using HTTP-based clients: + +```bash +ktx mcp start +``` + Use `--target` for one target: ```bash ktx setup --agents --target codex ``` -Use `--global` only with `claude-code` or `codex`. Claude Desktop always -generates a project-local plugin ZIP: +Use `--global` only with `claude-code` or `codex`. Claude Desktop always writes +global Claude Desktop config and generates project-local skill ZIPs: ```bash ktx setup --agents --target claude-code --global @@ -46,9 +46,9 @@ remove only files KTX installed. The interactive command asks two questions: ```txt -◆ How should client agents connect to this KTX project? -│ ○ MCP tools + analytics skill -│ ○ MCP tools + analytics skill + admin CLI skill +◆ What should agents be allowed to do with this KTX project? +│ ○ Ask data questions with KTX MCP +│ ○ Ask data questions + manage KTX with CLI commands └ ◆ Which agent targets should KTX install? @@ -66,21 +66,31 @@ also asks where to install supported agent config: ```txt ◆ Where should KTX install supported agent config? -│ ○ Project -│ ○ Global +│ +│ KTX project: /path/to/your/ktx-project +│ +│ ○ Project scope (KTX project directory) +│ ○ Global scope (user config) └ ``` ## Generated files -KTX writes MCP client configuration and an analytics skill by default. It writes -admin CLI skills only when you choose **MCP tools + analytics skill + admin CLI -skill**. +KTX writes MCP client configuration and analytics guidance by default. It writes +admin CLI guidance only when you choose **Ask data questions + manage KTX with +CLI commands**. -| Target | MCP tools + analytics skill | Adds with admin CLI skill | +After setup, KTX prints **Required before using agents**. Complete those steps +before opening the configured agent. If it shows `ktx mcp start --project-dir ...`, +run that command before using Claude Code, Codex, Cursor, OpenCode, or generic +MCP clients. The same output also prints the matching `ktx mcp stop` command +for when you want to stop MCP later. Claude Desktop uses its own launcher and +prints separate skill upload steps. + +| Target | Ask data questions with KTX MCP | Adds when agents can manage KTX with CLI | |--------|------------------------------|---------------------------| | Claude Code | `.mcp.json`, `.claude/skills/ktx-analytics/SKILL.md` | `.claude/skills/ktx/SKILL.md`, `.claude/rules/ktx.md` | -| Claude Desktop | `~/Library/Application Support/Claude/claude_desktop_config.json` stdio entry + `.ktx/agents/claude/ktx-plugin.zip` with analytics skill | Adds `skills/ktx/SKILL.md` inside the plugin ZIP | +| Claude Desktop | `~/Library/Application Support/Claude/claude_desktop_config.json` stdio entry + `.ktx/agents/claude/ktx-analytics.zip` upload | Adds `.ktx/agents/claude/ktx.zip` upload | | Codex | Printed snippet for `~/.codex/config.toml`, `.agents/skills/ktx-analytics/SKILL.md` | `.agents/skills/ktx/SKILL.md`, `.codex/instructions/ktx.md` | | Cursor | `.cursor/mcp.json`, `.cursor/rules/ktx-analytics.mdc` | `.cursor/rules/ktx.mdc` | | OpenCode | Printed snippet for `opencode.json`, `.opencode/commands/ktx-analytics.md` | `.opencode/commands/ktx.md` | @@ -144,7 +154,7 @@ During setup, select **Cursor** from the agent targets. KTX writes: | Mode | File | |------|------| -| MCP tools + analytics skill | `.cursor/mcp.json`, `.cursor/rules/ktx-analytics.mdc` | +| Ask data questions with KTX MCP | `.cursor/mcp.json`, `.cursor/rules/ktx-analytics.mdc` | | Admin CLI rules | `.cursor/rules/ktx.mdc` | Cursor supports project-scoped installation only. @@ -168,8 +178,8 @@ same markdown command definitions. ## Claude Desktop During setup, select **Claude Desktop** from the agent targets. KTX writes the -MCP server entry directly into Claude Desktop's config and generates a separate -plugin ZIP for the analytics skill: +MCP server entry directly into Claude Desktop's config and prepares uploadable +Claude Desktop skill packages for the KTX workflows: - `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%AppData%/Claude/claude_desktop_config.json` (Windows) gets an @@ -177,14 +187,22 @@ plugin ZIP for the analytics skill: launcher shim at `.ktx/agents/claude/ktx-plugin-runner.sh`. The shim locates a usable Node.js (Volta, NVM, Homebrew, system) so Claude Desktop can spawn the server without needing `node` in PATH. -- `.ktx/agents/claude/ktx-plugin.zip` contains the `ktx-analytics` skill (and - the admin `ktx` skill if you choose **MCP tools + analytics skill + admin - CLI skill**). Install the ZIP from Claude Desktop's plugin UI to load the - skill. +- `.ktx/agents/claude/ktx-analytics.zip` contains the `ktx-analytics` skill. + If you choose **Ask data questions + manage KTX with CLI commands**, KTX also + generates `.ktx/agents/claude/ktx.zip` with the admin `ktx` skill. Claude + Desktop requires each uploaded ZIP to contain exactly one skill folder. After `ktx setup`, restart Claude Desktop so it picks up the new MCP server -entry, then install the plugin ZIP. No daemon needs to be running — Claude -Desktop spawns the MCP server itself per session. +entry. No daemon needs to be running -- Claude Desktop spawns the MCP server +itself per session. + +Upload each generated skill ZIP from Claude Desktop: + +1. Open **Customize** > **Skills**. +2. Click **+** > **Create skill** > **Upload a skill**. +3. Upload `.ktx/agents/claude/ktx-analytics.zip`. +4. If generated, upload `.ktx/agents/claude/ktx.zip`. +5. Toggle the uploaded KTX skills on. Claude Desktop does not introspect local stdio MCP servers, so the per-tool "Connector"-style UI is not rendered for KTX. The tools are still callable @@ -192,7 +210,8 @@ from any Claude Desktop chat. If you move the KTX checkout or project directory, rerun `ktx setup --agents` to refresh the absolute paths in `claude_desktop_config.json` and the launcher -shim, then reinstall the regenerated plugin ZIP. +shim, regenerate the skill ZIPs, then restart Claude Desktop and upload the new +ZIPs. --- @@ -235,7 +254,7 @@ During setup, select **OpenCode** from the agent targets. KTX writes: | Mode | File | |------|------| -| MCP tools + analytics skill | Snippet for `opencode.json`, `.opencode/commands/ktx-analytics.md` | +| Ask data questions with KTX MCP | Snippet for `opencode.json`, `.opencode/commands/ktx-analytics.md` | | Admin CLI commands | `.opencode/commands/ktx.md` | OpenCode supports project-scoped installation only. @@ -266,7 +285,7 @@ Admin CLI skills call the same KTX CLI commands: | `ktx sl list --json` | List semantic-layer sources | | `ktx sl search --json` | Search semantic-layer sources | | `ktx sl validate --connection-id ` | Validate semantic source definitions | -| `ktx sl query --format json` | Execute a semantic-layer query when semantic compute is configured | +| `ktx sl query --format json` | Execute a Semantic Query when semantic compute is configured | ### Security constraints @@ -280,8 +299,8 @@ Admin CLI skills call the same KTX CLI commands: | | Claude Code | Claude Desktop | Cursor | Codex | OpenCode | |---|---|---|---|---|---| | MCP tools | Yes | Local stdio via `claude_desktop_config.json` | Yes | Snippet | Snippet | -| Analytics skill | `.claude/skills/ktx-analytics/SKILL.md` | Included in plugin ZIP | `.cursor/rules/ktx-analytics.mdc` | `.agents/skills/ktx-analytics/SKILL.md` | `.opencode/commands/ktx-analytics.md` | -| Admin CLI skills | Optional | Optional in plugin ZIP | Optional (.mdc) | Optional | Optional | -| Global install | Yes | Project-local ZIP | No | Yes | No | -| Rule or instruction file | `.claude/rules/ktx.md` | Plugin `SETUP.md` | `.cursor/rules/ktx.mdc` | `.codex/instructions/ktx.md` | `.opencode/commands/ktx.md` | -| Skill file | `.claude/skills/ktx/SKILL.md` | `skills/ktx/SKILL.md` in plugin ZIP | Not separate | `.agents/skills/ktx/SKILL.md` | Not separate | +| Analytics skill | `.claude/skills/ktx-analytics/SKILL.md` | Upload `.ktx/agents/claude/ktx-analytics.zip` | `.cursor/rules/ktx-analytics.mdc` | `.agents/skills/ktx-analytics/SKILL.md` | `.opencode/commands/ktx-analytics.md` | +| Admin CLI skills | Optional | Optional `.ktx/agents/claude/ktx.zip` upload | Optional (.mdc) | Optional | Optional | +| Global install | Yes | Claude Desktop config | No | Yes | No | +| Rule or instruction file | `.claude/rules/ktx.md` | Not separate | `.cursor/rules/ktx.mdc` | `.codex/instructions/ktx.md` | `.opencode/commands/ktx.md` | +| Skill file | `.claude/skills/ktx/SKILL.md` | `ktx/SKILL.md` inside `ktx.zip` | Not separate | `.agents/skills/ktx/SKILL.md` | Not separate | diff --git a/docs-site/content/docs/integrations/index.mdx b/docs-site/content/docs/integrations/index.mdx deleted file mode 100644 index 92a677aa..00000000 --- a/docs-site/content/docs/integrations/index.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: Integrations -description: Connect KTX to warehouses, analytics tools, and coding agents. ---- - -KTX integrations bring trusted context into an analytics project and make that -context available to coding agents through the CLI. Start with `ktx setup` when -you want the guided flow, then use the integration reference pages for exact -configuration fields, generated files, and manual setup. - -## Integration types - -| Type | What it connects | Start here | -|------|------------------|------------| -| Primary sources | Warehouses and databases that KTX scans for schemas, constraints, row counts, and optional query history | [Primary Sources](/docs/integrations/primary-sources) | -| Context sources | Existing analytics and knowledge tools such as dbt, MetricFlow, LookML, Metabase, Looker, and Notion | [Context Sources](/docs/integrations/context-sources) | -| Agent clients | Claude Code, Codex, Cursor, OpenCode, and universal `.agents` consumers | [Agent Clients](/docs/integrations/agent-clients) | - -## Recommended setup flow - -Use this order for a new project: - -1. Run `ktx setup` from the analytics project directory. -2. Configure an LLM backend and embeddings so KTX can enrich and search context. -3. Add at least one primary source connection. -4. Add optional context sources that describe the same warehouse or business domain. -5. Build context during setup, or run `ktx ingest ` later. -6. Install agent integration with `ktx setup --agents` when the context is ready. - -For repeatable setup, pass `--project-dir`, `--no-input`, and the relevant -automation flags documented in [`ktx setup`](/docs/cli-reference/ktx-setup). - -## What setup writes - -| Path | Purpose | -|------|---------| -| `ktx.yaml` | Main project configuration for providers, embeddings, connections, source mappings, and query history | -| `.ktx/secrets/*` | Local file-backed secrets when you choose file references during setup | -| `.ktx/setup/*` | Local setup progress and context-build state | -| `semantic-layer//` | YAML semantic sources generated by database and source ingestion | -| `wiki/` | Markdown business context, definitions, and ingested knowledge | -| `.ktx/agents/install-manifest.json` | Manifest of agent integration files installed by `ktx setup --agents` | -| Agent client files | Skills, rules, or commands that teach agents when and how to call KTX | - -## Common commands - -```bash -# Start or resume the guided flow -ktx setup - -# Add or refresh every configured integration -ktx ingest --all - -# Refresh one configured warehouse, source, or knowledge integration -ktx ingest warehouse - -# Install one project-scoped agent target -ktx setup --agents --target codex - -# Check whether integrations are ready -ktx status -``` - -## Choosing docs - -Read [Primary Sources](/docs/integrations/primary-sources) when you need -database driver fields, authentication formats, query history support, or -warehouse-specific notes. Read [Context Sources](/docs/integrations/context-sources) -when you need source adapter fields, repository authentication, BI tool mapping, -or Notion crawl options. Read [Agent Clients](/docs/integrations/agent-clients) -when you need generated file locations or manual agent configuration. diff --git a/docs-site/content/docs/integrations/meta.json b/docs-site/content/docs/integrations/meta.json index 20dc642f..70fe26ec 100644 --- a/docs-site/content/docs/integrations/meta.json +++ b/docs-site/content/docs/integrations/meta.json @@ -1,5 +1,5 @@ { "title": "Integrations", "defaultOpen": true, - "pages": ["index", "primary-sources", "context-sources", "agent-clients"] + "pages": ["primary-sources", "context-sources", "agent-clients"] } diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 00cc39aa..28ff4559 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -1,12 +1,17 @@ --- title: Primary Sources -description: Connect KTX to PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, or SQLite. +description: Connect KTX to PostgreSQL, Snowflake, BigQuery, MySQL, SQL Server, or SQLite. --- KTX connects to your data warehouse or database to build schema context, discover relationships, and execute semantic layer queries. Each connection is defined in `ktx.yaml` under the `connections` key. +For analytics tools and knowledge systems such as dbt, MetricFlow, LookML, +Metabase, Looker, and Notion, use [Context Sources](/docs/integrations/context-sources). +For Claude Code, Codex, Cursor, OpenCode, and other agent clients, use +[Agent Clients](/docs/integrations/agent-clients). + All connectors share these conventions: - Sensitive values support `env:VAR_NAME` (read from environment) and @@ -21,9 +26,9 @@ Agents should prefer environment or file references over literal secrets. | Field | Required | Applies to | Description | |-------|----------|------------|-------------| -| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `clickhouse`, `mysql`, `sqlserver`, or `sqlite` | +| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `sqlserver`, or `sqlite` | | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | -| `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, ClickHouse, SQL Server | Field-by-field connection values | +| `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | | `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it | | `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference | @@ -264,63 +269,6 @@ staged artifact shape as Postgres and Snowflake. --- -## ClickHouse - -Connects over HTTP (port 8123) or HTTPS (port 8443). Supports the ClickHouse native type system including `Nullable`, `LowCardinality`, and `Array` wrappers. - -### Connection config - -```yaml title="ktx.yaml" -connections: - my-clickhouse: - driver: clickhouse - url: http://localhost:8123/analytics -``` - -Or with individual fields: - -```yaml title="ktx.yaml" -connections: - my-clickhouse: - driver: clickhouse - host: clickhouse.internal - port: 8123 - database: analytics - username: default - password: env:CH_PASSWORD - ssl: false -``` - -### Authentication - -| Method | Config | -|--------|--------| -| Basic auth | `username` + `password` (HTTP basic auth) | -| No auth | Default user `default` with no password | -| HTTPS | Set `ssl: true` (uses port 8443 by default) | - -### Features - -| Feature | Supported | Notes | -|---------|-----------|-------| -| Tables & views | Yes | Via `system.tables`, engine-based detection | -| Primary keys | Yes | Via `system.columns` | -| Foreign keys | No | Not a ClickHouse concept | -| Row count estimates | Yes | Via `system.parts` aggregation | -| Column statistics | No | - | -| Query history | No | - | -| Table sampling | Yes | - | - -### Dialect notes - -- Parameter binding uses `{param:Type}` syntax (e.g., `{database:String}`) -- Detects views vs. tables by engine name (`View`, `MaterializedView`) -- Handles `Nullable(T)` and `LowCardinality(Nullable(T))` type wrappers -- Dictionary tables are excluded from scanning -- Results returned in JSONCompact or JSONEachRow format - ---- - ## MySQL Standard MySQL/MariaDB connector with full foreign key support and schema introspection. @@ -510,4 +458,4 @@ No authentication required - SQLite is file-based. The file must be readable by | Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | | Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest --query-history` or `ktx setup` | | Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on fast schema context | -| Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags | +| Semantic Query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags | diff --git a/docs-site/lib/agent-setup-markdown.ts b/docs-site/lib/agent-setup-markdown.ts new file mode 100644 index 00000000..5a42ea1f --- /dev/null +++ b/docs-site/lib/agent-setup-markdown.ts @@ -0,0 +1,12 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +export const agentSetupSlug = ["agents-setup"] as const; + +export function isAgentSetupSlug(slug: string[] | undefined) { + return slug?.length === 1 && slug[0] === agentSetupSlug[0]; +} + +export function readAgentSetupMarkdown() { + return readFile(join(process.cwd(), "content/agents-setup.md"), "utf8"); +} diff --git a/docs-site/lib/llm-docs.ts b/docs-site/lib/llm-docs.ts index 561f73e0..f821ca30 100644 --- a/docs-site/lib/llm-docs.ts +++ b/docs-site/lib/llm-docs.ts @@ -52,8 +52,8 @@ KTX provides semantic-layer files, warehouse scans, wiki pages, provenance, and ## Agent Entry Points -${link("/docs/ai-resources", "AI Resources", "Machine-readable docs, prompt recipes, and agent setup paths")} ${link("/docs/ai-resources/agent-quickstart", "Agent Quickstart", "Task-first route for coding assistants using KTX")} +${link("/docs/agents-setup", "Agent Setup", "Copy-pasteable prompt for agents installing and configuring KTX")} ${link("/docs/ai-resources/markdown-access", "Markdown Access", "Fetch KTX docs as llms.txt, llms-full.txt, or per-page Markdown")} ${link("/docs/ai-resources/agent-instructions", "Agent Instructions", "Suggested instructions for coding assistants that need to read and cite KTX docs")} @@ -73,6 +73,7 @@ ${link("/docs/guides/writing-context", "Writing Context", "Write semantic source ## CLI Reference +${link("/docs/cli-reference/ktx", "ktx", "Root command map and global options")} ${link("/docs/cli-reference/ktx-setup", "ktx setup", "Interactive project setup")} ${link("/docs/cli-reference/ktx-sl", "ktx sl", "Semantic-layer commands")} ${link("/docs/cli-reference/ktx-wiki", "ktx wiki", "Wiki page commands")} diff --git a/docs-site/next-env.d.ts b/docs-site/next-env.d.ts index 9edff1c7..c4b7818f 100644 --- a/docs-site/next-env.d.ts +++ b/docs-site/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/types/routes.d.ts"; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/docs-site/next.config.mjs b/docs-site/next.config.mjs index 3beb3073..30a96741 100644 --- a/docs-site/next.config.mjs +++ b/docs-site/next.config.mjs @@ -15,6 +15,12 @@ const config = { }, async redirects() { return [ + { + source: "/", + destination: "/ktx/docs/getting-started/introduction", + permanent: false, + basePath: false, + }, { source: "/docs", destination: "/docs/getting-started/introduction", diff --git a/docs-site/package.json b/docs-site/package.json index 229af1e3..4cf896ff 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -10,6 +10,7 @@ "test": "node --test tests/*.test.mjs" }, "dependencies": { + "@xyflow/react": "^12.10.2", "fumadocs-core": "16.8.10", "fumadocs-mdx": "15.0.4", "fumadocs-ui": "16.8.10", @@ -18,11 +19,11 @@ "react-dom": "19.2.6" }, "devDependencies": { + "@tailwindcss/postcss": "^4", "@types/node": "^25.7.0", "@types/react": "^19", "@types/react-dom": "^19", - "typescript": "^6.0", - "@tailwindcss/postcss": "^4", - "tailwindcss": "^4" + "tailwindcss": "^4", + "typescript": "^6.0" } } diff --git a/docs-site/middleware.ts b/docs-site/proxy.ts similarity index 96% rename from docs-site/middleware.ts rename to docs-site/proxy.ts index 1b892076..49d1c324 100644 --- a/docs-site/middleware.ts +++ b/docs-site/proxy.ts @@ -6,7 +6,7 @@ const markdownMimeTypes = new Set([ "application/markdown", ]); -export function middleware(request: NextRequest) { +export function proxy(request: NextRequest) { if (!isMarkdownPreferred(request.headers.get("accept"))) { return NextResponse.next(); } diff --git a/docs-site/public/images/ingestion-flow-transparent.svg b/docs-site/public/images/ingestion-flow-transparent.svg new file mode 100644 index 00000000..6fa1a40a --- /dev/null +++ b/docs-site/public/images/ingestion-flow-transparent.svg @@ -0,0 +1,211 @@ + + KTX ingestion flow + Source systems flow through source adapters, context builder, reconciliation, and validation to create wiki Markdown and semantic-layer YAML outputs. + + + + + + + + + + + + + + + + + + + + + + + + + Databases + Schemas, columns, keys, + row counts, and query + history. + + + PostgreSQL + + Snowflake + + BigQuery + + SQLite + + + + + + + BI tools + Dashboards, questions, + explores, usage, and trusted + examples. + + + Metabase + + Looker + + + + + + + Modeling code + Existing metrics, dimensions, + models, joins, and entities. + + + dbt + + LookML + + MetricFlow + + + + + + + Docs and notes + Policies, caveats, team + definitions, and analyst + context. + + + Notion + + Any text + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + Source adapters + Read each configured system in + its native shape. + + + + + + 2 + Context builder + Turn source evidence into + proposed context updates. + + + + + + 3 + Reconciliation + Merge new evidence with the + context that already exists. + + + + + + 4 + Validation + Check references and + semantics before agents rely on + them. + + + + + + + + wiki/*.md + Wiki + + + free-form + + auto-maintained + + Definitions, caveats, policies, analyst notes, and + business language that agents can search. + + + + + + semantic-layer/*.yaml + Semantic layer + + + structured + + executable + + auto-maintained + + Metrics, joins, tables, dimensions, filters, and + segments that KTX can validate and compile into + SQL. + + + + + references + + + diff --git a/docs-site/public/images/ingestion-flow.png b/docs-site/public/images/ingestion-flow.png new file mode 100644 index 00000000..49bc544f Binary files /dev/null and b/docs-site/public/images/ingestion-flow.png differ diff --git a/docs-site/source.config.ts b/docs-site/source.config.ts index 71c0f614..5af0fa2b 100644 --- a/docs-site/source.config.ts +++ b/docs-site/source.config.ts @@ -5,5 +5,13 @@ export const docs = defineDocs({ }); export default defineConfig({ - mdxOptions: {}, + mdxOptions: { + rehypeCodeOptions: { + addLanguageClass: true, + themes: { + light: "min-light", + dark: "github-dark", + }, + }, + }, }); diff --git a/docs-site/tests/docs-index-route.test.mjs b/docs-site/tests/docs-index-route.test.mjs index 7d1c62c0..fdd8ec81 100644 --- a/docs-site/tests/docs-index-route.test.mjs +++ b/docs-site/tests/docs-index-route.test.mjs @@ -111,3 +111,33 @@ test("/ktx/docs redirects to the docs introduction", async () => { `${docsBasePath}/docs/getting-started/introduction`, ); }); + +test("/ redirects into the /ktx docs site", async () => { + const response = await fetch(`${docsSiteUrl}/`, { + redirect: "manual", + }); + + assert.equal(response.status, 307); + assert.equal( + response.headers.get("location"), + `${docsBasePath}/docs/getting-started/introduction`, + ); +}); + +test("/ktx/api/search returns docs search results", async () => { + const response = await fetch( + `${docsSiteUrl}${docsBasePath}/api/search?query=setup`, + ); + + assert.equal(response.status, 200); + + const results = await response.json(); + assert.ok(Array.isArray(results), "search response should be an array"); + assert.ok( + results.some( + (result) => + typeof result.url === "string" && result.url.startsWith("/docs/"), + ), + "search should return at least one docs result", + ); +}); diff --git a/docs-site/tests/docs-search-behavior.test.mjs b/docs-site/tests/docs-search-behavior.test.mjs new file mode 100644 index 00000000..ece51477 --- /dev/null +++ b/docs-site/tests/docs-search-behavior.test.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { access, readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const docsSiteDir = join(dirname(fileURLToPath(import.meta.url)), ".."); + +async function readDocsFile(path) { + return readFile(join(docsSiteDir, path), "utf8"); +} + +test("root provider uses the base-path-aware search API", async () => { + const layout = await readDocsFile("app/layout.tsx"); + + assert.match(layout, /search=\{\{/); + assert.match(layout, /api:\s*"\/ktx\/api\/search"/); +}); + +test("metadata icons include the docs base path", async () => { + const layout = await readDocsFile("app/layout.tsx"); + + assert.match(layout, /icon:\s*"\/ktx\/brand\/ktx-mascot\.svg"/); + assert.match(layout, /shortcut:\s*"\/ktx\/brand\/ktx-mascot\.svg"/); + assert.doesNotMatch(layout, /:\s*"\/brand\/ktx-mascot\.svg"/); +}); + +test("markdown negotiation uses the Next proxy convention", async () => { + await assert.doesNotReject(access(join(docsSiteDir, "proxy.ts"))); + await assert.rejects(access(join(docsSiteDir, "middleware.ts"))); + + const proxy = await readDocsFile("proxy.ts"); + assert.match(proxy, /export function proxy/); + assert.doesNotMatch(proxy, /export function middleware/); +}); + +test("site background stacking does not target every body child", async () => { + const css = await readDocsFile("app/global.css"); + + assert.doesNotMatch(css, /body\s*>\s*\*\s*\{[^}]*z-index/s); + assert.match(css, /\.ktx-site-shell\s*\{[^}]*z-index:\s*2/s); +}); + +test("search lock relies on body overflow propagation, not html or sidebar overrides", async () => { + const css = await readDocsFile("app/global.css"); + + // Body still clips horizontal overflow defensively. + assert.match(css, /(^|\s)body\s*\{[^}]*overflow-x:\s*clip/s); + + // html must keep its default `visible` overflow so body's lock + // (`overflow: hidden` from react-remove-scroll-bar) propagates to the + // viewport. Locking html directly breaks `position: sticky` on the + // sidebar placeholder. + assert.doesNotMatch(css, /(^|\s)html\s*,?\s*\{[^}]*overflow(-y|\s*:)\s*(hidden|clip)/s); + assert.doesNotMatch( + css, + /html:has\(body\[data-scroll-locked\]\)[^{]*\{[^}]*overflow:\s*(hidden|clip)/s, + ); + + // No site-specific overrides to body's data-scroll-locked overflow or + // to the sidebar placeholder when locked. + assert.doesNotMatch( + css, + /html\s+body\[data-scroll-locked\][^{]*\{[^}]*overflow:/s, + ); + assert.doesNotMatch( + css, + /body\[data-scroll-locked\]\s+\[data-sidebar-placeholder\][^{]*\{[^}]*position:\s*fixed/s, + ); +}); diff --git a/docs-site/tests/product-mechanics-content.test.mjs b/docs-site/tests/product-mechanics-content.test.mjs index bf03abcc..d0f1cb0c 100644 --- a/docs-site/tests/product-mechanics-content.test.mjs +++ b/docs-site/tests/product-mechanics-content.test.mjs @@ -23,7 +23,7 @@ test("docs introduction frames the concept before showing product mechanics", as const heroIndex = introduction.indexOf("Make analytics context"); const whyIndex = introduction.indexOf("## Why KTX"); - const createsIndex = introduction.indexOf("## What KTX creates"); + const worksIndex = introduction.indexOf("## How KTX works"); const mechanicsIndex = introduction.indexOf(""); const useCaseIndex = introduction.indexOf("## Use it for"); const heroSource = introduction.slice(0, mechanicsIndex); @@ -34,12 +34,12 @@ test("docs introduction frames the concept before showing product mechanics", as "problem framing should appear after the hero", ); assert.ok( - createsIndex > whyIndex, - "artifact summary should appear after problem framing", + worksIndex > whyIndex, + "mechanics bridge should appear after problem framing", ); assert.ok( - mechanicsIndex > createsIndex, - "mechanics component should appear after the artifact summary", + mechanicsIndex > worksIndex, + "mechanics component should appear after the mechanics bridge", ); assert.ok( mechanicsIndex < useCaseIndex, @@ -49,49 +49,47 @@ test("docs introduction frames the concept before showing product mechanics", as assert.doesNotMatch(heroSource, /The Context Layer/); assert.doesNotMatch(heroSource, /Building Context/); assert.doesNotMatch(heroSource, /flex flex-wrap gap-3/); + assert.doesNotMatch(introduction, /raw-sources/); + assert.doesNotMatch(introduction, /\.ktx/); }); -test("product mechanics component covers source-specific context and SQL expansion", async () => { +test("product mechanics component explains ingestion outputs", async () => { const component = await readDocsFile("components/product-mechanics.tsx"); for (const expectedText of [ - "How KTX works", - "Build context from source evidence", - "Run agent requests through the model", - "Ingestion", - "Runtime", - "wiki/", - "semantic-layer/", - "raw-sources/", - ".ktx/", - "sl_refs", - "Database structure", - "BI and usage evidence", - "Semantic modeling", - "Company documentation", - "Notion pages", - "Sources", - "KTX transforms evidence", - "KTX builds the model", - "Outputs KTX writes", - "Postgres", + "How ingestion works", + "Ingestion flow", + "From scattered source systems to agent-ready context", + "wiki/*.md", + "semantic-layer/*.yaml", + "Wiki", + "Semantic layer", + "Databases", + "BI tools", + "Modeling code", + "Docs and notes", + "Source adapters", + "Context builder", + "Reconciliation", + "Validation", + "PostgreSQL", "Snowflake", "BigQuery", - "and many others", "Metabase", "Looker", + "dbt", "MetricFlow", "LookML", - "extract evidence", - "reconcile entities", - "validate references", - "semantic query plan", - "dialect SQL", - "bounded rows", - "provenance", - "measure: orders.total_revenue", - "dimension: customers.segment", - "select", + "Notion", + "Any text", + "compile into SQL", + '"use client"', + "@xyflow/react", + " **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `docs-site` discoverable and readable by coding agents through `llms.txt`, bundled markdown, per-page markdown routes, markdown negotiation, and stricter agent-friendly docs content. - -**Architecture:** Keep the existing Next 15 + Fumadocs app. Add a small `lib/llm-docs.ts` module that reads Fumadocs pages and builds machine-readable markdown responses, then expose those responses through route handlers and a markdown negotiation proxy. Rewrite existing MDX pages in place so the rendered UI and machine-readable routes share one source of truth. - -**Tech Stack:** Next.js 15 App Router, Fumadocs, MDX, TypeScript, pnpm, Node 22. - ---- - -### Task 1: Machine-Readable Docs Routes - -**Files:** -- Create: `docs-site/lib/llm-docs.ts` -- Create: `docs-site/app/llms.txt/route.ts` -- Create: `docs-site/app/llms-full.txt/route.ts` -- Create: `docs-site/app/llms.mdx/docs/[[...slug]]/route.ts` -- Modify: `docs-site/next.config.mjs` - -- [ ] **Step 1: Add the LLM docs utility** - -Create `docs-site/lib/llm-docs.ts` with functions that: - -```ts -import { source } from "@/lib/source"; - -const SITE_ORIGIN = "https://ktx.dev"; - -export type LlmDocsPage = { - title: string; - description?: string; - url: string; - markdownUrl: string; - slug: string[]; - getMarkdown: () => Promise; -}; - -export function getLlmDocsPages(): LlmDocsPage[] { - return source.getPages().map((page) => ({ - title: page.data.title, - description: page.data.description, - url: page.url, - markdownUrl: `${page.url}.md`, - slug: page.slugs, - getMarkdown: async () => normalizeMarkdown(await page.data.getText("raw")), - })); -} - -export function getLlmDocsPage(slug: string[] | undefined) { - const page = source.getPage(slug); - if (!page) return null; - - return { - title: page.data.title, - description: page.data.description, - url: page.url, - markdownUrl: `${page.url}.md`, - slug: page.slugs, - getMarkdown: async () => normalizeMarkdown(await page.data.getText("raw")), - } satisfies LlmDocsPage; -} - -export async function getPageMarkdown(page: LlmDocsPage) { - const body = await page.getMarkdown(); - const description = page.description ? `\n\n> ${page.description}` : ""; - - return `# ${page.title}${description}\n\nCanonical URL: ${page.url}\nMarkdown URL: ${page.markdownUrl}\n\n${body}`; -} - -export function buildLlmsTxt() { - const pages = getLlmDocsPages(); - const byUrl = new Map(pages.map((page) => [page.url, page])); - const link = (url: string, label: string, fallbackDescription: string) => { - const page = byUrl.get(url); - const description = page?.description ?? fallbackDescription; - return `- [${label}](${url}): ${description}`; - }; - - return `# KTX - -> Agent-native context layer for analytics engineering and database agents. - -KTX provides semantic-layer files, warehouse scans, knowledge pages, provenance, and agent-facing tools that help coding agents answer analytics questions without inventing metrics or joins. - -## Start Here - -${link("/docs/getting-started/introduction", "Introduction", "What KTX is and who it is for")} -${link("/docs/getting-started/quickstart", "Quickstart", "Set up KTX and build your first context")} -${link("/docs/guides/serving-agents", "Serving Agents", "Expose KTX context through MCP and CLI tools")} -${link("/docs/guides/writing-context", "Writing Context", "Write semantic sources and knowledge pages")} - -## Machine-Readable Documentation - -- [Full documentation](/llms-full.txt): All docs pages in one plain-text markdown response -- [Quickstart markdown](/docs/getting-started/quickstart.md): Raw markdown for the setup guide -- [Agent CLI markdown](/docs/cli-reference/ktx-agent.md): Raw markdown for machine-readable agent commands -- [Serving Agents markdown](/docs/guides/serving-agents.md): Raw markdown for MCP and CLI workflows - -## CLI Reference - -${link("/docs/cli-reference/ktx-setup", "ktx setup", "Interactive project setup")} -${link("/docs/cli-reference/ktx-agent", "ktx agent", "Machine-readable commands for coding agents")} -${link("/docs/cli-reference/ktx-sl", "ktx sl", "Semantic-layer commands")} -${link("/docs/cli-reference/ktx-wiki", "ktx wiki", "Knowledge page commands")} -${link("/docs/cli-reference/ktx-connection", "ktx connection", "Connection management commands")} - -## Integrations - -${link("/docs/integrations/agent-clients", "Agent Clients", "Configure Claude Code, Cursor, Codex, and OpenCode")} -${link("/docs/integrations/primary-sources", "Primary Sources", "Connect KTX to databases and warehouses")} -${link("/docs/integrations/context-sources", "Context Sources", "Ingest dbt, LookML, Metabase, Looker, MetricFlow, and Notion")} -`; -} - -export async function buildLlmsFullTxt() { - const pages = getLlmDocsPages(); - const rendered = await Promise.all(pages.map(getPageMarkdown)); - return [`# KTX Full Documentation`, `Source: ${SITE_ORIGIN}`, ...rendered].join("\n\n---\n\n"); -} - -function normalizeMarkdown(markdown: string) { - return markdown.trim().replace(/\n{3,}/g, "\n\n"); -} -``` - -- [ ] **Step 2: Add route handlers** - -Create route files: - -```ts -import { buildLlmsTxt } from "@/lib/llm-docs"; - -export const dynamic = "force-static"; - -export function GET() { - return new Response(buildLlmsTxt(), { - headers: { "Content-Type": "text/plain; charset=utf-8" }, - }); -} -``` - -```ts -import { buildLlmsFullTxt } from "@/lib/llm-docs"; - -export const dynamic = "force-static"; - -export async function GET() { - return new Response(await buildLlmsFullTxt(), { - headers: { "Content-Type": "text/plain; charset=utf-8" }, - }); -} -``` - -```ts -import { getLlmDocsPage, getPageMarkdown } from "@/lib/llm-docs"; -import { notFound } from "next/navigation"; - -export const dynamic = "force-static"; - -export async function GET( - _request: Request, - props: { params: Promise<{ slug?: string[] }> }, -) { - const params = await props.params; - const page = getLlmDocsPage(params.slug); - if (!page) notFound(); - - return new Response(await getPageMarkdown(page), { - headers: { "Content-Type": "text/markdown; charset=utf-8" }, - }); -} - -export function generateStaticParams() { - return getLlmDocsPages().map((page) => ({ slug: page.slug })); -} -``` - -- [ ] **Step 3: Add `.md` rewrite** - -Modify `docs-site/next.config.mjs`: - -```js -import { createMDX } from "fumadocs-mdx/next"; - -const withMDX = createMDX(); - -/** @type {import('next').NextConfig} */ -const config = { - async rewrites() { - return [ - { - source: "/docs/:path*.md", - destination: "/llms.mdx/docs/:path*", - }, - ]; - }, -}; - -export default withMDX(config); -``` - -- [ ] **Step 4: Build check** - -Run: `pnpm --filter ktx-docs build` - -Expected: Next build completes and static routes include `llms.txt`, `llms-full.txt`, and the LLM markdown route. - -### Task 2: Markdown Negotiation - -**Files:** -- Create: `docs-site/proxy.ts` - -- [ ] **Step 1: Add markdown negotiation proxy** - -Create `docs-site/proxy.ts`: - -```ts -import { isMarkdownPreferred, rewritePath } from "fumadocs-core/negotiation"; -import { NextResponse, type NextRequest } from "next/server"; - -const { rewrite } = rewritePath("/docs/*path", "/llms.mdx/docs/*path"); - -export function proxy(request: NextRequest) { - if (!isMarkdownPreferred(request)) { - return NextResponse.next(); - } - - const rewrittenPath = rewrite(request.nextUrl.pathname); - if (!rewrittenPath) { - return NextResponse.next(); - } - - return NextResponse.rewrite(new URL(rewrittenPath, request.nextUrl)); -} - -export const config = { - matcher: ["/docs/:path*"], -}; -``` - -- [ ] **Step 2: Verify build** - -Run: `pnpm --filter ktx-docs build` - -Expected: Build passes with the proxy included. - -### Task 3: Agent-Friendly High-Priority Guides - -**Files:** -- Modify: `docs-site/content/docs/getting-started/quickstart.mdx` -- Modify: `docs-site/content/docs/guides/serving-agents.mdx` -- Modify: `docs-site/content/docs/guides/writing-context.mdx` - -- [ ] **Step 1: Rewrite quickstart structure** - -Add sections for: - -- Workflow summary -- Generated files -- Common errors and recovery - -Keep existing setup detail, but make each command block copy-pasteable and each expected output complete enough for agents to recognize success. - -- [ ] **Step 2: Rewrite Serving Agents as API reference** - -Add tables for MCP tool inputs and CLI command inputs. Add workflows: - -- Answer an analytics question through MCP -- Answer an analytics question through CLI -- Safely execute SQL with row limits - -- [ ] **Step 3: Rewrite Writing Context with schemas and workflows** - -Add semantic-source field tables, knowledge-page field tables, and workflows: - -- Inspect a source -- Edit and validate a source -- Query through the semantic layer -- Write and search a knowledge page - -- [ ] **Step 4: Build check** - -Run: `pnpm --filter ktx-docs build` - -Expected: MDX compiles without syntax errors. - -### Task 4: CLI Reference Normalization - -**Files:** -- Modify: `docs-site/content/docs/cli-reference/*.mdx` - -- [ ] **Step 1: Normalize every CLI page** - -For each CLI reference page, ensure this structure exists: - -```md -## Command signature - -```bash -ktx [subcommand] [options] -``` - -## Subcommands - -| Subcommand | Description | -|---|---| - -## Options - -| Flag | Type | Required | Description | Default | -|---|---|---|---|---| - -## Examples - -```bash -ktx --real-flag realistic-value -``` - -## Output - -```text -complete expected output shape -``` - -## Common errors - -| Error | Cause | Recovery | -|---|---|---| -``` - -Only add sections that are relevant to the command; do not invent output for commands whose output is intentionally interactive. - -- [ ] **Step 2: Build check** - -Run: `pnpm --filter ktx-docs build` - -Expected: MDX compiles without syntax errors. - -### Task 5: Integration and Concept Page Polish - -**Files:** -- Modify: `docs-site/content/docs/integrations/agent-clients.mdx` -- Modify: `docs-site/content/docs/integrations/primary-sources.mdx` -- Modify: `docs-site/content/docs/integrations/context-sources.mdx` -- Modify: `docs-site/content/docs/concepts/*.mdx` -- Modify: `docs-site/content/docs/benchmarks/link-detection.mdx` - -- [ ] **Step 1: Normalize integrations** - -Add structured sections for supported values, config snippets, authentication, generated files, and recovery notes. Keep existing examples aligned with current KTX commands. - -- [ ] **Step 2: Add agent usage notes** - -For concept and benchmark pages, add a compact `## Agent usage notes` section that tells agents when the page is relevant and which concrete page to read next. - -- [ ] **Step 3: Build check** - -Run: `pnpm --filter ktx-docs build` - -Expected: MDX compiles without syntax errors. - -### Task 6: Route Verification and Final Checks - -**Files:** -- No required source changes unless verification finds a bug. - -- [ ] **Step 1: Run production build** - -Run: `pnpm --filter ktx-docs build` - -Expected: Build succeeds. - -- [ ] **Step 2: Run TypeScript check** - -Run: `pnpm --filter ktx-docs exec tsc --noEmit` - -Expected: TypeScript exits successfully. - -- [ ] **Step 3: Start local server** - -Run: `pnpm --filter ktx-docs start` - -Expected: Server starts on an available port. - -- [ ] **Step 4: Verify machine-readable routes** - -Run: - -```bash -curl -i http://localhost:3000/llms.txt -curl -i http://localhost:3000/llms-full.txt -curl -i http://localhost:3000/docs/getting-started/quickstart.md -curl -i -H "Accept: text/markdown" http://localhost:3000/docs/getting-started/quickstart -curl -i http://localhost:3000/docs/not-a-page.md -``` - -Expected: - -- `/llms.txt`: `200`, `Content-Type: text/plain; charset=utf-8` -- `/llms-full.txt`: `200`, `Content-Type: text/plain; charset=utf-8` -- `/docs/getting-started/quickstart.md`: `200`, `Content-Type: text/markdown; charset=utf-8` -- `/docs/getting-started/quickstart` with `Accept: text/markdown`: `200`, `Content-Type: text/markdown; charset=utf-8` -- `/docs/not-a-page.md`: `404` - -- [ ] **Step 5: Inspect final diff** - -Run: `git diff --stat && git diff --check` - -Expected: Diff contains only docs-site and plan changes, with no whitespace errors. diff --git a/docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md b/docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md deleted file mode 100644 index 5a523605..00000000 --- a/docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md +++ /dev/null @@ -1,1144 +0,0 @@ -# Bundled Python Runtime Wheel Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build and package one bundled `kaelio-ktx` Python wheel that contains -KTX-owned Python runtime code and keeps local embedding dependencies optional. - -**Architecture:** Add a deterministic Node assembly script that copies the -existing `semantic_layer` and `ktx_daemon` source trees into a temporary wheel -source tree, writes a runtime-only `pyproject.toml`, and builds one wheel with -`uv build`. Wire package artifacts so the CLI npm tarball includes the bundled -wheel plus a checksum manifest under `assets/python/`. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, `uv`, Hatchling, Python 3.13, -pnpm, TypeScript package artifacts. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. -There are no committed plan files under `docs/superpowers/plans/` in this -worktree or in git history for this spec. The spec itself is the only tracked -Superpowers document. - -The following pieces are already implemented: - -- `packages/context/src/daemon/semantic-layer-compute.ts` can invoke - `python -m ktx_daemon` for one-shot semantic-layer operations. -- `python/ktx-daemon` exposes `ktx-daemon` one-shot commands and an HTTP - `serve-http` daemon with `/health`. -- `scripts/package-artifacts.mjs` builds npm package tarballs and separate - `ktx-sl` and `ktx-daemon` Python artifacts. -- `scripts/package-artifacts.mjs` writes a checksummed artifact manifest. - -The following spec requirements are not implemented yet: - -- A single public `@kaelio/ktx` npm surface. -- One KTX-owned bundled Python wheel inside the npm package. -- A managed runtime root, installer, runtime manifest, and runtime command - family. -- Lazy `local-embeddings` installation that keeps `sentence-transformers` and - `torch` out of the default Python dependency set. - -This plan implements the bundled wheel prerequisite. Runtime install commands -must be planned after this lands because they need a real wheel payload and -checksum manifest to install. - -## File structure - -- Create `scripts/build-python-runtime-wheel.mjs`: assembles the temporary - runtime wheel source tree and runs `uv build`. -- Create `scripts/build-python-runtime-wheel.test.mjs`: tests source copying, - generated `pyproject.toml`, and the `uv build` command shape. -- Modify `scripts/package-artifacts.mjs`: builds the runtime wheel before npm - packing, copies it into `packages/cli/assets/python/`, includes it in the - artifact manifest, and installs it in artifact smoke tests. -- Modify `scripts/package-artifacts.test.mjs`: covers runtime wheel metadata, - manifest entries, install arguments, and CLI asset copy behavior. -- Modify `scripts/release-readiness.test.mjs`: expects `kaelio-ktx` in Python - release metadata and policy fixtures. -- Modify `release-policy.json`: lists `kaelio-ktx` as a CI-only Python - artifact. -- Modify `python/ktx-daemon/pyproject.toml`: moves - `sentence-transformers` and `torch` to a `local-embeddings` optional - dependency group. -- Modify `uv.lock`: records the dependency metadata change. -- Modify `.gitignore`: ignores generated `packages/cli/assets/python/` - contents. - -## Plan status - -No earlier plans were found for this spec. This is plan 1 for the spec. - -### Task 1: Add failing tests for the runtime wheel builder - -**Files:** - -- Create: `scripts/build-python-runtime-wheel.test.mjs` -- Test: `scripts/build-python-runtime-wheel.test.mjs` - -- [ ] **Step 1: Write the failing test file** - -Create `scripts/build-python-runtime-wheel.test.mjs` with this content: - -```javascript -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; - -import { - RUNTIME_WHEEL_DISTRIBUTION_NAME, - RUNTIME_WHEEL_PACKAGE_VERSION, - createRuntimeWheelBuildTree, - runtimeWheelBuildCommand, - runtimeWheelLayout, - runtimeWheelPyproject, -} from './build-python-runtime-wheel.mjs'; - -async function writeRuntimeSourceFixture(root) { - await mkdir(join(root, 'python', 'ktx-sl', 'semantic_layer'), { - recursive: true, - }); - await mkdir(join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), { - recursive: true, - }); - - await writeFile( - join(root, 'python', 'ktx-sl', 'semantic_layer', '__init__.py'), - 'SEMANTIC_LAYER_FIXTURE = True\n', - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__init__.py'), - 'KTX_DAEMON_FIXTURE = True\n', - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'src', 'ktx_daemon', '__main__.py'), - 'def main():\n return 0\n', - ); -} - -describe('runtimeWheelLayout', () => { - it('uses stable source, build, and output paths', () => { - const layout = runtimeWheelLayout('/repo/ktx'); - - assert.equal(layout.rootDir, '/repo/ktx'); - assert.equal(layout.semanticLayerSourceDir, '/repo/ktx/python/ktx-sl/semantic_layer'); - assert.equal(layout.daemonSourceDir, '/repo/ktx/python/ktx-daemon/src/ktx_daemon'); - assert.equal(layout.buildRoot, '/repo/ktx/dist/runtime-wheel-src'); - assert.equal(layout.outputDir, '/repo/ktx/dist/artifacts/python'); - }); -}); - -describe('runtimeWheelPyproject', () => { - it('describes one kaelio-ktx wheel with lazy local embeddings', () => { - const pyproject = runtimeWheelPyproject(); - - assert.match(pyproject, /name = "kaelio-ktx"/); - assert.match(pyproject, /version = "0\.1\.0"/); - assert.match(pyproject, /ktx-daemon = "ktx_daemon\.__main__:main"/); - assert.match(pyproject, /packages = \["semantic_layer", "ktx_daemon"\]/); - assert.match(pyproject, /\[project\.optional-dependencies\]/); - assert.match(pyproject, /local-embeddings = \[/); - assert.match(pyproject, /"sentence-transformers>=5\.1\.1"/); - assert.match(pyproject, /"torch>=2\.2\.0"/); - assert.doesNotMatch( - pyproject.match(/dependencies = \[[\s\S]*?\]/)?.[0] ?? '', - /sentence-transformers|torch/, - ); - }); -}); - -describe('createRuntimeWheelBuildTree', () => { - it('copies KTX-owned Python packages into the build tree', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-wheel-test-')); - try { - await writeRuntimeSourceFixture(root); - const layout = runtimeWheelLayout(root); - - await createRuntimeWheelBuildTree(layout); - - assert.equal( - await readFile(join(layout.buildRoot, 'semantic_layer', '__init__.py'), 'utf8'), - 'SEMANTIC_LAYER_FIXTURE = True\n', - ); - assert.equal( - await readFile(join(layout.buildRoot, 'ktx_daemon', '__main__.py'), 'utf8'), - 'def main():\n return 0\n', - ); - const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); - assert.match(pyproject, /name = "kaelio-ktx"/); - assert.match(pyproject, /local-embeddings = \[/); - const readme = await readFile(join(layout.buildRoot, 'README.md'), 'utf8'); - assert.match(readme, /Bundled Python runtime wheel for KTX/); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); - -describe('runtimeWheelBuildCommand', () => { - it('runs uv build against the generated build tree', () => { - const layout = runtimeWheelLayout('/repo/ktx'); - - assert.deepEqual(runtimeWheelBuildCommand(layout), { - command: 'uv', - args: [ - 'build', - '--wheel', - '--out-dir', - '/repo/ktx/dist/artifacts/python', - '/repo/ktx/dist/runtime-wheel-src', - ], - cwd: '/repo/ktx', - }); - assert.equal(RUNTIME_WHEEL_DISTRIBUTION_NAME, 'kaelio-ktx'); - assert.equal(RUNTIME_WHEEL_PACKAGE_VERSION, '0.1.0'); - }); -}); -``` - -- [ ] **Step 2: Run the failing test** - -Run: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs -``` - -Expected: FAIL with an import error for -`./build-python-runtime-wheel.mjs`. - -### Task 2: Implement the runtime wheel builder - -**Files:** - -- Create: `scripts/build-python-runtime-wheel.mjs` -- Test: `scripts/build-python-runtime-wheel.test.mjs` - -- [ ] **Step 1: Create the builder script** - -Create `scripts/build-python-runtime-wheel.mjs` with this content: - -```javascript -#!/usr/bin/env node - -import { execFile } from 'node:child_process'; -import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); - -export const RUNTIME_WHEEL_DISTRIBUTION_NAME = 'kaelio-ktx'; -export const RUNTIME_WHEEL_NORMALIZED_NAME = 'kaelio_ktx'; -export const RUNTIME_WHEEL_PACKAGE_VERSION = '0.1.0'; - -function scriptRootDir() { - return resolve(dirname(fileURLToPath(import.meta.url)), '..'); -} - -export function runtimeWheelLayout(rootDir = scriptRootDir()) { - return { - rootDir, - semanticLayerSourceDir: join(rootDir, 'python', 'ktx-sl', 'semantic_layer'), - daemonSourceDir: join(rootDir, 'python', 'ktx-daemon', 'src', 'ktx_daemon'), - buildRoot: join(rootDir, 'dist', 'runtime-wheel-src'), - outputDir: join(rootDir, 'dist', 'artifacts', 'python'), - }; -} - -export function runtimeWheelPyproject() { - return `[project] -name = "${RUNTIME_WHEEL_DISTRIBUTION_NAME}" -version = "${RUNTIME_WHEEL_PACKAGE_VERSION}" -description = "Bundled Python runtime payload for the KTX npm package" -readme = "README.md" -requires-python = ">=3.13" -license = "Apache-2.0" -dependencies = [ - "fastapi>=0.115.0", - "lkml>=1.3.7", - "numpy>=2.2.6", - "orjson>=3.11.4", - "pandas>=2.2.3", - "psycopg[binary]>=3.2.0", - "pydantic>=2.9.0", - "pyyaml>=6", - "requests>=2.32.0", - "sqlglot>=26", - "uvicorn[standard]>=0.32.0", -] - -[project.optional-dependencies] -local-embeddings = [ - "sentence-transformers>=5.1.1", - "torch>=2.2.0", -] - -[project.scripts] -ktx-daemon = "ktx_daemon.__main__:main" - -[project.urls] -Homepage = "https://github.com/kaelio/ktx" -Repository = "https://github.com/kaelio/ktx" -Issues = "https://github.com/kaelio/ktx/issues" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["semantic_layer", "ktx_daemon"] -`; -} - -export function runtimeWheelReadme() { - return `# kaelio-ktx Python runtime - -Bundled Python runtime wheel for KTX. - -This wheel is built from the repository's \`semantic_layer\` and -\`ktx_daemon\` source trees for inclusion in the npm package. It is not a -separate public PyPI release artifact. -`; -} - -export async function createRuntimeWheelBuildTree(layout = runtimeWheelLayout()) { - await rm(layout.buildRoot, { recursive: true, force: true }); - await mkdir(layout.buildRoot, { recursive: true }); - await cp(layout.semanticLayerSourceDir, join(layout.buildRoot, 'semantic_layer'), { - recursive: true, - }); - await cp(layout.daemonSourceDir, join(layout.buildRoot, 'ktx_daemon'), { - recursive: true, - }); - await writeFile(join(layout.buildRoot, 'pyproject.toml'), runtimeWheelPyproject()); - await writeFile(join(layout.buildRoot, 'README.md'), runtimeWheelReadme()); -} - -export function runtimeWheelBuildCommand(layout = runtimeWheelLayout()) { - return { - command: 'uv', - args: ['build', '--wheel', '--out-dir', layout.outputDir, layout.buildRoot], - cwd: layout.rootDir, - }; -} - -async function runCommand(command, args, options) { - const result = await execFileAsync(command, args, { - cwd: options.cwd, - encoding: 'utf8', - maxBuffer: 1024 * 1024 * 20, - }); - if (result.stdout) { - process.stdout.write(result.stdout); - } - if (result.stderr) { - process.stderr.write(result.stderr); - } -} - -export async function buildRuntimeWheel(layout = runtimeWheelLayout()) { - await mkdir(layout.outputDir, { recursive: true }); - await createRuntimeWheelBuildTree(layout); - const command = runtimeWheelBuildCommand(layout); - await runCommand(command.command, command.args, { cwd: command.cwd }); - const pyproject = await readFile(join(layout.buildRoot, 'pyproject.toml'), 'utf8'); - return { - buildRoot: layout.buildRoot, - outputDir: layout.outputDir, - pyproject, - }; -} - -async function main() { - await buildRuntimeWheel(runtimeWheelLayout()); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - try { - await main(); - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); - process.exitCode = 1; - } -} -``` - -- [ ] **Step 2: Run the builder test** - -Run: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 3: Commit the builder** - -Run: - -```bash -git add scripts/build-python-runtime-wheel.mjs scripts/build-python-runtime-wheel.test.mjs -git commit -m "build: add bundled python runtime wheel builder" -``` - -### Task 3: Move heavy local embedding dependencies behind an extra - -**Files:** - -- Modify: `python/ktx-daemon/pyproject.toml` -- Modify: `uv.lock` -- Test: `python/ktx-daemon/tests/test_embeddings.py` -- Test: `scripts/build-python-runtime-wheel.test.mjs` - -- [ ] **Step 1: Update daemon dependencies** - -In `python/ktx-daemon/pyproject.toml`, remove these two lines from -`[project].dependencies`: - -```toml - "sentence-transformers>=5.1.1", - "torch>=2.2.0", -``` - -Add this block immediately after `[project.scripts]`: - -```toml -[project.optional-dependencies] -local-embeddings = [ - "sentence-transformers>=5.1.1", - "torch>=2.2.0", -] -``` - -The relevant section must read: - -```toml -[project] -name = "ktx-daemon" -version = "0.1.0" -description = "Portable compute package for KTX semantic-layer operations" -readme = "README.md" -requires-python = ">=3.13" -license = "Apache-2.0" -dependencies = [ - "fastapi>=0.115.0", - "ktx-sl", - "lkml>=1.3.7", - "numpy>=2.2.6", - "orjson>=3.11.4", - "pandas>=2.2.3", - "psycopg[binary]>=3.2.0", - "pydantic>=2.9.0", - "requests>=2.32.0", - "sqlglot>=26", - "uvicorn[standard]>=0.32.0", -] - -[project.scripts] -ktx-daemon = "ktx_daemon.__main__:main" - -[project.optional-dependencies] -local-embeddings = [ - "sentence-transformers>=5.1.1", - "torch>=2.2.0", -] -``` - -- [ ] **Step 2: Refresh the uv lockfile** - -Run: - -```bash -uv lock -``` - -Expected: PASS and `uv.lock` records the `ktx-daemon` optional dependency -metadata. If the local `uv` version is older than `tool.uv.required-version`, -record the version mismatch and do not edit `pyproject.toml` to lower the pin. - -- [ ] **Step 3: Run Python tests that cover lazy embedding imports** - -Run: - -```bash -uv run pytest python/ktx-daemon/tests/test_embeddings.py -q -``` - -Expected: PASS. The tests use injected fake providers and do not require -`sentence-transformers` or `torch`. - -- [ ] **Step 4: Run the runtime wheel metadata test** - -Run: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs -``` - -Expected: PASS and the generated runtime `pyproject.toml` keeps -`sentence-transformers` and `torch` under `local-embeddings`. - -- [ ] **Step 5: Commit the dependency split** - -Run: - -```bash -git add python/ktx-daemon/pyproject.toml uv.lock -git commit -m "build: make local embedding dependencies optional" -``` - -### Task 4: Add artifact tests for the bundled runtime wheel - -**Files:** - -- Modify: `scripts/package-artifacts.test.mjs` -- Test: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Extend imports** - -In `scripts/package-artifacts.test.mjs`, extend the import from -`./package-artifacts.mjs` with these names: - -```javascript - CLI_PYTHON_ASSET_MANIFEST, - RUNTIME_WHEEL_DISTRIBUTION_NAME, - RUNTIME_WHEEL_NORMALIZED_NAME, - RUNTIME_WHEEL_PACKAGE_VERSION, - copyRuntimeWheelAssets, -``` - -- [ ] **Step 2: Update Python metadata fixtures** - -In `writeReleaseMetadataInputs`, keep the existing `ktx-sl` and `ktx-daemon` -fixture files and add no new on-disk Python package. The runtime wheel metadata -will come from constants exported by `package-artifacts.mjs`. - -- [ ] **Step 3: Update uploadable artifact fixtures** - -In `writeUploadableArtifactFixtures`, add this runtime wheel entry to -`fileContents`: - -```javascript - [ - join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - 'kaelio-ktx-runtime-wheel', - ], -``` - -- [ ] **Step 4: Update build command expectations** - -Replace the `buildArtifactCommands` expectations with these three assertions: - -```javascript - assert.deepEqual( - commands.slice(0, NPM_ARTIFACT_PACKAGES.length).map((command) => [command.command, command.args]), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => ['pnpm', ['--filter', packageInfo.name, 'run', 'build']]), - ); - assert.deepEqual( - commands - .slice(NPM_ARTIFACT_PACKAGES.length, NPM_ARTIFACT_PACKAGES.length + 3) - .map((command) => [command.command, command.args]), - [ - [ - process.execPath, - ['scripts/build-python-runtime-wheel.mjs'], - ], - [ - 'uv', - ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python'], - ], - [ - 'uv', - ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python'], - ], - ], - ); - assert.deepEqual( - commands.slice(NPM_ARTIFACT_PACKAGES.length + 3).map((command) => [command.command, command.args]), - NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ - 'pnpm', - ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], - ]), - ); -``` - -- [ ] **Step 5: Update release metadata expectations** - -In the `packageReleaseMetadata` test, add this Python metadata entry after -`ktx-daemon`: - -```javascript - { - ecosystem: 'python', - packageName: 'kaelio-ktx', - packageRoot: 'python/runtime-wheel', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, -``` - -- [ ] **Step 6: Update Python artifact discovery expectations** - -In the `findPythonArtifacts` test, create the runtime wheel fixture: - -```javascript - await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); -``` - -Then update the expected object: - -```javascript - assert.deepEqual(await findPythonArtifacts(root), { - runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - ktxSlWheel: join(root, 'ktx_sl-0.1.0-py3-none-any.whl'), - ktxSlSdist: join(root, 'ktx_sl-0.1.0.tar.gz'), - ktxDaemonWheel: join(root, 'ktx_daemon-0.1.0-py3-none-any.whl'), - ktxDaemonSdist: join(root, 'ktx_daemon-0.1.0.tar.gz'), - }); -``` - -- [ ] **Step 7: Update manifest file count expectations** - -In the `verifyArtifactManifest` test, replace: - -```javascript - assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 4); -``` - -with: - -```javascript - assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 5); -``` - -- [ ] **Step 8: Add CLI asset copy test** - -Add this test near the other artifact helper tests: - -```javascript -describe('copyRuntimeWheelAssets', () => { - it('copies the runtime wheel and checksum manifest into CLI assets', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-assets-test-')); - const layout = packageArtifactLayout(root); - try { - await mkdir(layout.pythonDir, { recursive: true }); - await writeFile( - join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - 'kaelio-ktx-runtime-wheel', - ); - - const assets = await copyRuntimeWheelAssets(layout, { - runtimeWheel: join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - }); - - assert.equal( - assets.wheelPath, - join(root, 'packages', 'cli', 'assets', 'python', 'kaelio_ktx-0.1.0-py3-none-any.whl'), - ); - assert.equal( - assets.manifestPath, - join(root, 'packages', 'cli', 'assets', 'python', CLI_PYTHON_ASSET_MANIFEST), - ); - const manifest = JSON.parse(await readFile(assets.manifestPath, 'utf8')); - assert.deepEqual(manifest, { - schemaVersion: 1, - distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, - normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, - version: RUNTIME_WHEEL_PACKAGE_VERSION, - wheel: { - file: 'kaelio_ktx-0.1.0-py3-none-any.whl', - sha256: createHash('sha256') - .update('kaelio-ktx-runtime-wheel') - .digest('hex'), - bytes: Buffer.byteLength('kaelio-ktx-runtime-wheel'), - }, - }); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); -``` - -- [ ] **Step 9: Update install argument test** - -Replace the `pythonArtifactInstallArgs` expectation with one runtime wheel: - -```javascript - assert.deepEqual(args, [ - 'pip', - 'install', - '--python', - '/tmp/smoke/.venv/bin/python', - '/repo/ktx/dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl', - ]); - assert.equal(args.includes('ktx-daemon'), false); - assert.equal(args.includes('ktx-sl'), false); - assert.equal(args.includes('--find-links'), false); -``` - -- [ ] **Step 10: Run the failing package artifact tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: FAIL with missing exports from `scripts/package-artifacts.mjs`. - -### Task 5: Wire the runtime wheel into artifact packaging - -**Files:** - -- Modify: `scripts/package-artifacts.mjs` -- Modify: `scripts/package-artifacts.test.mjs` -- Test: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Import runtime wheel builder constants** - -Add this import near the top of `scripts/package-artifacts.mjs`: - -```javascript -import { - RUNTIME_WHEEL_DISTRIBUTION_NAME, - RUNTIME_WHEEL_NORMALIZED_NAME, - RUNTIME_WHEEL_PACKAGE_VERSION, -} from './build-python-runtime-wheel.mjs'; -``` - -Then re-export those constants after the existing constants: - -```javascript -export { - RUNTIME_WHEEL_DISTRIBUTION_NAME, - RUNTIME_WHEEL_NORMALIZED_NAME, - RUNTIME_WHEEL_PACKAGE_VERSION, -}; -``` - -- [ ] **Step 2: Add CLI asset manifest constant** - -Add this constant after `PYTHON_PACKAGE_VERSION`: - -```javascript -export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; -``` - -- [ ] **Step 3: Change build command order** - -Replace `buildArtifactCommands(layout)` with this implementation: - -```javascript -export function buildArtifactCommands(layout) { - const npmBuildCommands = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - command: 'pnpm', - args: ['--filter', packageInfo.name, 'run', 'build'], - cwd: layout.rootDir, - })); - const npmPackCommands = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - command: 'pnpm', - args: ['--filter', packageInfo.name, 'pack', '--out', layout.npmTarballs[packageInfo.name]], - cwd: layout.rootDir, - })); - - return [ - ...npmBuildCommands, - { - command: process.execPath, - args: ['scripts/build-python-runtime-wheel.mjs'], - cwd: layout.rootDir, - }, - { - command: 'uv', - args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, - { - command: 'uv', - args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, - ...npmPackCommands, - ]; -} -``` - -- [ ] **Step 4: Discover the runtime wheel** - -Update `findPythonArtifacts(pythonDir)` to return `runtimeWheel`: - -```javascript -export async function findPythonArtifacts(pythonDir) { - const files = await readdir(pythonDir); - - return { - runtimeWheel: findOne( - files, - RUNTIME_WHEEL_DISTRIBUTION_NAME, - '.whl', - 'kaelio-ktx runtime wheel', - pythonDir, - RUNTIME_WHEEL_PACKAGE_VERSION, - ), - ktxSlWheel: findOne(files, 'ktx-sl', '.whl', 'ktx-sl wheel', pythonDir), - ktxSlSdist: findOne(files, 'ktx-sl', '.tar.gz', 'ktx-sl source distribution', pythonDir), - ktxDaemonWheel: findOne(files, 'ktx-daemon', '.whl', 'ktx-daemon wheel', pythonDir), - ktxDaemonSdist: findOne(files, 'ktx-daemon', '.tar.gz', 'ktx-daemon source distribution', pythonDir), - }; -} -``` - -Change `findOne` to accept an optional version: - -```javascript -function findOne(files, distributionName, suffix, label, pythonDir, version = PYTHON_PACKAGE_VERSION) { - const normalized = normalizePythonDistributionName(distributionName); - const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); - if (!found) { - throw new Error(`Missing Python artifact: ${label}`); - } - return join(pythonDir, found); -} -``` - -- [ ] **Step 5: Add runtime wheel release metadata** - -In `packageReleaseMetadata`, append this entry after `ktxDaemonPackage`: - -```javascript - releaseMetadataEntry({ - ecosystem: 'python', - packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, - packageRoot: 'python/runtime-wheel', - packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION, - privatePackage: false, - }), -``` - -- [ ] **Step 6: Add runtime wheel to artifact manifest records** - -In `artifactPackageRecords`, add this record after npm records: - -```javascript - { - artifactKind: 'wheel', - artifactPath: pythonArtifacts.runtimeWheel, - metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), - }, -``` - -- [ ] **Step 7: Add CLI Python asset copy helper** - -Add this function before `pythonArtifactInstallArgs`: - -```javascript -function runtimeWheelAssetName(runtimeWheelPath) { - return runtimeWheelPath.split(sep).at(-1); -} - -export async function copyRuntimeWheelAssets(layout, pythonArtifacts) { - const assetDir = join(layout.rootDir, 'packages', 'cli', 'assets', 'python'); - const wheelFile = runtimeWheelAssetName(pythonArtifacts.runtimeWheel); - if (!wheelFile) { - throw new Error(`Unable to determine runtime wheel filename: ${pythonArtifacts.runtimeWheel}`); - } - const wheelContents = await readFile(pythonArtifacts.runtimeWheel); - await rm(assetDir, { recursive: true, force: true }); - await mkdir(assetDir, { recursive: true }); - const wheelPath = join(assetDir, wheelFile); - const manifestPath = join(assetDir, CLI_PYTHON_ASSET_MANIFEST); - await writeFile(wheelPath, wheelContents); - await writeFile( - manifestPath, - `${JSON.stringify( - { - schemaVersion: 1, - distributionName: RUNTIME_WHEEL_DISTRIBUTION_NAME, - normalizedName: RUNTIME_WHEEL_NORMALIZED_NAME, - version: RUNTIME_WHEEL_PACKAGE_VERSION, - wheel: { - file: wheelFile, - sha256: createHash('sha256').update(wheelContents).digest('hex'), - bytes: wheelContents.byteLength, - }, - }, - null, - 2, - )}\n`, - ); - return { assetDir, wheelPath, manifestPath }; -} -``` - -- [ ] **Step 8: Install the runtime wheel in artifact smokes** - -Replace `pythonArtifactInstallArgs` with: - -```javascript -export function pythonArtifactInstallArgs(python, pythonArtifacts) { - return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel]; -} -``` - -Update `pythonVerifySource()` to assert `kaelio-ktx` metadata and keep module -imports: - -```javascript -export function pythonVerifySource() { - return ` -import importlib.metadata - -import semantic_layer -import ktx_daemon - -assert importlib.metadata.version("kaelio-ktx") == "0.1.0" -assert semantic_layer is not None -assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" -`; -} -``` - -- [ ] **Step 9: Copy runtime assets before npm packing** - -Replace the loop in `buildArtifacts(layout)` with these explicit phases: - -```javascript - const commands = buildArtifactCommands(layout); - const npmBuildCount = NPM_ARTIFACT_PACKAGES.length; - const npmPackStart = commands.length - NPM_ARTIFACT_PACKAGES.length; - - for (const command of commands.slice(0, npmBuildCount)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } - for (const command of commands.slice(npmBuildCount, npmPackStart)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } - const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); - await copyRuntimeWheelAssets(layout, pythonArtifacts); - for (const command of commands.slice(npmPackStart)) { - await runCommand(command.command, command.args, { cwd: command.cwd }); - } -``` - -- [ ] **Step 10: Run package artifact tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 11: Commit artifact wiring** - -Run: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -git commit -m "build: bundle python runtime wheel in cli artifacts" -``` - -### Task 6: Update release policy and generated asset ignores - -**Files:** - -- Modify: `release-policy.json` -- Modify: `.gitignore` -- Modify: `scripts/release-readiness.test.mjs` -- Test: `scripts/release-readiness.test.mjs` - -- [ ] **Step 1: Ignore generated CLI Python assets** - -Add this block to `.gitignore` after the `dist/` ignore: - -```gitignore -packages/cli/assets/python/ -``` - -- [ ] **Step 2: Add runtime wheel to release policy** - -Update `release-policy.json` so the Python packages list is: - -```json - "python": { - "publish": false, - "repository": null, - "packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"] - }, -``` - -- [ ] **Step 3: Update release readiness fixtures** - -In `scripts/release-readiness.test.mjs`, update fixture policy objects that -list Python packages from: - -```javascript -packages: ['ktx-sl', 'ktx-daemon'], -``` - -to: - -```javascript -packages: ['ktx-sl', 'ktx-daemon', 'kaelio-ktx'], -``` - -Update expected package name arrays to include `kaelio-ktx`: - -```javascript -packageNames: [ - ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => packageInfo.name), - 'ktx-sl', - 'ktx-daemon', - 'kaelio-ktx', -], -``` - -- [ ] **Step 4: Run release readiness tests** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 5: Commit policy updates** - -Run: - -```bash -git add .gitignore release-policy.json scripts/release-readiness.test.mjs -git commit -m "build: track bundled python runtime release artifact" -``` - -### Task 7: Verify the built runtime wheel end to end - -**Files:** - -- Build output: `dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl` -- Build output: `packages/cli/assets/python/manifest.json` -- Build output: - `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` - -- [ ] **Step 1: Run focused script tests** - -Run: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 2: Run Python package tests affected by dependency split** - -Run: - -```bash -uv run pytest python/ktx-daemon/tests -q -``` - -Expected: PASS. - -- [ ] **Step 3: Run package artifact check** - -Run: - -```bash -pnpm run artifacts:check -``` - -Expected: PASS. This command builds the runtime wheel, copies it into CLI -assets before npm packing, installs the packed npm packages in a clean smoke -project, installs the bundled runtime wheel with `uv pip install`, and verifies -`semantic_layer` plus `ktx_daemon` imports from the one `kaelio-ktx` wheel. - -- [ ] **Step 4: Inspect the generated CLI asset manifest** - -Run: - -```bash -node -e "const fs=require('node:fs'); const m=JSON.parse(fs.readFileSync('packages/cli/assets/python/manifest.json','utf8')); console.log(m.distributionName, m.version, m.wheel.file, m.wheel.sha256.length)" -``` - -Expected output: - -```text -kaelio-ktx 0.1.0 kaelio_ktx-0.1.0-py3-none-any.whl 64 -``` - -- [ ] **Step 5: Run pre-commit when configured** - -Run this only if `.pre-commit-config.yaml` exists: - -```bash -uv run pre-commit run --files python/ktx-daemon/pyproject.toml uv.lock pyproject.toml scripts/build-python-runtime-wheel.mjs scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs release-policy.json .gitignore -``` - -Expected: PASS. If no pre-commit config exists, record that no pre-commit -configuration exists in this repository and skip this command. - -- [ ] **Step 6: Commit verification-only updates if any** - -If verification required small code or test fixes, commit them: - -```bash -git add scripts/build-python-runtime-wheel.mjs scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs python/ktx-daemon/pyproject.toml uv.lock release-policy.json .gitignore -git commit -m "test: verify bundled python runtime wheel" -``` - -If no files changed after verification, do not create an empty commit. - -## Acceptance criteria - -- `dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl` is built by - `pnpm run artifacts:check`. -- The built CLI npm tarball includes - `assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and - `assets/python/manifest.json`. -- The asset manifest records the wheel filename, byte count, and SHA-256. -- Installing only the bundled runtime wheel exposes `semantic_layer`, - `ktx_daemon`, and the `ktx-daemon` console script. -- `sentence-transformers` and `torch` are absent from default dependencies and - present under the `local-embeddings` extra. -- Existing separate `ktx-sl` and `ktx-daemon` artifacts can remain CI artifacts - in this plan; the npm runtime payload uses `kaelio-ktx`. - -## Self-review - -Spec coverage: - -- Covers the package-model requirement for one bundled KTX-owned Python wheel. -- Covers the wheel checksum or runtime manifest requirement by adding the npm - asset manifest. -- Covers lazy local embedding dependencies by moving heavy packages into the - `local-embeddings` extra. -- Leaves managed runtime directories, install commands, daemon reuse, and - `@kaelio/ktx` npm renaming for later plans. - -Placeholder scan: - -- The plan contains no placeholder markers and no unspecified implementation - steps. - -Type and name consistency: - -- Runtime distribution name is consistently `kaelio-ktx`. -- Wheel filename prefix is consistently `kaelio_ktx`. -- Runtime version is consistently `0.1.0`. diff --git a/docs/superpowers/plans/2026-05-11-demo-guided-tour.md b/docs/superpowers/plans/2026-05-11-demo-guided-tour.md deleted file mode 100644 index 92b2c6c0..00000000 --- a/docs/superpowers/plans/2026-05-11-demo-guided-tour.md +++ /dev/null @@ -1,813 +0,0 @@ -# Demo Guided Tour Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the disconnected "Try KTX with packaged demo data" flow with a guided tour that walks users through the same setup wizard steps using pre-filled, read-only selections, then connects their agent to the populated demo project. - -**Architecture:** A new `setup-demo-tour.ts` module owns the demo tour flow. It renders read-only cards (database, sources), a simulated context build replay using the existing `renderContextBuildView` + `createRepainter` pipeline from `context-build-view.ts`, then hands off to the real `runKtxSetupAgentsStep`. The entry point in `setup.ts` (`runKtxSetupDemoFromEntryMenu`) is rewired to call this new module instead of `runKtxDemo`. - -**Tech Stack:** TypeScript (ESM), Node.js raw stdin for keypress handling, existing `@clack/prompts` visual patterns, vitest for tests. - ---- - -### Task 1: Create `setup-demo-tour.ts` with keypress utility and banner - -**Files:** -- Create: `packages/cli/src/setup-demo-tour.ts` -- Test: `packages/cli/src/setup-demo-tour.test.ts` - -- [ ] **Step 1: Write the failing test for `renderDemoBanner`** - -```typescript -// packages/cli/src/setup-demo-tour.test.ts -import { describe, expect, it } from 'vitest'; -import { renderDemoBanner } from './setup-demo-tour.js'; - -describe('renderDemoBanner', () => { - it('includes demo mode explanation', () => { - const output = renderDemoBanner(); - expect(output).toContain('Demo mode'); - expect(output).toContain('pre-processed'); - expect(output).toContain('read-only'); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL - module not found - -- [ ] **Step 3: Implement `renderDemoBanner` and `waitForDemoNavigation`** - -```typescript -// packages/cli/src/setup-demo-tour.ts -import type { KtxCliIo } from './cli-runtime.js'; -import { KtxSetupExitError } from './setup-interrupt.js'; - -const ESC = String.fromCharCode(0x1b); - -function cyan(text: string): string { - return `${ESC}[36m${text}${ESC}[39m`; -} - -function dim(text: string): string { - return `${ESC}[2m${text}${ESC}[22m`; -} - -export function renderDemoBanner(): string { - const lines = [ - '', - `┌ ${cyan('Demo mode')} - data has been pre-processed and KTX context is already built.`, - `│ This walkthrough illustrates the setup steps. Selections are pre-filled and read-only.`, - '', - ]; - return lines.join('\n'); -} - -export async function waitForDemoNavigation( - stdin: NodeJS.ReadStream = process.stdin, -): Promise<'forward' | 'back'> { - return new Promise((resolve, reject) => { - const wasRaw = stdin.isRaw; - if (stdin.setRawMode) stdin.setRawMode(true); - stdin.resume(); - - const onData = (data: Buffer) => { - const key = data.toString(); - if (key === '\r' || key === '\n') { - cleanup(); - resolve('forward'); - } else if (key === '\x1b') { - cleanup(); - resolve('back'); - } else if (key === '\x03') { - cleanup(); - reject(new KtxSetupExitError()); - } - }; - - const cleanup = () => { - stdin.off('data', onData); - if (stdin.setRawMode) stdin.setRawMode(wasRaw ?? false); - }; - - stdin.on('data', onData); - }); -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "feat(cli): add demo tour banner and keypress navigation utility" -``` - ---- - -### Task 2: Add `renderDemoCard` function - -**Files:** -- Modify: `packages/cli/src/setup-demo-tour.ts` -- Modify: `packages/cli/src/setup-demo-tour.test.ts` - -- [ ] **Step 1: Write the failing test for `renderDemoCard`** - -Append to the test file: - -```typescript -import { renderDemoCardContent } from './setup-demo-tour.js'; - -describe('renderDemoCardContent', () => { - it('renders a card with title and selections', () => { - const output = renderDemoCardContent('Database connection', ['PostgreSQL (demo warehouse)']); - expect(output).toContain('Database connection'); - expect(output).toContain('PostgreSQL (demo warehouse)'); - expect(output).toContain('Press Enter to continue'); - expect(output).toContain('Escape to go back'); - }); - - it('renders multiple selections', () => { - const output = renderDemoCardContent('Context sources', ['dbt', 'Metabase', 'Notion']); - expect(output).toContain('dbt'); - expect(output).toContain('Metabase'); - expect(output).toContain('Notion'); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL - `renderDemoCardContent` not exported - -- [ ] **Step 3: Implement `renderDemoCardContent` and `renderDemoCard`** - -Add to `setup-demo-tour.ts`: - -```typescript -export function renderDemoCardContent(title: string, selections: string[]): string { - const lines = [ - `┌ ${title}`, - '│', - ...selections.map((s) => `│ ${cyan('▸')} ${s}`), - '│', - `│ ${dim('Press Enter to continue, Escape to go back')}`, - '└', - '', - ]; - return lines.join('\n'); -} - -export async function renderDemoCard( - title: string, - selections: string[], - io: KtxCliIo, - stdin?: NodeJS.ReadStream, - waitNav?: (stdin?: NodeJS.ReadStream) => Promise<'forward' | 'back'>, -): Promise<'forward' | 'back'> { - io.stdout.write(renderDemoBanner()); - io.stdout.write(renderDemoCardContent(title, selections)); - const nav = waitNav ?? waitForDemoNavigation; - return nav(stdin); -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "feat(cli): add demo tour read-only card rendering" -``` - ---- - -### Task 3: Add demo context build replay animation - -**Files:** -- Modify: `packages/cli/src/setup-demo-tour.ts` -- Modify: `packages/cli/src/setup-demo-tour.test.ts` - -- [ ] **Step 1: Write the failing test for demo replay event sequence** - -Append to the test file: - -```typescript -import { buildDemoReplayTimeline, DEMO_REPLAY_TARGETS } from './setup-demo-tour.js'; - -describe('buildDemoReplayTimeline', () => { - it('produces events for all four demo targets', () => { - const events = buildDemoReplayTimeline(); - const connectionIds = new Set(events.map((e) => e.connectionId)); - expect(connectionIds).toEqual(new Set(['demo-warehouse', 'dbt', 'metabase', 'notion'])); - }); - - it('ends with all targets done', () => { - const events = buildDemoReplayTimeline(); - const lastByConnection = new Map(); - for (const e of events) { - lastByConnection.set(e.connectionId, e.status); - } - for (const status of lastByConnection.values()) { - expect(status).toBe('done'); - } - }); - - it('events are sorted by delayMs', () => { - const events = buildDemoReplayTimeline(); - for (let i = 1; i < events.length; i++) { - expect(events[i]!.delayMs).toBeGreaterThanOrEqual(events[i - 1]!.delayMs); - } - }); -}); - -describe('DEMO_REPLAY_TARGETS', () => { - it('has one primary source and three context sources', () => { - expect(DEMO_REPLAY_TARGETS.primarySources).toHaveLength(1); - expect(DEMO_REPLAY_TARGETS.contextSources).toHaveLength(3); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL - exports not found - -- [ ] **Step 3: Implement replay timeline and target definitions** - -Add to `setup-demo-tour.ts`: - -```typescript -import type { KtxPublicIngestPlanTarget } from './public-ingest.js'; -import type { ContextBuildTargetState, ContextBuildViewState } from './context-build-view.js'; - -export interface DemoReplayEvent { - delayMs: number; - connectionId: string; - status: 'running' | 'done'; - detailLine: string | null; - summaryText: string | null; -} - -function createDemoTarget(connectionId: string, operation: 'scan' | 'source-ingest', driver: string): KtxPublicIngestPlanTarget { - return { - connectionId, - driver, - operation, - debugCommand: `ktx ${operation === 'scan' ? 'scan' : 'ingest'} ${connectionId}`, - steps: operation === 'scan' ? ['scan'] : ['source-ingest'], - }; -} - -const primaryTarget = createDemoTarget('demo-warehouse', 'scan', 'postgres'); -const dbtTarget = createDemoTarget('dbt', 'source-ingest', 'dbt'); -const metabaseTarget = createDemoTarget('metabase', 'source-ingest', 'metabase'); -const notionTarget = createDemoTarget('notion', 'source-ingest', 'notion'); - -function createTargetState(target: KtxPublicIngestPlanTarget): ContextBuildTargetState { - return { - target, - status: 'queued', - detailLine: null, - summaryText: null, - startedAt: null, - elapsedMs: 0, - }; -} - -export const DEMO_REPLAY_TARGETS = { - primarySources: [primaryTarget], - contextSources: [dbtTarget, metabaseTarget, notionTarget], -}; - -export function buildDemoReplayTimeline(): DemoReplayEvent[] { - return [ - { delayMs: 0, connectionId: 'demo-warehouse', status: 'running', detailLine: 'scanning...', summaryText: null }, - { delayMs: 600, connectionId: 'demo-warehouse', status: 'running', detailLine: '[50%] scanning...', summaryText: null }, - { delayMs: 1200, connectionId: 'demo-warehouse', status: 'done', detailLine: null, summaryText: 'completed' }, - { delayMs: 1200, connectionId: 'dbt', status: 'running', detailLine: 'ingesting...', summaryText: null }, - { delayMs: 1800, connectionId: 'dbt', status: 'running', detailLine: '[60%] ingesting...', summaryText: null }, - { delayMs: 2200, connectionId: 'dbt', status: 'done', detailLine: null, summaryText: 'completed' }, - { delayMs: 2200, connectionId: 'metabase', status: 'running', detailLine: 'ingesting...', summaryText: null }, - { delayMs: 2800, connectionId: 'metabase', status: 'done', detailLine: null, summaryText: 'completed' }, - { delayMs: 2800, connectionId: 'notion', status: 'running', detailLine: 'ingesting...', summaryText: null }, - { delayMs: 3400, connectionId: 'notion', status: 'done', detailLine: null, summaryText: 'completed' }, - ]; -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: PASS - -- [ ] **Step 5: Implement `runDemoContextReplay` animation driver** - -Add to `setup-demo-tour.ts`: - -```typescript -import { renderContextBuildView, createRepainter } from './context-build-view.js'; - -export async function runDemoContextReplay( - io: KtxCliIo, - stdin?: NodeJS.ReadStream, -): Promise<'forward' | 'back'> { - const repainter = createRepainter(io); - const timeline = buildDemoReplayTimeline(); - - const state: ContextBuildViewState = { - primarySources: DEMO_REPLAY_TARGETS.primarySources.map((t) => createTargetState(t)), - contextSources: DEMO_REPLAY_TARGETS.contextSources.map((t) => createTargetState(t)), - frame: 0, - startedAt: Date.now(), - totalElapsedMs: 0, - }; - - const allTargets = [...state.primarySources, ...state.contextSources]; - const targetMap = new Map(allTargets.map((t) => [t.target.connectionId, t])); - let eventIndex = 0; - const startTime = Date.now(); - const FRAME_MS = 120; - - await new Promise((resolve) => { - const interval = setInterval(() => { - const elapsed = Date.now() - startTime; - state.frame += 1; - state.totalElapsedMs = elapsed; - - while (eventIndex < timeline.length && timeline[eventIndex]!.delayMs <= elapsed) { - const event = timeline[eventIndex]!; - const target = targetMap.get(event.connectionId); - if (target) { - target.status = event.status; - target.detailLine = event.detailLine; - target.summaryText = event.summaryText; - if (event.status === 'running' && target.startedAt === null) { - target.startedAt = Date.now(); - } - if (event.status === 'done') { - target.elapsedMs = target.startedAt ? Date.now() - target.startedAt : 0; - } - } - eventIndex += 1; - } - - for (const t of allTargets) { - if (t.status === 'running' && t.startedAt !== null) { - t.elapsedMs = Date.now() - t.startedAt; - } - } - - repainter.paint(renderContextBuildView(state, { styled: io.stdout.isTTY ?? false, showHint: false })); - - if (eventIndex >= timeline.length && allTargets.every((t) => t.status === 'done')) { - clearInterval(interval); - resolve(); - } - }, FRAME_MS); - }); - - io.stdout.write(renderDemoContextCompletionSummary()); - return waitForDemoNavigation(stdin); -} - -function renderDemoContextCompletionSummary(): string { - const lines = [ - '', - `${cyan('★')} KTX finished ingesting demo data`, - '', - ' Placeholder - final counts will come from pre-packaged demo results.', - '', - ` ${dim('Press Enter to continue, Escape to go back')}`, - '', - ]; - return lines.join('\n'); -} -``` - -Note: `renderDemoContextCompletionSummary` is a placeholder that will be updated when -the user provides the real pre-packaged demo data. The summary counts (business areas, -query definitions, knowledge pages) will be populated from those assets. - -- [ ] **Step 6: Run tests and type-check** - -Run: `pnpm --filter @ktx/cli run type-check && pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: PASS - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "feat(cli): add demo context build replay animation" -``` - ---- - -### Task 4: Add transition message and completion summary - -**Files:** -- Modify: `packages/cli/src/setup-demo-tour.ts` -- Modify: `packages/cli/src/setup-demo-tour.test.ts` - -- [ ] **Step 1: Write the failing tests** - -Append to test file: - -```typescript -import { renderDemoAgentTransition, renderDemoCompletionSummary } from './setup-demo-tour.js'; - -describe('renderDemoAgentTransition', () => { - it('includes transition message about connecting agent', () => { - const output = renderDemoAgentTransition(); - expect(output).toContain('Demo project is ready'); - expect(output).toContain('connect your agent'); - }); -}); - -describe('renderDemoCompletionSummary', () => { - it('includes project path and temp warning', () => { - const output = renderDemoCompletionSummary('/tmp/ktx-demo-abc123', true); - expect(output).toContain('/tmp/ktx-demo-abc123'); - expect(output).toContain('temporary'); - expect(output).toContain('ktx setup'); - }); - - it('shows manual agent instructions when agent not installed', () => { - const output = renderDemoCompletionSummary('/tmp/ktx-demo-abc123', false); - expect(output).toContain('ktx setup --agents'); - }); - - it('shows success message when agent installed', () => { - const output = renderDemoCompletionSummary('/tmp/ktx-demo-abc123', true); - expect(output).toContain('agent is connected'); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL - exports not found - -- [ ] **Step 3: Implement transition and completion rendering** - -Add to `setup-demo-tour.ts`: - -```typescript -export function renderDemoAgentTransition(): string { - const lines = [ - '', - `┌ Demo project is ready - let's connect your agent`, - '│', - '│ Your KTX context has been built with demo data.', - '│ Select an agent to start using it.', - '└', - '', - ]; - return lines.join('\n'); -} - -export function renderDemoCompletionSummary(projectDir: string, agentInstalled: boolean): string { - const lines = [ - '', - `${cyan('★')} KTX demo is ready`, - '', - ]; - - if (agentInstalled) { - lines.push(' Your agent is connected to a demo KTX project.'); - } else { - lines.push(' Demo project created. Connect an agent to start using it:'); - lines.push(` $ ktx setup --agents --project-dir ${projectDir}`); - } - - lines.push( - '', - ` ${dim('⚠')} This project is in a temporary directory and will be`, - ` cleaned up by your system. To set up KTX with your own`, - ' data, run: ktx setup', - '', - ` Project: ${projectDir}`, - '', - ); - return lines.join('\n'); -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "feat(cli): add demo tour transition and completion summary" -``` - ---- - -### Task 5: Implement `runDemoTour` orchestrator - -**Files:** -- Modify: `packages/cli/src/setup-demo-tour.ts` -- Modify: `packages/cli/src/setup-demo-tour.test.ts` - -- [ ] **Step 1: Write the failing test for the orchestrator** - -Append to test file: - -```typescript -import { vi } from 'vitest'; -import type { KtxSetupAgentsResult } from './setup-agents.js'; -import { runDemoTour } from './setup-demo-tour.js'; - -describe('runDemoTour', () => { - function createMockIo() { - const chunks: string[] = []; - return { - io: { - stdout: { isTTY: true, columns: 80, write: (chunk: string) => { chunks.push(chunk); } }, - stderr: { write: () => {} }, - }, - chunks, - }; - } - - it('returns 0 on successful tour with agent installed', async () => { - const { io } = createMockIo(); - const mockAgents = vi.fn<() => Promise>().mockResolvedValue({ - status: 'ready', - projectDir: '/tmp/test', - installs: [{ target: 'claude-code' as const, scope: 'project' as const, mode: 'both' as const }], - }); - - const navigation = vi.fn<() => Promise<'forward' | 'back'>>().mockResolvedValue('forward'); - - const result = await runDemoTour( - { inputMode: 'auto' }, - io, - { agents: mockAgents, waitForNavigation: navigation, skipReplayAnimation: true }, - ); - expect(result).toBe(0); - expect(mockAgents).toHaveBeenCalled(); - }); - - it('handles back navigation from first step', async () => { - const { io } = createMockIo(); - const navigation = vi.fn<() => Promise<'forward' | 'back'>>().mockResolvedValue('back'); - - const result = await runDemoTour( - { inputMode: 'auto' }, - io, - { waitForNavigation: navigation, skipReplayAnimation: true }, - ); - expect(result).toBe(0); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: FAIL - `runDemoTour` not exported or wrong signature - -- [ ] **Step 3: Implement `runDemoTour`** - -Add to `setup-demo-tour.ts`: - -```typescript -import { defaultDemoProjectDir, ensureSeededDemoProject } from './demo-assets.js'; -import type { KtxSetupAgentsResult } from './setup-agents.js'; -import { runKtxSetupAgentsStep } from './setup-agents.js'; - -type DemoStep = 'databases' | 'sources' | 'context' | 'agents'; - -const DEMO_STEPS: DemoStep[] = ['databases', 'sources', 'context', 'agents']; - -export interface DemoTourDeps { - agents?: (args: Parameters[0], io: KtxCliIo) => Promise; - waitForNavigation?: (stdin?: NodeJS.ReadStream) => Promise<'forward' | 'back'>; - ensureProject?: typeof ensureSeededDemoProject; - skipReplayAnimation?: boolean; -} - -export async function runDemoTour( - args: { inputMode: 'auto' | 'disabled' }, - io: KtxCliIo, - deps: DemoTourDeps = {}, -): Promise { - const waitNav = deps.waitForNavigation ?? waitForDemoNavigation; - const ensureProject = deps.ensureProject ?? ensureSeededDemoProject; - - const projectDir = defaultDemoProjectDir(); - await ensureProject({ projectDir }); - - let stepIndex = 0; - - while (stepIndex < DEMO_STEPS.length) { - const step = DEMO_STEPS[stepIndex]!; - let direction: 'forward' | 'back'; - - if (step === 'databases') { - direction = await renderDemoCard('Database connection', ['PostgreSQL (demo warehouse)'], io, undefined, waitNav); - } else if (step === 'sources') { - direction = await renderDemoCard('Context sources', ['dbt', 'Metabase', 'Notion'], io, undefined, waitNav); - } else if (step === 'context') { - io.stdout.write(renderDemoBanner()); - if (deps.skipReplayAnimation) { - direction = await waitNav(); - } else { - direction = await runDemoContextReplay(io); - } - } else { - io.stdout.write(renderDemoAgentTransition()); - const agentsRunner = deps.agents ?? runKtxSetupAgentsStep; - const agentsResult = await agentsRunner( - { - projectDir, - inputMode: args.inputMode, - yes: false, - agents: true, - scope: 'project', - mode: 'both', - skipAgents: false, - }, - io, - ); - const agentInstalled = agentsResult.status === 'ready'; - if (agentsResult.status === 'back') { - direction = 'back'; - } else { - io.stdout.write(renderDemoCompletionSummary(projectDir, agentInstalled)); - return 0; - } - } - - if (direction === 'back') { - if (stepIndex === 0) return 0; - stepIndex -= 1; - } else { - stepIndex += 1; - } - } - - return 0; -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `pnpm --filter @ktx/cli run test -- --testPathPattern setup-demo-tour` -Expected: PASS - -- [ ] **Step 5: Run type-check** - -Run: `pnpm --filter @ktx/cli run type-check` -Expected: PASS - all types align with existing interfaces - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "feat(cli): add runDemoTour orchestrator with step navigation" -``` - ---- - -### Task 6: Wire up in `setup.ts` - -**Files:** -- Modify: `packages/cli/src/setup.ts` - -- [ ] **Step 1: Read the current `runKtxSetupDemoFromEntryMenu` function** - -Read `packages/cli/src/setup.ts` and locate `runKtxSetupDemoFromEntryMenu` (around lines 218-233). - -Current implementation: -```typescript -async function runKtxSetupDemoFromEntryMenu( - args: Extract, - io: KtxCliIo, - deps: KtxSetupDeps, -): Promise { - const runner = deps.demo ?? (await import('./demo.js')).runKtxDemo; - return await runner( - { - command: 'seeded', - projectDir: defaultDemoProjectDir(), - outputMode: 'viz', - inputMode: args.inputMode, - }, - io, - ); -} -``` - -- [ ] **Step 2: Replace with demo tour call** - -Replace the function body to call `runDemoTour`: - -```typescript -async function runKtxSetupDemoFromEntryMenu( - args: Extract, - io: KtxCliIo, - deps: KtxSetupDeps, -): Promise { - const { runDemoTour } = await import('./setup-demo-tour.js'); - return await runDemoTour( - { inputMode: args.inputMode }, - io, - { agents: deps.agents }, - ); -} -``` - -- [ ] **Step 3: Update imports - remove unused `defaultDemoProjectDir` import if no longer needed elsewhere in setup.ts** - -Check if `defaultDemoProjectDir` is used elsewhere in `setup.ts`. If it's only used -in `runKtxSetupDemoFromEntryMenu`, remove the import. If used elsewhere, keep it. - -Also check if the `KtxDemoArgs` import is still needed. If `runKtxSetupDemoFromEntryMenu` -was the only consumer of `deps.demo` with that type, it may now be unused. Keep the -`demo` slot in `KtxSetupDeps` for backwards compatibility but it will no longer be -called from the entry menu path. - -- [ ] **Step 4: Run type-check and tests** - -Run: `pnpm --filter @ktx/cli run type-check && pnpm --filter @ktx/cli run test` -Expected: PASS - existing tests continue to work, demo tour is now wired in - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup.ts -git commit -m "feat(cli): wire demo tour into setup entry menu" -``` - ---- - -### Task 7: End-to-end verification - -**Files:** -- None (verification only) - -- [ ] **Step 1: Run full test suite** - -Run: `pnpm --filter @ktx/cli run test 2>&1 | tee /tmp/ktx-demo-tour-test.log` -Expected: All tests pass. Check the output for any regressions. - -- [ ] **Step 2: Run type-check across workspace** - -Run: `pnpm run type-check` -Expected: PASS - -- [ ] **Step 3: Run pre-commit checks if available** - -Run: `pnpm run check` (if configured) -Expected: PASS - -- [ ] **Step 4: Manual smoke test (if TTY available)** - -Run: `pnpm --filter @ktx/cli run build && node packages/cli/dist/cli.js setup` - -1. Select "Try KTX with packaged demo data" -2. Verify demo banner appears with full explanation text -3. Verify "Database connection" card shows with "PostgreSQL (demo warehouse)" -4. Press Enter → verify "Context sources" card shows with dbt, Metabase, Notion -5. Press Escape → verify you go back to database card -6. Press Enter twice → verify context build replay animation runs -7. Verify completion summary appears after replay -8. Press Enter → verify agents step prompt appears (interactive) -9. Press Escape all the way back → verify you return to entry menu - -- [ ] **Step 5: Final commit if any adjustments needed** - -```bash -git add -A -git commit -m "fix(cli): demo tour adjustments from smoke test" -``` - ---- - -## Open Seams for Demo Data - -When the user provides the real pre-packaged demo results, update these locations: - -1. **`renderDemoContextCompletionSummary()`** in `setup-demo-tour.ts` - replace placeholder text with actual counts (business areas, query definitions, knowledge pages) from the demo data -2. **`buildDemoReplayTimeline()`** in `setup-demo-tour.ts` - adjust timing and progress details to match the real ingestion profile -3. **`demo-assets.ts`** - update `REQUIRED_SEEDED_ASSET_PATHS` and `demoConfig()` if the demo dataset changes from SQLite/Orbit to Postgres/dbt/Metabase/Notion -4. **Pre-packaged asset files** in `packages/cli/assets/demo/` - replace with the new demo dataset diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md deleted file mode 100644 index 3fc3e496..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md +++ /dev/null @@ -1,1277 +0,0 @@ -# Historic SQL Cross-Dialect Readiness Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the redesigned historic-SQL adapter usable through the local CLI for Postgres, BigQuery, and Snowflake, with a truthful probe contract and Postgres doctor severity that matches the redesign. - -**Architecture:** Keep the unified hot path and skills/projection code intact. Normalize every historic-SQL reader to return a deterministic probe object, allow the local adapter factory to inject any `HistoricSqlReader` plus matching query client, and let the CLI choose the reader/query client from the configured connection dialect. Postgres `pg_stat_statements.max` becomes informational while `pg_stat_statements.track = none` remains a warning. - -**Tech Stack:** TypeScript ESM/NodeNext, zod 4, Vitest, existing KTX connector scan interfaces, existing managed daemon SQL-analysis port. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` - -Implemented status verified in this worktree: - -- `2026-05-11-historic-sql-foundations.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `SqlAnalysisPort.analyzeBatch()` in `packages/context/src/sql-analysis/ports.ts`, `/sql/analyze-batch` in `python/ktx-daemon/src/ktx_daemon/app.py`, `SemanticLayerSource.usage` in `packages/context/src/sl/types.ts`, and `mergeUsagePreservingExternal()` in `packages/context/src/ingest/adapters/live-database/manifest.ts`. -- `2026-05-11-historic-sql-search-enrichment.md` is implemented. Evidence: `packages/context/src/sl/sl-search.service.ts` indexes `source.usage`, `packages/context/src/sl/sqlite-sl-sources-index.ts` selects FTS snippets, and local/MCP list surfaces expose `frequencyTier` and `snippet`. -- `2026-05-11-historic-sql-unified-hot-path.md` is implemented for the shared stager/chunker and Postgres reader. Evidence: `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake `fetchAggregated()` methods, unified schemas, and exports exist. -- `2026-05-11-historic-sql-skills-projection-cutover.md` is implemented for the production adapter, skills, evidence tool, projection post-processor, and old code deletion. Evidence: `HistoricSqlSourceAdapter` uses `stageHistoricSqlAggregatedSnapshot()` and `chunkHistoricSqlUnifiedStagedDir()`, `packages/context/skills/historic_sql_table_digest/` and `packages/context/skills/historic_sql_patterns/` exist, `HistoricSqlProjectionPostProcessor` is wired in `local-bundle-runtime.ts`, and old `historic_sql_ingest` / `historic_sql_curator` skill directories are absent. - -Remaining core gaps from the spec: - -- `BigQueryHistoricSqlQueryHistoryReader.probe()` and `SnowflakeHistoricSqlQueryHistoryReader.probe()` return `void`, but `stageHistoricSqlAggregatedSnapshot()` reads `probe.warnings`. A BigQuery or Snowflake historic-SQL run would fail before staging. -- `createKtxCliLocalIngestAdapters()` only registers a historic-SQL adapter when the target connection is Postgres, while `ktx setup` can enable `historicSql` for BigQuery and Snowflake. -- `PostgresPgssReader.probe()` still reports low `pg_stat_statements.max` as a warning, but the spec says that check is informational after baseline tracking was removed. - -This plan does not update `examples/postgres-historic/README.md` or `examples/postgres-historic/scripts/smoke.sh`. Those still describe the legacy baseline/delta/reset behavior and should be handled in a separate documentation/acceptance plan after this cross-dialect code path is fixed. - -## File Structure - -Modify: - -- `packages/context/src/ingest/adapters/historic-sql/types.ts` - Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` - Moves low `pg_stat_statements.max` from `warnings` to `info`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` - Locks `track = none` as warning and low `max` as info. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` - Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` - Locks the BigQuery probe return object. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` - Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` - Locks the Snowflake probe return object. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` - Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` - Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/local-adapters.ts` - Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers. -- `packages/context/src/ingest/local-adapters.test.ts` - Verifies generic reader/query-client injection and the existing Postgres compatibility path. -- `packages/cli/src/local-adapters.ts` - Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection. -- `packages/cli/src/local-adapters.test.ts` - Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake. -- `packages/cli/src/historic-sql-doctor.ts` - Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings. -- `packages/cli/src/historic-sql-doctor.test.ts` - Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn. -- `packages/cli/src/doctor.test.ts` - Updates the project doctor integration expectation for the new info-only behavior. - -## Task 1: Normalize Historic-SQL Probe Results - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/types.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` - -- [ ] **Step 1: Update failing reader probe tests** - -In `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts`, replace the existing successful probe assertion: - -```typescript -await expect(reader.probe(client)).resolves.toBeUndefined(); -``` - -with: - -```typescript -await expect(reader.probe(client)).resolves.toEqual({ warnings: [], info: [] }); -``` - -In `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts`, replace the existing successful probe assertion: - -```typescript -await expect(reader.probe(client)).resolves.toBeUndefined(); -``` - -with: - -```typescript -await expect(reader.probe(client)).resolves.toEqual({ warnings: [], info: [] }); -``` - -In `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts`, change the successful probe expectation to include `info: []`: - -```typescript -await expect(reader.probe(client)).resolves.toEqual({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [], -}); -``` - -In the `returns a warning instead of failing when pg_stat_statements.track is none` test, change the expected object to: - -```typescript -await expect(reader.probe(client)).resolves.toEqual({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [ - 'pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config', - ], - info: [], -}); -``` - -Rename the low-max test from: - -```typescript -it('warns when pg_stat_statements.max is below the recommended floor', async () => { -``` - -to: - -```typescript -it('returns an info note when pg_stat_statements.max is below the recommended floor', async () => { -``` - -and change its expected object to: - -```typescript -await expect(reader.probe(client)).resolves.toEqual({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], -}); -``` - -In `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`, change the test reader probe from: - -```typescript -async probe() { - return { warnings: ['pg_stat_statements.max is low; aggregation still proceeds'] }; -}, -``` - -to: - -```typescript -async probe() { - return { warnings: ['pg_stat_statements.track is none; aggregation still proceeds'], info: [] }; -}, -``` - -and update the manifest expectation from: - -```typescript -probeWarnings: ['pg_stat_statements.max is low; aggregation still proceeds'], -``` - -to: - -```typescript -probeWarnings: ['pg_stat_statements.track is none; aggregation still proceeds'], -``` - -In `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`, replace every test reader probe result: - -```typescript -return { warnings: [] }; -``` - -with: - -```typescript -return { warnings: [], info: [] }; -``` - -- [ ] **Step 2: Run reader tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts \ - src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/stage-unified.test.ts \ - src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts -``` - -Expected: FAIL. The failure should show missing `info` fields and BigQuery/Snowflake probes resolving to `undefined`. - -- [ ] **Step 3: Update probe contracts and implementations** - -In `packages/context/src/ingest/adapters/historic-sql/types.ts`, replace: - -```typescript -export interface HistoricSqlProbeResult { - warnings: string[]; -} -``` - -with: - -```typescript -export interface HistoricSqlProbeResult { - warnings: string[]; - info?: string[]; -} -``` - -In the same file, replace: - -```typescript -export interface PostgresPgssProbeResult { - pgServerVersion: string; - warnings: string[]; -} -``` - -with: - -```typescript -export interface PostgresPgssProbeResult extends HistoricSqlProbeResult { - pgServerVersion: string; - warnings: string[]; - info: string[]; -} -``` - -In `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`, replace the warning construction block: - -```typescript -const warnings: string[] = []; -if (track === 'none') { - warnings.push('pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config'); -} -if (pgssMax !== null && pgssMax < RECOMMENDED_PGSS_MAX) { - warnings.push( - `pg_stat_statements.max is ${pgssMax}; set it to at least ${RECOMMENDED_PGSS_MAX} to reduce query-template eviction churn`, - ); -} - -return { pgServerVersion, warnings }; -``` - -with: - -```typescript -const warnings: string[] = []; -const info: string[] = []; -if (track === 'none') { - warnings.push('pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config'); -} -if (pgssMax !== null && pgssMax < RECOMMENDED_PGSS_MAX) { - info.push( - `pg_stat_statements.max is ${pgssMax}; set it to at least ${RECOMMENDED_PGSS_MAX} to reduce query-template eviction churn`, - ); -} - -return { pgServerVersion, warnings, info }; -``` - -In `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts`, replace the successful end of `probe()`: - -```typescript -if (result.error) { - throw grantsError(result.error); -} -``` - -with: - -```typescript -if (result.error) { - throw grantsError(result.error); -} -return { warnings: [], info: [] }; -``` - -and change the method signature from: - -```typescript -async probe(client: unknown): Promise { -``` - -to: - -```typescript -async probe(client: unknown): Promise<{ warnings: string[]; info: string[] }> { -``` - -In `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts`, make the same signature and return changes: - -```typescript -async probe(client: unknown): Promise<{ warnings: string[]; info: string[] }> { - let result: QueryResultLike; - try { - result = await queryClient(client).executeQuery(PROBE_SQL); - } catch (error) { - throw grantsError(error); - } - if (result.error) { - throw grantsError(result.error); - } - return { warnings: [], info: [] }; -} -``` - -- [ ] **Step 4: Run reader tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts \ - src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/stage-unified.test.ts \ - src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add \ - packages/context/src/ingest/adapters/historic-sql/types.ts \ - packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts \ - packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts \ - packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts \ - packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts \ - packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts \ - packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts \ - packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts \ - packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts -git commit -m "fix: normalize historic sql probe results" -``` - -## Task 2: Allow Generic Historic-SQL Reader Injection - -**Files:** -- Modify: `packages/context/src/ingest/local-adapters.ts` -- Modify: `packages/context/src/ingest/local-adapters.test.ts` - -- [ ] **Step 1: Write failing context adapter injection tests** - -In `packages/context/src/ingest/local-adapters.test.ts`, add `HistoricSqlReader` to the existing imports from `./adapters/historic-sql/types.js` if that import exists, or add this import near the other ingest imports: - -```typescript -import type { HistoricSqlReader } from './adapters/historic-sql/types.js'; -``` - -Add this test after `registers historic-sql locally when Postgres historic-SQL deps are provided`: - -```typescript -it('registers historic-sql with an injected non-Postgres reader and query client', () => { - const reader: HistoricSqlReader = { - async probe() { - return { warnings: [], info: [] }; - }, - async *fetchAggregated() {}, - }; - const queryClient = { executeQuery: async () => ({ headers: [], rows: [], totalRows: 0 }) }; - - const adapters = createDefaultLocalIngestAdapters(project, { - historicSql: { - sqlAnalysis: { - async analyzeForFingerprint(sql) { - return { - fingerprint: 'fp', - normalizedSql: sql, - tablesTouched: [], - literalSlots: [], - }; - }, - async analyzeBatch() { - return new Map(); - }, - }, - reader, - queryClient, - }, - }); - - const adapter = adapters.find((candidate) => candidate.source === 'historic-sql'); - expect(adapter).toBeDefined(); - expect(adapter?.fetch).toBeTypeOf('function'); -}); -``` - -Add this assertion inside the existing `registers historic-sql locally when Postgres historic-SQL deps are provided` test after the adapter lookup assertion: - -```typescript -expect(adapters.find((adapter) => adapter.source === 'historic-sql')?.skillNames).toEqual([ - 'historic_sql_table_digest', - 'historic_sql_patterns', -]); -``` - -- [ ] **Step 2: Run context adapter tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-adapters.test.ts -``` - -Expected: FAIL with TypeScript or runtime errors because `DefaultLocalIngestAdaptersOptions['historicSql']` does not accept `reader` or `queryClient`. - -- [ ] **Step 3: Update local adapter dependency shape** - -In `packages/context/src/ingest/local-adapters.ts`, add `HistoricSqlReader` to the historic-SQL type imports: - -```typescript -import { - HISTORIC_SQL_SOURCE_KEY, - historicSqlUnifiedPullConfigSchema, - type HistoricSqlReader, - type KtxPostgresQueryClient, -} from './adapters/historic-sql/types.js'; -``` - -Replace the `historicSql` option block in `DefaultLocalIngestAdaptersOptions`: - -```typescript -historicSql?: { - sqlAnalysis: SqlAnalysisPort; - postgresQueryClient: KtxPostgresQueryClient; - postgresBaselineRootDir?: string; - now?: () => Date; -}; -``` - -with: - -```typescript -historicSql?: { - sqlAnalysis: SqlAnalysisPort; - reader?: HistoricSqlReader; - queryClient?: unknown; - postgresQueryClient?: KtxPostgresQueryClient; - postgresBaselineRootDir?: string; - now?: () => Date; -}; -``` - -Replace the historic-SQL adapter construction block: - -```typescript -if (options.historicSql) { - adapters.push( - new HistoricSqlSourceAdapter({ - sqlAnalysis: options.historicSql.sqlAnalysis, - reader: new PostgresPgssReader(), - queryClient: options.historicSql.postgresQueryClient, - legacyPostgresBaselineRootDir: options.historicSql.postgresBaselineRootDir, - now: options.historicSql.now, - }), - ); -} -``` - -with: - -```typescript -if (options.historicSql) { - const queryClient = options.historicSql.queryClient ?? options.historicSql.postgresQueryClient; - if (!queryClient) { - throw new Error('Historic SQL local adapter requires queryClient or postgresQueryClient'); - } - adapters.push( - new HistoricSqlSourceAdapter({ - sqlAnalysis: options.historicSql.sqlAnalysis, - reader: options.historicSql.reader ?? new PostgresPgssReader(), - queryClient, - legacyPostgresBaselineRootDir: options.historicSql.postgresBaselineRootDir, - now: options.historicSql.now, - }), - ); -} -``` - -- [ ] **Step 4: Run context adapter tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/context/src/ingest/local-adapters.ts packages/context/src/ingest/local-adapters.test.ts -git commit -m "feat: allow generic historic sql readers locally" -``` - -## Task 3: Register BigQuery And Snowflake Historic SQL In The CLI - -**Files:** -- Create: `packages/cli/src/local-adapters.test.ts` -- Modify: `packages/cli/src/local-adapters.ts` - -- [ ] **Step 1: Write failing CLI local adapter tests** - -Create `packages/cli/src/local-adapters.test.ts`: - -```typescript -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { loadKtxProject } from '@ktx/context/project'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; - -function sqlAnalysisStub() { - return { - async analyzeForFingerprint(sql: string) { - return { - fingerprint: 'fp', - normalizedSql: sql, - tablesTouched: [], - literalSlots: [], - }; - }, - async analyzeBatch() { - return new Map(); - }, - }; -} - -async function writeProject(projectDir: string, body: string): Promise { - await writeFile(join(projectDir, 'ktx.yaml'), body, 'utf-8'); -} - -describe('CLI local ingest adapters', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-local-adapters-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('registers Postgres historic SQL from the requested connection', async () => { - await writeProject( - tempDir, - [ - 'project: warehouse', - 'connections:', - ' warehouse:', - ' driver: postgres', - ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', - ' historicSql:', - ' enabled: true', - ' dialect: postgres', - 'ingest:', - ' adapters:', - ' - historic-sql', - '', - ].join('\n'), - ); - const project = await loadKtxProject({ projectDir: tempDir }); - - const adapters = createKtxCliLocalIngestAdapters(project, { - historicSqlConnectionId: 'warehouse', - sqlAnalysis: sqlAnalysisStub(), - }); - - expect(adapters.find((adapter) => adapter.source === 'historic-sql')?.skillNames).toEqual([ - 'historic_sql_table_digest', - 'historic_sql_patterns', - ]); - }); - - it('registers BigQuery historic SQL from the requested connection', async () => { - await writeProject( - tempDir, - [ - 'project: warehouse', - 'connections:', - ' bq:', - ' driver: bigquery', - ' readonly: true', - ' dataset_id: analytics', - ' location: us', - ' credentials_json: \'{"project_id":"demo-project"}\'', - ' historicSql:', - ' enabled: true', - ' dialect: bigquery', - 'ingest:', - ' adapters:', - ' - historic-sql', - '', - ].join('\n'), - ); - const project = await loadKtxProject({ projectDir: tempDir }); - - const adapters = createKtxCliLocalIngestAdapters(project, { - historicSqlConnectionId: 'bq', - sqlAnalysis: sqlAnalysisStub(), - }); - - expect(adapters.find((adapter) => adapter.source === 'historic-sql')?.skillNames).toEqual([ - 'historic_sql_table_digest', - 'historic_sql_patterns', - ]); - }); - - it('registers Snowflake historic SQL from the requested connection', async () => { - await writeProject( - tempDir, - [ - 'project: warehouse', - 'connections:', - ' sf:', - ' driver: snowflake', - ' readonly: true', - ' account: acct', - ' warehouse: wh', - ' database: ANALYTICS', - ' schema_name: PUBLIC', - ' username: reader', - ' password: env:SNOWFLAKE_PASSWORD', - ' historicSql:', - ' enabled: true', - ' dialect: snowflake', - 'ingest:', - ' adapters:', - ' - historic-sql', - '', - ].join('\n'), - ); - const project = await loadKtxProject({ projectDir: tempDir }); - - const adapters = createKtxCliLocalIngestAdapters(project, { - historicSqlConnectionId: 'sf', - sqlAnalysis: sqlAnalysisStub(), - }); - - expect(adapters.find((adapter) => adapter.source === 'historic-sql')?.skillNames).toEqual([ - 'historic_sql_table_digest', - 'historic_sql_patterns', - ]); - }); -}); -``` - -- [ ] **Step 2: Run the new CLI adapter test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/local-adapters.test.ts -``` - -Expected: FAIL. BigQuery and Snowflake cases should not find a `historic-sql` adapter. - -- [ ] **Step 3: Add cross-dialect query clients and reader selection** - -In `packages/cli/src/local-adapters.ts`, replace the BigQuery import: - -```typescript -import { createBigQueryLiveDatabaseIntrospection, isKtxBigQueryConnectionConfig } from '@ktx/connector-bigquery'; -``` - -with: - -```typescript -import { - createBigQueryLiveDatabaseIntrospection, - isKtxBigQueryConnectionConfig, - KtxBigQueryScanConnector, - type KtxBigQueryConnectionConfig, -} from '@ktx/connector-bigquery'; -``` - -Replace the context ingest import block: - -```typescript -import { - createDaemonLiveDatabaseIntrospection, - createDefaultLocalIngestAdapters, - type DefaultLocalIngestAdaptersOptions, - type LiveDatabaseIntrospectionPort, - LiveDatabaseSourceAdapter, - type SourceAdapter, -} from '@ktx/context/ingest'; -``` - -with: - -```typescript -import { - BigQueryHistoricSqlQueryHistoryReader, - createDaemonLiveDatabaseIntrospection, - createDefaultLocalIngestAdapters, - type DefaultLocalIngestAdaptersOptions, - type HistoricSqlReader, - type LiveDatabaseIntrospectionPort, - LiveDatabaseSourceAdapter, - PostgresPgssReader, - SnowflakeHistoricSqlQueryHistoryReader, - type SourceAdapter, -} from '@ktx/context/ingest'; -``` - -Replace the SQL-analysis import: - -```typescript -import { createHttpSqlAnalysisPort } from '@ktx/context/sql-analysis'; -``` - -with: - -```typescript -import { createHttpSqlAnalysisPort, type SqlAnalysisPort } from '@ktx/context/sql-analysis'; -``` - -Add this top-level Snowflake type alias below `hasSnowflakeDriver()`: - -```typescript -type SnowflakeConnectorModule = typeof import('@ktx/connector-snowflake'); -``` - -Add an injectable SQL-analysis port to `KtxCliLocalIngestAdaptersOptions`: - -```typescript -export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { - historicSqlConnectionId?: string; - sqlAnalysis?: SqlAnalysisPort; - sqlAnalysisUrl?: string; - managedDaemon?: ManagedPythonCoreDaemonOptions; -} -``` - -Add this as the first branch in `ktxCliHistoricSqlAnalysis()`: - -```typescript -if (options.sqlAnalysis) { - return options.sqlAnalysis; -} -``` - -Replace `isEnabledPostgresHistoricSqlConnection()` with these helpers: - -```typescript -function historicSqlRecord(connection: unknown): Record | null { - if ( - connection && - typeof connection === 'object' && - 'historicSql' in connection && - typeof (connection as { historicSql?: unknown }).historicSql === 'object' && - (connection as { historicSql?: unknown }).historicSql !== null && - !Array.isArray((connection as { historicSql?: unknown }).historicSql) - ) { - return (connection as { historicSql: Record }).historicSql; - } - return null; -} - -function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery' | 'snowflake' | null { - const historicSql = historicSqlRecord(connection); - if (historicSql?.enabled !== true) { - return null; - } - const dialect = String(historicSql.dialect ?? '').toLowerCase(); - return dialect === 'postgres' || dialect === 'bigquery' || dialect === 'snowflake' ? dialect : null; -} -``` - -Keep `createEphemeralPostgresHistoricSqlClient()` and add these two query-client helpers below it: - -```typescript -function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) { - const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined; - if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); - } - return { - async executeQuery(query: string) { - const connector = new KtxBigQueryScanConnector({ - connectionId, - connection, - }); - try { - const result = await connector.executeReadOnly({ connectionId, sql: query }, {} as never); - return { - headers: result.headers, - rows: result.rows, - totalRows: result.totalRows, - }; - } finally { - await connector.cleanup(); - } - }, - }; -} - -async function createEphemeralSnowflakeHistoricSqlClient( - project: KtxLocalProject, - connectionId: string, - connectorModule: SnowflakeConnectorModule, -) { - const connection = project.config.connections[connectionId]; - if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`, - ); - } - return { - async executeQuery(query: string) { - const connector = new connectorModule.KtxSnowflakeScanConnector({ - connectionId, - connection, - }); - try { - const result = await connector.executeReadOnly({ connectionId, sql: query }, {} as never); - return { - headers: result.headers, - rows: result.rows, - totalRows: result.totalRows, - }; - } finally { - await connector.cleanup(); - } - }, - }; -} -``` - -Replace `historicSqlOptionsForLocalRun()` with: - -```typescript -function bigQueryProjectId(connection: KtxBigQueryConnectionConfig, env: NodeJS.ProcessEnv): string { - const raw = typeof connection.credentials_json === 'string' ? connection.credentials_json : ''; - const resolved = raw.startsWith('env:') ? env[raw.slice('env:'.length)] ?? '' : raw; - const parsed = JSON.parse(resolved) as { project_id?: unknown }; - if (typeof parsed.project_id !== 'string' || parsed.project_id.trim().length === 0) { - throw new Error('Historic SQL BigQuery connection requires credentials_json.project_id'); - } - return parsed.project_id; -} - -function bigQueryRegion(connection: KtxBigQueryConnectionConfig): string { - return typeof connection.location === 'string' && connection.location.trim().length > 0 - ? connection.location.trim() - : 'us'; -} - -function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCliLocalIngestAdaptersOptions) { - const connectionId = options.historicSqlConnectionId; - if (!connectionId) { - return undefined; - } - const connection = project.config.connections[connectionId]; - const dialect = enabledHistoricSqlDialect(connection); - if (!dialect) { - return undefined; - } - - const base = { - sqlAnalysis: ktxCliHistoricSqlAnalysis(options), - postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), - }; - - if (dialect === 'postgres') { - return { - ...base, - reader: new PostgresPgssReader() satisfies HistoricSqlReader, - queryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), - }; - } - - if (dialect === 'bigquery') { - if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); - } - return { - ...base, - reader: new BigQueryHistoricSqlQueryHistoryReader({ - projectId: bigQueryProjectId(connection, process.env), - region: bigQueryRegion(connection), - }) satisfies HistoricSqlReader, - queryClient: createEphemeralBigQueryHistoricSqlClient(project, connectionId), - }; - } - - return { - ...base, - reader: new SnowflakeHistoricSqlQueryHistoryReader() satisfies HistoricSqlReader, - queryClient: { - async executeQuery(query: string) { - const connectorModule = await import('@ktx/connector-snowflake'); - const client = await createEphemeralSnowflakeHistoricSqlClient(project, connectionId, connectorModule); - return client.executeQuery(query); - }, - }, - }; -} -``` - -- [ ] **Step 4: Run CLI adapter tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Run existing ingest wiring tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts -pnpm --filter @ktx/context exec vitest run src/ingest/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/src/local-adapters.ts packages/cli/src/local-adapters.test.ts -git commit -m "feat: wire historic sql readers for bigquery and snowflake" -``` - -## Task 4: Downgrade Low PGSS Max To Informational Doctor Output - -**Files:** -- Modify: `packages/cli/src/historic-sql-doctor.ts` -- Modify: `packages/cli/src/historic-sql-doctor.test.ts` -- Modify: `packages/cli/src/doctor.test.ts` - -- [ ] **Step 1: Write failing doctor severity tests** - -In `packages/cli/src/historic-sql-doctor.test.ts`, replace the existing low-max warning test with: - -```typescript -it('passes with an informational note when only pg_stat_statements.max is below the recommended floor', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], - }), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass', - detail: - 'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - }, - ]); -}); -``` - -Add this test immediately after it: - -```typescript -it('warns when pg_stat_statements tracking is disabled', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [ - 'pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config', - ], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], - }), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'warn', - detail: - 'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config; info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - fix: 'Update the Postgres parameter group or config, then rerun `ktx dev doctor --project-dir /tmp/ktx-project`', - }, - ]); -}); -``` - -In `packages/cli/src/doctor.test.ts`, replace the `includes Postgres historic-SQL readiness in project doctor output` test's fake historic-SQL check with a pass/info check: - -```typescript -const runHistoricSqlDoctorChecks = vi.fn(async () => [ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass' as const, - detail: - 'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - }, -]); -``` - -and replace the output assertions: - -```typescript -expect(testIo.stdout()).toContain('WARN Postgres Historic SQL (warehouse): pg_stat_statements ready'); -expect(testIo.stdout()).toContain('Fix: Update the Postgres parameter group or config'); -``` - -with: - -```typescript -expect(testIo.stdout()).toContain('PASS Postgres Historic SQL (warehouse): pg_stat_statements ready'); -expect(testIo.stdout()).toContain('info: pg_stat_statements.max is 1000'); -expect(testIo.stdout()).not.toContain('Fix: Update the Postgres parameter group or config'); -``` - -- [ ] **Step 2: Run doctor tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/historic-sql-doctor.test.ts src/doctor.test.ts -``` - -Expected: FAIL. The current doctor still treats any probe note as `warn`. - -- [ ] **Step 3: Update doctor probe and rendering logic** - -In `packages/cli/src/historic-sql-doctor.ts`, replace: - -```typescript -export interface PostgresHistoricSqlDoctorProbeResult { - pgServerVersion: string; - warnings: string[]; -} -``` - -with: - -```typescript -export interface PostgresHistoricSqlDoctorProbeResult { - pgServerVersion: string; - warnings: string[]; - info?: string[]; -} -``` - -Add this helper below `failureDetail()`: - -```typescript -function readinessDetail(result: PostgresHistoricSqlDoctorProbeResult): string { - const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : ''; - const info = result.info ?? []; - const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : ''; - return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`; -} -``` - -Replace this block: - -```typescript -if (result.warnings.length > 0) { - checks.push( - check( - 'warn', - checkId(connectionId), - label, - `pg_stat_statements ready (${result.pgServerVersion}) with warnings: ${result.warnings.join('; ')}`, - `Update the Postgres parameter group or config, then rerun \`ktx dev doctor --project-dir ${project.projectDir}\``, - ), - ); -} else { - checks.push( - check('pass', checkId(connectionId), label, `pg_stat_statements ready (${result.pgServerVersion})`), - ); -} -``` - -with: - -```typescript -if (result.warnings.length > 0) { - checks.push( - check( - 'warn', - checkId(connectionId), - label, - readinessDetail(result), - `Update the Postgres parameter group or config, then rerun \`ktx dev doctor --project-dir ${project.projectDir}\``, - ), - ); -} else { - checks.push(check('pass', checkId(connectionId), label, readinessDetail(result))); -} -``` - -- [ ] **Step 4: Run doctor tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/historic-sql-doctor.test.ts src/doctor.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/historic-sql-doctor.ts packages/cli/src/historic-sql-doctor.test.ts packages/cli/src/doctor.test.ts -git commit -m "fix: make pgss max advisory informational" -``` - -## Task 5: Final Verification - -**Files:** -- Verify: `packages/context/src/ingest/adapters/historic-sql/*` -- Verify: `packages/context/src/ingest/local-adapters.ts` -- Verify: `packages/cli/src/local-adapters.ts` -- Verify: `packages/cli/src/historic-sql-doctor.ts` - -- [ ] **Step 1: Run focused historic-SQL test suites** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/types.test.ts \ - src/ingest/adapters/historic-sql/buckets.test.ts \ - src/ingest/adapters/historic-sql/stage-unified.test.ts \ - src/ingest/adapters/historic-sql/chunk-unified.test.ts \ - src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts \ - src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts \ - src/ingest/local-adapters.test.ts -pnpm --filter @ktx/cli exec vitest run \ - src/local-adapters.test.ts \ - src/historic-sql-doctor.test.ts \ - src/doctor.test.ts \ - src/ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run the no-old-code grep** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline|historic_sql_ingest|historic_sql_curator|PostgresPgssQueryHistoryReader|historic_sql_template" packages/context packages/cli -``` - -Expected: no matches. - -- [ ] **Step 4: Run pre-commit for touched files** - -Run with the actual touched file list: - -```bash -uv run pre-commit run --files $(git diff --name-only) -``` - -Expected: PASS. If local `uv` refuses due the repo's exact uv pin, activate `.venv` and run the nearest available checks, then record the exact uv version mismatch in the implementation summary. - -- [ ] **Step 5: Confirm verification did not create unintended changes** - -Run: - -```bash -git status --short -``` - -Expected: the only changed files are the files committed in Tasks 1-4. If a verification command changed another tracked file, inspect it with `git diff -- ` and either commit it with the task that intentionally owns that file or revert only that verification-generated file after confirming it was not user-authored work. - -## Self-Review - -Spec coverage: - -- One pipeline across dialects: Task 1 fixes reader probe compatibility; Task 3 wires BigQuery and Snowflake into the CLI local adapter path. -- Unified reader interface: Task 1 makes every reader return the probe result shape consumed by the stager. -- Doctor command severity: Task 4 implements the spec's downgrade of low `pg_stat_statements.max` from warning to informational note. -- Hard cutover and old-code deletion: Task 5 keeps the no-old-code grep in verification. -- Search surfaces, skills, evidence projection, wiki pattern pages, and old skill deletion are already implemented by earlier plans and intentionally unchanged here. -- Postgres example smoke/docs are outside this plan because they are documentation/acceptance assets, not cross-dialect adapter plumbing. The next plan should update `examples/postgres-historic/scripts/smoke.sh`, `examples/postgres-historic/README.md`, `examples/README.md`, and `scripts/examples-docs.test.mjs` from legacy baseline/delta/reset assertions to unified `manifest.json`, `tables/*.json`, `patterns-input.json`, and no-WorkUnit idempotency assertions. - -Plan-quality scan: - -- No unresolved marker text from the forbidden-pattern list is present. -- Every code-changing task names exact files, includes concrete test snippets or replacement blocks, and specifies commands plus expected outcomes. - -Type consistency: - -- `HistoricSqlProbeResult.info` is optional for the generic reader interface. -- `PostgresPgssProbeResult.info` is required because the doctor consumes Postgres-specific info notes. -- `DefaultLocalIngestAdaptersOptions.historicSql.reader` and `.queryClient` align with `HistoricSqlSourceAdapterDeps`. -- CLI query-client helpers return the `headers`, `rows`, and `totalRows` shape already consumed by BigQuery and Snowflake historic-SQL readers. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md b/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md deleted file mode 100644 index 25df4e01..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md +++ /dev/null @@ -1,886 +0,0 @@ -# Historic SQL Docs Smoke And Config Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish the historic-SQL redesign follow-through by making setup emit the canonical config shape and replacing stale PGSS baseline/delta/reset example docs with unified artifact and no-WorkUnit idempotency checks. - -**Architecture:** This is the acceptance/documentation slice after the adapter cutover. Product code changes are limited to `ktx setup` Historic SQL config serialization; the Postgres example smoke remains a deterministic stage-only path that uses the real local adapter, managed daemon, Docker Postgres, and raw artifact diffing without requiring LLM credentials. Public docs are updated to match the unified Postgres, BigQuery, and Snowflake reader behavior already present in source. - -**Tech Stack:** TypeScript, Vitest, Bash, Node.js ESM, `node:test`, pnpm, Docker Compose, KTX local stage-only ingest, managed `ktx-daemon`. - ---- - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans already based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` - implemented in source: `skill-schemas.ts`, `SemanticLayerSource.usage`, `mergeUsagePreservingExternal()`, `/sql/analyze-batch`, and `SqlAnalysisPort.analyzeBatch()`. -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` - implemented in source: usage fields in `buildSemanticLayerSourceSearchText()`, SQLite FTS snippets, query-mode `score`, `frequencyTier`, and agent/MCP list propagation. -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` - implemented in source: unified config/types, bucket helpers, `stage-unified.ts`, aggregate readers, and `chunk-unified.ts`. -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` - implemented in source: replacement skills, evidence tool, projection, post-processor wiring, production adapter cutover, legacy source deletion, and `minExecutions` alias support. -- `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md` - implemented in source: cross-dialect CLI wiring, generic reader injection, probe result normalization, and PGSS max informational doctor output. - -Remaining gap this plan covers: - -- `examples/postgres-historic/scripts/smoke.sh`, `examples/postgres-historic/README.md`, `examples/README.md`, and `scripts/examples-docs.test.mjs` still describe the legacy baseline/delta/reset model. -- Public docs still mention `minCalls` and say BigQuery/Snowflake local CLI Historic SQL uses the Postgres path. -- `packages/cli/src/setup-databases.ts` still writes `serviceAccountUserPatterns` for new setup output even though the redesign's canonical runtime config is `filters.serviceAccounts`. - -## File Structure - -- Modify `packages/cli/src/setup-databases.ts`: write canonical `historicSql.filters.serviceAccounts` blocks from setup flags while keeping existing parser compatibility in `packages/context/src/ingest/adapters/historic-sql/types.ts`. -- Modify `packages/cli/src/setup-databases.test.ts`: assert generated YAML uses `filters` and no longer writes `serviceAccountUserPatterns`. -- Modify `scripts/examples-docs.test.mjs`: lock public example docs and smoke script to the unified artifact contract. -- Modify `examples/postgres-historic/scripts/smoke.sh`: assert `manifest.json`, `tables/*.json`, `patterns-input.json`, per-run `workUnitCount`, and stage-only runtime under 60 seconds after runtime warm-up. -- Modify `examples/postgres-historic/README.md`: replace baseline/delta/reset instructions with unified artifact, no-WorkUnit idempotency, and `minExecutions` language. -- Modify `examples/README.md`: replace the stale one-paragraph summary. -- Modify `docs/content/docs/integrations/primary-sources.mdx`: update Postgres, Snowflake, and BigQuery Historic SQL docs to the unified config and current support status. -- Modify `docs/content/docs/cli-reference/ktx-setup.mdx`: document `--historic-sql-min-executions` as primary and `--historic-sql-min-calls` as the one-release alias. - -### Task 1: Emit Canonical Historic SQL Setup Config - -**Files:** -- Modify: `packages/cli/src/setup-databases.test.ts` -- Modify: `packages/cli/src/setup-databases.ts` - -- [ ] **Step 1: Update failing setup config assertions** - -In `packages/cli/src/setup-databases.test.ts`, update the Snowflake expectation in `writes Historic SQL config for supported Snowflake databases after validation succeeds` to: - -```typescript - expect(config.connections.snowflake).toMatchObject({ - driver: 'snowflake', - authMethod: 'password', - historicSql: { - enabled: true, - dialect: 'snowflake', - windowDays: 30, - filters: { - dropTrivialProbes: true, - serviceAccounts: { - patterns: ['^svc_'], - mode: 'exclude', - }, - }, - redactionPatterns: ['(?i)secret'], - }, - }); - expect(config.connections.snowflake.historicSql).not.toHaveProperty('serviceAccountUserPatterns'); -``` - -In the same file, update the Postgres expectation in `writes Postgres Historic SQL config with minExecutions and ignores window/redaction output` to: - -```typescript - expect(config.connections.warehouse).toMatchObject({ - driver: 'postgres', - url: 'env:DATABASE_URL', - schemas: ['public'], - historicSql: { - enabled: true, - dialect: 'postgres', - minExecutions: 12, - filters: { - dropTrivialProbes: true, - serviceAccounts: { - patterns: ['^svc_'], - mode: 'exclude', - }, - }, - }, - }); - expect(config.connections.warehouse.historicSql).not.toHaveProperty('minCalls'); - expect(config.connections.warehouse.historicSql).not.toHaveProperty('windowDays'); - expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns'); - expect(config.connections.warehouse.historicSql).not.toHaveProperty('serviceAccountUserPatterns'); -``` - -Update the existing BigQuery connection expectation in `writes Historic SQL config for supported existing database connections` to: - -```typescript - expect(config.connections.analytics).toMatchObject({ - historicSql: { - enabled: true, - dialect: 'bigquery', - windowDays: 45, - filters: { - dropTrivialProbes: true, - }, - redactionPatterns: [], - }, - }); - expect(config.connections.analytics.historicSql).not.toHaveProperty('serviceAccountUserPatterns'); -``` - -Update the existing Postgres connection expectation in `enables Historic SQL on an existing Postgres connection` to: - -```typescript - expect(config.connections.warehouse).toMatchObject({ - historicSql: { - enabled: true, - dialect: 'postgres', - minExecutions: 8, - filters: { - dropTrivialProbes: true, - }, - }, - }); - expect(config.connections.warehouse.historicSql).not.toHaveProperty('serviceAccountUserPatterns'); -``` - -- [ ] **Step 2: Run setup tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts --testNamePattern "Historic SQL" -``` - -Expected: FAIL because `historicSql.serviceAccountUserPatterns` is still written and `historicSql.filters` is missing from generated setup YAML. - -- [ ] **Step 3: Write canonical setup config** - -In `packages/cli/src/setup-databases.ts`, add this helper near `maybeApplyHistoricSqlConfig()`: - -```typescript -function historicSqlFiltersForSetup(patterns: string[] | undefined) { - const serviceAccountPatterns = patterns ?? []; - return { - dropTrivialProbes: true, - ...(serviceAccountPatterns.length > 0 - ? { - serviceAccounts: { - patterns: serviceAccountPatterns, - mode: 'exclude' as const, - }, - } - : {}), - }; -} -``` - -Then replace the `common` object inside `maybeApplyHistoricSqlConfig()` with: - -```typescript - const common: Record = { - ...existing, - enabled: true, - dialect, - filters: historicSqlFiltersForSetup(input.args.historicSqlServiceAccountPatterns), - }; - delete common.serviceAccountUserPatterns; -``` - -Keep the existing `minExecutions`, `windowDays`, and `redactionPatterns` branches unchanged after this object replacement. - -- [ ] **Step 4: Run setup tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts --testNamePattern "Historic SQL" -``` - -Expected: PASS for all Historic SQL setup tests in `src/setup-databases.test.ts`. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "fix: write canonical historic sql setup filters" -``` - -### Task 2: Lock Example Docs To Unified Historic SQL Terms - -**Files:** -- Modify: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Update the failing example docs test** - -Replace the `documents the Postgres historic SQL smoke example` test body in `scripts/examples-docs.test.mjs` with: - -```javascript - it('documents the Postgres historic SQL smoke example', async () => { - const examples = await readText('examples/README.md'); - const readme = await readText('examples/postgres-historic/README.md'); - const compose = await readText('examples/postgres-historic/docker-compose.yml'); - const initSql = await readText('examples/postgres-historic/init/001-schema.sql'); - const workload = await readText('examples/postgres-historic/scripts/generate-workload.sh'); - const smoke = await readText('examples/postgres-historic/scripts/smoke.sh'); - - assert.match(examples, /postgres-historic/); - assert.match(examples, /unified Historic SQL artifacts/); - assert.match(readme, /--enable-historic-sql/); - assert.match(readme, /--historic-sql-min-executions 2/); - assert.match(readme, /ktx dev doctor --project-dir/); - assert.match(readme, /Postgres Historic SQL/); - assert.match(readme, /manifest\.json/); - assert.match(readme, /tables\/\*\.json/); - assert.match(readme, /patterns-input\.json/); - assert.match(readme, /workUnitCount: 0/); - assert.match(compose, /postgres:14/); - assert.match(compose, /shared_preload_libraries=pg_stat_statements/); - assert.match(compose, /pg_stat_statements.track=top/); - assert.match(initSql, /CREATE EXTENSION IF NOT EXISTS pg_stat_statements/); - assert.match(initSql, /GRANT pg_read_all_stats TO ktx_reader/); - assert.match(workload, /JOIN customers/); - assert.match(workload, /app_user/); - assert.match(workload, /etl_user/); - assert.match(smoke, /assert_unified_snapshot/); - assert.match(smoke, /assert_stage_record "\$UNCHANGED_RECORD" unchanged zero/); - assert.match(smoke, /--historic-sql-min-executions 2/); - assert.match(smoke, /KTX_RUNTIME_ROOT/); - assert.match(smoke, /managedDaemon/); - assert.match(smoke, /installPolicy: 'auto'/); - assert.match(smoke, /getKtxCliPackageInfo/); - assert.doesNotMatch(smoke, /python-service/); - assert.doesNotMatch(smoke, /PYTHON_SERVICE/); - assert.doesNotMatch(smoke, /uvicorn app\.main:app/); - assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/); - assert.doesNotMatch(smoke, /baselineFirstRun|degraded|statsResetAt|assert_manifest/); - assert.doesNotMatch(readme, /python-service/); - assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/); - assert.doesNotMatch(readme, /baselineFirstRun|degraded: true|statsResetAt|fresh PGSS baseline|delta-only/); - assert.doesNotMatch(readme, /--historic-sql-min-calls/); - }); -``` - -- [ ] **Step 2: Run the docs test to verify it fails** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL because the current README and smoke script still mention `--historic-sql-min-calls`, `baselineFirstRun`, `degraded`, and the legacy `assert_manifest` helper. - -- [ ] **Step 3: Commit the failing test** - -```bash -git add scripts/examples-docs.test.mjs -git commit -m "test: expect unified historic sql example docs" -``` - -### Task 3: Rewrite The Postgres Historic SQL Smoke - -**Files:** -- Modify: `examples/postgres-historic/scripts/smoke.sh` - -- [ ] **Step 1: Replace the smoke script with unified artifact assertions** - -Replace `examples/postgres-historic/scripts/smoke.sh` with: - -```bash -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXAMPLE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -KTX_ROOT="$(cd "$EXAMPLE_DIR/../.." && pwd)" -COMPOSE_FILE="$EXAMPLE_DIR/docker-compose.yml" -PROJECT_PARENT="${KTX_POSTGRES_HISTORIC_PROJECT_PARENT:-$(mktemp -d)}" -PROJECT_DIR="$PROJECT_PARENT/postgres-historic-ktx" -KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" -MAX_STAGE_SECONDS="${KTX_POSTGRES_HISTORIC_MAX_STAGE_SECONDS:-60}" -export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime" -unset KTX_DAEMON_URL -unset KTX_SQL_ANALYSIS_URL - -cleanup() { - if [[ -f "$KTX_BIN" ]]; then - node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true - fi - if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then - docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true - fi -} -trap cleanup EXIT - -latest_manifest() { - find "$PROJECT_DIR/raw-sources/warehouse/historic-sql" -name manifest.json | sort | tail -n 1 -} - -assert_unified_snapshot() { - local manifest_path="$1" - node - "$manifest_path" <<'NODE' -const { dirname, join } = require('node:path'); -const { readFileSync, readdirSync } = require('node:fs'); - -const manifestPath = process.argv[2]; -const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); -function assert(condition, message) { - if (!condition) throw new Error(message); -} - -assert(manifest.source === 'historic-sql', `Expected source historic-sql, got ${manifest.source}`); -assert(manifest.dialect === 'postgres', `Expected dialect postgres, got ${manifest.dialect}`); -assert(Number.isInteger(manifest.snapshotRowCount) && manifest.snapshotRowCount > 0, 'Expected snapshotRowCount > 0'); -assert(Number.isInteger(manifest.touchedTableCount) && manifest.touchedTableCount > 0, 'Expected touchedTableCount > 0'); -assert(Number.isInteger(manifest.parseFailures), 'Expected numeric parseFailures'); -assert(Array.isArray(manifest.warnings), 'Expected warnings array'); -assert(Array.isArray(manifest.probeWarnings), 'Expected probeWarnings array'); -for (const legacyKey of ['degraded', 'baselineFirstRun', 'pgServerVersion', 'statsResetAt', 'templates']) { - assert(!(legacyKey in manifest), `Legacy manifest key is still present: ${legacyKey}`); -} - -const root = dirname(manifestPath); -const tableDir = join(root, 'tables'); -const tableFiles = readdirSync(tableDir).filter((file) => file.endsWith('.json')).sort(); -assert(tableFiles.length === manifest.touchedTableCount, `Expected ${manifest.touchedTableCount} table files, got ${tableFiles.length}`); - -const firstTable = JSON.parse(readFileSync(join(tableDir, tableFiles[0]), 'utf8')); -assert(typeof firstTable.table === 'string' && firstTable.table.length > 0, 'Expected staged table name'); -assert(firstTable.stats && typeof firstTable.stats.executionsBucket === 'string', 'Expected bucketed table stats'); -assert(firstTable.columnsByClause && typeof firstTable.columnsByClause === 'object', 'Expected columnsByClause object'); -assert(Array.isArray(firstTable.observedJoins), 'Expected observedJoins array'); -assert(Array.isArray(firstTable.topTemplates) && firstTable.topTemplates.length > 0, 'Expected topTemplates'); - -const patterns = JSON.parse(readFileSync(join(root, 'patterns-input.json'), 'utf8')); -assert(Array.isArray(patterns.templates) && patterns.templates.length > 0, 'Expected patterns-input templates'); -assert( - patterns.templates.every((template) => Array.isArray(template.tablesTouched) && template.tablesTouched.length > 0), - 'Expected every pattern template to have touched tables', -); -NODE -} - -assert_stage_record() { - local record_path="$1" - local label="$2" - local expected_work_units="$3" - node - "$record_path" "$label" "$expected_work_units" "$MAX_STAGE_SECONDS" <<'NODE' -const { readFileSync } = require('node:fs'); - -const record = JSON.parse(readFileSync(process.argv[2], 'utf8')); -const label = process.argv[3]; -const expectedWorkUnits = process.argv[4]; -const maxSeconds = Number(process.argv[5]); -function assert(condition, message) { - if (!condition) throw new Error(message); -} - -assert(record.status === 'done', `${label}: expected status done, got ${record.status}`); -assert(record.adapter === 'historic-sql', `${label}: expected historic-sql adapter`); -assert(record.connectionId === 'warehouse', `${label}: expected warehouse connection`); -assert(record.rawFileCount >= 3, `${label}: expected manifest, patterns input, and at least one table file`); -assert(Array.isArray(record.errors) && record.errors.length === 0, `${label}: expected no errors`); - -if (expectedWorkUnits === 'zero') { - assert(record.workUnitCount === 0, `${label}: expected zero WorkUnits, got ${record.workUnitCount}`); - assert(Array.isArray(record.workUnits) && record.workUnits.length === 0, `${label}: expected empty workUnits`); -} else if (expectedWorkUnits === 'nonzero') { - assert(record.workUnitCount > 0, `${label}: expected nonzero WorkUnits`); - assert(record.workUnits.some((unit) => unit.unitKey === 'historic-sql-patterns'), `${label}: expected patterns WorkUnit`); - assert(record.workUnits.some((unit) => unit.unitKey.startsWith('historic-sql-table-')), `${label}: expected table WorkUnit`); -} else { - throw new Error(`${label}: unknown expected work unit mode ${expectedWorkUnits}`); -} - -const elapsedMs = Date.parse(record.completedAt) - Date.parse(record.startedAt); -assert(Number.isFinite(elapsedMs) && elapsedMs >= 0, `${label}: invalid elapsed time`); -assert(elapsedMs <= maxSeconds * 1000, `${label}: stage-only ingest took ${elapsedMs}ms, over ${maxSeconds}s`); -NODE -} - -run_historic_stage_only() { - local job_id="$1" - local record_path="$2" - node - "$KTX_ROOT" "$PROJECT_DIR" "$job_id" "$record_path" <<'NODE' -const { writeFile } = await import('node:fs/promises'); -const { join } = await import('node:path'); - -const ktxRoot = process.argv[2]; -const projectDir = process.argv[3]; -const jobId = process.argv[4]; -const recordPath = process.argv[5]; -const { loadKtxProject } = await import(join(ktxRoot, 'packages/context/dist/project/index.js')); -const { runLocalStageOnlyIngest } = await import(join(ktxRoot, 'packages/context/dist/ingest/index.js')); -const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); -const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js')); - -const project = await loadKtxProject({ projectDir }); -const cliVersion = getKtxCliPackageInfo().version; -const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr }; -const adapters = createKtxCliLocalIngestAdapters(project, { - historicSqlConnectionId: 'warehouse', - managedDaemon: { - cliVersion, - installPolicy: 'auto', - io: managedRuntimeIo, - }, -}); -const adapter = adapters.find((candidate) => candidate.source === 'historic-sql'); -if (!adapter) throw new Error('historic-sql adapter was not registered for local run'); -const record = await runLocalStageOnlyIngest({ - project, - adapters, - adapter: 'historic-sql', - connectionId: 'warehouse', - trigger: 'manual_resync', - jobId, -}); -await writeFile(recordPath, `${JSON.stringify(record, null, 2)}\n`, 'utf8'); -console.log(`${record.syncId} workUnits=${record.workUnitCount}`); -NODE -} - -cd "$KTX_ROOT" -pnpm --filter @ktx/context run build -pnpm --filter @ktx/cli run build - -docker compose -f "$COMPOSE_FILE" up -d --wait -"$EXAMPLE_DIR/scripts/generate-workload.sh" base - -export WAREHOUSE_DATABASE_URL="${WAREHOUSE_DATABASE_URL:-postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics}" # pragma: allowlist secret -node "$KTX_BIN" --project-dir "$PROJECT_DIR" setup \ - --new \ - --skip-agents \ - --skip-llm \ - --skip-embeddings \ - --skip-sources \ - --database postgres \ - --new-database-connection-id warehouse \ - --database-url env:WAREHOUSE_DATABASE_URL \ - --database-schema public \ - --enable-historic-sql \ - --historic-sql-min-executions 2 \ - --yes \ - --no-input - -node "$KTX_BIN" runtime install --yes -node "$KTX_BIN" runtime start - -FIRST_RECORD="$PROJECT_PARENT/first-record.json" -run_historic_stage_only "historic-first-$$" "$FIRST_RECORD" -FIRST_MANIFEST="$(latest_manifest)" -assert_unified_snapshot "$FIRST_MANIFEST" -assert_stage_record "$FIRST_RECORD" first nonzero - -UNCHANGED_RECORD="$PROJECT_PARENT/unchanged-record.json" -run_historic_stage_only "historic-unchanged-$$" "$UNCHANGED_RECORD" -UNCHANGED_MANIFEST="$(latest_manifest)" -assert_unified_snapshot "$UNCHANGED_MANIFEST" -assert_stage_record "$UNCHANGED_RECORD" unchanged zero - -"$EXAMPLE_DIR/scripts/generate-workload.sh" extra -CHANGED_RECORD="$PROJECT_PARENT/changed-record.json" -run_historic_stage_only "historic-changed-$$" "$CHANGED_RECORD" -CHANGED_MANIFEST="$(latest_manifest)" -assert_unified_snapshot "$CHANGED_MANIFEST" -assert_stage_record "$CHANGED_RECORD" changed nonzero - -echo "Postgres historic SQL smoke passed" -echo "Project dir: $PROJECT_DIR" -``` - -- [ ] **Step 2: Run the docs test to verify smoke-script assertions now pass or expose remaining README failures** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL remains because `examples/postgres-historic/README.md`, `examples/README.md`, and public docs have not been rewritten yet. The smoke-specific assertions for `assert_unified_snapshot`, `assert_stage_record`, and `--historic-sql-min-executions 2` should pass. - -- [ ] **Step 3: Commit** - -```bash -git add examples/postgres-historic/scripts/smoke.sh -git commit -m "test: assert unified postgres historic sql smoke" -``` - -### Task 4: Update Example And Public Docs - -**Files:** -- Modify: `examples/postgres-historic/README.md` -- Modify: `examples/README.md` -- Modify: `docs/content/docs/integrations/primary-sources.mdx` -- Modify: `docs/content/docs/cli-reference/ktx-setup.mdx` - -- [ ] **Step 1: Replace the Postgres historic README** - -Replace `examples/postgres-historic/README.md` with: - -````markdown -# Postgres Historic SQL Example - -This example is a manual smoke for the redesigned Postgres historic-SQL ingest -path through `pg_stat_statements`. It starts Postgres 14 with the extension -preloaded, generates query workload under separate users, runs `ktx setup` with -`--enable-historic-sql`, and verifies the unified staged artifacts: - -- `manifest.json` -- `tables/*.json` -- `patterns-input.json` - -The smoke also runs the same workload twice and verifies the second stage-only -run has `workUnitCount: 0`, which proves unchanged bucketed table and pattern -inputs do not schedule LLM work. - -## Prerequisites - -- Docker with Compose v2 -- Node and pnpm matching the KTX workspace -- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled - runtime wheel - -## Run - -From the KTX repository root: - -```bash -examples/postgres-historic/scripts/smoke.sh -``` - -The smoke creates a temporary KTX project, isolates the managed Python runtime -under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and -uses this connection URL: - -```bash -postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics # pragma: allowlist secret -``` - -Set `KTX_POSTGRES_HISTORIC_KEEP_DOCKER=1` to leave the container running after -the script exits. - -The smoke validates the historic-SQL raw snapshot path without requiring LLM -credentials. It uses KTX's local stage-only ingest API after `ktx setup`, so the -deterministic reader, batch SQL parser, stable artifact writer, and diff-based -WorkUnit planning are checked independently from curation. - -## Manual Commands - -Start Postgres and generate the base workload: - -```bash -docker compose -f examples/postgres-historic/docker-compose.yml up -d --wait -examples/postgres-historic/scripts/generate-workload.sh base -``` - -Create a project and enable historic SQL: - -```bash -export WAREHOUSE_DATABASE_URL=postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics # pragma: allowlist secret -pnpm --filter @ktx/cli run build -node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic setup \ - --new \ - --skip-agents \ - --skip-llm \ - --skip-embeddings \ - --skip-sources \ - --database postgres \ - --new-database-connection-id warehouse \ - --database-url env:WAREHOUSE_DATABASE_URL \ - --database-schema public \ - --enable-historic-sql \ - --historic-sql-min-executions 2 \ - --yes \ - --no-input -``` - -### Readiness check - -```bash -pnpm run ktx -- dev doctor --project-dir /tmp/ktx-postgres-historic --no-input -``` - -The installed CLI form is: - -```bash -ktx dev doctor --project-dir /tmp/ktx-postgres-historic --no-input -``` - -Expected output includes `PASS Postgres Historic SQL (warehouse)` when -`pg_stat_statements` is installed, `pg_read_all_stats` is granted, and tracking -is enabled. A low `pg_stat_statements.max` value is reported as an informational -note, not a warning. - -Run local historic-SQL ingest: - -```bash -pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \ - --connection-id warehouse \ - --adapter historic-sql \ - --plain \ - --yes \ - --no-input -``` - -The full `dev ingest run` path also runs curation WorkUnits, so it requires a -configured LLM provider. - -Inspect the latest manifest: - -```bash -find /tmp/ktx-postgres-historic/raw-sources/warehouse/historic-sql -name manifest.json | sort | tail -n 1 -``` - -The manifest should have `source: "historic-sql"`, `dialect: "postgres"`, -positive `snapshotRowCount`, positive `touchedTableCount`, numeric -`parseFailures`, `warnings`, and `probeWarnings`. The same directory should -contain `patterns-input.json` and one `tables/*.json` file per touched table. - -## Troubleshooting - -- Missing extension: confirm `shared_preload_libraries=pg_stat_statements` and - `CREATE EXTENSION pg_stat_statements;` both happened in the `analytics` - database. -- Missing grants: confirm `GRANT pg_read_all_stats TO ktx_reader;`. -- Empty snapshot: rerun `scripts/generate-workload.sh base` and keep - `--historic-sql-min-executions 2` for the smoke. -- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX - repository root and confirm `uv`, the bundled Python wheel, and the managed - runtime all pass. -```` - -- [ ] **Step 2: Update the examples index paragraph** - -In `examples/README.md`, replace the `postgres-historic` paragraph with: - -```markdown -## postgres-historic - -`postgres-historic/` is a manual Docker-backed smoke for Postgres -historic-SQL ingest via `pg_stat_statements`. It verifies setup, unified -Historic SQL artifacts, managed daemon batch SQL analysis, and no-WorkUnit -idempotency for unchanged bucketed table and pattern inputs. -``` - -- [ ] **Step 3: Update the setup CLI reference** - -In `docs/content/docs/cli-reference/ktx-setup.mdx`, replace the Historic SQL flag rows with: - -```markdown -| `--enable-historic-sql` | Enable Historic SQL when the selected database supports it | `false` | -| `--disable-historic-sql` | Disable Historic SQL for the selected database | `false` | -| `--historic-sql-window-days ` | Historic SQL query-history window in days | - | -| `--historic-sql-min-executions ` | Minimum executions for a Historic SQL template | - | -| `--historic-sql-min-calls ` | Alias for `--historic-sql-min-executions` for one release | - | -| `--historic-sql-service-account-pattern ` | Historic SQL service-account regex; repeatable | - | -| `--historic-sql-redaction-pattern ` | Historic SQL SQL-literal redaction regex; repeatable | - | -``` - -- [ ] **Step 4: Update primary source Historic SQL docs** - -In `docs/content/docs/integrations/primary-sources.mdx`, replace the Postgres Historic SQL config block with: - -````markdown -```yaml -historicSql: - enabled: true - dialect: postgres - minExecutions: 5 - filters: - dropTrivialProbes: true -``` -```` - -Replace the Snowflake Historic SQL feature row with: - -```markdown -| Historic SQL | Yes | Via `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY` when enabled | -``` - -Replace the Snowflake Historic SQL paragraph and config block with: - -````markdown -Snowflake Historic SQL reads aggregated query-history templates from -`SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY` and feeds the same unified staged -artifact shape as Postgres and BigQuery. - -```yaml -historicSql: - enabled: true - dialect: snowflake - windowDays: 90 - minExecutions: 5 - filters: - dropTrivialProbes: true - serviceAccounts: - patterns: ['^svc_'] - mode: exclude - redactionPatterns: [] -``` -```` - -Replace the BigQuery Historic SQL feature row with: - -```markdown -| Historic SQL | Yes | Via region-scoped `INFORMATION_SCHEMA.JOBS_BY_PROJECT` when enabled | -``` - -Replace the BigQuery Historic SQL paragraph and config block with: - -````markdown -BigQuery Historic SQL reads aggregated query-history templates from -region-scoped `INFORMATION_SCHEMA.JOBS_BY_PROJECT` and feeds the same unified -staged artifact shape as Postgres and Snowflake. - -```yaml -historicSql: - enabled: true - dialect: bigquery - windowDays: 90 - minExecutions: 5 - filters: - dropTrivialProbes: true - serviceAccounts: - patterns: ['@bot\\.'] - mode: exclude - redactionPatterns: [] -``` -```` - -- [ ] **Step 5: Run docs tests to verify they pass** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. The Postgres historic example test now sees unified artifact language and no legacy baseline/delta/reset wording. - -- [ ] **Step 6: Commit** - -```bash -git add examples/postgres-historic/README.md examples/README.md docs/content/docs/integrations/primary-sources.mdx docs/content/docs/cli-reference/ktx-setup.mdx -git commit -m "docs: refresh historic sql setup and smoke docs" -``` - -### Task 5: Final Verification - -**Files:** -- Verify: `packages/cli/src/setup-databases.ts` -- Verify: `packages/cli/src/setup-databases.test.ts` -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `examples/postgres-historic/scripts/smoke.sh` -- Verify: `examples/postgres-historic/README.md` -- Verify: `examples/README.md` -- Verify: `docs/content/docs/integrations/primary-sources.mdx` -- Verify: `docs/content/docs/cli-reference/ktx-setup.mdx` - -- [ ] **Step 1: Run focused setup tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts --testNamePattern "Historic SQL" -``` - -Expected: PASS. - -- [ ] **Step 2: Run example docs tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 3: Run CLI type check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run grep checks for stale legacy wording** - -Run: - -```bash -rg -n "baselineFirstRun|fresh PGSS baseline|delta-only|--historic-sql-min-calls 2|local CLI Historic SQL ingest currently uses the Postgres path" examples docs/content scripts packages/cli/src/setup-databases.test.ts -``` - -Expected: no matches. - -Run: - -```bash -rg -n "serviceAccountUserPatterns" packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts docs/content examples -``` - -Expected: no matches. Existing runtime compatibility in `packages/context/src/ingest/adapters/historic-sql/types.ts` must remain untouched, so do not run this grep across `packages/context`. - -- [ ] **Step 5: Run the Docker-backed smoke when Docker is available** - -Run: - -```bash -examples/postgres-historic/scripts/smoke.sh -``` - -Expected: PASS with `Postgres historic SQL smoke passed`. If Docker is not running or unavailable, record the exact Docker error and still run Steps 1-4. - -- [ ] **Step 6: Run pre-commit for touched files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/cli/src/setup-databases.ts \ - packages/cli/src/setup-databases.test.ts \ - scripts/examples-docs.test.mjs \ - examples/postgres-historic/scripts/smoke.sh \ - examples/postgres-historic/README.md \ - examples/README.md \ - docs/content/docs/integrations/primary-sources.mdx \ - docs/content/docs/cli-reference/ktx-setup.mdx -``` - -Expected: PASS when pre-commit is configured. If pre-commit is not configured or this workspace lacks the required hook environment, keep the output and rely on Steps 1-5 plus `git diff --check`. - -- [ ] **Step 7: Run whitespace check** - -Run: - -```bash -git diff --check -``` - -Expected: no output. - -- [ ] **Step 8: Commit verification fixes only if verification changed files** - -If any verification step required an edit, commit the exact touched files: - -```bash -git add packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts scripts/examples-docs.test.mjs examples/postgres-historic/scripts/smoke.sh examples/postgres-historic/README.md examples/README.md docs/content/docs/integrations/primary-sources.mdx docs/content/docs/cli-reference/ktx-setup.mdx -git commit -m "test: verify historic sql docs and smoke cleanup" -``` - -If verification made no edits, do not create an empty commit. - -## Self-Review - -Spec coverage: - -- Spec §8 setup config is covered by Task 1 and Task 4. -- Spec §10.3 docs and setup wizard updates are covered by Tasks 1 and 4. -- Spec §10.4 demo DB acceptance is covered by Task 3 and Task 5. -- The prior implemented plans already cover daemon batch analysis, unified staging, skills/projection, search enrichment, old-code deletion, and cross-dialect local adapter wiring. - -Placeholder scan: - -- This plan contains concrete file paths, exact replacement snippets, exact commands, and expected outcomes for every step. - -Type consistency: - -- `filters.dropTrivialProbes`, `filters.serviceAccounts.patterns`, and `filters.serviceAccounts.mode` match `historicSqlUnifiedPullConfigSchema`. -- `workUnitCount`, `rawFileCount`, `startedAt`, and `completedAt` match `LocalIngestRunRecord`. -- `manifest.json`, `tables/*.json`, and `patterns-input.json` match the unified staged artifact names from `stage-unified.ts`. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md deleted file mode 100644 index 106131ed..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md +++ /dev/null @@ -1,452 +0,0 @@ -# Historic SQL End-To-End Retrieval Acceptance Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add one focused regression test that proves the redesigned historic-SQL pipeline reaches both agent retrieval surfaces after a real scheduled local ingest run. - -**Architecture:** All historic-SQL redesign implementation slices are already present. This plan adds acceptance coverage around the existing production `HistoricSqlSourceAdapter`: a fake aggregate reader and fake batch SQL analysis drive the deterministic hot path, a fake `AgentRunnerService` emits typed table and pattern evidence through `emit_historic_sql_evidence`, and the normal local ingest runner performs projection, squash, wiki indexing, and semantic-layer reindexing. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, YAML, SQLite FTS5 local search, existing local ingest runner, existing historic-SQL adapter. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md` - -Implemented status verified from this worktree: - -- `2026-05-11-historic-sql-foundations.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `packages/context/src/sql-analysis/ports.ts` exposes `analyzeBatch()`, `python/ktx-daemon/src/ktx_daemon/app.py` registers `/sql/analyze-batch`, `packages/context/src/sl/types.ts` has `SemanticLayerSource.usage`, and `packages/context/src/ingest/adapters/live-database/manifest.ts` has `mergeUsagePreservingExternal()`. -- `2026-05-11-historic-sql-search-enrichment.md` is implemented. Evidence: `packages/context/src/sl/sl-search.service.ts` indexes `source.usage`, `packages/context/src/sl/sqlite-sl-sources-index.ts` selects FTS snippets, and local/MCP list surfaces expose `frequencyTier` and `snippet`. -- `2026-05-11-historic-sql-unified-hot-path.md` is implemented. Evidence: `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake `fetchAggregated()` methods, unified schemas, and package exports exist. -- `2026-05-11-historic-sql-skills-projection-cutover.md` is implemented. Evidence: `HistoricSqlSourceAdapter` uses the unified stager/chunker, `packages/context/skills/historic_sql_table_digest/` and `packages/context/skills/historic_sql_patterns/` exist, `emit_historic_sql_evidence` exists, `HistoricSqlProjectionPostProcessor` is wired in `packages/context/src/ingest/local-bundle-runtime.ts`, and legacy skill names no longer grep in `packages/context` or `packages/cli`. -- `2026-05-11-historic-sql-cross-dialect-readiness.md` is implemented. Evidence: `packages/cli/src/local-adapters.test.ts` covers Postgres, BigQuery, and Snowflake historic-SQL registration, and `packages/cli/src/historic-sql-doctor.test.ts` covers low `pg_stat_statements.max` as informational output. -- `2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` is implemented. Evidence: `packages/cli/src/setup-databases.test.ts` expects canonical `historicSql.filters.serviceAccounts`, `examples/postgres-historic/scripts/smoke.sh` asserts unified `manifest.json`, `tables/*.json`, `patterns-input.json`, and zero WorkUnits on the unchanged run, and public docs use `minExecutions`. -- `2026-05-11-historic-sql-projection-archive-hardening.md` is implemented. Evidence: `projection.ts` has `isArchivedPatternPage()`, excludes archived pages from active slug matching, and `projection.test.ts` covers reappearing archived patterns, stable archived pages, stale table marking, and legacy query-page deletion. - -Remaining acceptance gap this plan covers: - -- The current Postgres example smoke is intentionally stage-only, so it verifies raw artifacts and zero unchanged WorkUnits but does not prove table/pattern evidence projection and retrieval. -- `packages/context/src/ingest/local-bundle-ingest.test.ts` verifies the historic-SQL post-processor with a source-dir test adapter, but it does not exercise the production `HistoricSqlSourceAdapter` scheduled-pull path or the `historic_sql_patterns` WorkUnit. -- Existing SL and wiki search tests prove the search layers independently, but no single regression proves spec §7's retrieval chain after historic-SQL ingest writes `_schema` usage and `knowledge/global/historic-sql/*.md`. - -## File Structure - -Create: - -- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` - Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider. - -## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` - -- [ ] **Step 1: Verify the acceptance test does not exist yet** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -``` - -Expected: FAIL with "No test files found" because no end-to-end historic-SQL retrieval acceptance test exists yet. - -- [ ] **Step 2: Write the acceptance test** - -Create `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`: - -```typescript -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import YAML from 'yaml'; -import { AgentRunnerService } from '../../../agent/index.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../project/index.js'; -import { type SqlAnalysisPort } from '../../../sql-analysis/index.js'; -import { searchLocalSlSources } from '../../../sl/local-sl.js'; -import { searchLocalKnowledgePages } from '../../../wiki/local-knowledge.js'; -import { runLocalIngest } from '../../local-ingest.js'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js'; -import type { AggregatedTemplate, HistoricSqlReader, HistoricSqlUnifiedPullConfig } from './types.js'; - -class AcceptanceHistoricSqlReader implements HistoricSqlReader { - async probe() { - return { warnings: [], info: [] }; - } - - async *fetchAggregated( - _client: unknown, - _window: { start: Date; end: Date }, - _config: HistoricSqlUnifiedPullConfig, - ): AsyncIterable { - yield { - templateId: 'pg:orders-lifecycle', - canonicalSql: - 'select o.status, c.segment, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.status = $1 group by o.status, c.segment', - dialect: 'postgres', - stats: { - executions: 42, - distinctUsers: 4, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 18, - p95RuntimeMs: 84, - errorRate: 0, - rowsProduced: 420, - }, - topUsers: [{ user: 'analyst@example.test', executions: 42 }], - }; - } -} - -class HistoricSqlAcceptanceAgentRunner extends AgentRunnerService { - override runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags?.operationName !== 'ingest-bundle-wu') { - return { stopReason: 'natural' as const }; - } - - const emitEvidence = params.toolSet.emit_historic_sql_evidence; - if (!emitEvidence?.execute) { - throw new Error('emit_historic_sql_evidence tool was not available to the historic-SQL WorkUnit'); - } - - if (params.telemetryTags.unitKey === 'historic-sql-table-public-orders') { - const result = await emitEvidence.execute( - { - kind: 'table_usage', - table: 'public.orders', - rawPath: 'tables/public.orders.json', - usage: { - narrative: 'Analysts repeatedly inspect paid order lifecycle by customer segment.', - frequencyTier: 'high', - commonFilters: ['status'], - commonGroupBys: ['status', 'segment'], - commonJoins: [{ table: 'public.customers', on: ['customer_id', 'id'] }], - staleSince: null, - }, - }, - { toolCallId: 'historic-sql-orders-usage' }, - ); - if (!String(result).includes('Recorded historic-SQL table_usage evidence')) { - throw new Error(`Unexpected orders evidence result: ${String(result)}`); - } - } - - if (params.telemetryTags.unitKey === 'historic-sql-table-public-customers') { - const result = await emitEvidence.execute( - { - kind: 'table_usage', - table: 'public.customers', - rawPath: 'tables/public.customers.json', - usage: { - narrative: 'Customers provide segment context for paid order lifecycle analysis.', - frequencyTier: 'mid', - commonFilters: [], - commonGroupBys: ['segment'], - commonJoins: [{ table: 'public.orders', on: ['id', 'customer_id'] }], - staleSince: null, - }, - }, - { toolCallId: 'historic-sql-customers-usage' }, - ); - if (!String(result).includes('Recorded historic-SQL table_usage evidence')) { - throw new Error(`Unexpected customers evidence result: ${String(result)}`); - } - } - - if (params.telemetryTags.unitKey === 'historic-sql-patterns') { - const result = await emitEvidence.execute( - { - kind: 'pattern', - rawPath: 'patterns-input.json', - pattern: { - slug: 'paid-order-lifecycle', - title: 'Paid Order Lifecycle', - narrative: 'Analysts join orders and customers to compare paid order lifecycle by segment.', - definitionSql: - 'select o.status, c.segment, count(*) from public.orders o join public.customers c on c.id = o.customer_id group by o.status, c.segment', - tablesInvolved: ['public.orders', 'public.customers'], - slRefs: ['orders', 'customers'], - constituentTemplateIds: ['pg:orders-lifecycle'], - }, - }, - { toolCallId: 'historic-sql-pattern' }, - ); - if (!String(result).includes('Recorded historic-SQL pattern evidence')) { - throw new Error(`Unexpected pattern evidence result: ${String(result)}`); - } - } - - return { stopReason: 'natural' as const }; - }); - - constructor() { - super({ llmProvider: { getModel: () => ({}) as never } as never }); - } -} - -function acceptanceSqlAnalysis(): SqlAnalysisPort { - return { - analyzeForFingerprint: async () => { - throw new Error('analyzeForFingerprint should not be used by unified historic-SQL ingest'); - }, - analyzeBatch: vi.fn(async (items) => { - return new Map( - items.map((item) => [ - item.id, - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: ['status', 'segment'], - where: ['status'], - join: ['customer_id', 'id'], - groupBy: ['status', 'segment'], - }, - }, - ]), - ); - }), - }; -} - -async function writeHistoricSqlProject(project: KtxLocalProject): Promise { - await writeFile( - join(project.projectDir, 'ktx.yaml'), - [ - 'project: warehouse', - 'connections:', - ' warehouse:', - ' driver: postgres', - ' historicSql:', - ' enabled: true', - ' dialect: postgres', - ' minExecutions: 2', - 'ingest:', - ' adapters:', - ' - historic-sql', - ' embeddings:', - ' backend: deterministic', - 'storage:', - ' state: sqlite', - ' search: sqlite-fts5', - ' git:', - ' auto_commit: false', - ' author: KTX Test ', - '', - ].join('\n'), - 'utf-8', - ); - - const loaded = await loadKtxProject({ projectDir: project.projectDir }); - await loaded.fileStore.writeFile( - 'semantic-layer/warehouse/_schema/public.yaml', - YAML.stringify({ - tables: { - orders: { - table: 'public.orders', - columns: [ - { name: 'id', type: 'string' }, - { name: 'status', type: 'string' }, - { name: 'customer_id', type: 'string' }, - ], - }, - customers: { - table: 'public.customers', - columns: [ - { name: 'id', type: 'string' }, - { name: 'segment', type: 'string' }, - ], - }, - }, - }), - 'KTX Test', - 'system@ktx.local', - 'Seed schema shard', - ); - return loaded; -} - -describe('historic-SQL local ingest retrieval acceptance', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-historic-sql-acceptance-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('projects table and pattern evidence into semantic-layer and wiki retrieval surfaces', async () => { - const initialized = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - const project = await writeHistoricSqlProject(initialized); - const sqlAnalysis = acceptanceSqlAnalysis(); - const agentRunner = new HistoricSqlAcceptanceAgentRunner(); - const adapter = new HistoricSqlSourceAdapter({ - reader: new AcceptanceHistoricSqlReader(), - queryClient: {}, - sqlAnalysis, - now: () => new Date('2026-05-11T00:00:00.000Z'), - }); - - const result = await runLocalIngest({ - project, - adapters: [adapter], - adapter: 'historic-sql', - connectionId: 'warehouse', - jobId: 'historic-sql-retrieval-acceptance', - agentRunner, - }); - - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledTimes(1); - expect(result.result.failedWorkUnits).toEqual([]); - expect(result.result.workUnitCount).toBe(3); - expect(agentRunner.runLoop).toHaveBeenCalledTimes(3); - expect(result.report.body.postProcessor).toMatchObject({ - sourceKey: 'historic-sql', - status: 'success', - result: { - tableUsageMerged: 2, - patternPagesWritten: 1, - }, - touchedSources: [ - { connectionId: 'warehouse', sourceName: 'customers' }, - { connectionId: 'warehouse', sourceName: 'orders' }, - ], - }); - - await expect(readFile(join(project.projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves - .toContain('Analysts repeatedly inspect paid order lifecycle by customer segment.'); - await expect(readFile(join(project.projectDir, 'knowledge/global/historic-sql/paid-order-lifecycle.md'), 'utf-8')) - .resolves.toContain('Paid Order Lifecycle'); - - const reloaded = await loadKtxProject({ projectDir: project.projectDir }); - await expect( - searchLocalSlSources(reloaded, { connectionId: 'warehouse', query: 'paid order lifecycle', limit: 5 }), - ).resolves.toEqual([ - expect.objectContaining({ - name: 'orders', - frequencyTier: 'high', - snippet: expect.stringContaining(''), - matchReasons: expect.arrayContaining(['lexical']), - }), - ]); - await expect( - searchLocalKnowledgePages(reloaded, { query: 'paid order lifecycle', userId: 'local', limit: 5 }), - ).resolves.toEqual([ - expect.objectContaining({ - key: 'historic-sql/paid-order-lifecycle', - summary: 'Paid Order Lifecycle', - matchReasons: expect.arrayContaining(['lexical']), - }), - ]); - }); -}); -``` - -- [ ] **Step 3: Run the focused acceptance test after creating the file** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -``` - -Expected: PASS. The output reports one passing test and `sqlAnalysis.analyzeBatch` is called exactly once by the test assertion. - -- [ ] **Step 4: Commit the acceptance test** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -git commit -m "test: cover historic sql retrieval acceptance" -``` - -## Task 2: Run Adjacent Historic-SQL Regression Checks - -**Files:** -- Verify: `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` -- Verify: `packages/context/src/sl/local-sl.test.ts` -- Verify: `packages/context/src/wiki/local-knowledge.test.ts` - -- [ ] **Step 1: Run the new acceptance test with the adjacent historic-SQL unit tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts \ - src/ingest/adapters/historic-sql/projection.test.ts \ - src/ingest/adapters/historic-sql/stage-unified.test.ts \ - src/ingest/adapters/historic-sql/chunk-unified.test.ts \ - src/sl/local-sl.test.ts \ - src/wiki/local-knowledge.test.ts -``` - -Expected: PASS. These suites cover the new acceptance chain plus the deterministic projection, stager, chunker, SL search, and wiki search layers it depends on. - -- [ ] **Step 2: Run pre-commit for the new test file** - -Run: - -```bash -uv run pre-commit run --files packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -``` - -Expected: PASS. If `uv` refuses to run because the local binary does not satisfy the repo pin, activate `.venv` and run the closest TypeScript checks instead: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -``` - -- [ ] **Step 3: Confirm no unrelated files are included** - -Run: - -```bash -git status --short -``` - -Expected: either an empty status after the Task 1 commit, or only intentionally changed plan/test files if the worker is preserving an uncommitted plan handoff. - -## Self-Review - -Spec coverage: - -- Spec §4 hot path is covered because the test uses `HistoricSqlSourceAdapter.fetch()` with `stageHistoricSqlAggregatedSnapshot()`, a fake `HistoricSqlReader.fetchAggregated()`, and one `SqlAnalysisPort.analyzeBatch()` call. -- Spec §5 cold path is covered because the fake agent emits `table_usage` and `pattern` evidence through `emit_historic_sql_evidence`, and the normal `HistoricSqlProjectionPostProcessor` projects that evidence. -- Spec §6 and §7 retrieval surfaces are covered because the same test verifies `searchLocalSlSources()` returns `frequencyTier` and an FTS snippet and `searchLocalKnowledgePages()` returns `historic-sql/paid-order-lifecycle`. -- Spec §10.4 search retrieval acceptance is covered without requiring a live warehouse or LLM credentials. - -Placeholder scan: - -- The placeholder scan is clean, and the plan contains concrete file paths, code, commands, and expected outputs. -- The only fallback in the plan is the explicit `uv` version-mismatch path required by repository instructions. - -Type consistency: - -- `HistoricSqlReader`, `HistoricSqlUnifiedPullConfig`, `SqlAnalysisPort`, `HistoricSqlSourceAdapter`, `runLocalIngest`, `searchLocalSlSources`, and `searchLocalKnowledgePages` match existing exported APIs. -- Evidence payloads match `emit_historic_sql_evidence` input schemas: table evidence omits `connectionId` because the tool injects it; projected persisted evidence includes it. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md deleted file mode 100644 index 6705d56f..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md +++ /dev/null @@ -1,1477 +0,0 @@ -# Historic SQL Foundations Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build the foundation slice for the historic SQL redesign: shared usage schemas, semantic-layer usage plumbing, scan-safe usage preservation, and batch SQL analysis across the Python daemon and TypeScript port. - -**Architecture:** Keep the existing historic-SQL adapter behavior unchanged in this slice. Add the additive contracts from the redesign first so later adapter, skill, projection, and search work can depend on stable types and daemon APIs. The Python daemon owns SQL parsing through `sqlglot`; TypeScript owns HTTP mapping, semantic-layer schema acceptance, and manifest projection. - -**Tech Stack:** TypeScript ESM/NodeNext, zod 4, Vitest, FastAPI, Pydantic v2, sqlglot, pytest, uv. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Existing plans derived from this spec: none found. A repo search found only managed-runtime plans that mention historic-SQL smoke commands or `pg_stat_statements`; those plans are not based on the redesign spec and do not implement the redesign architecture. - -Current implementation state: - -- `packages/context/src/sql-analysis/ports.ts` exposes only `analyzeForFingerprint()`. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` only calls `/api/sql/analyze-for-fingerprint`. -- `python/ktx-daemon/src/ktx_daemon/app.py` has no `/sql/analyze-batch` endpoint. -- `packages/context/src/sl/types.ts` has no `SemanticLayerSource.usage`. -- `packages/context/src/sl/schemas.ts` is strict and rejects top-level `usage`. -- `packages/context/src/sl/semantic-layer.service.ts` does not project `_schema` manifest `usage`. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` does not preserve usage through live database scan rewrites. -- The old historic-SQL code path is still present (`stage-pgss.ts`, `stagePgStatStatementsTemplates`, `pgss-baseline`, slot classification, per-template wiki page staging). - -This plan implements only the foundation ordering item from spec §10.3: - -- Daemon `analyze-batch` endpoint. -- `SqlAnalysisPort.analyzeBatch()`. -- `SemanticLayerSource.usage`. -- `LiveDatabaseManifestTableEntry.usage`. -- `mergeUsagePreservingExternal()` plus tests. - -The next plan after this one should cover search enrichment from spec §6.2.3-§6.2.5. - -## File Structure - -Create: - -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` - Owns the shared zod schemas for historic-SQL LLM outputs. -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` - Locks schema acceptance, JSON schema generation, and future-key tolerance. -- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` - Implements batch sqlglot parsing for table and clause-level column extraction. -- `python/ktx-daemon/tests/test_sql_analysis.py` - Tests batch parser behavior without FastAPI. - -Modify: - -- `packages/context/src/ingest/index.ts` - Exports the new historic-SQL skill schemas. -- `packages/context/src/sl/types.ts` - Adds `usage?: TableUsageOutput` to `SemanticLayerSource`. -- `packages/context/src/sl/schemas.ts` - Accepts `usage` in standalone and overlay semantic-layer source validation. -- `packages/context/src/sl/semantic-layer.service.ts` - Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally. -- `packages/context/src/sl/semantic-layer.service.test.ts` - Tests source schema acceptance, manifest projection, and overlay composition. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` - Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`. -- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` - Tests scan-managed usage replacement while preserving external keys. -- `packages/context/src/scan/local-enrichment-artifacts.ts` - Loads existing manifest usage and passes it through scan manifest rebuilds. -- `packages/context/src/scan/local-enrichment-artifacts.test.ts` - Tests that structural scan rewrites preserve existing usage. -- `python/ktx-daemon/src/ktx_daemon/app.py` - Registers `/sql/analyze-batch`. -- `python/ktx-daemon/tests/test_app.py` - Tests the FastAPI endpoint. -- `packages/context/src/sql-analysis/ports.ts` - Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`. -- `packages/context/src/sql-analysis/index.ts` - Exports the new batch analysis types. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` - Maps `/sql/analyze-batch` request and response payloads. -- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` - Tests HTTP mapping and malformed response rejection. -- `packages/cli/src/managed-python-http.test.ts` - Verifies the managed daemon wrapper routes `analyzeBatch()`. -- Existing test files with `SqlAnalysisPort` object literals - Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`. - -## Task 1: Add Historic SQL Skill Schemas - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Write the failing schema tests** - -Create `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { z } from 'zod'; -import { - patternOutputSchema, - patternsArraySchema, - tableUsageOutputSchema, -} from './skill-schemas.js'; - -describe('historic-sql skill schemas', () => { - it('accepts table usage output and preserves future keys', () => { - const parsed = tableUsageOutputSchema.parse({ - narrative: 'Orders are queried for paid/refunded lifecycle analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonGroupBys: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - staleSince: null, - analystNote: 'preserve me', - }); - - expect(parsed).toMatchObject({ - narrative: 'Orders are queried for paid/refunded lifecycle analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonGroupBys: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - staleSince: null, - analystNote: 'preserve me', - }); - }); - - it('rejects invalid frequency tiers', () => { - const result = tableUsageOutputSchema.safeParse({ - narrative: 'Orders are queried often.', - frequencyTier: 'sometimes', - commonFilters: [], - commonJoins: [], - }); - - expect(result.success).toBe(false); - }); - - it('accepts pattern outputs used for wiki projection', () => { - const parsed = patternsArraySchema.parse([ - { - slug: 'order-lifecycle-analysis', - title: 'Order Lifecycle Analysis', - narrative: 'Teams inspect order status by customer and month.', - definitionSql: 'select status, count(*) from public.orders group by status', - tablesInvolved: ['public.orders', 'public.customers'], - slRefs: ['orders', 'customers'], - constituentTemplateIds: ['template_1', 'template_2'], - }, - ]); - - expect(parsed[0]).toEqual({ - slug: 'order-lifecycle-analysis', - title: 'Order Lifecycle Analysis', - narrative: 'Teams inspect order status by customer and month.', - definitionSql: 'select status, count(*) from public.orders group by status', - tablesInvolved: ['public.orders', 'public.customers'], - slRefs: ['orders', 'customers'], - constituentTemplateIds: ['template_1', 'template_2'], - }); - }); - - it('exports zod schemas that can produce JSON schema for prompt prefixes', () => { - const tableUsageJsonSchema = z.toJSONSchema(tableUsageOutputSchema); - const patternJsonSchema = z.toJSONSchema(patternOutputSchema); - - expect(tableUsageJsonSchema).toMatchObject({ type: 'object' }); - expect(patternJsonSchema).toMatchObject({ type: 'object' }); - }); -}); -``` - -- [ ] **Step 2: Run the schema test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/skill-schemas.test.ts -``` - -Expected: FAIL with an import error for `./skill-schemas.js`. - -- [ ] **Step 3: Add the schema implementation** - -Create `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`: - -```typescript -import { z } from 'zod'; - -export const tableUsageOutputSchema = z - .object({ - narrative: z.string(), - frequencyTier: z.enum(['high', 'mid', 'low', 'unused']), - commonFilters: z.array(z.string()), - commonGroupBys: z.array(z.string()).optional(), - commonJoins: z.array( - z.object({ - table: z.string(), - on: z.array(z.string()), - }), - ), - staleSince: z.iso.datetime().nullable().optional(), - }) - .passthrough(); -export type TableUsageOutput = z.infer; - -export const patternOutputSchema = z.object({ - slug: z.string(), - title: z.string(), - narrative: z.string(), - definitionSql: z.string(), - tablesInvolved: z.array(z.string()), - slRefs: z.array(z.string()), - constituentTemplateIds: z.array(z.string()), -}); -export type PatternOutput = z.infer; - -export const patternsArraySchema = z.array(patternOutputSchema); -``` - -- [ ] **Step 4: Export the schemas from the ingest barrel** - -Add this export block to `packages/context/src/ingest/index.ts` near the other historic-SQL exports: - -```typescript -export { - patternOutputSchema, - patternsArraySchema, - tableUsageOutputSchema, -} from './adapters/historic-sql/skill-schemas.js'; -export type { - PatternOutput, - TableUsageOutput, -} from './adapters/historic-sql/skill-schemas.js'; -``` - -- [ ] **Step 5: Run the schema test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/skill-schemas.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts packages/context/src/ingest/index.ts -git commit -m "feat: add historic sql skill schemas" -``` - -## Task 2: Add `usage` to Semantic Layer Sources - -**Files:** -- Modify: `packages/context/src/sl/types.ts` -- Modify: `packages/context/src/sl/schemas.ts` -- Modify: `packages/context/src/sl/semantic-layer.service.ts` -- Test: `packages/context/src/sl/semantic-layer.service.test.ts` - -- [ ] **Step 1: Write failing semantic-layer usage tests** - -In `packages/context/src/sl/semantic-layer.service.test.ts`, extend the import from `./semantic-layer.service.js`: - -```typescript -import { - composeOverlay, - enrichColumnsFromManifest, - findDanglingSegmentRefs, - projectManifestEntry, - SemanticLayerService, -} from './semantic-layer.service.js'; -``` - -Add this test inside `describe('composeOverlay', ...)` after the descriptions test: - -```typescript - it('replaces manifest usage only when an overlay explicitly provides usage', () => { - const baseWithUsage: SemanticLayerSource = { - ...baseTable, - usage: { - narrative: 'Orders are commonly queried by lifecycle status.', - frequencyTier: 'high', - commonFilters: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - }, - }; - - expect(composeOverlay(baseWithUsage, { name: 'fct_labs', measures: [] }).usage).toEqual(baseWithUsage.usage); - - const composed = composeOverlay(baseWithUsage, { - name: 'fct_labs', - usage: { - narrative: 'Overlay-curated usage note.', - frequencyTier: 'mid', - commonFilters: ['created_at'], - commonGroupBys: ['created_at'], - commonJoins: [], - }, - }); - - expect(composed.usage).toEqual({ - narrative: 'Overlay-curated usage note.', - frequencyTier: 'mid', - commonFilters: ['created_at'], - commonGroupBys: ['created_at'], - commonJoins: [], - }); - }); -``` - -Add this test inside `describe('sourceDefinitionSchema', ...)`: - -```typescript - it('accepts historic SQL usage on standalone sources', () => { - const result = sourceDefinitionSchema.safeParse({ - name: 'orders', - table: 'public.orders', - grain: ['id'], - columns: [{ name: 'id', type: 'string' }], - joins: [], - measures: [], - usage: { - narrative: 'Orders are queried for fulfillment and revenue analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - externalOwner: 'analytics', - }, - }); - - expect(result.success).toBe(true); - if (!result.success) { - return; - } - expect(result.data.usage).toMatchObject({ - narrative: 'Orders are queried for fulfillment and revenue analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - externalOwner: 'analytics', - }); - }); -``` - -Add a new describe block before `describe('findManifestEntryByTableRef', ...)`: - -```typescript -describe('projectManifestEntry', () => { - it('projects manifest usage onto the semantic-layer source', () => { - const source = projectManifestEntry('orders', { - table: 'public.orders', - usage: { - narrative: 'Orders are frequently filtered by status.', - frequencyTier: 'high', - commonFilters: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - }, - columns: [ - { name: 'id', type: 'string', pk: true }, - { name: 'status', type: 'string' }, - ], - }); - - expect(source.usage).toEqual({ - narrative: 'Orders are frequently filtered by status.', - frequencyTier: 'high', - commonFilters: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - }); - }); -}); -``` - -- [ ] **Step 2: Run the semantic-layer tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/semantic-layer.service.test.ts -``` - -Expected: FAIL because `usage` is rejected by strict schemas and not projected from manifest entries. - -- [ ] **Step 3: Add `usage` to the TypeScript source type** - -In `packages/context/src/sl/types.ts`, add this import at the top: - -```typescript -import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js'; -``` - -Add this field to `SemanticLayerSource` after `freshness`: - -```typescript - usage?: TableUsageOutput; -``` - -- [ ] **Step 4: Add `usage` to zod validation** - -In `packages/context/src/sl/schemas.ts`, add this import after the existing zod import: - -```typescript -import { tableUsageOutputSchema } from '../ingest/adapters/historic-sql/skill-schemas.js'; -``` - -Add this field to `sourceDefinitionSchema` near `freshness`: - -```typescript - usage: tableUsageOutputSchema.optional(), -``` - -Add this field to `sourceOverlaySchema` near `default_time_dimension`: - -```typescript - usage: tableUsageOutputSchema.optional(), -``` - -- [ ] **Step 5: Project and compose usage intentionally** - -In `packages/context/src/sl/semantic-layer.service.ts`, add this type import: - -```typescript -import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js'; -``` - -Add this field to `ManifestTableEntry`: - -```typescript - usage?: TableUsageOutput; -``` - -In `projectManifestEntry()`, add `usage` to the returned object: - -```typescript - ...(entry.usage ? { usage: entry.usage } : {}), -``` - -Add `'usage'` to `COMPOSE_KNOWN_KEYS`: - -```typescript - 'usage', -``` - -In `composeOverlay()`, add this block after the descriptions merge and before column filtering: - -```typescript - if (normalizedOverlay.usage !== undefined) { - result.usage = normalizedOverlay.usage as SemanticLayerSource['usage']; - } -``` - -- [ ] **Step 6: Run the semantic-layer tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/semantic-layer.service.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/src/sl/types.ts packages/context/src/sl/schemas.ts packages/context/src/sl/semantic-layer.service.ts packages/context/src/sl/semantic-layer.service.test.ts -git commit -m "feat: carry historic sql usage in semantic sources" -``` - -## Task 3: Preserve Manifest Usage Through Scan Rewrites - -**Files:** -- Modify: `packages/context/src/ingest/adapters/live-database/manifest.ts` -- Test: `packages/context/src/ingest/adapters/live-database/manifest.test.ts` -- Modify: `packages/context/src/scan/local-enrichment-artifacts.ts` -- Test: `packages/context/src/scan/local-enrichment-artifacts.test.ts` - -- [ ] **Step 1: Write failing manifest-builder test** - -In `packages/context/src/ingest/adapters/live-database/manifest.test.ts`, add this test inside `describe('buildLiveDatabaseManifestShards', ...)`: - -```typescript - it('preserves external usage keys while replacing historic SQL managed keys', () => { - const existingUsage = new Map([ - [ - 'orders', - { - narrative: 'Old generated usage narrative.', - frequencyTier: 'low' as const, - commonFilters: ['old_status'], - commonJoins: [], - ownerNote: 'Pinned analyst note', - }, - ], - ]); - - const result = buildLiveDatabaseManifestShards({ - connectionType: 'POSTGRESQL', - mapColumnType: (nativeType) => nativeType.toLowerCase(), - existingUsage, - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - usage: { - narrative: 'Fresh generated usage narrative.', - frequencyTier: 'high', - commonFilters: ['status'], - commonGroupBys: ['created_at'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - }, - columns: [{ name: 'id', type: 'INTEGER' }], - }, - ], - joins: [], - }); - - expect(shardObject(result.shards)).toEqual({ - public: { - tables: { - orders: { - table: 'public.orders', - usage: { - ownerNote: 'Pinned analyst note', - narrative: 'Fresh generated usage narrative.', - frequencyTier: 'high', - commonFilters: ['status'], - commonGroupBys: ['created_at'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - }, - columns: [{ name: 'id', type: 'integer' }], - }, - }, - }, - }); - }); -``` - -- [ ] **Step 2: Run the manifest test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/live-database/manifest.test.ts -``` - -Expected: FAIL because `existingUsage` and table input `usage` are not supported. - -- [ ] **Step 3: Add usage types and merge helper** - -In `packages/context/src/ingest/adapters/live-database/manifest.ts`, add this import at the top: - -```typescript -import type { TableUsageOutput } from '../historic-sql/skill-schemas.js'; -``` - -Add this constant after `SCAN_MANAGED_DESCRIPTION_KEYS`: - -```typescript -const HISTORIC_SQL_MANAGED_USAGE_KEYS = new Set([ - 'narrative', - 'frequencyTier', - 'commonFilters', - 'commonGroupBys', - 'commonJoins', - 'staleSince', -]); -``` - -Add `usage` to `LiveDatabaseManifestTableEntry`: - -```typescript - usage?: TableUsageOutput; -``` - -Add `usage` to `LiveDatabaseManifestTableData`: - -```typescript - usage?: TableUsageOutput; -``` - -Add `existingUsage` to `BuildLiveDatabaseManifestShardsInput`: - -```typescript - existingUsage?: Map; -``` - -Add this exported helper after `mergeDescriptionsPreservingExternal()`: - -```typescript -export function mergeUsagePreservingExternal( - existing: TableUsageOutput | undefined, - incoming: TableUsageOutput | undefined, -): TableUsageOutput | undefined { - if (!existing && !incoming) { - return undefined; - } - const result: Record = {}; - if (existing) { - for (const [key, value] of Object.entries(existing)) { - if (!HISTORIC_SQL_MANAGED_USAGE_KEYS.has(key)) { - result[key] = value; - } - } - } - if (incoming) { - Object.assign(result, incoming); - } - return Object.keys(result).length > 0 ? (result as TableUsageOutput) : undefined; -} -``` - -In `buildLiveDatabaseManifestShards()`, add this block after table descriptions are set: - -```typescript - const usage = mergeUsagePreservingExternal(input.existingUsage?.get(table.name), table.usage); - if (usage) { - entry.usage = usage; - } -``` - -- [ ] **Step 4: Run the manifest test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/live-database/manifest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing scan-preservation test** - -In `packages/context/src/scan/local-enrichment-artifacts.test.ts`, inside the existing structural manifest shard test, extend the seeded YAML under `orders` with this block: - -```yaml - usage: - narrative: Orders are commonly filtered by lifecycle status. - frequencyTier: high - commonFilters: - - status - commonJoins: - - table: public.customers - on: - - customer_id - ownerNote: Preserve analyst note -``` - -Extend the parsed manifest type in that test: - -```typescript - usage?: Record; -``` - -Add this assertion after the descriptions assertions: - -```typescript - expect(manifest.tables.orders.usage).toEqual({ - narrative: 'Orders are commonly filtered by lifecycle status.', - frequencyTier: 'high', - commonFilters: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - ownerNote: 'Preserve analyst note', - }); -``` - -- [ ] **Step 6: Run the scan-preservation test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/scan/local-enrichment-artifacts.test.ts -``` - -Expected: FAIL because `loadExistingManifestState()` does not capture usage and scan rewrites drop it. - -- [ ] **Step 7: Preserve usage in local enrichment artifact writes** - -In `packages/context/src/scan/local-enrichment-artifacts.ts`, add `TableUsageOutput` to the ingest import: - -```typescript - type TableUsageOutput, -``` - -Add `usage` to `ExistingManifestState`: - -```typescript - usage: Map; -``` - -Initialize it in `loadExistingManifestState()`: - -```typescript - const usage = new Map(); -``` - -Update the early catch return: - -```typescript - return { descriptions, preservedJoins, usage }; -``` - -Inside the `for (const [tableName, entry] of Object.entries(shard.tables))` loop, after descriptions are captured, add: - -```typescript - if (entry.usage) { - usage.set(tableName, { ...entry.usage }); - } -``` - -Update the final return: - -```typescript - return { descriptions, preservedJoins, usage }; -``` - -Pass usage into `buildLiveDatabaseManifestShards()`: - -```typescript - existingUsage: existing.usage, -``` - -- [ ] **Step 8: Run scan-preservation test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/scan/local-enrichment-artifacts.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Commit** - -```bash -git add packages/context/src/ingest/adapters/live-database/manifest.ts packages/context/src/ingest/adapters/live-database/manifest.test.ts packages/context/src/scan/local-enrichment-artifacts.ts packages/context/src/scan/local-enrichment-artifacts.test.ts -git commit -m "feat: preserve historic sql usage in manifest shards" -``` - -## Task 4: Add Python Batch SQL Analysis - -**Files:** -- Create: `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` -- Create: `python/ktx-daemon/tests/test_sql_analysis.py` -- Modify: `python/ktx-daemon/src/ktx_daemon/app.py` -- Test: `python/ktx-daemon/tests/test_app.py` - -- [ ] **Step 1: Write failing parser tests** - -Create `python/ktx-daemon/tests/test_sql_analysis.py`: - -```python -from __future__ import annotations - -from ktx_daemon.sql_analysis import ( - AnalyzeSqlBatchItem, - AnalyzeSqlBatchRequest, - analyze_sql_batch_response, -) - - -def test_analyze_sql_batch_extracts_tables_and_clause_columns() -> None: - response = analyze_sql_batch_response( - AnalyzeSqlBatchRequest( - dialect="postgres", - items=[ - AnalyzeSqlBatchItem( - id="orders_by_customer", - sql=( - "select o.status, count(*) " - "from public.orders o " - "join public.customers c on o.customer_id = c.id " - "where o.created_at >= current_date - interval '30 day' " - "group by o.status" - ), - ) - ], - max_workers=1, - ) - ) - - result = response.results["orders_by_customer"] - assert result.error is None - assert result.tables_touched == ["public.orders", "public.customers"] - assert result.columns_by_clause == { - "select": ["status"], - "where": ["created_at"], - "join": ["customer_id", "id"], - "groupBy": ["status"], - } - - -def test_analyze_sql_batch_returns_per_item_parse_errors() -> None: - response = analyze_sql_batch_response( - AnalyzeSqlBatchRequest( - dialect="postgres", - items=[AnalyzeSqlBatchItem(id="broken", sql="select * from where")], - max_workers=1, - ) - ) - - result = response.results["broken"] - assert result.tables_touched == [] - assert result.columns_by_clause == {} - assert result.error is not None -``` - -- [ ] **Step 2: Run parser tests to verify they fail** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py -q -``` - -Expected: FAIL with `ModuleNotFoundError: No module named 'ktx_daemon.sql_analysis'`. - -- [ ] **Step 3: Add the batch parser module** - -Create `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`: - -```python -from __future__ import annotations - -import os -from concurrent.futures import ProcessPoolExecutor -from typing import Literal - -import sqlglot -from pydantic import BaseModel, ConfigDict, Field -from sqlglot import exp - -SqlAnalysisClause = Literal["select", "where", "join", "groupBy", "having", "orderBy"] - - -class AnalyzeSqlBatchItem(BaseModel): - id: str - sql: str - - -class AnalyzeSqlBatchRequest(BaseModel): - dialect: str - items: list[AnalyzeSqlBatchItem] - max_workers: int | None = Field(default=None, ge=1, le=32) - - -class AnalyzeSqlBatchResult(BaseModel): - model_config = ConfigDict(populate_by_name=True) - - tables_touched: list[str] = Field(default_factory=list) - columns_by_clause: dict[SqlAnalysisClause, list[str]] = Field(default_factory=dict) - error: str | None = None - - -class AnalyzeSqlBatchResponse(BaseModel): - results: dict[str, AnalyzeSqlBatchResult] - - -def _ordered_unique(values: list[str]) -> list[str]: - seen: set[str] = set() - result: list[str] = [] - for value in values: - if value and value not in seen: - seen.add(value) - result.append(value) - return result - - -def _table_ref(table: exp.Table) -> str: - parts: list[str] = [] - catalog = table.args.get("catalog") - db = table.args.get("db") - if catalog is not None and getattr(catalog, "name", None): - parts.append(str(catalog.name)) - if db is not None and getattr(db, "name", None): - parts.append(str(db.name)) - if table.name: - parts.append(str(table.name)) - return ".".join(parts) - - -def _column_name(column: exp.Column) -> str: - return str(column.name) - - -def _columns_from_nodes(nodes: list[exp.Expression | None]) -> list[str]: - names: list[str] = [] - for node in nodes: - if node is None: - continue - names.extend(_column_name(column) for column in node.find_all(exp.Column)) - return _ordered_unique(names) - - -def _columns_by_clause(tree: exp.Expression) -> dict[SqlAnalysisClause, list[str]]: - result: dict[SqlAnalysisClause, list[str]] = {} - - select_columns = _columns_from_nodes(list(tree.expressions)) - if select_columns: - result["select"] = select_columns - - where_columns = _columns_from_nodes([tree.args.get("where")]) - if where_columns: - result["where"] = where_columns - - join_columns = _columns_from_nodes([join.args.get("on") for join in tree.args.get("joins") or []]) - if join_columns: - result["join"] = join_columns - - group = tree.args.get("group") - group_columns = _columns_from_nodes(list(group.expressions) if group is not None else []) - if group_columns: - result["groupBy"] = group_columns - - having_columns = _columns_from_nodes([tree.args.get("having")]) - if having_columns: - result["having"] = having_columns - - order = tree.args.get("order") - order_columns = _columns_from_nodes(list(order.expressions) if order is not None else []) - if order_columns: - result["orderBy"] = order_columns - - return result - - -def _analyze_one(item_id: str, sql: str, dialect: str) -> tuple[str, AnalyzeSqlBatchResult]: - try: - tree = sqlglot.parse_one(sql, read=dialect) - except sqlglot.errors.SQLGlotError as exc: - return item_id, AnalyzeSqlBatchResult(error=str(exc)) - - cte_names = {cte.alias_or_name.lower() for cte in tree.find_all(exp.CTE)} - table_refs = [ - table_ref - for table_ref in (_table_ref(table) for table in tree.find_all(exp.Table)) - if table_ref and table_ref.split(".")[-1].lower() not in cte_names - ] - - return item_id, AnalyzeSqlBatchResult( - tables_touched=_ordered_unique(table_refs), - columns_by_clause=_columns_by_clause(tree), - error=None, - ) - - -def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]: - item_id, sql, dialect = payload - return _analyze_one(item_id, sql, dialect) - - -def _worker_count(request: AnalyzeSqlBatchRequest) -> int: - if len(request.items) <= 1: - return 1 - if request.max_workers is not None: - return min(request.max_workers, len(request.items)) - return min(os.cpu_count() or 1, len(request.items), 8) - - -def analyze_sql_batch_response(request: AnalyzeSqlBatchRequest) -> AnalyzeSqlBatchResponse: - payloads = [(item.id, item.sql, request.dialect) for item in request.items] - if _worker_count(request) == 1: - analyzed = [_analyze_payload(payload) for payload in payloads] - else: - with ProcessPoolExecutor(max_workers=_worker_count(request)) as executor: - analyzed = list(executor.map(_analyze_payload, payloads)) - - return AnalyzeSqlBatchResponse(results={item_id: result for item_id, result in analyzed}) -``` - -- [ ] **Step 4: Run parser tests to verify they pass** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py -q -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing FastAPI endpoint test** - -In `python/ktx-daemon/tests/test_app.py`, add this test after `test_sql_parse_table_identifier_endpoint()`: - -```python -def test_sql_analyze_batch_endpoint_returns_per_item_results() -> None: - client = TestClient(create_app()) - - response = client.post( - "/sql/analyze-batch", - json={ - "dialect": "postgres", - "max_workers": 1, - "items": [ - { - "id": "orders", - "sql": "select status from public.orders where created_at is not null", - }, - {"id": "broken", "sql": "select * from where"}, - ], - }, - ) - - assert response.status_code == 200 - body = response.json() - assert body["results"]["orders"]["tables_touched"] == ["public.orders"] - assert body["results"]["orders"]["columns_by_clause"] == { - "select": ["status"], - "where": ["created_at"], - } - assert body["results"]["orders"]["error"] is None - assert body["results"]["broken"]["tables_touched"] == [] - assert body["results"]["broken"]["columns_by_clause"] == {} - assert body["results"]["broken"]["error"] is not None -``` - -- [ ] **Step 6: Run the endpoint test to verify it fails** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py::test_sql_analyze_batch_endpoint_returns_per_item_results -q -``` - -Expected: FAIL with HTTP 404. - -- [ ] **Step 7: Register the daemon endpoint** - -In `python/ktx-daemon/src/ktx_daemon/app.py`, add this import block with the other daemon imports: - -```python -from ktx_daemon.sql_analysis import ( - AnalyzeSqlBatchRequest, - AnalyzeSqlBatchResponse, - analyze_sql_batch_response, -) -``` - -Add this route after `/sql/parse-table-identifier`: - -```python - @app.post("/sql/analyze-batch", response_model=AnalyzeSqlBatchResponse) - async def sql_analyze_batch( - request: AnalyzeSqlBatchRequest, - ) -> AnalyzeSqlBatchResponse: - try: - return analyze_sql_batch_response(request) - except Exception as error: - logger.exception("SQL batch analysis failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"SQL batch analysis failed: {error}", - ) from error -``` - -- [ ] **Step 8: Run Python tests to verify the daemon slice passes** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py::test_sql_analyze_batch_endpoint_returns_per_item_results -q -``` - -Expected: PASS. - -- [ ] **Step 9: Check Python formatting/lint hook availability** - -Run: - -```bash -test -f .pre-commit-config.yaml && source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py || printf 'pre-commit config missing\n' -``` - -Expected in this workspace: prints `pre-commit config missing`. - -- [ ] **Step 10: Commit** - -```bash -git add python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -git commit -m "feat: add daemon sql batch analysis" -``` - -## Task 5: Add TypeScript Batch SQL Analysis Port - -**Files:** -- Modify: `packages/context/src/sql-analysis/ports.ts` -- Modify: `packages/context/src/sql-analysis/index.ts` -- Modify: `packages/context/src/sql-analysis/http-sql-analysis-port.ts` -- Test: `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` -- Test: `packages/cli/src/managed-python-http.test.ts` -- Modify: legacy `SqlAnalysisPort` mocks found by `rg -n "const .*SqlAnalysis|sqlAnalysis: \\{" packages/context packages/cli` - -- [ ] **Step 1: Write failing HTTP port tests** - -In `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`, add these tests before the malformed daemon response test: - -```typescript - it('calls the SQL batch endpoint and maps snake_case response fields into a Map', async () => { - const requestJson = vi.fn(async () => ({ - results: { - orders: { - tables_touched: ['public.orders', 'public.customers'], - columns_by_clause: { - select: ['status'], - where: ['created_at'], - join: ['customer_id', 'id'], - }, - error: null, - }, - broken: { - tables_touched: [], - columns_by_clause: {}, - error: 'Invalid expression / Unexpected token', - }, - }, - })); - const port = createHttpSqlAnalysisPort({ baseUrl: 'http://python.test', requestJson }); - - await expect( - port.analyzeBatch( - [ - { id: 'orders', sql: 'select status from public.orders' }, - { id: 'broken', sql: 'select * from where' }, - ], - 'postgres', - ), - ).resolves.toEqual( - new Map([ - [ - 'orders', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: ['status'], - where: ['created_at'], - join: ['customer_id', 'id'], - }, - error: null, - }, - ], - [ - 'broken', - { - tablesTouched: [], - columnsByClause: {}, - error: 'Invalid expression / Unexpected token', - }, - ], - ]), - ); - - expect(requestJson).toHaveBeenCalledWith('/sql/analyze-batch', { - dialect: 'postgres', - items: [ - { id: 'orders', sql: 'select status from public.orders' }, - { id: 'broken', sql: 'select * from where' }, - ], - }); - }); - - it('rejects malformed SQL batch responses instead of inventing defaults', async () => { - const requestJson = vi.fn(async () => ({ - results: { - orders: { - tables_touched: ['public.orders'], - columns_by_clause: { select: ['status'], where: [42] }, - error: null, - }, - }, - })); - const port = createHttpSqlAnalysisPort({ baseUrl: 'http://python.test', requestJson }); - - await expect(port.analyzeBatch([{ id: 'orders', sql: 'select status from public.orders' }], 'postgres')).rejects - .toThrow('sql analysis response is missing string[] field columns_by_clause.where'); - }); -``` - -- [ ] **Step 2: Run HTTP port tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts -``` - -Expected: FAIL because `analyzeBatch` is not defined. - -- [ ] **Step 3: Add batch types to the port** - -In `packages/context/src/sql-analysis/ports.ts`, add these types after `SqlAnalysisFingerprintResult`: - -```typescript -export type SqlAnalysisClause = 'select' | 'where' | 'join' | 'groupBy' | 'having' | 'orderBy' | (string & {}); - -export interface SqlAnalysisBatchItem { - id: string; - sql: string; -} - -export interface SqlAnalysisBatchResult { - tablesTouched: string[]; - columnsByClause: Partial>; - error?: string | null; -} -``` - -Update `SqlAnalysisPort`: - -```typescript -export interface SqlAnalysisPort { - analyzeForFingerprint(sql: string, dialect: SqlAnalysisDialect): Promise; - analyzeBatch( - items: SqlAnalysisBatchItem[], - dialect: SqlAnalysisDialect, - ): Promise>; -} -``` - -In `packages/context/src/sql-analysis/index.ts`, export the new types: - -```typescript - SqlAnalysisBatchItem, - SqlAnalysisBatchResult, - SqlAnalysisClause, -``` - -- [ ] **Step 4: Map the HTTP batch response** - -In `packages/context/src/sql-analysis/http-sql-analysis-port.ts`, add the new type imports: - -```typescript - SqlAnalysisBatchItem, - SqlAnalysisBatchResult, -``` - -Add this helper after `requiredStringArray()`: - -```typescript -function requiredObject(raw: Record, field: string): Record { - const value = raw[field]; - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`sql analysis response is missing object field ${field}`); - } - return value as Record; -} -``` - -Add this helper after `mapResult()`: - -```typescript -function mapColumnsByClause(raw: Record): SqlAnalysisBatchResult['columnsByClause'] { - const value = requiredObject(raw, 'columns_by_clause'); - const result: SqlAnalysisBatchResult['columnsByClause'] = {}; - for (const [clause, columns] of Object.entries(value)) { - if (!Array.isArray(columns) || columns.some((item) => typeof item !== 'string')) { - throw new Error(`sql analysis response is missing string[] field columns_by_clause.${clause}`); - } - result[clause] = columns; - } - return result; -} - -function mapBatchResult(raw: Record): SqlAnalysisBatchResult { - const error = optionalString(raw, 'error'); - return { - tablesTouched: requiredStringArray(raw, 'tables_touched'), - columnsByClause: mapColumnsByClause(raw), - ...(error !== undefined ? { error } : {}), - }; -} - -function mapBatchResponse(raw: Record): Map { - const results = requiredObject(raw, 'results'); - return new Map( - Object.entries(results).map(([id, value]) => { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`sql analysis response contains invalid batch result ${id}`); - } - return [id, mapBatchResult(value as Record)]; - }), - ); -} -``` - -Add `analyzeBatch()` to the object returned by `createHttpSqlAnalysisPort()`: - -```typescript - async analyzeBatch(items: SqlAnalysisBatchItem[], dialect: SqlAnalysisDialect) { - const raw = await requestJson('/sql/analyze-batch', { - dialect, - items, - }); - return mapBatchResponse(raw); - }, -``` - -- [ ] **Step 5: Run HTTP port tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Update managed-daemon wrapper test** - -In `packages/cli/src/managed-python-http.test.ts`, add this test after the existing SQL analysis port test: - -```typescript - it('routes SQL batch analysis through the managed daemon runner', async () => { - const requestJson = vi.fn(async () => ({ - results: { - orders: { - tables_touched: ['public.orders'], - columns_by_clause: { select: ['status'] }, - error: null, - }, - }, - })); - const sqlAnalysis = createManagedDaemonSqlAnalysisPort({ requestJson }); - - await expect(sqlAnalysis.analyzeBatch([{ id: 'orders', sql: 'select status from public.orders' }], 'postgres')) - .resolves.toEqual( - new Map([ - [ - 'orders', - { - tablesTouched: ['public.orders'], - columnsByClause: { select: ['status'] }, - error: null, - }, - ], - ]), - ); - expect(requestJson).toHaveBeenCalledWith('/sql/analyze-batch', { - dialect: 'postgres', - items: [{ id: 'orders', sql: 'select status from public.orders' }], - }); - }); -``` - -- [ ] **Step 7: Update legacy `SqlAnalysisPort` mocks** - -Run: - -```bash -rg -n "SqlAnalysisPort|sqlAnalysis: \\{|analyzeForFingerprint" packages/context/src/ingest packages/cli/src -``` - -For every object literal typed as `SqlAnalysisPort` or passed into a typed `sqlAnalysis` dependency, add: - -```typescript - async analyzeBatch() { - return new Map(); - }, -``` - -Known files from the current workspace: - -- `packages/context/src/ingest/local-adapters.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage-pgss-golden.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage-pgss.test.ts` - -- [ ] **Step 8: Run CLI wrapper and context type checks** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/managed-python-http.test.ts -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. If type-check reports a `SqlAnalysisPort` mock missing `analyzeBatch`, add the no-op method from Step 7 and rerun the failing type-check command. - -- [ ] **Step 9: Commit** - -```bash -git add packages/context/src/sql-analysis/ports.ts packages/context/src/sql-analysis/index.ts packages/context/src/sql-analysis/http-sql-analysis-port.ts packages/context/src/sql-analysis/http-sql-analysis-port.test.ts packages/cli/src/managed-python-http.test.ts packages/context/src/ingest/local-adapters.test.ts packages/context/src/ingest/adapters/historic-sql/stage.test.ts packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts packages/context/src/ingest/adapters/historic-sql/stage-pgss-golden.test.ts packages/context/src/ingest/adapters/historic-sql/stage-pgss.test.ts -git commit -m "feat: add sql analysis batch port" -``` - -## Task 6: Final Verification - -**Files:** -- Read-only verification across TypeScript and Python. - -- [ ] **Step 1: Run focused TypeScript tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/skill-schemas.test.ts src/sl/semantic-layer.service.test.ts src/ingest/adapters/live-database/manifest.test.ts src/scan/local-enrichment-artifacts.test.ts src/sql-analysis/http-sql-analysis-port.test.ts -pnpm --filter @ktx/cli exec vitest run src/managed-python-http.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused Python tests** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py::test_sql_analyze_batch_endpoint_returns_per_item_results -q -``` - -Expected: PASS. - -- [ ] **Step 3: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run Python pre-commit check if configured** - -Run: - -```bash -test -f .pre-commit-config.yaml && source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py || printf 'pre-commit config missing\n' -``` - -Expected in this workspace: prints `pre-commit config missing`. - -- [ ] **Step 5: Confirm the old adapter was not cut over in this slice** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline" packages/context/src/ingest/adapters/historic-sql packages/context/src/ingest/index.ts -``` - -Expected: matches still exist. This confirms the foundation slice did not silently perform the hard cutover from spec §10.1. - -- [ ] **Step 6: Commit verification notes if code changed during verification** - -If verification required edits, commit only those files: - -```bash -git status --short -git add packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts packages/context/src/ingest/index.ts packages/context/src/sl/types.ts packages/context/src/sl/schemas.ts packages/context/src/sl/semantic-layer.service.ts packages/context/src/sl/semantic-layer.service.test.ts packages/context/src/ingest/adapters/live-database/manifest.ts packages/context/src/ingest/adapters/live-database/manifest.test.ts packages/context/src/scan/local-enrichment-artifacts.ts packages/context/src/scan/local-enrichment-artifacts.test.ts python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py packages/context/src/sql-analysis/ports.ts packages/context/src/sql-analysis/index.ts packages/context/src/sql-analysis/http-sql-analysis-port.ts packages/context/src/sql-analysis/http-sql-analysis-port.test.ts packages/cli/src/managed-python-http.test.ts packages/context/src/ingest/local-adapters.test.ts packages/context/src/ingest/adapters/historic-sql/stage.test.ts packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts packages/context/src/ingest/adapters/historic-sql/stage-pgss-golden.test.ts packages/context/src/ingest/adapters/historic-sql/stage-pgss.test.ts -git commit -m "test: finish historic sql foundations verification" -``` - -If verification required no edits, do not create an empty commit. - -## Self-Review - -**Spec coverage:** This plan covers the foundation item in spec §10.3. It intentionally does not cover search enrichment (§6.2.3-§6.2.5), the unified reader and staged artifacts (§4), skills and projection (§5), legacy cleanup (§10.2), or setup/doctor docs (§8). Those should be separate plans because each produces a testable subsystem and avoids one oversized cutover plan. - -**Placeholder scan:** The plan contains exact file paths, test code, implementation snippets, commands, expected failures, expected passes, and commit commands. It does not use placeholder markers or deferred implementation text. - -**Type consistency:** `TableUsageOutput` is created in `skill-schemas.ts`, then reused by `SemanticLayerSource`, `ManifestTableEntry`, and `LiveDatabaseManifestTableEntry`. `SqlAnalysisPort.analyzeBatch()` returns `Map` consistently across `ports.ts`, `http-sql-analysis-port.ts`, and `managed-python-http.test.ts`. The Python daemon response uses snake_case fields that the TypeScript HTTP port maps to camelCase. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md deleted file mode 100644 index b5382ff4..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md +++ /dev/null @@ -1,407 +0,0 @@ -# Historic SQL Pattern Shard Smoke Docs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Align the Postgres historic-SQL smoke and example docs with sharded pattern WorkUnits. - -**Architecture:** The runtime already writes the full `patterns-input.json` audit file and bounded `patterns-input/part-0001.json` style shards. This plan updates the example acceptance assets so they verify the sharded contract instead of the pre-sharding root `historic-sql-patterns` WorkUnit. - -**Tech Stack:** Bash, Node.js built-in test runner, pnpm workspace scripts, KTX local stage-only ingest. - ---- - -## Spec And Existing Plan Status - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans derived from this spec and implemented in this worktree: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` - implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `packages/context/src/sql-analysis/ports.ts`, daemon `/sql/analyze-batch`, `SemanticLayerSource.usage`, and `mergeUsagePreservingExternal()`. -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` - implemented. Evidence: usage-aware SL search text, SQLite FTS snippets, and local/MCP result fields `frequencyTier` plus `snippet`. -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` - implemented. Evidence: `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake readers, unified schemas, and package exports. -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` - implemented. Evidence: `HistoricSqlSourceAdapter`, `historic_sql_table_digest`, `historic_sql_patterns`, `emit_historic_sql_evidence`, `HistoricSqlProjectionPostProcessor`, and legacy skill removal from runtime code. -- `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md` - implemented. Evidence: local adapter registration tests for Postgres, BigQuery, and Snowflake plus PG doctor coverage for informational `pg_stat_statements.max`. -- `docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` - implemented at the time it was written, but its smoke assertions predate pattern shard WorkUnits. -- `docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md` - implemented. Evidence: `isArchivedPatternPage()`, archive exclusion from slug matching, stale table tests, and legacy query-page cleanup coverage. -- `docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md` - implemented. Evidence: `local-ingest-acceptance.test.ts` proves production adapter output reaches SL search and wiki search. -- `docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md` - implemented. Evidence: `redaction.ts`, `redaction.test.ts`, and staged artifact redaction coverage in `stage-unified.test.ts`. -- `docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md` - implemented. Evidence: `pattern-inputs.ts`, `pattern-inputs.test.ts`, `stage-unified.ts` writes `patterns-input/part-*.json`, `chunk-unified.ts` emits `historic-sql-patterns-part-*`, `historic_sql_patterns` reads shards, and acceptance tests use `rawPath: 'patterns-input/part-0001.json'`. - -No existing spec-derived implementation plan is currently unimplemented in this worktree. - -Remaining gap this plan fixes: - -- `examples/postgres-historic/scripts/smoke.sh` still asserts a WorkUnit with `unitKey === 'historic-sql-patterns'`. -- Current runtime emits pattern WorkUnits with keys like `historic-sql-patterns-part-0001` and raw files like `patterns-input/part-0001.json`. -- The same smoke only validates the audit file `patterns-input.json`; it does not assert that the bounded shard files exist or contain only cross-table candidates. -- `examples/postgres-historic/README.md` and `examples/README.md` describe unchanged "pattern inputs" but do not explain that `patterns-input.json` is now audit-only and `patterns-input/part-*.json` drives pattern WorkUnits. -- `scripts/examples-docs.test.mjs` does not pin the sharded smoke/doc contract, so the stale root WorkUnit assertion can regress silently. - -## File Structure - -- Modify `scripts/examples-docs.test.mjs` - Pins docs and smoke script to the sharded pattern WorkUnit contract. -- Modify `examples/postgres-historic/scripts/smoke.sh` - Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits. -- Modify `examples/postgres-historic/README.md` - Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input. -- Modify `examples/README.md` - Updates the short example catalog entry with the same audit-vs-shard wording. - -### Task 1: Pin Example Tests To Pattern Shards - -**Files:** -- Modify: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Add failing assertions for sharded pattern smoke/docs** - -In `scripts/examples-docs.test.mjs`, inside `it('documents the Postgres historic SQL smoke example', ...)`, add these assertions immediately after the existing `assert.match(readme, /patterns-input\.json/);` line: - -```javascript - assert.match(readme, /patterns-input\/part-\*\.json/); - assert.match(readme, /full audit input/); - assert.match(readme, /bounded pattern WorkUnit shards/); -``` - -In the same test, add these assertions immediately after the existing `assert.match(smoke, /assert_stage_record "\$UNCHANGED_RECORD" unchanged zero/);` line: - -```javascript - assert.match(smoke, /assertPatternShards/); - assert.match(smoke, /historic-sql-patterns-part-/); - assert.match(smoke, /patterns-input\/part-/); - assert.doesNotMatch(smoke, /unitKey === 'historic-sql-patterns'/); -``` - -- [ ] **Step 2: Run the example docs test to verify it fails** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL. The test should report missing `patterns-input/part-*.json`, `full audit input`, `bounded pattern WorkUnit shards`, `assertPatternShards`, or it should fail because `smoke.sh` still contains `unitKey === 'historic-sql-patterns'`. - -- [ ] **Step 3: Commit the failing test** - -Run: - -```bash -git add scripts/examples-docs.test.mjs -git commit -m "test: expect historic sql pattern shard smoke docs" -``` - -### Task 2: Update The Postgres Historic Smoke - -**Files:** -- Modify: `examples/postgres-historic/scripts/smoke.sh` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Import `existsSync` in the embedded snapshot assertion** - -In `examples/postgres-historic/scripts/smoke.sh`, inside `assert_unified_snapshot()`, replace this line: - -```javascript -const { readFileSync, readdirSync } = require('node:fs'); -``` - -with: - -```javascript -const { existsSync, readFileSync, readdirSync } = require('node:fs'); -``` - -- [ ] **Step 2: Add shard validation to `assert_unified_snapshot()`** - -In `examples/postgres-historic/scripts/smoke.sh`, inside the embedded Node script in `assert_unified_snapshot()`, add this function after the `legacyKeys` loop: - -```javascript -function assertPatternShards(root) { - const shardDir = join(root, 'patterns-input'); - assert(existsSync(shardDir), 'Expected patterns-input shard directory'); - const shardFiles = readdirSync(shardDir) - .filter((file) => /^part-\d{4}\.json$/.test(file)) - .sort() - .map((file) => `patterns-input/${file}`); - assert(shardFiles.length > 0, 'Expected at least one pattern shard file'); - - for (const shardFile of shardFiles) { - const shard = JSON.parse(readFileSync(join(root, shardFile), 'utf8')); - assert(Array.isArray(shard.templates), `${shardFile}: expected templates array`); - assert(shard.templates.length > 0, `${shardFile}: expected at least one template`); - assert( - shard.templates.every((template) => Array.isArray(template.tablesTouched) && template.tablesTouched.length >= 2), - `${shardFile}: expected only cross-table pattern candidates`, - ); - } - - return shardFiles; -} -``` - -- [ ] **Step 3: Assert the full audit input and bounded shards** - -In the same embedded Node script, replace the current `patterns` block: - -```javascript -const patterns = JSON.parse(readFileSync(join(root, 'patterns-input.json'), 'utf8')); -assert(Array.isArray(patterns.templates) && patterns.templates.length > 0, 'Expected patterns-input templates'); -assert( - patterns.templates.every((template) => Array.isArray(template.tablesTouched) && template.tablesTouched.length > 0), - 'Expected every pattern template to have touched tables', -); -``` - -with: - -```javascript -const patterns = JSON.parse(readFileSync(join(root, 'patterns-input.json'), 'utf8')); -assert(Array.isArray(patterns.templates) && patterns.templates.length > 0, 'Expected patterns-input audit templates'); -assert( - patterns.templates.every((template) => Array.isArray(template.tablesTouched) && template.tablesTouched.length > 0), - 'Expected every audit pattern template to have touched tables', -); -const shardFiles = assertPatternShards(root); -assert( - shardFiles.length <= patterns.templates.length, - `Expected shard count ${shardFiles.length} to be no greater than audit template count ${patterns.templates.length}`, -); -``` - -- [ ] **Step 4: Update the stage record WorkUnit assertions** - -In `examples/postgres-historic/scripts/smoke.sh`, inside the embedded Node script in `assert_stage_record()`, replace: - -```javascript -assert(record.rawFileCount >= 3, `${label}: expected manifest, patterns input, and at least one table file`); -``` - -with: - -```javascript -assert(record.rawFileCount >= 4, `${label}: expected manifest, audit patterns input, pattern shard, and at least one table file`); -``` - -Then replace this nonzero WorkUnit block: - -```javascript -} else if (expectedWorkUnits === 'nonzero') { - assert(record.workUnitCount > 0, `${label}: expected nonzero WorkUnits`); - assert(record.workUnits.some((unit) => unit.unitKey === 'historic-sql-patterns'), `${label}: expected patterns WorkUnit`); - assert(record.workUnits.some((unit) => unit.unitKey.startsWith('historic-sql-table-')), `${label}: expected table WorkUnit`); -} else { -``` - -with: - -```javascript -} else if (expectedWorkUnits === 'nonzero') { - assert(record.workUnitCount > 0, `${label}: expected nonzero WorkUnits`); - const patternUnits = record.workUnits.filter((unit) => /^historic-sql-patterns-part-\d{4}$/.test(unit.unitKey)); - assert(patternUnits.length > 0, `${label}: expected sharded patterns WorkUnit`); - for (const unit of patternUnits) { - assert( - unit.rawFiles.some((rawFile) => /^patterns-input\/part-\d{4}\.json$/.test(rawFile)), - `${label}: expected ${unit.unitKey} to read a pattern shard`, - ); - assert( - !unit.rawFiles.includes('patterns-input.json'), - `${label}: expected ${unit.unitKey} not to schedule the full audit patterns input`, - ); - } - assert(record.workUnits.some((unit) => unit.unitKey.startsWith('historic-sql-table-')), `${label}: expected table WorkUnit`); -} else { -``` - -- [ ] **Step 5: Run shell syntax and the docs test** - -Run: - -```bash -bash -n examples/postgres-historic/scripts/smoke.sh -node --test scripts/examples-docs.test.mjs -``` - -Expected: `bash -n` exits 0. The docs test still fails until the README files are updated in Task 3. - -- [ ] **Step 6: Commit the smoke update** - -Run: - -```bash -git add examples/postgres-historic/scripts/smoke.sh -git commit -m "test: assert historic sql pattern shard smoke" -``` - -### Task 3: Update Example Documentation - -**Files:** -- Modify: `examples/postgres-historic/README.md` -- Modify: `examples/README.md` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Update the artifact list in the Postgres historic README** - -In `examples/postgres-historic/README.md`, replace this list: - -```markdown -- `manifest.json` -- `tables/*.json` -- `patterns-input.json` -``` - -with: - -```markdown -- `manifest.json` -- `tables/*.json` -- `patterns-input.json` as the full audit input -- `patterns-input/part-*.json` as bounded pattern WorkUnit shards -``` - -- [ ] **Step 2: Update the idempotency wording** - -In `examples/postgres-historic/README.md`, replace this paragraph: - -```markdown -The smoke also runs the same workload twice and verifies the second stage-only -run has `workUnitCount: 0`, which proves unchanged bucketed table and pattern -inputs do not schedule LLM work. -``` - -with: - -```markdown -The smoke also runs the same workload twice and verifies the second stage-only -run has `workUnitCount: 0`, which proves unchanged bucketed table inputs and -unchanged bounded pattern shards do not schedule LLM work. -``` - -- [ ] **Step 3: Update the manifest inspection wording** - -In `examples/postgres-historic/README.md`, replace this paragraph: - -```markdown -The manifest should have `source: "historic-sql"`, `dialect: "postgres"`, -positive `snapshotRowCount`, positive `touchedTableCount`, numeric -`parseFailures`, `warnings`, and `probeWarnings`. The same directory should -contain `patterns-input.json` and one `tables/*.json` file per touched table. -``` - -with: - -```markdown -The manifest should have `source: "historic-sql"`, `dialect: "postgres"`, -positive `snapshotRowCount`, positive `touchedTableCount`, numeric -`parseFailures`, `warnings`, and `probeWarnings`. The same directory should -contain `patterns-input.json`, at least one `patterns-input/part-*.json` pattern -shard for cross-table candidates, and one `tables/*.json` file per touched -table. -``` - -- [ ] **Step 4: Update the examples catalog entry** - -In `examples/README.md`, replace this paragraph: - -```markdown -`postgres-historic/` is a manual Docker-backed smoke for Postgres historic-SQL -ingest via `pg_stat_statements`. It verifies setup, unified Historic SQL artifacts, -managed daemon batch SQL analysis, and no-WorkUnit idempotency for unchanged -bucketed table and pattern inputs. -``` - -with: - -```markdown -`postgres-historic/` is a manual Docker-backed smoke for Postgres historic-SQL -ingest via `pg_stat_statements`. It verifies setup, unified Historic SQL artifacts, -managed daemon batch SQL analysis, bounded pattern WorkUnit shards, and -no-WorkUnit idempotency for unchanged bucketed table inputs and pattern shards. -``` - -- [ ] **Step 5: Run the example docs test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the docs update** - -Run: - -```bash -git add examples/postgres-historic/README.md examples/README.md -git commit -m "docs: explain historic sql pattern shards" -``` - -### Task 4: Verify The Smoke Contract - -**Files:** -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `examples/postgres-historic/scripts/smoke.sh` -- Verify: `examples/postgres-historic/README.md` -- Verify: `examples/README.md` - -- [ ] **Step 1: Run focused local checks** - -Run: - -```bash -bash -n examples/postgres-historic/scripts/smoke.sh -node --test scripts/examples-docs.test.mjs -``` - -Expected: both commands pass. - -- [ ] **Step 2: Run the Docker-backed Postgres historic smoke** - -Run: - -```bash -examples/postgres-historic/scripts/smoke.sh -``` - -Expected: PASS with `Postgres historic SQL smoke passed`. The stage-only records should include pattern WorkUnits with keys like `historic-sql-patterns-part-0001`, each reading `patterns-input/part-0001.json`, and the unchanged run should report `workUnitCount: 0`. - -- [ ] **Step 3: Run the drift grep** - -Run: - -```bash -rg -n "unitKey === 'historic-sql-patterns'|expected patterns WorkUnit|patterns-input\\.json\\` and one \\`tables|unchanged bucketed table and pattern inputs" examples scripts -``` - -Expected: no matches. - -- [ ] **Step 4: Commit verification metadata if any test-only wording changed** - -Run: - -```bash -git status --short -``` - -Expected: no unstaged files. If a previous step required a wording fix, commit only the touched files: - -```bash -git add scripts/examples-docs.test.mjs examples/postgres-historic/scripts/smoke.sh examples/postgres-historic/README.md examples/README.md -git commit -m "test: verify historic sql sharded smoke docs" -``` - -## Self-Review - -**Spec coverage:** This plan follows spec section 5.2's deterministic pattern sharding and preserves section 4.6's full `patterns-input.json` audit artifact. It updates the smoke and docs around the already implemented sharded runtime contract. - -**Placeholder scan:** The plan contains exact file paths, exact snippets, commands, expected outcomes, and commit commands. - -**Type consistency:** The plan uses the implemented runtime names consistently: `patterns-input.json` for the audit file, `patterns-input/part-*.json` for bounded shards, and `historic-sql-patterns-part-0001` style WorkUnit keys for pattern curation. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md deleted file mode 100644 index c67f6d78..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md +++ /dev/null @@ -1,943 +0,0 @@ -# Historic SQL Pattern WorkUnit Sharding Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Keep historic-SQL pattern WorkUnit inputs under the raw-file and prompt-size limits by writing deterministic bounded pattern shards while preserving `patterns-input.json` as the full audit artifact. - -**Architecture:** The stager continues to write full `patterns-input.json` for audit and diff visibility, then writes bounded `patterns-input/part-0001.json` style shards that contain only cross-table pattern candidates. The chunker emits one `historic_sql_patterns` WorkUnit per changed shard and never asks the skill to read the full audit file. Pattern projection is unchanged because emitted evidence already carries a free-form `rawPath`. - -**Tech Stack:** TypeScript, Node.js filesystem APIs, Zod, Vitest, pnpm workspace commands. - ---- - -## Spec And Existing Plan Status - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans derived from this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` - implemented. Current evidence includes `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `SqlAnalysisPort.analyzeBatch()`, daemon `/sql/analyze-batch`, `SemanticLayerSource.usage`, and `mergeUsagePreservingExternal()`. -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` - implemented. Current evidence includes usage-aware `buildSemanticLayerSourceSearchText()`, FTS snippets in `sqlite-sl-sources-index.ts`, and list surfaces exposing `frequencyTier` plus `snippet`. -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` - implemented. Current evidence includes `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake readers, unified schemas, and package exports. -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` - implemented. Current evidence includes production adapter cutover, `historic_sql_table_digest`, `historic_sql_patterns`, `emit_historic_sql_evidence`, `HistoricSqlProjectionPostProcessor`, and removal of legacy skill names from runtime code. -- `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md` - implemented. Current evidence includes local adapter registration tests for Postgres, BigQuery, and Snowflake plus PG doctor coverage for informational `pg_stat_statements.max`. -- `docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` - implemented. Current evidence includes canonical setup config tests, docs using `minExecutions`, and the Postgres historic smoke script asserting unified staged artifacts and unchanged-run idempotency. -- `docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md` - implemented. Current evidence includes `isArchivedPatternPage()`, archive exclusion from active slug matching, stale table tests, and legacy query-page cleanup coverage. -- `docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md` - implemented. Current evidence includes `local-ingest-acceptance.test.ts` proving production adapter output reaches SL search and wiki search. -- `docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md` - implemented. Current evidence includes `redaction.ts`, `redaction.test.ts`, and `stage-unified.test.ts` proving original SQL is analyzed while staged artifacts contain `[REDACTED]`. - -No existing spec-derived plan is currently unimplemented in this worktree. This plan covers the next uncovered implementation gap from spec section 5.2: `historic_sql_patterns` may need "a small handful" of deterministic chunks when `patterns-input.json` exceeds the LLM context budget. Current code always emits one WorkUnit with raw file `patterns-input.json`; `read_raw_file` rejects files larger than 120,000 bytes and WorkUnit prompt construction rejects prompts larger than 240,000 characters. - -## File Structure - -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` - Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection. -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` - Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` - Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` - Adds a regression for audit file preservation and sharded WorkUnit input creation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` - Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` - Updates root-file expectations and adds multi-shard diff behavior. -- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` - Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`. -- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` - Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`. -- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` - Keeps packaged skill assertions aligned with sharded pattern file guidance. - -## Task 1: Add Pattern Input Sharding Helper - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` - -- [ ] **Step 1: Write the failing helper tests** - -Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { - HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES, - isHistoricSqlPatternInputShardPath, - serializedStagedPatternsInputByteLength, - splitHistoricSqlPatternInputs, -} from './pattern-inputs.js'; -import type { StagedPatternsInput } from './types.js'; - -type PatternTemplate = StagedPatternsInput['templates'][number]; - -function template(id: string, tablesTouched: string[], canonicalSql = 'select 1'): PatternTemplate { - return { - id, - canonicalSql, - tablesTouched, - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }; -} - -describe('historic-SQL pattern input sharding', () => { - it('keeps the audit input complete while sharding only cross-table pattern candidates', () => { - const largeSql = `select * from public.orders join public.customers on true where marker = '${'x'.repeat(260)}'`; - const input: StagedPatternsInput = { - templates: [ - template('single-table-orders', ['public.orders']), - template('orders-customers-2', ['public.orders', 'public.customers'], largeSql), - template('orders-customers-1', ['public.customers', 'public.orders'], largeSql), - template('orders-customers-payments', ['public.orders', 'public.customers', 'public.payments'], largeSql), - ], - }; - - const result = splitHistoricSqlPatternInputs(input, { maxBytes: 760 }); - - expect(result.auditInput.templates.map((entry) => entry.id)).toEqual([ - 'orders-customers-1', - 'orders-customers-2', - 'orders-customers-payments', - 'single-table-orders', - ]); - expect(result.shards.length).toBeGreaterThan(1); - expect(result.shards.map((shard) => shard.path)).toEqual(['patterns-input/part-0001.json', 'patterns-input/part-0002.json', 'patterns-input/part-0003.json']); - expect(result.shards.flatMap((shard) => shard.input.templates.map((entry) => entry.id))).toEqual([ - 'orders-customers-payments', - 'orders-customers-1', - 'orders-customers-2', - ]); - expect(result.shards.every((shard) => shard.byteLength <= 760)).toBe(true); - expect(result.shards.flatMap((shard) => shard.input.templates).some((entry) => entry.id === 'single-table-orders')).toBe(false); - expect(result.warnings).toEqual([]); - }); - - it('omits a single oversized template from shards and reports a manifest warning', () => { - const input: StagedPatternsInput = { - templates: [ - template( - 'oversized-cross-table', - ['public.orders', 'public.customers'], - `select * from public.orders join public.customers on true where payload = '${'x'.repeat(500)}'`, - ), - ], - }; - - const result = splitHistoricSqlPatternInputs(input, { maxBytes: 240 }); - - expect(result.auditInput.templates.map((entry) => entry.id)).toEqual(['oversized-cross-table']); - expect(result.shards).toEqual([]); - expect(result.warnings).toEqual(['patterns_input_template_too_large:oversized-cross-table']); - }); - - it('recognizes only generated pattern shard paths', () => { - expect(isHistoricSqlPatternInputShardPath('patterns-input/part-0001.json')).toBe(true); - expect(isHistoricSqlPatternInputShardPath('patterns-input/part-0012.json')).toBe(true); - expect(isHistoricSqlPatternInputShardPath('patterns-input.json')).toBe(false); - expect(isHistoricSqlPatternInputShardPath('patterns-input/part-1.json')).toBe(false); - expect(isHistoricSqlPatternInputShardPath('patterns-input/readme.md')).toBe(false); - }); - - it('uses a production byte budget below read_raw_file maximum size', () => { - expect(HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES).toBeLessThan(120_000); - expect(serializedStagedPatternsInputByteLength({ templates: [] })).toBeGreaterThan(0); - }); -}); -``` - -- [ ] **Step 2: Run helper tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/pattern-inputs.test.ts -``` - -Expected: FAIL because `./pattern-inputs.js` does not exist. - -- [ ] **Step 3: Add the sharding helper** - -Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts`: - -```typescript -import { Buffer } from 'node:buffer'; -import type { StagedPatternsInput } from './types.js'; - -export const HISTORIC_SQL_PATTERN_WORKUNIT_DIR = 'patterns-input'; -export const HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES = 110_000; -export const HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE = /^patterns-input\/part-\d{4}\.json$/; - -type PatternTemplate = StagedPatternsInput['templates'][number]; - -export interface HistoricSqlPatternInputShard { - path: string; - input: StagedPatternsInput; - byteLength: number; -} - -export interface HistoricSqlPatternInputSplitResult { - auditInput: StagedPatternsInput; - shards: HistoricSqlPatternInputShard[]; - warnings: string[]; -} - -export interface HistoricSqlPatternInputSplitOptions { - maxBytes?: number; -} - -export function isHistoricSqlPatternInputShardPath(path: string): boolean { - return HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE.test(path); -} - -export function serializeStagedPatternsInput(input: StagedPatternsInput): string { - return `${JSON.stringify(input, null, 2)}\n`; -} - -export function serializedStagedPatternsInputByteLength(input: StagedPatternsInput): number { - return Buffer.byteLength(serializeStagedPatternsInput(input), 'utf-8'); -} - -function sortedAuditTemplates(templates: readonly PatternTemplate[]): PatternTemplate[] { - return [...templates].sort((left, right) => left.id.localeCompare(right.id)); -} - -function sortedPatternCandidates(templates: readonly PatternTemplate[]): PatternTemplate[] { - return [...templates] - .filter((template) => template.tablesTouched.length >= 2) - .map((template) => ({ ...template, tablesTouched: [...template.tablesTouched].sort() })) - .sort((left, right) => { - const cardinality = right.tablesTouched.length - left.tablesTouched.length; - if (cardinality !== 0) return cardinality; - const tableSignature = left.tablesTouched.join('\0').localeCompare(right.tablesTouched.join('\0')); - if (tableSignature !== 0) return tableSignature; - return left.id.localeCompare(right.id); - }); -} - -function shardPath(index: number): string { - return `${HISTORIC_SQL_PATTERN_WORKUNIT_DIR}/part-${String(index).padStart(4, '0')}.json`; -} - -export function splitHistoricSqlPatternInputs( - input: StagedPatternsInput, - options: HistoricSqlPatternInputSplitOptions = {}, -): HistoricSqlPatternInputSplitResult { - const maxBytes = options.maxBytes ?? HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES; - const auditInput: StagedPatternsInput = { templates: sortedAuditTemplates(input.templates) }; - const warnings: string[] = []; - const shards: HistoricSqlPatternInputShard[] = []; - let current: PatternTemplate[] = []; - - const flush = () => { - if (current.length === 0) { - return; - } - const shardInput: StagedPatternsInput = { templates: current }; - shards.push({ - path: shardPath(shards.length + 1), - input: shardInput, - byteLength: serializedStagedPatternsInputByteLength(shardInput), - }); - current = []; - }; - - for (const template of sortedPatternCandidates(input.templates)) { - const singleInput: StagedPatternsInput = { templates: [template] }; - if (serializedStagedPatternsInputByteLength(singleInput) > maxBytes) { - warnings.push(`patterns_input_template_too_large:${template.id}`); - continue; - } - - const nextInput: StagedPatternsInput = { templates: [...current, template] }; - if (current.length > 0 && serializedStagedPatternsInputByteLength(nextInput) > maxBytes) { - flush(); - } - current.push(template); - } - - flush(); - return { auditInput, shards, warnings }; -} -``` - -- [ ] **Step 4: Run helper tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/pattern-inputs.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the helper** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts -git commit -m "feat: shard historic sql pattern inputs" -``` - -## Task 2: Write Pattern Shards During Staging - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Test: `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` - -- [ ] **Step 1: Add the failing stager regression** - -Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapshot', ...)` block in `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`: - -```typescript - it('preserves full patterns audit input and writes bounded cross-table pattern shards', async () => { - const stagedDir = await tempDir(); - const largeSql = `select * from public.orders o join public.customers c on c.id = o.customer_id where payload = '${'x'.repeat(8000)}'`; - const reader: HistoricSqlReader = { - async probe() { - return { warnings: [], info: [] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'orders-customers-a', - canonicalSql: largeSql, - stats: { - executions: 25, - distinctUsers: 4, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 15, - p95RuntimeMs: 90, - errorRate: 0, - rowsProduced: 250, - }, - }); - yield aggregate({ - templateId: 'orders-customers-b', - canonicalSql: largeSql.replace('payload', 'payload_b'), - stats: { - executions: 22, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 20, - p95RuntimeMs: 95, - errorRate: 0, - rowsProduced: 220, - }, - }); - yield aggregate({ - templateId: 'orders-single-table', - canonicalSql: 'select count(*) from public.orders', - stats: { - executions: 30, - distinctUsers: 2, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 10, - p95RuntimeMs: 20, - errorRate: 0, - rowsProduced: 30, - }, - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'orders-customers-a', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: [], - where: ['payload'], - join: ['customer_id', 'id'], - groupBy: [], - }, - }, - ], - [ - 'orders-customers-b', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: [], - where: ['payload_b'], - join: ['customer_id', 'id'], - groupBy: [], - }, - }, - ], - [ - 'orders-single-table', - { - tablesTouched: ['public.orders'], - columnsByClause: { - select: [], - where: [], - join: [], - groupBy: [], - }, - }, - ], - ])), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { dialect: 'postgres' }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - const audit = await readJson>(stagedDir, 'patterns-input.json'); - expect(audit.templates.map((entry: any) => entry.id)).toEqual([ - 'orders-customers-a', - 'orders-customers-b', - 'orders-single-table', - ]); - - const firstShard = await readJson>(stagedDir, 'patterns-input/part-0001.json'); - expect(firstShard.templates.map((entry: any) => entry.id)).toEqual(['orders-customers-a', 'orders-customers-b']); - expect(firstShard.templates.some((entry: any) => entry.id === 'orders-single-table')).toBe(false); - - const manifest = await readJson>(stagedDir, 'manifest.json'); - expect(manifest.warnings).toEqual([]); - }); -``` - -- [ ] **Step 2: Run the stager regression to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: FAIL because `patterns-input/part-0001.json` is not written. - -- [ ] **Step 3: Import the sharding helper in the stager** - -In `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`, add this import below the bucket import block: - -```typescript -import { splitHistoricSqlPatternInputs } from './pattern-inputs.js'; -``` - -- [ ] **Step 4: Write the audit input and shard files** - -In `stageHistoricSqlAggregatedSnapshot()` in `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`, replace this block: - -```typescript - await writeJson(input.stagedDir, 'patterns-input.json', toPatternsInput(parsedTemplates)); - await writeJson(input.stagedDir, 'manifest.json', { - source: HISTORIC_SQL_SOURCE_KEY, - connectionId: input.connectionId, - dialect: config.dialect, - fetchedAt: now.toISOString(), - windowStart: windowStart.toISOString(), - windowEnd: now.toISOString(), - snapshotRowCount, - touchedTableCount: byTable.size, - parseFailures: warnings.filter((warning) => warning.startsWith('parse_failed:')).length, - warnings, - probeWarnings: probe.warnings, - staleArchiveAfterDays: config.staleArchiveAfterDays, - }); -``` - -with this code: - -```typescript - const patternsInput = toPatternsInput(parsedTemplates); - const patternInputSplit = splitHistoricSqlPatternInputs(patternsInput); - const allWarnings = [...warnings, ...patternInputSplit.warnings]; - await writeJson(input.stagedDir, 'patterns-input.json', patternInputSplit.auditInput); - for (const shard of patternInputSplit.shards) { - await writeJson(input.stagedDir, shard.path, shard.input); - } - await writeJson(input.stagedDir, 'manifest.json', { - source: HISTORIC_SQL_SOURCE_KEY, - connectionId: input.connectionId, - dialect: config.dialect, - fetchedAt: now.toISOString(), - windowStart: windowStart.toISOString(), - windowEnd: now.toISOString(), - snapshotRowCount, - touchedTableCount: byTable.size, - parseFailures: allWarnings.filter((warning) => warning.startsWith('parse_failed:')).length, - warnings: allWarnings, - probeWarnings: probe.warnings, - staleArchiveAfterDays: config.staleArchiveAfterDays, - }); -``` - -- [ ] **Step 5: Run helper and stager tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/pattern-inputs.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit stager shard writing** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts -git commit -m "feat: write historic sql pattern shards" -``` - -## Task 3: Emit Pattern WorkUnits From Shards - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` -- Test: `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` - -- [ ] **Step 1: Update chunk tests for sharded pattern WorkUnits** - -In `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts`, replace the `patterns-input.json` write inside `writeUnifiedStagedDir()` with these writes: - -```typescript - await writeJson(root, 'patterns-input.json', { - templates: [ - { - id: 'orders', - canonicalSql: 'select * from public.orders join public.customers on true', - tablesTouched: ['public.orders', 'public.customers'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ], - }); - await writeJson(root, 'patterns-input/part-0001.json', { - templates: [ - { - id: 'orders', - canonicalSql: 'select * from public.orders join public.customers on true', - tablesTouched: ['public.orders', 'public.customers'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ], - }); -``` - -In the first test, replace the patterns WorkUnit expectation with: - -```typescript - expect.objectContaining({ - unitKey: 'historic-sql-patterns-part-0001', - displayLabel: 'Historic SQL cross-table patterns: part-0001', - rawFiles: ['patterns-input/part-0001.json'], - dependencyPaths: ['manifest.json'], - notes: expect.stringContaining('patterns-input/part-0001.json'), - }), -``` - -In the diff-set test, replace the second expectation with: - -```typescript - await expect( - chunkHistoricSqlUnifiedStagedDir(stagedDir, { - added: [], - modified: ['patterns-input/part-0001.json'], - deleted: [], - unchanged: ['manifest.json', 'patterns-input.json', 'tables/public.orders.json'], - }), - ).resolves.toMatchObject({ - workUnits: [expect.objectContaining({ unitKey: 'historic-sql-patterns-part-0001' })], - }); - - await expect( - chunkHistoricSqlUnifiedStagedDir(stagedDir, { - added: [], - modified: ['patterns-input.json'], - deleted: [], - unchanged: ['manifest.json', 'patterns-input/part-0001.json', 'tables/public.orders.json'], - }), - ).resolves.toMatchObject({ - workUnits: [], - }); -``` - -In the scope test, add these expectations: - -```typescript - expect(scope.isPathInScope('patterns-input/part-0001.json')).toBe(true); - expect(scope.isPathInScope('patterns-input/part-1.json')).toBe(false); -``` - -Append this additional test inside the same `describe` block: - -```typescript - it('emits one patterns WorkUnit per changed shard', async () => { - const stagedDir = await tempDir(); - await writeUnifiedStagedDir(stagedDir); - await writeJson(stagedDir, 'patterns-input/part-0002.json', { - templates: [ - { - id: 'line-items', - canonicalSql: 'select * from public.orders join public.line_items on true', - tablesTouched: ['public.orders', 'public.line_items'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ], - }); - - const result = await chunkHistoricSqlUnifiedStagedDir(stagedDir, { - added: ['patterns-input/part-0002.json'], - modified: ['patterns-input/part-0001.json'], - deleted: [], - unchanged: ['manifest.json', 'patterns-input.json', 'tables/public.orders.json'], - }); - - expect(result.workUnits.map((unit) => unit.unitKey)).toEqual([ - 'historic-sql-patterns-part-0001', - 'historic-sql-patterns-part-0002', - ]); - expect(result.workUnits.map((unit) => unit.rawFiles)).toEqual([ - ['patterns-input/part-0001.json'], - ['patterns-input/part-0002.json'], - ]); - }); -``` - -- [ ] **Step 2: Run chunk tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/chunk-unified.test.ts -``` - -Expected: FAIL because `chunkHistoricSqlUnifiedStagedDir()` still emits `historic-sql-patterns` from root `patterns-input.json`. - -- [ ] **Step 3: Import shard path helpers in the chunker** - -In `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`, add this import below the existing type imports: - -```typescript -import { isHistoricSqlPatternInputShardPath } from './pattern-inputs.js'; -``` - -- [ ] **Step 4: Emit WorkUnits from shard paths** - -In `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`, replace the root `patterns-input.json` WorkUnit block: - -```typescript - if (files.includes('patterns-input.json') && touchedPath('patterns-input.json', touched)) { - stagedPatternsInputSchema.parse(await readJson(stagedDir, 'patterns-input.json')); - workUnits.push({ - unitKey: 'historic-sql-patterns', - displayLabel: 'Historic SQL cross-table patterns', - rawFiles: ['patterns-input.json'], - dependencyPaths: ['manifest.json'], - peerFileIndex: files.filter((file) => file !== 'patterns-input.json' && file !== 'manifest.json').sort(), - notes: - 'Use historic_sql_patterns. Read patterns-input.json and emit pattern objects with emit_historic_sql_evidence. Do not call wiki_write or sl_write_source.', - }); - } -``` - -with this code: - -```typescript - for (const path of files.filter(isHistoricSqlPatternInputShardPath)) { - if (!touchedPath(path, touched)) { - continue; - } - stagedPatternsInputSchema.parse(await readJson(stagedDir, path)); - const shardLabel = path.replace(/^patterns-input\//, '').replace(/\.json$/, ''); - workUnits.push({ - unitKey: `historic-sql-patterns-${safeUnitKey(shardLabel)}`, - displayLabel: `Historic SQL cross-table patterns: ${shardLabel}`, - rawFiles: [path], - dependencyPaths: ['manifest.json'], - peerFileIndex: files.filter((file) => file !== path && file !== 'manifest.json').sort(), - notes: - `Use historic_sql_patterns. Read ${path} and emit pattern objects with emit_historic_sql_evidence using rawPath "${path}". Do not call wiki_write or sl_write_source.`, - }); - } -``` - -- [ ] **Step 5: Update eviction and scope matching** - -In `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`, replace the deleted-path filter: - -```typescript - const deleted = diffSet?.deleted.filter((path) => path === 'patterns-input.json' || /^tables\/.+\.json$/.test(path)).sort(); -``` - -with: - -```typescript - const deleted = diffSet?.deleted - .filter((path) => isHistoricSqlPatternInputShardPath(path) || /^tables\/.+\.json$/.test(path)) - .sort(); -``` - -In `describeHistoricSqlUnifiedScope()`, replace the scope predicate: - -```typescript - isPathInScope: (rawPath) => - rawPath === 'manifest.json' || rawPath === 'patterns-input.json' || /^tables\/.+\.json$/.test(rawPath), -``` - -with: - -```typescript - isPathInScope: (rawPath) => - rawPath === 'manifest.json' || - rawPath === 'patterns-input.json' || - isHistoricSqlPatternInputShardPath(rawPath) || - /^tables\/.+\.json$/.test(rawPath), -``` - -- [ ] **Step 6: Run helper, stage, and chunk tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/pattern-inputs.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts src/ingest/adapters/historic-sql/chunk-unified.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit chunker shard WorkUnits** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts -git commit -m "feat: emit historic sql pattern shard work units" -``` - -## Task 4: Update Skill Guidance And Acceptance Coverage - -**Files:** -- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md` -- Modify: `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` -- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Update the packaged historic SQL patterns skill** - -Replace `packages/context/skills/historic_sql_patterns/SKILL.md` with: - -````markdown ---- -name: historic_sql_patterns -description: Identify recurring cross-table historic-SQL analytical intents from a bounded pattern shard and emit typed pattern evidence for deterministic wiki projection. -callers: [memory_agent] ---- - -# Historic SQL Patterns - -Use this skill when the WorkUnit raw file is a `patterns-input/part-0001.json` style shard from the `historic-sql` adapter. Older staged bundles may still provide root `patterns-input.json`; when that is the WorkUnit raw file, read it the same way. - -## Required Workflow - -1. Read the WorkUnit notes first. -2. Find the single pattern input file listed under the WorkUnit `rawFiles` section. -3. Call `read_raw_file` for that exact raw file path. -4. Identify recurring analytical intents that span at least two tables and have repeated usage signal. -5. Emit one `pattern` evidence object per durable cross-table intent by calling `emit_historic_sql_evidence`. -6. Set each evidence object's `rawPath` to the exact raw file path read in step 3. -7. Stop after all pattern evidence has been emitted. - -## Evidence Shape - -Each call to `emit_historic_sql_evidence` must use this shape: - -```json -{ - "kind": "pattern", - "rawPath": "patterns-input/part-0001.json", - "pattern": { - "slug": "order-lifecycle-analysis", - "title": "Order Lifecycle Analysis", - "narrative": "Analysts compare order statuses with customer segments to understand lifecycle movement.", - "definitionSql": "select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id group by o.status", - "tablesInvolved": ["public.orders", "public.customers"], - "slRefs": ["orders", "customers"], - "constituentTemplateIds": ["pg:1", "pg:2"] - } -} -``` - -The `pattern` object must match `patternOutputSchema`; multiple calls together must form `patternsArraySchema`. - -## Pattern Selection Rules - -- Prefer patterns that involve two or more tables. -- Prefer templates with `executionsBucket` at least `10-100` and `distinctUsersBucket` above solo usage. -- Merge templates into one pattern only when the business intent is the same. -- Use a stable kebab-case slug based on intent, not a template id. -- Set `definitionSql` to the clearest representative SQL from a constituent template. -- Set `slRefs` to source names when the source name is obvious from table names; omit uncertain refs rather than guessing. -- Treat each pattern shard independently; do not read peer shard files from `peerFileIndex`. - -## Boundaries - -- Do not call wiki_write. -- Do not call sl_write_source. -- Do not call sl_edit_source. -- Do not call context_candidate_write. -- Do not create single-table pattern pages. -- Do not copy credentials, tokens, user emails, or unredacted literals into evidence. -```` - -- [ ] **Step 2: Update runtime asset assertions** - -In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, replace this assertion: - -```typescript - expect(body).toContain('patterns-input.json'); -``` - -with: - -```typescript - expect(body).toContain('patterns-input/part-0001.json'); -``` - -- [ ] **Step 3: Update the local ingest acceptance fake agent** - -In `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`, replace this block: - -```typescript - if (params.telemetryTags.unitKey === 'historic-sql-patterns') { - const result = await emitEvidence.execute( - { - kind: 'pattern', - rawPath: 'patterns-input.json', - pattern: { -``` - -with: - -```typescript - if (params.telemetryTags.unitKey === 'historic-sql-patterns-part-0001') { - const result = await emitEvidence.execute( - { - kind: 'pattern', - rawPath: 'patterns-input/part-0001.json', - pattern: { -``` - -The rest of the pattern object stays unchanged. - -- [ ] **Step 4: Run skill and acceptance tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-runtime-assets.test.ts src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit skill and acceptance updates** - -```bash -git add packages/context/skills/historic_sql_patterns/SKILL.md packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts -git commit -m "test: align historic sql pattern skill with shards" -``` - -## Task 5: Final Verification - -**Files:** -- Verify: `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` -- Verify: `packages/context/skills/historic_sql_patterns/SKILL.md` - -- [ ] **Step 1: Run focused historic SQL tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/pattern-inputs.test.ts \ - src/ingest/adapters/historic-sql/stage-unified.test.ts \ - src/ingest/adapters/historic-sql/chunk-unified.test.ts \ - src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts \ - src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run context package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Verify no legacy historic SQL code path returned** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline|historic_sql_ingest|historic_sql_curator|PostgresPgssQueryHistoryReader|historic_sql_template" packages/context packages/cli -``` - -Expected: no matches in runtime or test source. Matches inside `docs/superpowers/plans/` are acceptable when searching docs separately, but this command does not search docs. - -- [ ] **Step 4: Run pre-commit on changed files if configured** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts \ - packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts \ - packages/context/src/ingest/adapters/historic-sql/stage-unified.ts \ - packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts \ - packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts \ - packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts \ - packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts \ - packages/context/src/ingest/ingest-runtime-assets.test.ts \ - packages/context/skills/historic_sql_patterns/SKILL.md -``` - -Expected: PASS. If the repository has no pre-commit config or the local `uv` version cannot satisfy the project pin, record the exact error and rely on the focused tests plus type-check above. - -- [ ] **Step 5: Commit verification-only adjustments if any were needed** - -If any test or type-check step required small follow-up edits, commit them: - -```bash -git add packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/skills/historic_sql_patterns/SKILL.md -git commit -m "test: verify historic sql pattern shard work units" -``` - -If there were no follow-up edits, do not create an empty commit. - -## Self-Review - -**Spec coverage:** This plan covers spec section 5.2's allowance for multiple deterministic pattern WorkUnits when `patterns-input.json` exceeds a context budget. It preserves section 4.6's full `patterns-input.json` audit artifact, keeps section 4.7's changed-file DiffSet behavior, and does not alter deterministic projection from section 5.3. - -**Placeholder scan:** The plan contains concrete files, commands, expected outcomes, code snippets, and commit commands. It has no deferred implementation markers. - -**Type consistency:** `StagedPatternsInput`, `splitHistoricSqlPatternInputs()`, `isHistoricSqlPatternInputShardPath()`, `HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES`, and `serializedStagedPatternsInputByteLength()` are introduced in Task 1 and imported with the same names in later tasks. Pattern shard raw paths use `patterns-input/part-0001.json` consistently in the stager, chunker, skill, and acceptance test. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md b/docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md deleted file mode 100644 index 655d9568..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md +++ /dev/null @@ -1,444 +0,0 @@ -# Historic SQL Projection Archive Hardening Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Keep historic-SQL archived pattern pages stable across runs and add projection regression coverage for archive, stale-table, and legacy-page behavior from the redesign spec. - -**Architecture:** The redesigned historic-SQL pipeline is already cut over. This plan only hardens the deterministic projection step by treating `knowledge/global/historic-sql/_archived/*.md` pages as historical records, not active candidates for slug reuse or stale/archive processing. Tests stay in the existing projection unit suite because the behavior is pure filesystem projection. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, YAML, local filesystem fixtures. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` - -Implemented status verified from this worktree: - -- `2026-05-11-historic-sql-foundations.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `packages/context/src/sql-analysis/ports.ts` exposes `analyzeBatch()`, `python/ktx-daemon/src/ktx_daemon/app.py` registers `/sql/analyze-batch`, `packages/context/src/sl/types.ts` has `SemanticLayerSource.usage`, and `packages/context/src/ingest/adapters/live-database/manifest.ts` has `mergeUsagePreservingExternal()`. -- `2026-05-11-historic-sql-search-enrichment.md` is implemented. Evidence: `packages/context/src/sl/sl-search.service.ts` indexes `source.usage`, `packages/context/src/sl/sqlite-sl-sources-index.ts` selects FTS snippets, and local/MCP list surfaces expose `frequencyTier` and `snippet`. -- `2026-05-11-historic-sql-unified-hot-path.md` is implemented. Evidence: `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake `fetchAggregated()` methods, unified schemas, and exports exist. -- `2026-05-11-historic-sql-skills-projection-cutover.md` is implemented. Evidence: `HistoricSqlSourceAdapter` uses the unified stager/chunker, `packages/context/skills/historic_sql_table_digest/` and `packages/context/skills/historic_sql_patterns/` exist, `emit_historic_sql_evidence` exists, `HistoricSqlProjectionPostProcessor` is wired in `packages/context/src/ingest/local-bundle-runtime.ts`, and legacy skill names no longer grep in `packages/context` or `packages/cli`. -- `2026-05-11-historic-sql-cross-dialect-readiness.md` is implemented. Evidence: `packages/cli/src/local-adapters.test.ts` covers Postgres, BigQuery, and Snowflake historic-SQL registration, and `packages/cli/src/historic-sql-doctor.test.ts` covers low `pg_stat_statements.max` as informational output. -- `2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` is implemented. Evidence: `packages/cli/src/setup-databases.test.ts` expects canonical `historicSql.filters.serviceAccounts`, `examples/postgres-historic/scripts/smoke.sh` asserts `manifest.json`, `tables/*.json`, `patterns-input.json`, and zero WorkUnits on the unchanged run, and public docs use `minExecutions`. - -Remaining issue this plan fixes: - -- `packages/context/src/ingest/adapters/historic-sql/projection.ts` recursively loads every markdown page below `knowledge/global/historic-sql`, including pages already under `_archived/`. -- Because archived pages still have `source: historic-sql` and tags `['historic-sql', 'pattern', 'archived']`, they are currently active candidates for slug reuse and stale/archive processing. -- A reappearing pattern can be written back to `_archived/.md` instead of active `historic-sql/.md`. -- A later no-pattern run can move an already archived page to `_archived/_archived/.md`. -- `projection.test.ts` does not cover stale table marking, legacy query-page deletion, or the archived-page stability behavior required by spec §5.3 and §10.2. - -## File Structure - -- Modify `packages/context/src/ingest/adapters/historic-sql/projection.ts`: add an archived-page predicate and exclude archived pages from active pattern slug matching and stale/archive loops. -- Modify `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`: add failing tests for archived-page stability, active slug restoration after a pattern reappears, stale table marking, and legacy query-page cleanup. - -### Task 1: Add Archived Pattern Projection Regression Tests - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` - -- [ ] **Step 1: Add failing tests for archived page handling** - -Append these tests inside the existing `describe('projectHistoricSqlEvidence', ...)` block in `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`: - -```typescript - it('writes a reappearing pattern to the active slug instead of reusing an archived page key', async () => { - const workdir = await tempWorkdir(); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 2, - touchedTableCount: 2, - parseFailures: 0, - warnings: [], - probeWarnings: [], - staleArchiveAfterDays: 30, - }); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.orders.json', { table: 'public.orders' }); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.customers.json', { table: 'public.customers' }); - await writeText( - workdir, - 'knowledge/global/historic-sql/_archived/order-lifecycle-analysis.md', - [ - '---', - YAML.stringify({ - summary: 'Archived order lifecycle page', - tags: ['historic-sql', 'pattern', 'archived'], - refs: [], - sl_refs: ['orders'], - usage_mode: 'auto', - source: 'historic-sql', - tables: ['public.orders', 'public.customers'], - fingerprints: ['pg:1'], - stale_since: '2026-01-01T00:00:00.000Z', - }).trimEnd(), - '---', - '', - 'Archived body', - '', - ].join('\n'), - ); - await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/pattern.json', { - kind: 'pattern', - connectionId: 'warehouse', - rawPath: 'patterns-input.json', - pattern: { - slug: 'order-lifecycle-analysis', - title: 'Order Lifecycle Analysis', - narrative: 'Analysts compare order status with customer segment again.', - definitionSql: 'select * from public.orders join public.customers on customers.id = orders.customer_id', - tablesInvolved: ['public.orders', 'public.customers'], - slRefs: ['orders', 'customers'], - constituentTemplateIds: ['pg:1', 'pg:2'], - }, - }); - - const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); - - expect(result.patternPagesWritten).toBe(1); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/order-lifecycle-analysis.md'), 'utf-8')).resolves.toContain( - 'Order Lifecycle Analysis', - ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/_archived/order-lifecycle-analysis.md'), 'utf-8')).resolves.toContain( - 'Archived body', - ); - await expect( - readFile(join(workdir, 'knowledge/global/historic-sql/_archived/_archived/order-lifecycle-analysis.md'), 'utf-8'), - ).rejects.toMatchObject({ code: 'ENOENT' }); - }); - - it('leaves already archived pattern pages stable when they are still absent', async () => { - const workdir = await tempWorkdir(); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 0, - touchedTableCount: 0, - parseFailures: 0, - warnings: [], - probeWarnings: [], - staleArchiveAfterDays: 30, - }); - await writeText( - workdir, - 'knowledge/global/historic-sql/_archived/retired-pattern.md', - [ - '---', - YAML.stringify({ - summary: 'Retired pattern', - tags: ['historic-sql', 'pattern', 'archived'], - refs: [], - sl_refs: [], - usage_mode: 'auto', - source: 'historic-sql', - tables: ['public.tickets'], - fingerprints: ['pg:9'], - stale_since: '2026-01-01T00:00:00.000Z', - }).trimEnd(), - '---', - '', - 'Archived retired body', - '', - ].join('\n'), - ); - - const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); - - expect(result.archivedPatternPages).toBe(0); - expect(result.stalePatternPagesMarked).toBe(0); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/_archived/retired-pattern.md'), 'utf-8')).resolves.toContain( - 'Archived retired body', - ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/_archived/_archived/retired-pattern.md'), 'utf-8')).rejects.toMatchObject({ - code: 'ENOENT', - }); - }); -``` - -- [ ] **Step 2: Run projection tests to verify the archived-page tests fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/projection.test.ts -``` - -Expected: FAIL. The first new test should fail because `knowledge/global/historic-sql/order-lifecycle-analysis.md` is not written. The second new test should fail because `result.archivedPatternPages` is `1` or `_archived/_archived/retired-pattern.md` exists. - -### Task 2: Exclude Archived Pages From Active Projection Processing - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/projection.ts` -- Test: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` - -- [ ] **Step 1: Add the archived-page predicate** - -In `packages/context/src/ingest/adapters/historic-sql/projection.ts`, add this function after `isLegacyQueryPage()`: - -```typescript -function isArchivedPatternPage(page: HistoricSqlPatternPage): boolean { - const tags = Array.isArray(page.frontmatter.tags) ? page.frontmatter.tags : []; - return page.key.startsWith('_archived/') || tags.includes('archived'); -} -``` - -- [ ] **Step 2: Use only active pattern pages for slug matching and stale/archive processing** - -In `projectHistoricSqlEvidence()`, replace: - -```typescript - const allPages = await loadPatternPages(wikiRoot); - const patternPages = allPages.filter(isHistoricPatternPage); -``` - -with: - -```typescript - const allPages = await loadPatternPages(wikiRoot); - const activePages = allPages.filter((page) => !isArchivedPatternPage(page)); - const patternPages = activePages.filter(isHistoricPatternPage); -``` - -- [ ] **Step 3: Run projection tests to verify the archived-page fix passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/projection.test.ts -``` - -Expected: PASS. All projection tests pass, including the two archived-page tests from Task 1. - -- [ ] **Step 4: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/projection.ts packages/context/src/ingest/adapters/historic-sql/projection.test.ts -git commit -m "fix: keep historic sql archived patterns stable" -``` - -### Task 3: Add Stale Table And Legacy Page Cleanup Regression Coverage - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` - -- [ ] **Step 1: Add projection coverage for table drift and legacy query-page cleanup** - -Append this test inside the existing `describe('projectHistoricSqlEvidence', ...)` block in `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`: - -```typescript - it('marks missing table usage stale and deletes legacy historic SQL query pages', async () => { - const workdir = await tempWorkdir(); - await writeText( - workdir, - 'semantic-layer/warehouse/_schema/public.yaml', - YAML.stringify({ - tables: { - orders: { - table: 'public.orders', - usage: { - narrative: 'Orders were active before.', - frequencyTier: 'high', - commonFilters: ['status'], - commonGroupBys: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - ownerNote: 'keep analyst annotation', - }, - columns: [{ name: 'id', type: 'string' }], - }, - }, - }), - ); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 0, - touchedTableCount: 0, - parseFailures: 0, - warnings: [], - probeWarnings: [], - staleArchiveAfterDays: 90, - }); - await writeText( - workdir, - 'knowledge/global/historic-sql/legacy-template.md', - [ - '---', - YAML.stringify({ - summary: 'Legacy template page', - tags: ['historic-sql', 'query-pattern'], - refs: [], - sl_refs: ['orders'], - usage_mode: 'auto', - source: 'historic-sql', - tables: ['public.orders'], - fingerprints: ['legacy:1'], - }).trimEnd(), - '---', - '', - 'Legacy body', - '', - ].join('\n'), - ); - - const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); - - expect(result.staleTablesMarked).toBe(1); - expect(result.legacyPagesDeleted).toBe(1); - expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]); - const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')); - expect(shard.tables.orders.usage).toEqual({ - ownerNote: 'keep analyst annotation', - narrative: 'No recent historic SQL usage was observed in the latest snapshot.', - frequencyTier: 'unused', - commonFilters: [], - commonGroupBys: [], - commonJoins: [], - staleSince: '2026-05-11T00:00:00.000Z', - }); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/legacy-template.md'), 'utf-8')).rejects.toMatchObject({ - code: 'ENOENT', - }); - }); -``` - -- [ ] **Step 2: Run projection tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/projection.test.ts -``` - -Expected: PASS. The new regression test should pass with the current implementation after Task 2, proving stale table drift and legacy query-page cleanup stay covered. - -- [ ] **Step 3: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/projection.test.ts -git commit -m "test: cover historic sql projection cleanup" -``` - -### Task 4: Final Verification - -**Files:** -- Verify: `packages/context/src/ingest/adapters/historic-sql/projection.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` - -- [ ] **Step 1: Run the focused projection test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/projection.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the focused historic-SQL adapter test group** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/evidence.test.ts \ - src/ingest/adapters/historic-sql/evidence-tool.test.ts \ - src/ingest/adapters/historic-sql/projection.test.ts \ - src/ingest/adapters/historic-sql/post-processor.test.ts \ - src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run context type check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Confirm old historic-SQL code paths remain absent** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|historic_sql_ingest|historic_sql_curator|PostgresPgssQueryHistoryReader|historic_sql_template" packages/context packages/cli -``` - -Expected: no output and exit code 1. - -- [ ] **Step 5: Run whitespace check** - -Run: - -```bash -git diff --check -``` - -Expected: no output. - -- [ ] **Step 6: Commit verification fixes only if verification changed files** - -If verification required an edit, commit the exact touched files: - -```bash -git add packages/context/src/ingest/adapters/historic-sql/projection.ts packages/context/src/ingest/adapters/historic-sql/projection.test.ts -git commit -m "test: verify historic sql projection archive hardening" -``` - -If verification made no edits, do not create an empty commit. - -## Self-Review - -Spec coverage: - -- Spec §5.3 stale pattern handling is covered by Task 1 and Task 2: archived pages are historical records and are not repeatedly archived or reused as active slug targets. -- Spec §10.2 legacy wiki page cleanup is covered by Task 3. -- Spec §10.4 drift behavior is covered by Task 3: a table absent from the latest snapshot receives `usage.staleSince` while external usage keys remain intact. -- Spec §10.6 slug churn and user-edited usage risks are covered by Task 1 and Task 3. - -Placeholder scan: - -- The plan contains no unresolved marker text from the forbidden-pattern list. -- Every code-changing step names exact files, exact inserted or replacement code, exact commands, and expected outcomes. - -Type consistency: - -- `staleSince`, `frequencyTier`, `commonFilters`, `commonGroupBys`, and `commonJoins` match `tableUsageOutputSchema`. -- `stale_since`, `tags`, `tables`, and `fingerprints` match the existing wiki frontmatter shape used in `projection.ts`. -- `archivedPatternPages`, `stalePatternPagesMarked`, `staleTablesMarked`, and `legacyPagesDeleted` match `HistoricSqlProjectionResult`. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md deleted file mode 100644 index e59e164b..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md +++ /dev/null @@ -1,441 +0,0 @@ -# Historic SQL Redaction Hardening Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `historicSql.redactionPatterns` actually redact sensitive SQL substrings from historic-SQL staged artifacts and WorkUnit inputs. - -**Architecture:** Keep the unified hot path parseable by sending original SQL to the local deterministic SQL-analysis daemon, then redact only the SQL text that is written to `tables/*.json` and `patterns-input.json`. Add a focused redaction helper so regex compatibility and error messages are tested independently from staging, then add a stager regression proving raw sensitive values do not reach files consumed by LLM skills. - -**Tech Stack:** TypeScript ESM/NodeNext, zod 4, Vitest, existing historic-SQL unified stager. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-projection-archive-hardening.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md` - -Implemented status verified from this worktree: - -- `2026-05-11-historic-sql-foundations.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `packages/context/src/sql-analysis/ports.ts` exposes `analyzeBatch()`, `python/ktx-daemon/src/ktx_daemon/app.py` registers `/sql/analyze-batch`, `packages/context/src/sl/types.ts` has `SemanticLayerSource.usage`, and `packages/context/src/ingest/adapters/live-database/manifest.ts` has `mergeUsagePreservingExternal()`. -- `2026-05-11-historic-sql-search-enrichment.md` is implemented. Evidence: `packages/context/src/sl/sl-search.service.ts` indexes `source.usage`, `packages/context/src/sl/sqlite-sl-sources-index.ts` selects FTS snippets, and local/MCP list surfaces expose `frequencyTier` and `snippet`. -- `2026-05-11-historic-sql-unified-hot-path.md` is implemented. Evidence: `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake `fetchAggregated()` methods, unified schemas, and package exports exist. -- `2026-05-11-historic-sql-skills-projection-cutover.md` is implemented. Evidence: `HistoricSqlSourceAdapter` uses the unified stager/chunker, `packages/context/skills/historic_sql_table_digest/` and `packages/context/skills/historic_sql_patterns/` exist, `emit_historic_sql_evidence` exists, `HistoricSqlProjectionPostProcessor` is wired in `packages/context/src/ingest/local-bundle-runtime.ts`, and legacy skill names no longer grep in `packages/context` or `packages/cli`. -- `2026-05-11-historic-sql-cross-dialect-readiness.md` is implemented. Evidence: `packages/cli/src/local-adapters.test.ts` covers Postgres, BigQuery, and Snowflake historic-SQL registration, and `packages/cli/src/historic-sql-doctor.test.ts` covers low `pg_stat_statements.max` as informational output. -- `2026-05-11-historic-sql-docs-smoke-and-config-cleanup.md` is implemented. Evidence: `packages/cli/src/setup-databases.test.ts` expects canonical `historicSql.filters.serviceAccounts`, `examples/postgres-historic/scripts/smoke.sh` asserts unified `manifest.json`, `tables/*.json`, `patterns-input.json`, and zero WorkUnits on the unchanged run, and public docs use `minExecutions`. -- `2026-05-11-historic-sql-projection-archive-hardening.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/projection.ts` has `isArchivedPatternPage()`, excludes archived pages from active slug matching, and `projection.test.ts` covers reappearing archived patterns, stable archived pages, stale table marking, and legacy query-page deletion. -- `2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` exercises the production `HistoricSqlSourceAdapter`, fake `emit_historic_sql_evidence` calls, projection, semantic-layer search, and wiki search. - -Focused verification before writing this plan: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts src/ingest/adapters/historic-sql/projection.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts src/ingest/adapters/historic-sql/types.test.ts -``` - -Observed: 4 files passed, 10 tests passed. - -Remaining spec gap this plan covers: - -- Spec §8 exposes `historicSql.redactionPatterns`, and setup/docs already write that field. -- `packages/context/src/ingest/adapters/historic-sql/types.ts` parses `redactionPatterns`, but `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` never applies them. -- Staged `tables/{schema}.{table}.json` and `patterns-input.json` currently copy `AggregatedTemplate.canonicalSql` unchanged into `topTemplates[].canonicalSql` and `templates[].canonicalSql`. -- Those staged files are WorkUnit inputs for `historic_sql_table_digest` and `historic_sql_patterns`, so sensitive substrings can reach LLM prompts even when the user configured redaction. - -## File Structure - -Create: - -- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` - Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs. -- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` - Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics. - -Modify: - -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` - Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` - Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL. - -## Task 1: Add Historic SQL Redaction Helper - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/redaction.ts` - -- [ ] **Step 1: Write the failing redaction helper test** - -Create `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { compileHistoricSqlRedactionPatterns, redactHistoricSqlText } from './redaction.js'; - -describe('historic-SQL redaction', () => { - it('redacts regex matches and supports the (?i) case-insensitive prefix', () => { - const redactors = compileHistoricSqlRedactionPatterns([ - 'sk_live_[A-Za-z0-9]+', - '(?i)secret_token_[a-z0-9]+', - ]); - - const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret - - expect(redactHistoricSqlText(sql, redactors)).toBe( - "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", - ); - }); - - it('returns the original SQL text when no redaction patterns are configured', () => { - const sql = "select * from public.orders where status = 'paid'"; - - expect(redactHistoricSqlText(sql, compileHistoricSqlRedactionPatterns([]))).toBe(sql); - }); - - it('throws a config-focused error for invalid redaction regex patterns', () => { - expect(() => compileHistoricSqlRedactionPatterns(['[broken'])).toThrow( - 'Invalid historicSql.redactionPatterns entry "[broken"', - ); - }); - - it('throws a config-focused error for empty redaction regex patterns', () => { - expect(() => compileHistoricSqlRedactionPatterns([' '])).toThrow( - 'Invalid historicSql.redactionPatterns entry " "', - ); - }); -}); -``` - -- [ ] **Step 2: Run the redaction helper test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/redaction.test.ts -``` - -Expected: FAIL because `./redaction.js` does not exist. - -- [ ] **Step 3: Add the redaction helper implementation** - -Create `packages/context/src/ingest/adapters/historic-sql/redaction.ts`: - -```typescript -export interface HistoricSqlRedactionPattern { - pattern: string; - expression: RegExp; -} - -const CASE_INSENSITIVE_PREFIX = '(?i)'; -const REDACTION_TOKEN = '[REDACTED]'; - -export function compileHistoricSqlRedactionPatterns(patterns: readonly string[]): HistoricSqlRedactionPattern[] { - return patterns.map((pattern) => { - const trimmed = pattern.trim(); - const caseInsensitive = trimmed.startsWith(CASE_INSENSITIVE_PREFIX); - const source = caseInsensitive ? trimmed.slice(CASE_INSENSITIVE_PREFIX.length) : trimmed; - if (source.length === 0) { - throw new Error(`Invalid historicSql.redactionPatterns entry "${pattern}": pattern must not be empty`); - } - - try { - return { - pattern, - expression: new RegExp(source, caseInsensitive ? 'gi' : 'g'), - }; - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - throw new Error(`Invalid historicSql.redactionPatterns entry "${pattern}": ${reason}`); - } - }); -} - -export function redactHistoricSqlText(text: string, redactors: readonly HistoricSqlRedactionPattern[]): string { - let next = text; - for (const redactor of redactors) { - redactor.expression.lastIndex = 0; - next = next.replace(redactor.expression, REDACTION_TOKEN); - } - return next; -} -``` - -- [ ] **Step 4: Run the redaction helper test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/redaction.test.ts -``` - -Expected: PASS. The output reports 1 test file passed and 4 tests passed. - -- [ ] **Step 5: Commit the redaction helper** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/redaction.ts packages/context/src/ingest/adapters/historic-sql/redaction.test.ts -git commit -m "feat: add historic sql redaction helper" -``` - -## Task 2: Apply Redaction To Unified Staged Artifacts - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/redaction.ts` - -- [ ] **Step 1: Add the failing staged-artifact redaction test** - -Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapshot', ...)` block in `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`: - -```typescript - it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { - const stagedDir = await tempDir(); - const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret - const reader: HistoricSqlReader = { - async probe() { - return { warnings: [], info: [] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'api-events-with-secret', - canonicalSql: originalSql, - stats: { - executions: 15, - distinctUsers: 2, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 12, - p95RuntimeMs: 25, - errorRate: 0, - rowsProduced: 15, - }, - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'api-events-with-secret', - { - tablesTouched: ['public.api_events'], - columnsByClause: { - select: [], - where: ['api_key', 'note'], - join: [], - groupBy: [], - }, - }, - ], - ])), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { - dialect: 'postgres', - redactionPatterns: ['sk_live_[A-Za-z0-9]+', '(?i)secret_token_[a-z0-9]+'], - }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( - [{ id: 'api-events-with-secret', sql: originalSql }], - 'postgres', - ); - - const tableJson = await readFile(join(stagedDir, 'tables/public.api_events.json'), 'utf-8'); - const patternsJson = await readFile(join(stagedDir, 'patterns-input.json'), 'utf-8'); - expect(tableJson).not.toContain('sk_live_abc123'); - expect(tableJson).not.toContain('Secret_Token_9f'); - expect(patternsJson).not.toContain('sk_live_abc123'); - expect(patternsJson).not.toContain('Secret_Token_9f'); - expect(tableJson).toContain('[REDACTED]'); - expect(patternsJson).toContain('[REDACTED]'); - }); -``` - -- [ ] **Step 2: Run the staged-artifact test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: FAIL because `tables/public.api_events.json` and `patterns-input.json` still contain `sk_live_abc123` and `Secret_Token_9f`. - -- [ ] **Step 3: Import the redaction helper in the stager** - -In `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`, add this import below the existing `./buckets.js` import block: - -```typescript -import { - compileHistoricSqlRedactionPatterns, - redactHistoricSqlText, - type HistoricSqlRedactionPattern, -} from './redaction.js'; -``` - -- [ ] **Step 4: Add a small template redaction helper** - -In `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`, add this helper after `shouldDropTemplate()`: - -```typescript -function redactTemplateSql( - template: AggregatedTemplate, - redactors: readonly HistoricSqlRedactionPattern[], -): AggregatedTemplate { - if (redactors.length === 0) { - return template; - } - return { - ...template, - canonicalSql: redactHistoricSqlText(template.canonicalSql, redactors), - }; -} -``` - -- [ ] **Step 5: Compile redaction patterns once per staged snapshot** - -In `stageHistoricSqlAggregatedSnapshot()` in `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`, replace this opening block: - -```typescript - const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig); - const now = input.now ?? new Date(); - const windowStart = new Date(now.getTime() - config.windowDays * 24 * 60 * 60 * 1000); -``` - -with: - -```typescript - const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig); - const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns); - const now = input.now ?? new Date(); - const windowStart = new Date(now.getTime() - config.windowDays * 24 * 60 * 60 * 1000); -``` - -- [ ] **Step 6: Store redacted SQL only after batch analysis has used original SQL** - -In `stageHistoricSqlAggregatedSnapshot()` in `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`, replace this `parsedTemplates.push()` block: - -```typescript - parsedTemplates.push({ - template, - tablesTouched, - columnsByClause: Object.fromEntries( - Object.entries(parsed.columnsByClause).map(([clause, columns]) => [clause, [...new Set(columns)].sort()]), - ), - }); -``` - -with: - -```typescript - parsedTemplates.push({ - template: redactTemplateSql(template, redactors), - tablesTouched, - columnsByClause: Object.fromEntries( - Object.entries(parsed.columnsByClause).map(([clause, columns]) => [clause, [...new Set(columns)].sort()]), - ), - }); -``` - -- [ ] **Step 7: Run staged-artifact and redaction tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/redaction.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: PASS. The output reports 2 test files passed and the staged-artifact test confirms both raw sensitive substrings are absent. - -- [ ] **Step 8: Commit the stager redaction** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts -git commit -m "feat: redact historic sql staged artifacts" -``` - -## Task 3: Run Focused Historic-SQL Regression Checks - -**Files:** -- Verify: `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` -- Verify: `packages/context/src/ingest/adapters/historic-sql/types.test.ts` - -- [ ] **Step 1: Run focused historic-SQL tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/redaction.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts src/ingest/adapters/historic-sql/projection.test.ts src/ingest/adapters/historic-sql/types.test.ts -``` - -Expected: PASS. The output reports 5 test files passed. - -- [ ] **Step 2: Run the context package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS with TypeScript completing without diagnostics. - -- [ ] **Step 3: Confirm the implementation did not reintroduce legacy historic-SQL codepaths** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline|historic_sql_ingest|historic_sql_curator" packages/context/src packages/context/skills packages/cli/src -``` - -Expected: no matches. - -- [ ] **Step 4: Commit verification-only adjustments if any were required** - -If Task 3 required a source or test correction, commit the verified files: - -```bash -git add packages/context/src/ingest/adapters/historic-sql/redaction.ts packages/context/src/ingest/adapters/historic-sql/redaction.test.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts -git commit -m "test: verify historic sql redaction hardening" -``` - -If Task 3 did not require changes, leave the existing commits from Task 1 and Task 2 unchanged. - -## Self-Review - -**Spec coverage:** This plan covers the remaining practical gap in spec §8's `redactionPatterns` config by applying it before SQL text reaches staged artifacts and LLM WorkUnit inputs. It does not alter reader SQL, projection, search enrichment, or setup output because those slices are already implemented. - -**Placeholder scan:** The plan contains no `TBD`, no `TODO`, and no missing code bodies. Every code-writing step includes the exact test or implementation block to add. - -**Type consistency:** `HistoricSqlRedactionPattern`, `compileHistoricSqlRedactionPatterns()`, and `redactHistoricSqlText()` are defined in Task 1 and imported with the same names in Task 2. `redactTemplateSql()` returns `AggregatedTemplate`, preserving the existing `ParsedTemplate.template` type. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-redesign-manual-test-plan.md b/docs/superpowers/plans/2026-05-11-historic-sql-redesign-manual-test-plan.md deleted file mode 100644 index ba539195..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-redesign-manual-test-plan.md +++ /dev/null @@ -1,459 +0,0 @@ -# External Hosted Postgres Discovery Manual Test Plan - -This plan tests KTX from the point of view of a new external user who discovers -the public CLI and connects the hosted Kaelio demo Postgres database as the -source. It starts with the credential-free seeded demo, then creates a real KTX -project that reads from `start.kaelio.com`. - -The plan avoids writing the database password into this repository. Keep the -password in a local environment variable and configure KTX with -`env:KTX_DEMO_DATABASE_URL`. - -## Scope - -Use this plan when the goal is to test KTX as an external user with the hosted -demo database. The commands use the published package shape through -`npx @kaelio/ktx`. If you are testing from this repository, you can replace -`npx @kaelio/ktx` with the local `ktx` alias. - -The required checks cover: - -- Running the packaged seeded demo without credentials. -- Creating a new project that points to the hosted Postgres demo source. -- Verifying the connection through the public CLI. -- Running public ingest against the hosted database. -- Searching semantic-layer sources through `agent sl list --query`. -- Running the Postgres historic-SQL readiness doctor. -- Running the historic-SQL adapter when the demo database exposes query - history and local LLM configuration is available. -- Searching generated historic-SQL usage and pattern pages when historic-SQL - ingest runs. - -## Prerequisites - -Prepare a clean terminal before starting. The required path needs Node and -network access to `start.kaelio.com:5432`. The optional historic-SQL ingest path -also needs `uv` and an LLM provider configured for KTX. - -1. Confirm Node 22 or newer is available: - - ```bash - node --version - ``` - - Expected: the version is `v22` or newer. - -2. Confirm the hosted Postgres endpoint is reachable from your network: - - ```bash - nc -vz start.kaelio.com 5432 - ``` - - Expected: the command reports that the TCP connection succeeds. If `nc` is - unavailable, continue and let `ktx connection test` perform the real check. - -3. Create an isolated test parent: - - ```bash - export KTX_EXTERNAL_PARENT="$(mktemp -d)" - export KTX_SEEDED_PROJECT="$KTX_EXTERNAL_PARENT/seeded-demo" - export KTX_HOSTED_PROJECT="$KTX_EXTERNAL_PARENT/hosted-postgres" - export KTX_RUNTIME_ROOT="$KTX_EXTERNAL_PARENT/managed-runtime" - ``` - - Expected: every file created by this test stays under - `$KTX_EXTERNAL_PARENT`. - -4. Set the hosted database URL without committing the password: - - ```bash - read -rsp "Demo database password: " KTX_DEMO_DB_PASSWORD - printf '\n' - export KTX_DEMO_DATABASE_URL="postgresql://kaelio_demo:${KTX_DEMO_DB_PASSWORD}" - export KTX_DEMO_DATABASE_URL="${KTX_DEMO_DATABASE_URL}@start.kaelio.com:5432/demo?sslmode=prefer" - unset KTX_DEMO_DB_PASSWORD - ``` - - Expected: `KTX_DEMO_DATABASE_URL` is set only in your shell. The project - config will store `env:KTX_DEMO_DATABASE_URL`, not the literal URL. - - The hosted demo endpoint uses libpq-style `sslmode=prefer`, which means - "try SSL, then fall back to non-SSL." KTX handles this mode explicitly for - the Node Postgres connector so the setup check can connect to the hosted - demo database. - -5. Verify the required shell variables before running any `ktx` commands: - - ```bash - : "${KTX_EXTERNAL_PARENT:?Run prerequisite step 3 in this shell first}" - : "${KTX_SEEDED_PROJECT:?Run prerequisite step 3 in this shell first}" - : "${KTX_HOSTED_PROJECT:?Run prerequisite step 3 in this shell first}" - : "${KTX_RUNTIME_ROOT:?Run prerequisite step 3 in this shell first}" - : "${KTX_DEMO_DATABASE_URL:?Run prerequisite step 4 in this shell first}" - ``` - - Expected: the command prints nothing and exits zero. If it prints a shell - error, rerun the referenced prerequisite in the same terminal before - continuing. - -## Step 1: Run the packaged seeded demo - -Start with the shortest public path. The seeded demo uses packaged data and -prebuilt context, so it must not ask for an LLM key. - -1. Run the seeded demo: - - ```bash - npx @kaelio/ktx setup demo \ - --project-dir "$KTX_SEEDED_PROJECT" \ - --plain \ - --no-input - ``` - - Expected: output includes `Mode: seeded`, `Source: packaged demo project`, - and `LLM calls: none`. - -2. Inspect the seeded demo: - - ```bash - npx @kaelio/ktx setup demo inspect \ - --project-dir "$KTX_SEEDED_PROJECT" \ - --json > "$KTX_EXTERNAL_PARENT/seeded-inspect.json" - ``` - - Expected: the JSON reports seeded mode, semantic-layer sources, knowledge - pages, and `reports/seeded-demo-report.json`. - -3. Search seeded semantic-layer sources: - - ```bash - npx @kaelio/ktx agent sl list \ - --project-dir "$KTX_SEEDED_PROJECT" \ - --json \ - --query "revenue" \ - > "$KTX_EXTERNAL_PARENT/seeded-sl-search.json" - ``` - - Expected: the command exits zero and returns at least one source with a - numeric `score`. - -## Step 2: Create a hosted Postgres project - -Create a new KTX project that uses the hosted demo database as the warehouse -source. This step enables historic SQL in the config, but it does not require -LLM credentials yet. - -If an earlier setup attempt failed after creating `$KTX_HOSTED_PROJECT/ktx.yaml`, -start a fresh test project before rerunning the `--new` command: - -```bash -export KTX_HOSTED_PROJECT="$KTX_EXTERNAL_PARENT/hosted-postgres-retry" -``` - -1. Create the project and connection: - - ```bash - npx @kaelio/ktx setup \ - --project-dir "${KTX_HOSTED_PROJECT:?Run prerequisite step 3 first}" \ - --new \ - --skip-llm \ - --skip-embeddings \ - --skip-sources \ - --skip-agents \ - --database postgres \ - --new-database-connection-id warehouse \ - --database-url env:KTX_DEMO_DATABASE_URL \ - --database-schema public \ - --enable-historic-sql \ - --historic-sql-min-executions 2 \ - --yes \ - --no-input - ``` - - Expected: `$KTX_HOSTED_PROJECT/ktx.yaml` exists and contains a `warehouse` - Postgres connection whose URL is `env:KTX_DEMO_DATABASE_URL`. - -2. Confirm the password was not written to disk: - - ```bash - grep -R "start.kaelio.com:5432/demo" "$KTX_HOSTED_PROJECT" || true - ``` - - Expected: no matches are printed. - -3. Inspect the generated connection config: - - ```bash - sed -n '1,120p' "$KTX_HOSTED_PROJECT/ktx.yaml" - ``` - - Expected: the `warehouse` connection has `driver: postgres`, - `url: env:KTX_DEMO_DATABASE_URL` or an equivalent URL reference, and - `historicSql.enabled: true`. - -## Step 3: Test the hosted connection - -Run the public connection check before ingest. This verifies that the external -user can reach and introspect the hosted source. - -1. Test the connection: - - ```bash - npx @kaelio/ktx connection test warehouse \ - --project-dir "$KTX_HOSTED_PROJECT" - ``` - - Expected: output includes `Driver: postgres` and a positive table count. - -2. List configured connections: - - ```bash - npx @kaelio/ktx connection list \ - --project-dir "$KTX_HOSTED_PROJECT" - ``` - - Expected: output includes the `warehouse` connection. - -## Step 4: Run public ingest - -Run the public ingest command. For warehouse connections, this performs the -database scan path and writes local context files that agent search can use. - -1. Run ingest: - - ```bash - npx @kaelio/ktx ingest warehouse \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --no-input - ``` - - Expected: output reports that ingest finished and that the `scan` step is - `done`. - -2. Inspect the latest public ingest status: - - ```bash - npx @kaelio/ktx ingest status \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --no-input - ``` - - Expected: the status references the hosted `warehouse` source and a - completed scan. - -3. Confirm semantic-layer files exist: - - ```bash - find "$KTX_HOSTED_PROJECT/semantic-layer/warehouse" \ - -name '*.yaml' -print | head - ``` - - Expected: at least one semantic-layer YAML file is printed. - -## Step 5: Search the hosted database context - -Use the agent-facing semantic-layer search command after ingest. This validates -the discovery path that agents use for database analysis. - -1. Run semantic-layer search: - - ```bash - npx @kaelio/ktx agent sl list \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --connection-id warehouse \ - --json \ - --query "orders revenue customers" \ - > "$KTX_EXTERNAL_PARENT/hosted-sl-search.json" - ``` - - Expected: the command exits zero. - -2. Validate search metadata: - - ```bash - node - "$KTX_EXTERNAL_PARENT/hosted-sl-search.json" <<'NODE' - const { readFileSync } = require('node:fs'); - const result = JSON.parse(readFileSync(process.argv[2], 'utf8')); - const assert = (ok, message) => { - if (!ok) throw new Error(message); - }; - assert(Array.isArray(result.sources), 'sources missing'); - assert(result.sources.length > 0, 'no semantic-layer hits'); - assert(Number.isFinite(result.sources[0].score), 'score missing'); - console.log('hosted semantic-layer search ok'); - NODE - ``` - - Expected: the script prints `hosted semantic-layer search ok`. - -3. Read the top source: - - ```bash - node - "$KTX_EXTERNAL_PARENT/hosted-sl-search.json" \ - > "$KTX_EXTERNAL_PARENT/hosted-top-source-name.txt" <<'NODE' - const { readFileSync } = require('node:fs'); - const result = JSON.parse(readFileSync(process.argv[2], 'utf8')); - process.stdout.write(result.sources[0].name); - NODE - - npx @kaelio/ktx agent sl read \ - "$(cat "$KTX_EXTERNAL_PARENT/hosted-top-source-name.txt")" \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --connection-id warehouse \ - --json \ - > "$KTX_EXTERNAL_PARENT/hosted-sl-read.json" - ``` - - Expected: the JSON includes the full semantic-layer source. - -## Step 6: Check historic-SQL readiness - -Run the Postgres historic-SQL doctor. This determines whether the hosted demo -database exposes the query-history prerequisites needed for the redesign's -historic-SQL adapter. - -1. Run doctor: - - ```bash - npx @kaelio/ktx dev doctor \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --no-input - ``` - - Expected: output includes a `Postgres Historic SQL (warehouse)` check. - -2. Interpret the result: - - - `PASS` means the hosted source is ready for the optional historic-SQL - ingest path. - - `WARN` or `FAIL` means the external discovery test still covers scan and - semantic-layer search, but historic-SQL query-history ingestion is blocked - by database permissions or configuration. - -## Step 7: Optional historic-SQL ingest - -Run this section only when the doctor passes and the KTX project has an LLM -provider configured. Historic-SQL table and pattern curation uses LLM-backed -skills, so this path is not credential-free. - -1. Configure LLM and embeddings if you skipped them during setup: - - ```bash - npx @kaelio/ktx setup \ - --project-dir "$KTX_HOSTED_PROJECT" - ``` - - Expected: `npx @kaelio/ktx setup status --project-dir "$KTX_HOSTED_PROJECT"` - reports that LLM and embedding setup are ready. - -2. Run historic-SQL ingest: - - ```bash - npx @kaelio/ktx dev ingest run \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --connection-id warehouse \ - --adapter historic-sql \ - --plain \ - --yes \ - --no-input - ``` - - Expected: the command exits zero and schedules `historic-sql-table-` and - `historic-sql-patterns-` WorkUnits when the database has qualifying query - history. - -3. Locate the latest historic-SQL manifest: - - ```bash - find "$KTX_HOSTED_PROJECT/raw-sources/warehouse/historic-sql" \ - -name manifest.json -print | sort | tail -n 1 - ``` - - Expected: a manifest path is printed. - -4. Search for generated usage: - - ```bash - npx @kaelio/ktx agent sl list \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --connection-id warehouse \ - --json \ - --query "common filters joins usage" \ - > "$KTX_EXTERNAL_PARENT/historic-sl-search.json" - ``` - - Expected: hits produced from historic-SQL usage include `score`, and hits - with projected usage include `frequencyTier` and `snippet`. - -5. Search for generated pattern pages: - - ```bash - npx @kaelio/ktx agent wiki search "historic sql pattern" \ - --project-dir "$KTX_HOSTED_PROJECT" \ - --json \ - --limit 10 \ - > "$KTX_EXTERNAL_PARENT/historic-wiki-search.json" - ``` - - Expected: results include pages whose keys start with `historic-sql/` when - the run produced cross-table patterns. - -## Step 8: Record results - -Capture the result in a way that separates the external discovery path from the -optional historic-SQL path. - -1. Save useful outputs: - - ```bash - mkdir -p "$KTX_EXTERNAL_PARENT/results" - cp "$KTX_EXTERNAL_PARENT/seeded-inspect.json" \ - "$KTX_EXTERNAL_PARENT/results/" 2>/dev/null || true - cp "$KTX_EXTERNAL_PARENT/hosted-sl-search.json" \ - "$KTX_EXTERNAL_PARENT/results/" 2>/dev/null || true - cp "$KTX_EXTERNAL_PARENT/hosted-sl-read.json" \ - "$KTX_EXTERNAL_PARENT/results/" 2>/dev/null || true - cp "$KTX_EXTERNAL_PARENT/historic-sl-search.json" \ - "$KTX_EXTERNAL_PARENT/results/" 2>/dev/null || true - cp "$KTX_EXTERNAL_PARENT/historic-wiki-search.json" \ - "$KTX_EXTERNAL_PARENT/results/" 2>/dev/null || true - ``` - - Expected: the results directory contains the JSON outputs created during the - run. - -2. Mark these areas as pass, fail, or blocked: - - - Public package discovery through `npx @kaelio/ktx`. - - Seeded demo without credentials. - - Hosted Postgres project setup. - - Hosted Postgres connection test. - - Public ingest scan. - - Semantic-layer search and read. - - Historic-SQL doctor. - - Historic-SQL ingest, if doctor and LLM setup allow it. - - Historic-SQL usage search, if ingest ran. - - Historic-SQL wiki pattern search, if ingest ran. - - Expected: every required external discovery area passes. Historic-SQL ingest - is pass, fail, or blocked based on the doctor result and local LLM - configuration. - -## Cleanup - -Remove the disposable project after collecting results. Keep it only when you -need the files for debugging. - -1. Stop the managed runtime: - - ```bash - npx @kaelio/ktx runtime stop || true - ``` - -2. Remove the test parent: - - ```bash - rm -rf "$KTX_EXTERNAL_PARENT" - ``` - - Expected: temporary projects and runtime files are removed. diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md deleted file mode 100644 index cafc234b..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md +++ /dev/null @@ -1,778 +0,0 @@ -# Historic SQL Search Enrichment Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make historic-SQL table usage searchable through semantic-layer search and return lean query-mode context with `frequencyTier` and an FTS snippet. - -**Architecture:** This is the second slice of the historic SQL redesign, covering spec §6.2.3-§6.2.5 and the search-hit tier in §7. It builds on the already implemented foundation slice: `SemanticLayerSource.usage` is the source of truth, the SL search text builder indexes usage narrative and structured usage fields, SQLite FTS returns snippets from indexed search text, and local/MCP list responses hydrate `frequencyTier` from the source while keeping the full `usage` block available through `agent sl read`. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, better-sqlite3 FTS5, zod-backed TypeScript types. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` - -Implemented status: - -- `2026-05-11-historic-sql-foundations.md` is implemented in this worktree. Evidence in code: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `SemanticLayerSource.usage` in `packages/context/src/sl/types.ts`, `mergeUsagePreservingExternal()` in `packages/context/src/ingest/adapters/live-database/manifest.ts`, `SqlAnalysisPort.analyzeBatch()` in `packages/context/src/sql-analysis/ports.ts`, and `/sql/analyze-batch` in `python/ktx-daemon/src/ktx_daemon/app.py`. -- Focused TypeScript foundation verification passed: `pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/skill-schemas.test.ts src/sl/semantic-layer.service.test.ts src/ingest/adapters/live-database/manifest.test.ts src/scan/local-enrichment-artifacts.test.ts src/sql-analysis/http-sql-analysis-port.test.ts` reported 5 files and 53 tests passed. -- `uv run pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -q` is blocked by the repo's exact uv pin: required `==0.11.11`, local `0.11.13`. Closest available check after activating `.venv` passed: `source .venv/bin/activate && python -m pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -q` reported 20 passed. - -Not yet implemented: - -- `buildSemanticLayerSourceSearchText()` in `packages/context/src/sl/sl-search.service.ts` does not include `source.usage`. -- `SqliteSlSourcesIndex` does not select `snippet(local_sl_sources_fts, ...)`. -- `LocalSlSourceSearchResult` and `KtxSemanticLayerSourceSummary` do not expose `frequencyTier` or `snippet`. -- `createLocalProjectMcpContextPorts().semanticLayer.listSources()` drops any future snippet/frequency metadata. - -This plan does not rewrite the historic-SQL adapter, readers, skills, projection, or cleanup path. The next plan after this one should cover the new adapter hot path from spec §4 and §10.3 step 3. - -## File Structure - -Modify: - -- `packages/context/src/sl/sl-search.service.ts` - Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers. -- `packages/context/src/sl/sl-search.service.test.ts` - Tests usage search-text content and direct service snippet pass-through. -- `packages/context/src/sl/ports.ts` - Extends `SlSourcesIndexPort.search()` rows with optional `snippet`. -- `packages/context/src/sl/sqlite-sl-sources-index.ts` - Adds FTS5 `snippet()` selection to lexical candidate search and direct index search. -- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` - Locks snippet behavior for both direct search and lexical lane candidates. -- `packages/context/src/sl/local-sl.ts` - Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`. -- `packages/context/src/sl/local-sl.test.ts` - Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet. -- `packages/context/src/sl/pglite-sl-search-prototype.ts` - Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful. -- `packages/context/src/mcp/types.ts` - Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`. -- `packages/context/src/mcp/local-project-ports.ts` - Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output. -- `packages/context/src/mcp/local-project-ports.test.ts` - Tests the agent/MCP-facing list response. - -## Task 1: Index Historic SQL Usage In SL Search Text - -**Files:** -- Modify: `packages/context/src/sl/sl-search.service.test.ts` -- Modify: `packages/context/src/sl/sl-search.service.ts` - -- [ ] **Step 1: Write the failing usage search-text test** - -Add this test at the end of the existing `describe('SlSearchService', ...)` block in `packages/context/src/sl/sl-search.service.test.ts`: - -```typescript - it('includes historic SQL usage in semantic-layer search text', () => { - const source: SemanticLayerSource = { - name: 'orders', - descriptions: { user: 'Customer orders' }, - table: 'public.orders', - grain: ['order_id'], - columns: [{ name: 'order_id', type: 'string' }], - joins: [], - measures: [], - usage: { - narrative: 'Analysts inspect paid and refunded order lifecycle trends by customer segment.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonGroupBys: ['customer_segment'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - staleSince: '2026-05-01T00:00:00.000Z', - }, - }; - - const text = buildSemanticLayerSourceSearchText(source); - - expect(text).toContain('usage: Analysts inspect paid and refunded order lifecycle trends by customer segment.'); - expect(text).toContain('frequency: high'); - expect(text).toContain('commonly filtered by: status, created_at'); - expect(text).toContain('commonly grouped by: customer_segment'); - expect(text).toContain('commonly joined to public.customers on customer_id'); - expect(text).toContain('stale since 2026-05-01T00:00:00.000Z'); - }); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/sl-search.service.test.ts -``` - -Expected: FAIL because the search text does not contain `usage: Analysts inspect paid and refunded order lifecycle trends by customer segment.` - -- [ ] **Step 3: Add usage fields to the canonical search text** - -In `packages/context/src/sl/sl-search.service.ts`, insert this block after the existing `freshness` block and before `return parts.join('. ');`: - -```typescript - if (source.usage) { - const usage = source.usage; - parts.push(`usage: ${usage.narrative}`); - parts.push(`frequency: ${usage.frequencyTier}`); - if (usage.commonFilters.length > 0) { - parts.push(`commonly filtered by: ${usage.commonFilters.join(', ')}`); - } - if (usage.commonGroupBys?.length) { - parts.push(`commonly grouped by: ${usage.commonGroupBys.join(', ')}`); - } - for (const join of usage.commonJoins) { - parts.push(`commonly joined to ${join.table} on ${join.on.join(',')}`); - } - if (usage.staleSince) { - parts.push(`stale since ${usage.staleSince}`); - } - } -``` - -- [ ] **Step 4: Run the search-text test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/sl-search.service.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/context/src/sl/sl-search.service.ts packages/context/src/sl/sl-search.service.test.ts -git commit -m "feat: index historic sql usage in sl search text" -``` - -## Task 2: Return SQLite FTS Snippets From SL Search - -**Files:** -- Modify: `packages/context/src/sl/ports.ts` -- Modify: `packages/context/src/sl/sqlite-sl-sources-index.ts` -- Modify: `packages/context/src/sl/sqlite-sl-sources-index.test.ts` -- Modify: `packages/context/src/sl/sl-search.service.ts` -- Modify: `packages/context/src/sl/sl-search.service.test.ts` - -- [ ] **Step 1: Write failing SQLite snippet assertions** - -Replace the existing `creates SQLite tables and searches indexed source text` test in `packages/context/src/sl/sqlite-sl-sources-index.test.ts` with: - -```typescript - it('creates SQLite tables and searches indexed source text with FTS snippets', async () => { - const index = new SqliteSlSourcesIndex({ dbPath }); - - await index.upsertSources('warehouse', [ - { - sourceName: 'orders', - searchText: 'orders table: public.orders measure: total_revenue sum(revenue) gross revenue', - embedding: null, - }, - { - sourceName: 'tickets', - searchText: 'tickets table: public.tickets measure: ticket_count count(*) support queue', - embedding: null, - }, - ]); - - await expect(access(dbPath)).resolves.toBeUndefined(); - - const directResults = await index.search('warehouse', null, 'gross revenue', 10); - expect(directResults).toEqual([ - expect.objectContaining({ - sourceName: 'orders', - rrfScore: expect.any(Number), - snippet: expect.stringContaining(''), - }), - ]); - expect(directResults[0]?.snippet).toContain('revenue'); - - const lexicalCandidates = await index.searchLexicalCandidates({ queryText: 'gross revenue', limit: 10 }); - expect(lexicalCandidates).toEqual([ - expect.objectContaining({ - id: 'warehouse/orders', - connectionId: 'warehouse', - sourceName: 'orders', - snippet: expect.stringContaining(''), - }), - ]); - }); -``` - -- [ ] **Step 2: Write the failing direct service snippet test** - -Add this test at the end of `packages/context/src/sl/sl-search.service.test.ts`: - -```typescript - it('preserves FTS snippets returned by the source index', async () => { - const service = new SlSearchService( - { - maxBatchSize: 16, - computeEmbedding: vi.fn(async () => [1, 0]), - computeEmbeddingsBulk: vi.fn(), - }, - { - upsertSources: vi.fn(), - getExistingSearchTexts: vi.fn(), - deleteStale: vi.fn(), - deleteByConnection: vi.fn(), - deleteByConnectionAndName: vi.fn(), - search: vi.fn(async () => [ - { - sourceName: 'orders', - rrfScore: 0.75, - snippet: 'usage: paid order lifecycle', - }, - ]), - }, - ); - - await expect(service.search('warehouse', 'order lifecycle', 10)).resolves.toEqual([ - { - sourceName: 'orders', - score: 0.75, - snippet: 'usage: paid order lifecycle', - }, - ]); - }); -``` - -- [ ] **Step 3: Run the snippet tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/sqlite-sl-sources-index.test.ts src/sl/sl-search.service.test.ts -``` - -Expected: FAIL because `snippet` is missing from SQLite search rows and `SlSearchService.search()` drops repository snippets. - -- [ ] **Step 4: Extend the index port result type** - -In `packages/context/src/sl/ports.ts`, replace the `search()` return type in `SlSourcesIndexPort` with: - -```typescript - search( - connectionId: string, - queryEmbedding: number[] | null, - queryText: string, - limit: number, - minRrfScore?: number, - ): Promise>; -``` - -- [ ] **Step 5: Add snippet fields and SQL selection in the SQLite index** - -In `packages/context/src/sl/sqlite-sl-sources-index.ts`, replace the `SearchRow` type with: - -```typescript -type SearchRow = { - connection_id?: string; - source_name: string; - rank: number; - snippet?: string | null; -}; -``` - -In the `SlSqliteLaneCandidate` interface, add the optional snippet property: - -```typescript -export interface SlSqliteLaneCandidate { - id: string; - connectionId: string; - sourceName: string; - rank: number; - rawScore: number; - snippet?: string; -} -``` - -In `searchLexicalCandidates()`, replace the SELECT list with: - -```sql - SELECT - connection_id, - source_name, - bm25(local_sl_sources_fts) AS rank, - snippet(local_sl_sources_fts, 2, '', '', '...', 12) AS snippet - FROM local_sl_sources_fts -``` - -Then replace the returned row mapping in `searchLexicalCandidates()` with: - -```typescript - return rows.map((row, index) => ({ - id: candidateId(row.connection_id, row.source_name), - connectionId: row.connection_id, - sourceName: row.source_name, - rank: index + 1, - rawScore: Number(row.rank), - ...(typeof row.snippet === 'string' && row.snippet.length > 0 ? { snippet: row.snippet } : {}), - })); -``` - -In the direct `search()` method, replace the SELECT list with: - -```sql - SELECT - source_name, - bm25(local_sl_sources_fts) AS rank, - snippet(local_sl_sources_fts, 2, '', '', '...', 12) AS snippet - FROM local_sl_sources_fts -``` - -Then replace the direct `search()` return mapping with: - -```typescript - return rows - .map((row) => ({ - sourceName: row.source_name, - rrfScore: scoreFromRank(row.rank), - ...(typeof row.snippet === 'string' && row.snippet.length > 0 ? { snippet: row.snippet } : {}), - })) - .filter((row) => row.rrfScore >= minRrfScore); -``` - -- [ ] **Step 6: Preserve snippets in direct `SlSearchService.search()` results** - -In `packages/context/src/sl/sl-search.service.ts`, replace the `search()` method signature and final return with: - -```typescript - async search( - connectionId: string, - query: string, - limit = 15, - minRrfScore = 0, - ): Promise> { - let queryEmbedding: number[] | null = null; - try { - queryEmbedding = await this.embeddingService.computeEmbedding(query); - } catch (error) { - this.logger.warn( - `Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`, - ); - } - - const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore); - return results.map((result) => ({ - sourceName: result.sourceName, - score: result.rrfScore, - ...(result.snippet ? { snippet: result.snippet } : {}), - })); - } -``` - -- [ ] **Step 7: Run the snippet tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/sqlite-sl-sources-index.test.ts src/sl/sl-search.service.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add packages/context/src/sl/ports.ts packages/context/src/sl/sqlite-sl-sources-index.ts packages/context/src/sl/sqlite-sl-sources-index.test.ts packages/context/src/sl/sl-search.service.ts packages/context/src/sl/sl-search.service.test.ts -git commit -m "feat: return sl search snippets" -``` - -## Task 3: Hydrate Query-Mode SL Results With Frequency And Snippet - -**Files:** -- Modify: `packages/context/src/sl/local-sl.ts` -- Modify: `packages/context/src/sl/local-sl.test.ts` -- Modify: `packages/context/src/sl/pglite-sl-search-prototype.ts` - -- [ ] **Step 1: Write the failing local search hydration test** - -Add this test after `searches local semantic-layer source text through SQLite FTS` in `packages/context/src/sl/local-sl.test.ts`: - -```typescript - it('searches historic SQL usage and returns frequency tier plus FTS snippet', async () => { - await project.fileStore.writeFile( - 'semantic-layer/warehouse/_schema/public.yaml', - `tables: - orders: - table: public.orders - usage: - narrative: Analysts inspect paid order lifecycle by customer segment. - frequencyTier: high - commonFilters: - - status - - created_at - commonGroupBys: - - customer_segment - commonJoins: - - table: public.customers - on: - - customer_id - columns: - - name: order_id - type: string - - name: status - type: string -`, - 'ktx', - 'ktx@example.com', - 'Add usage-backed manifest shard', - ); - - const results = await searchLocalSlSources(project, { - connectionId: 'warehouse', - query: 'paid lifecycle customer segment', - }); - - expect(results).toEqual([ - expect.objectContaining({ - connectionId: 'warehouse', - name: 'orders', - path: 'semantic-layer/warehouse/_schema/public.yaml#orders', - frequencyTier: 'high', - snippet: expect.stringContaining(''), - matchReasons: expect.arrayContaining(['lexical']), - }), - ]); - expect(results[0]?.snippet).toContain('lifecycle'); - }); -``` - -- [ ] **Step 2: Run the local search test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/local-sl.test.ts -``` - -Expected: FAIL because the query cannot match usage text yet if Task 1 is not present, and because `frequencyTier` and `snippet` are not hydrated into `LocalSlSourceSearchResult`. - -- [ ] **Step 3: Extend the local search result type** - -In `packages/context/src/sl/local-sl.ts`, replace the `LocalSlSourceSearchResult` interface with: - -```typescript -export interface LocalSlSourceSearchResult extends LocalSlSourceSummary { - score: number; - frequencyTier?: NonNullable['frequencyTier']; - snippet?: string; - matchReasons?: SlSearchMatchReason[]; - dictionaryMatches?: SlDictionaryMatch[]; - lanes?: SlSearchLaneSummary[]; -} -``` - -Then add this helper after `candidateKey()`: - -```typescript -function searchResultUsageFields(source: SemanticLayerSource): Pick { - return source.usage?.frequencyTier ? { frequencyTier: source.usage.frequencyTier } : {}; -} -``` - -- [ ] **Step 4: Include frequency tier in the non-SQLite token fallback** - -In `searchLocalSlSources()`, inside the `project.config.storage.search !== 'sqlite-fts5'` branch, replace the final mapped object with: - -```typescript - .map((result) => ({ - ...result.candidate.summary, - score: result.score, - matchReasons: ['token'], - ...searchResultUsageFields(result.candidate.source), - })) -``` - -- [ ] **Step 5: Collect lexical snippets during hybrid search** - -In `searchLocalSlSources()`, after `const dictionaryEvidence = new Map();`, add: - -```typescript - const lexicalSnippets = new Map(); -``` - -Inside the lexical generator, immediately after `const rows = await index.searchLexicalCandidates({ ... });`, add: - -```typescript - for (const row of rows) { - if (row.snippet) { - lexicalSnippets.set(row.id, row.snippet); - } - } -``` - -- [ ] **Step 6: Hydrate frequency tier and snippet in SQLite hybrid results** - -In the final hydration loop in `searchLocalSlSources()`, replace the `hydrated.push({ ... })` block with: - -```typescript - const dictionaryMatches = dictionaryEvidence.get(fused.id); - const snippet = lexicalSnippets.get(fused.id); - hydrated.push({ - ...candidate.summary, - score: fused.score, - ...searchResultUsageFields(candidate.source), - ...(snippet ? { snippet } : {}), - matchReasons: fused.matchReasons as SlSearchMatchReason[], - ...(dictionaryMatches && dictionaryMatches.length > 0 ? { dictionaryMatches } : {}), - lanes: result.lanes, - }); -``` - -- [ ] **Step 7: Propagate frequency tier in the PGlite prototype backend** - -In `packages/context/src/sl/pglite-sl-search-prototype.ts`, inside the final hydration loop, replace the `hydrated.push({ ... })` block with: - -```typescript - const dictionaryMatches = dictionaryEvidence.get(result.id); - const frequencyTier = candidate.source.usage?.frequencyTier; - hydrated.push({ - ...candidate.summary, - score: result.score, - ...(frequencyTier ? { frequencyTier } : {}), - matchReasons: result.matchReasons as SlSearchMatchReason[], - ...(dictionaryMatches && dictionaryMatches.length > 0 ? { dictionaryMatches } : {}), - lanes: fused.lanes, - }); -``` - -- [ ] **Step 8: Run the local search test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/local-sl.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Commit** - -```bash -git add packages/context/src/sl/local-sl.ts packages/context/src/sl/local-sl.test.ts packages/context/src/sl/pglite-sl-search-prototype.ts -git commit -m "feat: hydrate sl search usage metadata" -``` - -## Task 4: Expose Frequency And Snippet Through Agent/MCP SL List - -**Files:** -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write the failing agent-facing list test** - -Add this test after `returns semantic-layer hybrid search metadata through local project ports` in `packages/context/src/mcp/local-project-ports.test.ts`: - -```typescript - it('returns historic SQL usage frequency and snippet through semantic-layer list search', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - await project.fileStore.writeFile( - 'semantic-layer/warehouse/_schema/public.yaml', - `tables: - orders: - table: public.orders - usage: - narrative: Analysts inspect paid order lifecycle by customer segment. - frequencyTier: high - commonFilters: - - status - commonGroupBys: - - customer_segment - commonJoins: - - table: public.customers - on: - - customer_id - columns: - - name: order_id - type: string - - name: status - type: string -`, - 'ktx', - 'ktx@example.com', - 'Seed usage-backed manifest shard', - ); - - const ports = createLocalProjectMcpContextPorts(project); - await expect( - ports.semanticLayer?.listSources({ connectionId: 'warehouse', query: 'paid order lifecycle' }), - ).resolves.toEqual({ - sources: [ - expect.objectContaining({ - connectionId: 'warehouse', - connectionName: 'warehouse', - name: 'orders', - frequencyTier: 'high', - snippet: expect.stringContaining(''), - score: expect.any(Number), - matchReasons: expect.arrayContaining(['lexical']), - }), - ], - totalSources: 1, - }); - }); -``` - -- [ ] **Step 2: Run the local project ports test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -``` - -Expected: FAIL because `frequencyTier` and `snippet` are missing from `semanticLayer.listSources()` responses. - -- [ ] **Step 3: Add fields to the MCP summary type** - -In `packages/context/src/mcp/types.ts`, replace the ingest import with: - -```typescript -import type { IngestReportSnapshot, MemoryFlowReplayInput, TableUsageOutput } from '../ingest/index.js'; -``` - -Then add these optional fields to `KtxSemanticLayerSourceSummary` after `joinCount`: - -```typescript - frequencyTier?: TableUsageOutput['frequencyTier']; - snippet?: string; -``` - -- [ ] **Step 4: Pass fields through local project ports** - -In `packages/context/src/mcp/local-project-ports.ts`, inside the object built in `semanticLayer.listSources()`, add these two spread lines after `joinCount: source.joinCount,`: - -```typescript - ...(hasSlSearchMetadata(source) && source.frequencyTier ? { frequencyTier: source.frequencyTier } : {}), - ...(hasSlSearchMetadata(source) && source.snippet ? { snippet: source.snippet } : {}), -``` - -- [ ] **Step 5: Run the agent-facing list test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat: expose sl search usage snippets" -``` - -## Task 5: Final Verification - -**Files:** -- Verify: `packages/context/src/sl/sl-search.service.ts` -- Verify: `packages/context/src/sl/sqlite-sl-sources-index.ts` -- Verify: `packages/context/src/sl/local-sl.ts` -- Verify: `packages/context/src/mcp/local-project-ports.ts` - -- [ ] **Step 1: Run all focused tests from this plan** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/sl-search.service.test.ts src/sl/sqlite-sl-sources-index.test.ts src/sl/local-sl.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the context type check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Confirm the adapter rewrite is still untouched** - -Run: - -```bash -git diff -- packages/context/src/ingest/adapters/historic-sql/stage.ts packages/context/src/ingest/adapters/historic-sql/stage-pgss.ts packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts -``` - -Expected: no diff output. - -- [ ] **Step 4: Confirm no placeholder text remains in the plan** - -Run: - -```bash -node - <<'NODE' -import { readFileSync } from 'node:fs'; - -const path = 'docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md'; -const text = readFileSync(path, 'utf8'); -const redFlags = [ - 'T' + 'BD', - 'TO' + 'DO', - 'implement ' + 'later', - 'fill in ' + 'details', - 'Add appropriate ' + 'error handling', - 'add ' + 'validation', - 'handle edge ' + 'cases', - 'Write tests for ' + 'the above', - 'Similar to ' + 'Task', -]; - -let failed = false; -for (const flag of redFlags) { - if (text.includes(flag)) { - console.error(`${path}: contains red-flag placeholder text: ${flag}`); - failed = true; - } -} -process.exit(failed ? 1 : 0); -NODE -``` - -Expected: exits 0 with no output. - -- [ ] **Step 5: Commit verification notes if a verification-only edit was needed** - -If Step 1 or Step 2 required a code correction, commit only those corrected files: - -```bash -git status --short -git add packages/context/src/sl/sl-search.service.ts packages/context/src/sl/sl-search.service.test.ts packages/context/src/sl/ports.ts packages/context/src/sl/sqlite-sl-sources-index.ts packages/context/src/sl/sqlite-sl-sources-index.test.ts packages/context/src/sl/local-sl.ts packages/context/src/sl/local-sl.test.ts packages/context/src/sl/pglite-sl-search-prototype.ts packages/context/src/mcp/types.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "test: verify historic sql search enrichment" -``` - -If Step 1 and Step 2 pass without changes, skip this commit. - -## Self-Review - -Spec coverage: - -- Spec §6.2.3 is covered by Task 1: usage fields are included in `buildSemanticLayerSourceSearchText()`. -- Spec §6.2.4 is already covered by the foundation behavior in `SlSearchService.indexSources()`, which compares search text before re-embedding; Task 1 makes usage changes part of that search-text drift. -- Spec §6.2.5 is covered by Tasks 2-4: SQLite FTS snippets are selected and exposed through query-mode list results, and `frequencyTier` is hydrated from the source. -- Spec §7 search-hit tier is covered by Tasks 3-4: query-mode results carry name, table summary counts, description, score, frequency tier, and snippet. Full `usage` remains available through source read because the foundation plan added `SemanticLayerSource.usage`. - -Placeholder scan: - -- This plan contains no deferred implementation markers or unspecified code steps. - -Type consistency: - -- `frequencyTier` uses `TableUsageOutput['frequencyTier']` at the MCP boundary and `NonNullable['frequencyTier']` in local SL search results. -- `snippet` is consistently optional because lexical FTS may not contribute to every hybrid result. diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md deleted file mode 100644 index a7494e2d..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md +++ /dev/null @@ -1,1890 +0,0 @@ -# Historic SQL Skills Projection Cutover Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Cut the production `historic-sql` adapter over to the unified staged shape, add the two replacement LLM skills, project their evidence into `_schema` usage and pattern wiki pages, and delete the legacy per-template code path. - -**Architecture:** The deterministic fetch/chunk hot path is already present and remains LLM-free. WorkUnit skills emit typed evidence through a source-specific tool into ignored run-local files; a deterministic ingest post-processor reads those evidence files before the squash commit and writes `_schema` usage plus `knowledge/global/historic-sql/*.md` pattern pages. The existing `onPullSucceeded()` hook runs after the squash commit in this repo, so projection uses `IngestBundlePostProcessorPort`, which is the current pre-squash deterministic import hook. - -**Tech Stack:** TypeScript ESM/NodeNext, zod 4, Vitest, YAML, existing ingest WorkUnit runner, existing semantic-layer and wiki file layouts. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md` - -Implemented status verified in this worktree: - -- `2026-05-11-historic-sql-foundations.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `SqlAnalysisPort.analyzeBatch()` in `packages/context/src/sql-analysis/ports.ts`, `/sql/analyze-batch` in `python/ktx-daemon/src/ktx_daemon/app.py`, `SemanticLayerSource.usage` in `packages/context/src/sl/types.ts`, and `mergeUsagePreservingExternal()` in `packages/context/src/ingest/adapters/live-database/manifest.ts`. -- `2026-05-11-historic-sql-search-enrichment.md` is implemented. Evidence: `packages/context/src/sl/sl-search.service.ts` indexes `source.usage`, `packages/context/src/sl/sqlite-sl-sources-index.ts` selects FTS snippets, and local/MCP list surfaces expose `frequencyTier` and `snippet`. -- `2026-05-11-historic-sql-unified-hot-path.md` is implemented as helper code. Evidence: `stageHistoricSqlAggregatedSnapshot()`, `chunkHistoricSqlUnifiedStagedDir()`, `PostgresPgssReader`, aggregate BigQuery/Snowflake reader methods, unified schemas, and exports exist. - -Verification already run before writing this plan: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/skill-schemas.test.ts src/sl/semantic-layer.service.test.ts src/ingest/adapters/live-database/manifest.test.ts src/scan/local-enrichment-artifacts.test.ts src/sql-analysis/http-sql-analysis-port.test.ts src/sl/sl-search.service.test.ts src/sl/sqlite-sl-sources-index.test.ts src/sl/local-sl.test.ts src/mcp/local-project-ports.test.ts src/ingest/adapters/historic-sql/types.test.ts src/ingest/adapters/historic-sql/buckets.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts src/ingest/adapters/historic-sql/chunk-unified.test.ts src/package-exports.test.ts -``` - -Expected and observed: 17 files passed, 119 tests passed. - -```bash -source .venv/bin/activate && python -m pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -q -``` - -Expected and observed: 20 passed. - -Still not implemented: - -- `HistoricSqlSourceAdapter` still calls `stagePgStatStatementsTemplates()` or `stageHistoricSqlTemplates()` and advertises `historic_sql_ingest` / `historic_sql_curator`. -- Old skills still exist: `packages/context/skills/historic_sql_ingest/SKILL.md` and `packages/context/skills/historic_sql_curator/SKILL.md`. -- Old template staging and PGSS baseline files still exist: `stage.ts`, `stage-pgss.ts`, `chunk.ts`, `postgres-pgss-query-history-reader.ts`, related tests/fixtures. -- CLI doctor/setup code still imports `PostgresPgssQueryHistoryReader`. -- Runtime asset tests and page-triage prompts still mention `historic_sql_template`, `historic_sql_ingest`, and `historic_sql_curator`. - -## File Structure - -Create: - -- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` - Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` - Tests evidence schema validation, path normalization, and loader rejection of malformed evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` - Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` - Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions. -- `packages/context/src/ingest/adapters/historic-sql/projection.ts` - Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages. -- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` - Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` - Implements `IngestBundlePostProcessorPort` for the deterministic projection phase. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` - Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`. -- `packages/context/skills/historic_sql_table_digest/SKILL.md` - Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object. -- `packages/context/skills/historic_sql_patterns/SKILL.md` - Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent. - -Modify: - -- `packages/context/src/ingest/adapters/historic-sql/types.ts` - Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` - Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` - Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` - Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` - Rewrite around unified staging and new skills. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` - Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted. -- `packages/context/src/ingest/local-adapters.ts` - Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config. -- `packages/context/src/ingest/local-bundle-runtime.ts` - Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` - Replace old skill asset assertions with the two new skills. -- `packages/context/src/memory/memory-runtime-assets.test.ts` - Replace old historic-SQL skill heading with the two new skill headings. -- `packages/context/src/package-exports.test.ts` - Remove legacy export assertions and add evidence/projection export assertions. -- `packages/context/src/ingest/index.ts` - Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports. -- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` - Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`. -- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` - Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release. -- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` - Remove historic-SQL template triage examples because the new adapter no longer uses page triage. - -Delete: - -- `packages/context/src/ingest/adapters/historic-sql/stage.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage-pgss.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage-pgss.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/stage-pgss-golden.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/__fixtures__/postgres/` -- `packages/context/src/ingest/adapters/historic-sql/chunk.ts` -- `packages/context/src/ingest/adapters/historic-sql/chunk.test.ts` -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-query-history-reader.ts` -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-query-history-reader.test.ts` -- `packages/context/skills/historic_sql_ingest/SKILL.md` -- `packages/context/skills/historic_sql_curator/SKILL.md` - -## Task 1: Add Typed Historic-SQL Evidence Emission - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/evidence.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` -- Modify: `packages/context/src/ingest/index.ts` -- Modify: `packages/context/src/package-exports.test.ts` - -- [ ] **Step 1: Write failing evidence schema tests** - -Create `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { - historicSqlEvidenceEnvelopeSchema, - historicSqlEvidencePath, - historicSqlTableUsageEvidenceSchema, -} from './evidence.js'; - -describe('historic-sql evidence contracts', () => { - it('validates table usage evidence emitted by table digest WorkUnits', () => { - const parsed = historicSqlTableUsageEvidenceSchema.parse({ - kind: 'table_usage', - connectionId: 'warehouse', - table: 'public.orders', - rawPath: 'tables/public.orders.json', - usage: { - narrative: 'Orders are repeatedly queried for paid/refunded lifecycle analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonGroupBys: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - staleSince: null, - }, - }); - - expect(parsed.table).toBe('public.orders'); - expect(parsed.usage.frequencyTier).toBe('high'); - }); - - it('validates pattern evidence emitted by the patterns WorkUnit', () => { - const parsed = historicSqlEvidenceEnvelopeSchema.parse({ - kind: 'pattern', - connectionId: 'warehouse', - rawPath: 'patterns-input.json', - pattern: { - slug: 'order-lifecycle-analysis', - title: 'Order Lifecycle Analysis', - narrative: 'Analysts compare order status changes by customer segment.', - definitionSql: 'select status, count(*) from public.orders group by status', - tablesInvolved: ['public.orders', 'public.customers'], - slRefs: ['orders', 'customers'], - constituentTemplateIds: ['pg:1', 'pg:2'], - }, - }); - - expect(parsed.kind).toBe('pattern'); - expect(parsed.pattern.slug).toBe('order-lifecycle-analysis'); - }); - - it('builds a stable ignored evidence path from run and WorkUnit identity', () => { - expect(historicSqlEvidencePath('run-1', 'historic-sql-table-public-orders')).toBe( - '.ktx/ingest-evidence/historic-sql/run-1/historic-sql-table-public-orders.json', - ); - }); -}); -``` - -- [ ] **Step 2: Run the schema tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/evidence.test.ts -``` - -Expected: FAIL with an import error for `./evidence.js`. - -- [ ] **Step 3: Add evidence schemas and path helpers** - -Create `packages/context/src/ingest/adapters/historic-sql/evidence.ts`: - -```typescript -import { z } from 'zod'; -import { patternOutputSchema, tableUsageOutputSchema } from './skill-schemas.js'; - -function safeEvidenceSegment(value: string): string { - const segment = value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, ''); - if (!segment) { - throw new Error(`Invalid historic-SQL evidence path segment: ${value}`); - } - return segment; -} - -export const historicSqlTableUsageEvidenceSchema = z.object({ - kind: z.literal('table_usage'), - connectionId: z.string().min(1), - table: z.string().min(1), - rawPath: z.string().min(1), - usage: tableUsageOutputSchema, -}); -export type HistoricSqlTableUsageEvidence = z.infer; - -export const historicSqlPatternEvidenceSchema = z.object({ - kind: z.literal('pattern'), - connectionId: z.string().min(1), - rawPath: z.string().min(1), - pattern: patternOutputSchema, -}); -export type HistoricSqlPatternEvidence = z.infer; - -export const historicSqlEvidenceEnvelopeSchema = z.discriminatedUnion('kind', [ - historicSqlTableUsageEvidenceSchema, - historicSqlPatternEvidenceSchema, -]); -export type HistoricSqlEvidenceEnvelope = z.infer; - -export function historicSqlEvidencePath(runId: string, unitKey: string): string { - return `.ktx/ingest-evidence/historic-sql/${safeEvidenceSegment(runId)}/${safeEvidenceSegment(unitKey)}.json`; -} - -export function serializeHistoricSqlEvidence(evidence: HistoricSqlEvidenceEnvelope): string { - return `${JSON.stringify(historicSqlEvidenceEnvelopeSchema.parse(evidence), null, 2)}\n`; -} -``` - -- [ ] **Step 4: Write failing tool tests** - -Create `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts`: - -```typescript -import { describe, expect, it, vi } from 'vitest'; -import { createEmitHistoricSqlEvidenceTool } from './evidence-tool.js'; - -describe('emit_historic_sql_evidence tool', () => { - it('writes table usage evidence to the ignored run evidence directory', async () => { - const writeFile = vi.fn(async () => ({ success: true, commitHash: null })); - const tool = createEmitHistoricSqlEvidenceTool(); - - const result = await tool.execute!( - { - kind: 'table_usage', - table: 'public.orders', - rawPath: 'tables/public.orders.json', - usage: { - narrative: 'Orders are repeatedly queried by paid status.', - frequencyTier: 'high', - commonFilters: ['status'], - commonJoins: [], - staleSince: null, - }, - }, - { - toolCallId: 'call-1', - messages: [], - abortSignal: new AbortController().signal, - experimental_context: { - connectionId: 'warehouse', - session: { - ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'historic-sql' }, - configService: { writeFile }, - }, - }, - } as never, - ); - - expect(result).toBe('Recorded historic-SQL table_usage evidence for public.orders.'); - expect(writeFile).toHaveBeenCalledWith( - '.ktx/ingest-evidence/historic-sql/run-1/historic-sql-table-public-orders.json', - expect.stringContaining('"kind": "table_usage"'), - 'System User', - 'system@example.com', - 'Record historic-SQL evidence: historic-sql-table-public-orders', - { skipLock: true }, - ); - }); - - it('rejects non-historic ingest sessions', async () => { - const tool = createEmitHistoricSqlEvidenceTool(); - - await expect( - tool.execute!( - { - kind: 'pattern', - rawPath: 'patterns-input.json', - pattern: { - slug: 'orders', - title: 'Orders', - narrative: 'Orders pattern.', - definitionSql: 'select * from public.orders', - tablesInvolved: ['public.orders'], - slRefs: ['orders'], - constituentTemplateIds: ['pg:1'], - }, - }, - { - toolCallId: 'call-1', - messages: [], - abortSignal: new AbortController().signal, - experimental_context: { - connectionId: 'warehouse', - session: { - ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'notion' }, - configService: { writeFile: vi.fn() }, - }, - }, - } as never, - ), - ).resolves.toContain('Error: emit_historic_sql_evidence is only available during historic-sql ingest'); - }); -}); -``` - -- [ ] **Step 5: Run the tool tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/evidence-tool.test.ts -``` - -Expected: FAIL with an import error for `./evidence-tool.js`. - -- [ ] **Step 6: Add the evidence tool** - -Create `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts`: - -```typescript -import { tool } from 'ai'; -import { z } from 'zod'; -import { historicSqlEvidencePath, serializeHistoricSqlEvidence } from './evidence.js'; -import { patternOutputSchema, tableUsageOutputSchema } from './skill-schemas.js'; - -const SYSTEM_AUTHOR = 'System User'; -const SYSTEM_EMAIL = 'system@example.com'; - -function unitKeyForEvidence(input: { kind: string; table?: string; pattern?: { slug: string } }): string { - if (input.kind === 'table_usage') { - return `historic-sql-table-${String(input.table).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`; - } - return `historic-sql-pattern-${String(input.pattern?.slug).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`; -} - -export function createEmitHistoricSqlEvidenceTool() { - return tool({ - description: - 'Record typed historic-SQL evidence for deterministic projection. Use this instead of wiki_write, sl_write_source, sl_edit_source, or context_candidate_write during historic-SQL WorkUnits.', - inputSchema: z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('table_usage'), - table: z.string().min(1), - rawPath: z.string().min(1), - usage: tableUsageOutputSchema, - }), - z.object({ - kind: z.literal('pattern'), - rawPath: z.string().min(1), - pattern: patternOutputSchema, - }), - ]), - execute: async (input, options): Promise => { - const context = options.experimental_context as - | { - connectionId?: string | null; - session?: { - ingest?: { runId: string; sourceKey: string }; - configService?: { - writeFile( - path: string, - content: string, - author: string, - authorEmail: string, - commitMessage: string, - options?: { skipLock?: boolean }, - ): Promise; - }; - }; - } - | undefined; - const ingest = context?.session?.ingest; - const configService = context?.session?.configService; - if (!ingest || ingest.sourceKey !== 'historic-sql' || !configService || !context?.connectionId) { - return 'Error: emit_historic_sql_evidence is only available during historic-sql ingest.'; - } - - const unitKey = unitKeyForEvidence(input); - const content = serializeHistoricSqlEvidence({ ...input, connectionId: context.connectionId }); - await configService.writeFile( - historicSqlEvidencePath(ingest.runId, unitKey), - content, - SYSTEM_AUTHOR, - SYSTEM_EMAIL, - `Record historic-SQL evidence: ${unitKey}`, - { skipLock: true }, - ); - const label = input.kind === 'table_usage' ? input.table : input.pattern.slug; - return `Recorded historic-SQL ${input.kind} evidence for ${label}.`; - }, - }); -} -``` - -- [ ] **Step 7: Export evidence helpers and verify tests pass** - -Add these exports to `packages/context/src/ingest/index.ts`: - -```typescript -export { - historicSqlEvidenceEnvelopeSchema, - historicSqlEvidencePath, - historicSqlPatternEvidenceSchema, - historicSqlTableUsageEvidenceSchema, - serializeHistoricSqlEvidence, -} from './adapters/historic-sql/evidence.js'; -export type { - HistoricSqlEvidenceEnvelope, - HistoricSqlPatternEvidence, - HistoricSqlTableUsageEvidence, -} from './adapters/historic-sql/evidence.js'; -export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js'; -``` - -Add these assertions to the historic-SQL block in `packages/context/src/package-exports.test.ts`: - -```typescript - expect(ingest.historicSqlEvidenceEnvelopeSchema).toBeDefined(); - expect(ingest.historicSqlEvidencePath).toBeTypeOf('function'); - expect(ingest.createEmitHistoricSqlEvidenceTool).toBeTypeOf('function'); -``` - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/evidence.test.ts src/ingest/adapters/historic-sql/evidence-tool.test.ts src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/evidence.ts packages/context/src/ingest/adapters/historic-sql/evidence.test.ts packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts packages/context/src/ingest/index.ts packages/context/src/package-exports.test.ts -git commit -m "feat: add historic sql evidence emission" -``` - -## Task 2: Add Replacement Historic-SQL Skills - -**Files:** -- Create: `packages/context/skills/historic_sql_table_digest/SKILL.md` -- Create: `packages/context/skills/historic_sql_patterns/SKILL.md` -- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` -- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` - -- [ ] **Step 1: Write failing runtime asset tests for the new skills** - -In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, replace `historic_sql_ingest` with `historic_sql_table_digest` and `historic_sql_patterns` in `adapterSkillNames`, and remove `historic_sql_curator` from `adapterReconcileSkillNames`. - -Replace the two historic-SQL skill tests with: - -```typescript - it('packages historic-SQL table digest guidance from KTX assets', async () => { - const registry = new SkillsRegistryService({ skillsDir }); - const skills = await registry.listSkills(['historic_sql_table_digest'], 'memory_agent'); - - expect(skills.map((skill) => skill.name)).toEqual(['historic_sql_table_digest']); - - const body = await readFile(join(skills[0]!.path, 'SKILL.md'), 'utf-8'); - expect(body).toContain('# Historic SQL Table Digest'); - expect(body).toContain('tables/..json'); - expect(body).toContain('tableUsageOutputSchema'); - expect(body).toContain('emit_historic_sql_evidence'); - expect(body).toContain('Do not call wiki_write'); - expect(body).toContain('Do not call sl_write_source'); - expect(body).not.toMatch(forbiddenProductPattern()); - }); - - it('packages historic-SQL patterns guidance from KTX assets', async () => { - const registry = new SkillsRegistryService({ skillsDir }); - const skills = await registry.listSkills(['historic_sql_patterns'], 'memory_agent'); - - expect(skills.map((skill) => skill.name)).toEqual(['historic_sql_patterns']); - - const body = await readFile(join(skills[0]!.path, 'SKILL.md'), 'utf-8'); - expect(body).toContain('# Historic SQL Patterns'); - expect(body).toContain('patterns-input.json'); - expect(body).toContain('patternsArraySchema'); - expect(body).toContain('emit_historic_sql_evidence'); - expect(body).toContain('cross-table'); - expect(body).not.toMatch(forbiddenProductPattern()); - }); -``` - -In `packages/context/src/memory/memory-runtime-assets.test.ts`, change `expectedAdapterSkillHeadings` to include: - -```typescript - historic_sql_patterns: '# Historic SQL Patterns', - historic_sql_table_digest: '# Historic SQL Table Digest', -``` - -and remove: - -```typescript - historic_sql_ingest: '# Historic SQL Ingest', -``` - -- [ ] **Step 2: Run runtime asset tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts -``` - -Expected: FAIL because `historic_sql_table_digest` and `historic_sql_patterns` skill directories do not exist yet. - -- [ ] **Step 3: Add the table digest skill** - -Create `packages/context/skills/historic_sql_table_digest/SKILL.md`: - -```markdown ---- -name: historic_sql_table_digest -description: Convert one changed historic-SQL table usage bucket into typed table usage evidence for deterministic _schema projection. -callers: [memory_agent] ---- - -# Historic SQL Table Digest - -Use this skill when the WorkUnit raw file is one `tables/..json` file from the `historic-sql` adapter. - -## Required Workflow - -1. Read the WorkUnit notes first. -2. Call `read_raw_file` for the single `tables/..json` raw file. -3. Read `manifest.json` only if the table JSON omits the dialect or the WorkUnit notes are unclear. -4. Produce one concise usage narrative for this table from the staged table JSON. -5. Call `emit_historic_sql_evidence` exactly once with `kind: "table_usage"`. -6. Stop after the evidence tool succeeds. - -## Evidence Shape - -Call `emit_historic_sql_evidence` with this shape: - -```json -{ - "kind": "table_usage", - "table": "public.orders", - "rawPath": "tables/public.orders.json", - "usage": { - "narrative": "Orders are repeatedly queried for paid/refunded lifecycle analysis and customer-level rollups.", - "frequencyTier": "high", - "commonFilters": ["status", "created_at"], - "commonGroupBys": ["status"], - "commonJoins": [{ "table": "public.customers", "on": ["customer_id"] }], - "staleSince": null - } -} -``` - -The `usage` object must match `tableUsageOutputSchema`. - -## Interpretation Rules - -- Treat `columnsByClause.where` as common filters. -- Treat `columnsByClause.groupBy` as common group-bys. -- Treat `observedJoins` as common joins. -- Use `stats.executionsBucket`, `stats.distinctUsersBucket`, and `stats.recencyBucket` to choose `frequencyTier`. -- Use `frequencyTier: "high"` only when executions and distinct users are both broad. -- Use `frequencyTier: "mid"` for repeated team usage that is not broad enough for high. -- Use `frequencyTier: "low"` for low-volume but present usage. -- Use `frequencyTier: "unused"` only when the table input explicitly says the table is stale or has no recent templates. -- Keep `narrative` short and concrete. - -## Boundaries - -- Do not call `wiki_write`. -- Do not call `sl_write_source`. -- Do not call `sl_edit_source`. -- Do not call `context_candidate_write`. -- Do not emit more than one table usage evidence object. -- Do not invent columns, joins, or tables that are absent from the staged JSON. -``` - -- [ ] **Step 4: Add the patterns skill** - -Create `packages/context/skills/historic_sql_patterns/SKILL.md`: - -```markdown ---- -name: historic_sql_patterns -description: Identify recurring cross-table historic-SQL analytical intents and emit typed pattern evidence for deterministic wiki projection. -callers: [memory_agent] ---- - -# Historic SQL Patterns - -Use this skill when the WorkUnit raw file is `patterns-input.json` from the `historic-sql` adapter. - -## Required Workflow - -1. Read the WorkUnit notes first. -2. Call `read_raw_file` for `patterns-input.json`. -3. Identify recurring analytical intents that span at least two tables and have repeated usage signal. -4. Emit one `pattern` evidence object per durable cross-table intent by calling `emit_historic_sql_evidence`. -5. Stop after all pattern evidence has been emitted. - -## Evidence Shape - -Each call to `emit_historic_sql_evidence` must use this shape: - -```json -{ - "kind": "pattern", - "rawPath": "patterns-input.json", - "pattern": { - "slug": "order-lifecycle-analysis", - "title": "Order Lifecycle Analysis", - "narrative": "Analysts compare order statuses with customer segments to understand lifecycle movement.", - "definitionSql": "select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id group by o.status", - "tablesInvolved": ["public.orders", "public.customers"], - "slRefs": ["orders", "customers"], - "constituentTemplateIds": ["pg:1", "pg:2"] - } -} -``` - -The `pattern` object must match `patternOutputSchema`; multiple calls together must form `patternsArraySchema`. - -## Pattern Selection Rules - -- Prefer patterns that involve two or more tables. -- Prefer templates with `executionsBucket` at least `10-100` and `distinctUsersBucket` above solo usage. -- Merge templates into one pattern only when the business intent is the same. -- Use a stable kebab-case slug based on intent, not a template id. -- Set `definitionSql` to the clearest representative SQL from a constituent template. -- Set `slRefs` to source names when the source name is obvious from table names; omit uncertain refs rather than guessing. - -## Boundaries - -- Do not call `wiki_write`. -- Do not call `sl_write_source`. -- Do not call `sl_edit_source`. -- Do not call `context_candidate_write`. -- Do not create single-table pattern pages. -- Do not copy credentials, tokens, user emails, or unredacted literals into evidence. -``` - -- [ ] **Step 5: Run runtime asset tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/skills/historic_sql_table_digest/SKILL.md packages/context/skills/historic_sql_patterns/SKILL.md packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/src/memory/memory-runtime-assets.test.ts -git commit -m "feat: add historic sql evidence skills" -``` - -## Task 3: Project Evidence Into _schema Usage And Pattern Wiki Pages - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/projection.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/types.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Modify: `packages/context/src/wiki/types.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Extend staged manifest with stale archive policy** - -In `packages/context/src/ingest/adapters/historic-sql/types.test.ts`, add `staleArchiveAfterDays: 90` to the manifest fixture and assert: - -```typescript - expect( - stagedManifestSchema.parse({ - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 2, - touchedTableCount: 1, - parseFailures: 1, - warnings: ['parse_failed:bad'], - probeWarnings: [], - staleArchiveAfterDays: 90, - }).staleArchiveAfterDays, - ).toBe(90); -``` - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/types.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: FAIL because `staleArchiveAfterDays` is not in `stagedManifestSchema` or written by staging. - -- [ ] **Step 2: Implement staged manifest policy field** - -Add this field to `stagedManifestSchema` in `packages/context/src/ingest/adapters/historic-sql/types.ts`: - -```typescript - staleArchiveAfterDays: z.number().int().positive().default(90), -``` - -Add this property to the manifest object written by `stageHistoricSqlAggregatedSnapshot()` in `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`: - -```typescript - staleArchiveAfterDays: config.staleArchiveAfterDays, -``` - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/types.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Write failing projection tests** - -Create `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`: - -```typescript -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import YAML from 'yaml'; -import { describe, expect, it } from 'vitest'; -import { projectHistoricSqlEvidence } from './projection.js'; - -async function tempWorkdir(): Promise { - return mkdtemp(join(tmpdir(), 'historic-sql-projection-')); -} - -async function writeText(root: string, relPath: string, content: string): Promise { - const target = join(root, relPath); - await mkdir(join(target, '..'), { recursive: true }); - await writeFile(target, content, 'utf-8'); -} - -async function writeJson(root: string, relPath: string, value: unknown): Promise { - await writeText(root, relPath, `${JSON.stringify(value, null, 2)}\n`); -} - -describe('projectHistoricSqlEvidence', () => { - it('merges table usage into matching _schema shards and preserves external usage keys', async () => { - const workdir = await tempWorkdir(); - await writeText( - workdir, - 'semantic-layer/warehouse/_schema/public.yaml', - YAML.stringify({ - tables: { - orders: { - table: 'public.orders', - usage: { - narrative: 'Old generated usage.', - frequencyTier: 'low', - commonFilters: ['old_status'], - commonJoins: [], - ownerNote: 'keep me', - }, - columns: [{ name: 'id', type: 'string' }], - }, - }, - }), - ); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 1, - touchedTableCount: 1, - parseFailures: 0, - warnings: [], - probeWarnings: [], - staleArchiveAfterDays: 90, - }); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.orders.json', { table: 'public.orders' }); - await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/orders.json', { - kind: 'table_usage', - connectionId: 'warehouse', - table: 'public.orders', - rawPath: 'tables/public.orders.json', - usage: { - narrative: 'Orders are repeatedly queried for lifecycle analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonGroupBys: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - staleSince: null, - }, - }); - - const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); - - expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]); - const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')); - expect(shard.tables.orders.usage).toEqual({ - ownerNote: 'keep me', - narrative: 'Orders are repeatedly queried for lifecycle analysis.', - frequencyTier: 'high', - commonFilters: ['status', 'created_at'], - commonGroupBys: ['status'], - commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], - staleSince: null, - }); - }); - - it('writes pattern pages, reuses similar slugs, and marks missing old pattern pages stale', async () => { - const workdir = await tempWorkdir(); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 2, - touchedTableCount: 2, - parseFailures: 0, - warnings: [], - probeWarnings: [], - staleArchiveAfterDays: 90, - }); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.orders.json', { table: 'public.orders' }); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.customers.json', { table: 'public.customers' }); - await writeText( - workdir, - 'knowledge/global/historic-sql/old-order-lifecycle.md', - [ - '---', - YAML.stringify({ - summary: 'Old order lifecycle page', - tags: ['historic-sql', 'pattern'], - refs: [], - sl_refs: ['orders'], - usage_mode: 'auto', - source: 'historic-sql', - tables: ['public.orders', 'public.customers'], - fingerprints: ['pg:1'], - }).trimEnd(), - '---', - '', - 'Old body', - '', - ].join('\n'), - ); - await writeText( - workdir, - 'knowledge/global/historic-sql/retired-pattern.md', - [ - '---', - YAML.stringify({ - summary: 'Retired pattern', - tags: ['historic-sql', 'pattern'], - refs: [], - sl_refs: [], - usage_mode: 'auto', - source: 'historic-sql', - tables: ['public.tickets'], - fingerprints: ['pg:9'], - }).trimEnd(), - '---', - '', - 'Retired body', - '', - ].join('\n'), - ); - await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/pattern.json', { - kind: 'pattern', - connectionId: 'warehouse', - rawPath: 'patterns-input.json', - pattern: { - slug: 'order-lifecycle-analysis', - title: 'Order Lifecycle Analysis', - narrative: 'Analysts compare order status with customer segment.', - definitionSql: 'select * from public.orders join public.customers on customers.id = orders.customer_id', - tablesInvolved: ['public.orders', 'public.customers'], - slRefs: ['orders', 'customers'], - constituentTemplateIds: ['pg:1', 'pg:2'], - }, - }); - - const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); - - expect(result.patternPagesWritten).toBe(1); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/old-order-lifecycle.md'), 'utf-8')).resolves.toContain( - 'Order Lifecycle Analysis', - ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql/retired-pattern.md'), 'utf-8')).resolves.toContain( - 'stale_since: "2026-05-11T00:00:00.000Z"', - ); - }); -}); -``` - -- [ ] **Step 4: Run projection tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/projection.test.ts -``` - -Expected: FAIL with an import error for `./projection.js`. - -- [ ] **Step 5: Implement projection helpers** - -Create `packages/context/src/ingest/adapters/historic-sql/projection.ts` with these exported shapes and functions: - -```typescript -import { access, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, relative } from 'node:path'; -import YAML from 'yaml'; -import { rawSourcesDirForSync } from '../../raw-sources-paths.js'; -import { mergeUsagePreservingExternal } from '../live-database/manifest.js'; -import { historicSqlEvidenceEnvelopeSchema, type HistoricSqlEvidenceEnvelope } from './evidence.js'; -import { stagedManifestSchema } from './types.js'; - -export interface HistoricSqlProjectionInput { - workdir: string; - connectionId: string; - syncId: string; - runId: string; -} - -export interface HistoricSqlProjectionResult { - tableUsageMerged: number; - staleTablesMarked: number; - patternPagesWritten: number; - stalePatternPagesMarked: number; - archivedPatternPages: number; - legacyPagesDeleted: number; - touchedSources: Array<{ connectionId: string; sourceName: string }>; - warnings: string[]; -} - -interface ManifestShard { - tables?: Record; columns?: unknown[]; [key: string]: unknown }>; -} - -function safeKnowledgeSlug(value: string): string { - return value.toLowerCase().replace(/[^a-z0-9/-]+/g, '-').replace(/^-+|-+$/g, ''); -} - -async function pathExists(path: string): Promise { - try { - await access(path); - return true; - } catch { - return false; - } -} - -async function walkFiles(root: string): Promise { - if (!(await pathExists(root))) return []; - const entries = await readdir(root, { withFileTypes: true, recursive: true }); - return entries - .filter((entry) => entry.isFile()) - .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/')) - .sort(); -} - -async function readJson(path: string): Promise { - return JSON.parse(await readFile(path, 'utf-8')) as unknown; -} - -async function writeYamlAtomic(path: string, value: unknown): Promise { - await mkdir(dirname(path), { recursive: true }); - const tmp = `${path}.tmp`; - await writeFile(tmp, YAML.stringify(value, { indent: 2, lineWidth: 0 }), 'utf-8'); - await rename(tmp, path); -} - -function tableSourceName(tableRef: string): string { - return tableRef.split('.').filter(Boolean).at(-1) ?? tableRef; -} - -function staleUsage(fetchedAt: string) { - return { - narrative: 'No recent historic SQL usage was observed in the latest snapshot.', - frequencyTier: 'unused' as const, - commonFilters: [], - commonGroupBys: [], - commonJoins: [], - staleSince: fetchedAt, - }; -} - -async function loadEvidence(workdir: string, runId: string): Promise { - const root = join(workdir, '.ktx/ingest-evidence/historic-sql', runId); - const files = await walkFiles(root); - const evidence: HistoricSqlEvidenceEnvelope[] = []; - for (const file of files.filter((candidate) => candidate.endsWith('.json'))) { - evidence.push(historicSqlEvidenceEnvelopeSchema.parse(await readJson(join(root, file)))); - } - return evidence; -} - -function renderPatternMarkdown(pattern: HistoricSqlEvidenceEnvelope & { kind: 'pattern' }): string { - return [ - `# ${pattern.pattern.title}`, - '', - pattern.pattern.narrative, - '', - '## Representative SQL', - '', - '```sql', - pattern.pattern.definitionSql, - '```', - '', - '## Tables', - '', - ...pattern.pattern.tablesInvolved.map((table) => `- ${table}`), - '', - '## Constituent Templates', - '', - ...pattern.pattern.constituentTemplateIds.map((id) => `- ${id}`), - '', - ].join('\n'); -} - -function overlapRatio(left: string[], right: string[]): number { - const rightSet = new Set(right); - const intersection = left.filter((value) => rightSet.has(value)).length; - return left.length === 0 ? 0 : intersection / left.length; -} -``` - -In the same file, implement `projectHistoricSqlEvidence()` with this behavior: - -- Read `manifest.json` from `join(workdir, rawSourcesDirForSync(connectionId, 'historic-sql', syncId), 'manifest.json')` and parse with `stagedManifestSchema`. -- Read every current table file under `raw-sources//historic-sql//tables/*.json` and build a `Set` of current staged table refs. -- Load every evidence JSON file from `.ktx/ingest-evidence/historic-sql/`. -- For each `_schema/*.yaml` shard in `semantic-layer//_schema`: - - Parse the shard as YAML. - - For each table entry, match table evidence where `evidence.table === entry.table` or `tableSourceName(evidence.table) === tableName`. - - Merge evidence usage with `mergeUsagePreservingExternal(entry.usage, evidence.usage)`. - - If an entry has `usage` and its table ref is absent from the current staged table set, replace historic-SQL managed usage with `staleUsage(manifest.fetchedAt)` while preserving external keys through `mergeUsagePreservingExternal`. - - Write the shard atomically only when serialized YAML changes. -- For patterns: - - Read current pages under `knowledge/global/historic-sql/*.md`. - - Treat pages with frontmatter `tags` containing both `historic-sql` and `pattern` as historic-SQL pattern pages. - - For each pattern evidence, reuse an existing page key when overlap of `tables + constituentTemplateIds` against existing `tables + fingerprints` is at least `0.6`; otherwise write `historic-sql/`. - - Write frontmatter with `summary`, `tags: ['historic-sql', 'pattern']`, `refs`, `sl_refs`, `usage_mode: 'auto'`, `source: 'historic-sql'`, `tables`, `representative_sql`, and `fingerprints`. - - For existing pattern pages not written this run, add tag `stale` and `stale_since: manifest.fetchedAt`. - - If an existing stale page has `stale_since` older than `manifest.staleArchiveAfterDays`, move it under `knowledge/global/historic-sql/_archived/.md` and add tag `archived`. -- Delete legacy old per-template pages whose frontmatter has `source: historic-sql`, tag `query-pattern`, and lacks tag `pattern`. -- Return counts and touched source names for every `_schema` entry whose usage changed. - -- [ ] **Step 6: Extend wiki frontmatter type for stale pattern metadata** - -In `packages/context/src/wiki/types.ts`, add: - -```typescript - stale_since?: string; -``` - -to `WikiFrontmatter`. - -- [ ] **Step 7: Export projection and run tests** - -Add this export to `packages/context/src/ingest/index.ts`: - -```typescript -export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js'; -export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js'; -``` - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/projection.test.ts src/ingest/adapters/historic-sql/types.test.ts src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/projection.ts packages/context/src/ingest/adapters/historic-sql/projection.test.ts packages/context/src/ingest/adapters/historic-sql/types.ts packages/context/src/ingest/adapters/historic-sql/types.test.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts packages/context/src/wiki/types.ts packages/context/src/ingest/index.ts -git commit -m "feat: project historic sql evidence" -``` - -## Task 4: Wire The Projection Post-Processor And Evidence Tool Runtime - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Modify: `packages/context/src/ingest/local-bundle-ingest.test.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Write failing post-processor tests** - -Create `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts`: - -```typescript -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import YAML from 'yaml'; -import { describe, expect, it } from 'vitest'; -import { HistoricSqlProjectionPostProcessor } from './post-processor.js'; - -async function tempWorkdir(): Promise { - return mkdtemp(join(tmpdir(), 'historic-sql-post-processor-')); -} - -async function writeJson(root: string, relPath: string, value: unknown): Promise { - const target = join(root, relPath); - await mkdir(join(target, '..'), { recursive: true }); - await writeFile(target, `${JSON.stringify(value, null, 2)}\n`, 'utf-8'); -} - -describe('HistoricSqlProjectionPostProcessor', () => { - it('projects current run evidence before the ingest squash commit', async () => { - const workdir = await tempWorkdir(); - await mkdir(join(workdir, 'semantic-layer/warehouse/_schema'), { recursive: true }); - await writeFile( - join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), - YAML.stringify({ tables: { orders: { table: 'public.orders', columns: [{ name: 'id', type: 'string' }] } } }), - 'utf-8', - ); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 1, - touchedTableCount: 1, - parseFailures: 0, - warnings: [], - probeWarnings: [], - staleArchiveAfterDays: 90, - }); - await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.orders.json', { table: 'public.orders' }); - await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/orders.json', { - kind: 'table_usage', - connectionId: 'warehouse', - table: 'public.orders', - rawPath: 'tables/public.orders.json', - usage: { - narrative: 'Orders are repeatedly queried by lifecycle status.', - frequencyTier: 'high', - commonFilters: ['status'], - commonJoins: [], - staleSince: null, - }, - }); - - const result = await new HistoricSqlProjectionPostProcessor().run({ - connectionId: 'warehouse', - sourceKey: 'historic-sql', - syncId: 'sync-1', - jobId: 'job-1', - runId: 'run-1', - workdir, - parseArtifacts: null, - }); - - expect(result.errors).toEqual([]); - expect(result.warnings).toEqual([]); - expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]); - expect(result.result).toMatchObject({ tableUsageMerged: 1 }); - await expect(readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves.toContain( - 'Orders are repeatedly queried by lifecycle status.', - ); - }); -}); -``` - -- [ ] **Step 2: Run the post-processor test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/post-processor.test.ts -``` - -Expected: FAIL with an import error for `./post-processor.js`. - -- [ ] **Step 3: Implement the post-processor** - -Create `packages/context/src/ingest/adapters/historic-sql/post-processor.ts`: - -```typescript -import type { IngestBundlePostProcessorInput, IngestBundlePostProcessorPort, IngestBundlePostProcessorResult } from '../../ports.js'; -import { projectHistoricSqlEvidence } from './projection.js'; - -export class HistoricSqlProjectionPostProcessor implements IngestBundlePostProcessorPort { - async run(input: IngestBundlePostProcessorInput): Promise { - const projection = await projectHistoricSqlEvidence({ - workdir: input.workdir, - connectionId: input.connectionId, - syncId: input.syncId, - runId: input.runId, - }); - return { - result: projection, - warnings: projection.warnings, - errors: [], - touchedSources: projection.touchedSources, - }; - } -} -``` - -- [ ] **Step 4: Add the evidence tool and post-processor to local ingest runtime** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, import: - -```typescript -import { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js'; -import { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js'; -``` - -In `LocalIngestToolsetFactory.createIngestWuToolset()`, return the historic-SQL evidence tool only for historic-SQL ingest sessions: - -```typescript - createIngestWuToolset(session: ToolSession, options?: { includeContextEvidenceTools?: boolean }): IngestToolsetLike { - const sourceTools = session.ingest?.sourceKey === 'historic-sql' ? [createEmitHistoricSqlEvidenceTool()] : []; - return new LocalIngestToolSet( - options?.includeContextEvidenceTools - ? [...this.baseTools, ...this.contextTools, ...sourceTools] - : [...this.baseTools, ...sourceTools], - ); - } -``` - -In the `deps` object passed to `new IngestBundleRunner(deps)`, add: - -```typescript - postProcessors: { - 'historic-sql': new HistoricSqlProjectionPostProcessor(), - }, -``` - -- [ ] **Step 5: Add runtime integration assertions** - -In `packages/context/src/ingest/local-bundle-ingest.test.ts`, add a test using an injected `agentRunner` that calls `emit_historic_sql_evidence` for a planned historic-SQL WorkUnit and asserts the report `postProcessor` result contains `tableUsageMerged: 1`. Use the existing local-bundle ingest test patterns for injected tool execution; the key assertion is: - -```typescript -await expect(readFile(join(projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves.toContain( - 'Orders are repeatedly queried by lifecycle status.', -); -``` - -- [ ] **Step 6: Export post-processor and verify tests pass** - -Add this export to `packages/context/src/ingest/index.ts`: - -```typescript -export { HistoricSqlProjectionPostProcessor } from './adapters/historic-sql/post-processor.js'; -``` - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/post-processor.test.ts src/ingest/local-bundle-ingest.test.ts src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/post-processor.ts packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts packages/context/src/ingest/local-bundle-runtime.ts packages/context/src/ingest/local-bundle-ingest.test.ts packages/context/src/ingest/index.ts packages/context/src/package-exports.test.ts -git commit -m "feat: run historic sql deterministic projection" -``` - -## Task 5: Switch Production Adapter To Unified Hot Path - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/types.ts` -- Modify: `packages/context/src/ingest/local-adapters.ts` -- Modify: `packages/context/src/ingest/local-adapters.test.ts` - -- [ ] **Step 1: Write failing adapter metadata and fetch tests** - -In `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`, replace the metadata test expectations with: - -```typescript - expect(adapter.skillNames).toEqual(['historic_sql_table_digest', 'historic_sql_patterns']); - expect(adapter.reconcileSkillNames).toEqual([]); - expect(adapter.evidenceIndexing).toBeUndefined(); - expect(adapter.triageSupported).toBe(false); -``` - -Replace the legacy fetch tests with a unified fetch test: - -```typescript - it('fetches a unified aggregate snapshot and emits unified WorkUnits', async () => { - const stagedDir = await tempDir(); - const reader = { - async probe() { - return { warnings: [] }; - }, - async *fetchAggregated() { - yield { - templateId: 'pg:1', - canonicalSql: 'select status, count(*) from public.orders group by status', - dialect: 'postgres', - stats: { - executions: 25, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 10, - p95RuntimeMs: 20, - errorRate: 0, - rowsProduced: 10, - }, - topUsers: [{ user: 'analyst', executions: 25 }], - }; - }, - }; - const sqlAnalysis = { - async analyzeForFingerprint() { - throw new Error('legacy analyzeForFingerprint must not be used'); - }, - async analyzeBatch() { - return new Map([ - [ - 'pg:1', - { - tablesTouched: ['public.orders'], - columnsByClause: { select: ['status'], groupBy: ['status'] }, - }, - ], - ]); - }, - }; - const adapter = new HistoricSqlSourceAdapter({ - sqlAnalysis, - reader, - queryClient: {}, - now: () => new Date('2026-05-11T00:00:00.000Z'), - }); - - await adapter.fetch({ dialect: 'postgres', minExecutions: 5 }, stagedDir, { - connectionId: 'warehouse', - sourceKey: 'historic-sql', - }); - - await expect(adapter.detect(stagedDir)).resolves.toBe(true); - await expect(adapter.chunk(stagedDir)).resolves.toMatchObject({ - workUnits: [ - { unitKey: 'historic-sql-table-public-orders' }, - { unitKey: 'historic-sql-patterns' }, - ], - }); - }); -``` - -- [ ] **Step 2: Run adapter tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts -``` - -Expected: FAIL because the adapter still advertises legacy skills and calls legacy staging. - -- [ ] **Step 3: Update adapter dependency types** - -In `packages/context/src/ingest/adapters/historic-sql/types.ts`, change `HistoricSqlSourceAdapterDeps` to: - -```typescript -export interface HistoricSqlSourceAdapterDeps { - sqlAnalysis: SqlAnalysisPort; - reader: HistoricSqlReader; - queryClient: unknown; - legacyPostgresBaselineRootDir?: string; - now?: () => Date; -} -``` - -Extend `historicSqlUnifiedPullConfigSchema` preprocessing to map existing local config keys: - -```typescript - const next: Record = { ...value }; - if (next.minExecutions === undefined && typeof next.minCalls === 'number') { - next.minExecutions = next.minCalls; - } - if (!next.filters && Array.isArray(next.serviceAccountUserPatterns)) { - next.filters = { - serviceAccounts: { patterns: next.serviceAccountUserPatterns, mode: 'exclude' }, - dropTrivialProbes: true, - }; - } - return next; -``` - -- [ ] **Step 4: Replace adapter implementation** - -In `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts`, remove legacy imports and use: - -```typescript -import { rm } from 'node:fs/promises'; -import { join } from 'node:path'; -import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js'; -import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js'; -import { detectHistoricSqlStagedDir } from './detect.js'; -import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js'; -import { type HistoricSqlSourceAdapterDeps } from './types.js'; - -export class HistoricSqlSourceAdapter implements SourceAdapter { - readonly source = 'historic-sql'; - readonly skillNames = ['historic_sql_table_digest', 'historic_sql_patterns']; - readonly reconcileSkillNames: string[] = []; - readonly triageSupported = false; - - constructor(private readonly deps: HistoricSqlSourceAdapterDeps) {} - - detect(stagedDir: string): Promise { - return detectHistoricSqlStagedDir(stagedDir); - } - - async fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise { - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: ctx.connectionId, - queryClient: this.deps.queryClient, - reader: this.deps.reader, - sqlAnalysis: this.deps.sqlAnalysis, - pullConfig, - now: this.deps.now?.(), - }); - if (this.deps.legacyPostgresBaselineRootDir) { - await rm(join(this.deps.legacyPostgresBaselineRootDir, ctx.connectionId, 'pgss-baseline.json'), { - force: true, - }); - } - } - - chunk(stagedDir: string, diffSet?: DiffSet): Promise { - return chunkHistoricSqlUnifiedStagedDir(stagedDir, diffSet); - } - - describeScope(stagedDir: string): Promise { - return describeHistoricSqlUnifiedScope(stagedDir); - } -} -``` - -- [ ] **Step 5: Update WorkUnit notes to mention the evidence tool** - -In `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`, update notes to contain: - -```typescript -'Use historic_sql_table_digest. Read this table usage JSON and emit exactly one table_usage object with emit_historic_sql_evidence. Do not call wiki_write or sl_write_source.' -``` - -and: - -```typescript -'Use historic_sql_patterns. Read patterns-input.json and emit pattern objects with emit_historic_sql_evidence. Do not call wiki_write or sl_write_source.' -``` - -Update `chunk-unified.test.ts` assertions to check `emit_historic_sql_evidence`. - -- [ ] **Step 6: Update local adapter wiring** - -In `packages/context/src/ingest/local-adapters.ts`, import: - -```typescript -import { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js'; -``` - -Remove the `PostgresPgssQueryHistoryReader` import. Construct the local historic-SQL adapter as: - -```typescript - adapters.push( - new HistoricSqlSourceAdapter({ - sqlAnalysis: options.historicSql.sqlAnalysis, - reader: new PostgresPgssReader(), - queryClient: options.historicSql.postgresQueryClient, - legacyPostgresBaselineRootDir: options.historicSql.postgresBaselineRootDir, - now: options.historicSql.now, - }), - ); -``` - -In `localPullConfigForAdapter()`, parse with `historicSqlUnifiedPullConfigSchema` instead of `historicSqlPullConfigSchema`. - -- [ ] **Step 7: Run adapter/local tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts src/ingest/adapters/historic-sql/chunk-unified.test.ts src/ingest/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts packages/context/src/ingest/adapters/historic-sql/types.ts packages/context/src/ingest/local-adapters.ts packages/context/src/ingest/local-adapters.test.ts -git commit -m "feat: cut over historic sql adapter" -``` - -## Task 6: Delete Legacy Historic-SQL Code Path - -**Files:** -- Modify: `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/detect.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/detect.test.ts` -- Modify: `packages/context/src/ingest/index.ts` -- Modify: `packages/context/src/package-exports.test.ts` -- Modify: `packages/cli/src/setup-databases.ts` -- Modify: `packages/cli/src/historic-sql-doctor.ts` -- Delete the legacy files listed in the File Structure section. - -- [ ] **Step 1: Move PGSS probe behavior into `PostgresPgssReader`** - -Update `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` so the existing probe tests import `PostgresPgssReader` from `./postgres-pgss-reader.js` and assert the same probe warnings/errors now covered by `postgres-pgss-query-history-reader.test.ts`. - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts -``` - -Expected: FAIL until probe SQL and error mapping are inlined. - -- [ ] **Step 2: Inline the probe logic** - -In `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`, remove: - -```typescript -import { PostgresPgssQueryHistoryReader } from './postgres-pgss-query-history-reader.js'; -``` - -Remove: - -```typescript - private readonly legacyReader = new PostgresPgssQueryHistoryReader(); - - probe(client: unknown): Promise { - return this.legacyReader.probe(client); - } -``` - -Add the probe SQL and mapping currently used by `PostgresPgssQueryHistoryReader` into this file, and make `probe(client)` return `PostgresPgssProbeResult` directly. Preserve the existing doctor-facing checks for extension presence, grants, server version, `pg_stat_statements.track`, and informational `pg_stat_statements.max`. - -- [ ] **Step 3: Update CLI doctor/setup imports** - -In `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts`, replace dynamic imports of `PostgresPgssQueryHistoryReader` with `PostgresPgssReader`: - -```typescript -const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('@ktx/context/ingest'), import('./postgres-query-client.js')]); -``` - -Replace `new PostgresPgssQueryHistoryReader().probe(client)` with: - -```typescript -new PostgresPgssReader().probe(client) -``` - -- [ ] **Step 4: Simplify detection to the unified manifest shape** - -In `packages/context/src/ingest/adapters/historic-sql/detect.ts`, keep manifest-source detection and replace the old `templates/*/{metadata.json,page.md}` fallback with unified structural detection: - -```typescript - try { - await readFile(join(stagedDir, 'patterns-input.json'), 'utf-8'); - const entries = await readdir(join(stagedDir, 'tables'), { withFileTypes: true }); - return entries.some((entry) => entry.isFile() && entry.name.endsWith('.json')); - } catch { - return false; - } -``` - -Update `detect.test.ts` to use `stagedManifestSchema` and remove tests for legacy `historicSqlManifestSchema`, `historicSqlMetadataSchema`, and `historicSqlUsageSchema`. - -- [ ] **Step 5: Remove legacy exports and assertions** - -In `packages/context/src/ingest/index.ts`, delete exports for: - -```typescript -chunkHistoricSqlStagedDir -describeHistoricSqlScope -PostgresPgssQueryHistoryReader -stageHistoricSqlTemplates -stagePgStatStatementsTemplates -PgssBaseline -StagePgStatStatementsTemplatesResult -historicSqlManifestSchema -historicSqlMetadataSchema -historicSqlPullConfigSchema -historicSqlUsageSchema -``` - -In `packages/context/src/package-exports.test.ts`, remove assertions for those exports. - -- [ ] **Step 6: Delete legacy files and old skills** - -Run: - -```bash -rm -rf packages/context/src/ingest/adapters/historic-sql/__fixtures__/postgres -rm packages/context/src/ingest/adapters/historic-sql/stage.ts -rm packages/context/src/ingest/adapters/historic-sql/stage.test.ts -rm packages/context/src/ingest/adapters/historic-sql/stage-pgss.ts -rm packages/context/src/ingest/adapters/historic-sql/stage-pgss.test.ts -rm packages/context/src/ingest/adapters/historic-sql/stage-pgss-golden.test.ts -rm packages/context/src/ingest/adapters/historic-sql/chunk.ts -rm packages/context/src/ingest/adapters/historic-sql/chunk.test.ts -rm packages/context/src/ingest/adapters/historic-sql/postgres-pgss-query-history-reader.ts -rm packages/context/src/ingest/adapters/historic-sql/postgres-pgss-query-history-reader.test.ts -rm -rf packages/context/skills/historic_sql_ingest -rm -rf packages/context/skills/historic_sql_curator -``` - -Expected: files are removed from the worktree. Do not delete unified files: `stage-unified.ts`, `chunk-unified.ts`, `postgres-pgss-reader.ts`, `bigquery-query-history-reader.ts`, `snowflake-query-history-reader.ts`, `types.ts`, `skill-schemas.ts`, `evidence.ts`, `projection.ts`, and `post-processor.ts`. - -- [ ] **Step 7: Remove page-triage historic-SQL prompt references** - -In `packages/context/prompts/skills/page_triage_classifier.md`, remove the historic-SQL-specific block for `signals.objectType === "historic_sql_template"`. Update these tests to stop asserting that prompt text: - -- `packages/context/src/ingest/page-triage/page-triage.service.test.ts` -- `packages/context/src/ingest/ingest-prompts.test.ts` -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 8: Run no-old-code grep** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline|historic_sql_ingest|historic_sql_curator|PostgresPgssQueryHistoryReader|historic_sql_template" packages/context packages/cli -``` - -Expected: no matches in `packages/context` or `packages/cli`. - -- [ ] **Step 9: Run focused deletion tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/package-exports.test.ts -pnpm --filter @ktx/cli exec vitest run src/historic-sql-doctor.test.ts src/setup-databases.test.ts -``` - -Expected: PASS. - -- [ ] **Step 10: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql packages/context/skills packages/context/src/ingest/index.ts packages/context/src/package-exports.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/src/memory/memory-runtime-assets.test.ts packages/context/prompts/skills/page_triage_classifier.md packages/context/src/ingest/page-triage/page-triage.service.test.ts packages/context/src/ingest/ingest-prompts.test.ts packages/cli/src/setup-databases.ts packages/cli/src/historic-sql-doctor.ts packages/cli/src/historic-sql-doctor.test.ts packages/cli/src/setup-databases.test.ts -git commit -m "refactor: remove legacy historic sql pipeline" -``` - -## Task 7: Rename Setup Config To minExecutions - -**Files:** -- Modify: `packages/cli/src/commands/setup-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Modify: `packages/cli/src/setup-databases.ts` -- Modify: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Write failing setup CLI assertions** - -In `packages/cli/src/index.test.ts`, update setup help assertions so both flags are accepted: - -```typescript -expect(output).toContain('--historic-sql-min-executions'); -expect(output).toContain('--historic-sql-min-calls'); -``` - -In setup output/config tests, assert generated YAML uses: - -```yaml -historicSql: - enabled: true - dialect: postgres - minExecutions: 7 -``` - -and does not write `minCalls`. - -- [ ] **Step 2: Run setup tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/setup-databases.test.ts -``` - -Expected: FAIL because the CLI still writes `minCalls`. - -- [ ] **Step 3: Add the new flag and preserve the old alias** - -In `packages/cli/src/commands/setup-commands.ts`, add: - -```typescript - .option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template', positiveInteger) -``` - -Keep `--historic-sql-min-calls` with help text: - -```typescript - .option('--historic-sql-min-calls ', 'Alias for --historic-sql-min-executions', positiveInteger) -``` - -When building setup options, resolve: - -```typescript -const historicSqlMinExecutions = opts.historicSqlMinExecutions ?? opts.historicSqlMinCalls; -``` - -In `packages/cli/src/setup-databases.ts`, write `minExecutions` to config. Do not write `minCalls`. - -- [ ] **Step 4: Run setup tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/setup-databases.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/commands/setup-commands.ts packages/cli/src/index.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "feat: rename historic sql setup threshold" -``` - -## Task 8: Final Verification - -**Files:** -- Verify: historic-SQL adapter, CLI setup/doctor, runtime assets, exports, Python daemon batch endpoint. - -- [ ] **Step 1: Run all historic-SQL context tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql src/ingest/local-adapters.test.ts src/ingest/local-bundle-ingest.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI setup and doctor tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/historic-sql-doctor.test.ts src/setup-databases.test.ts src/index.test.ts src/ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run Python daemon SQL analysis tests** - -Run: - -```bash -source .venv/bin/activate && python -m pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -q -``` - -Expected: PASS. - -- [ ] **Step 4: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 5: Run no-old-code grep** - -Run: - -```bash -rg -n "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline|historic_sql_ingest|historic_sql_curator|PostgresPgssQueryHistoryReader|historic_sql_template" packages/context packages/cli -``` - -Expected: no matches. - -- [ ] **Step 6: Run pre-commit for touched files** - -Run with the actual touched file list from `git diff --name-only`: - -```bash -uv run pre-commit run --files $(git diff --name-only) -``` - -Expected: PASS. If local `uv` refuses due the repo's exact uv pin, activate `.venv` for Python checks and report the uv version mismatch exactly. - -- [ ] **Step 7: Commit final verification notes if test snapshots changed** - -If verification updated tracked snapshots or generated checked-in fixtures, commit only those intended files: - -```bash -git add -git commit -m "test: verify historic sql cutover" -``` - -Expected: either a small verification commit is created, or no commit is needed because `git status --short` is clean. - -## Self-Review - -Spec coverage: - -- New skills `historic_sql_table_digest` and `historic_sql_patterns`: Task 2. -- LLM skills emit evidence instead of direct writes: Task 1 and Task 2. -- Deterministic projection of table usage into `_schema` shards: Task 3 and Task 4. -- Pattern wiki pages under `knowledge/global/historic-sql/{slug}.md`: Task 3 and Task 4. -- Slug stability and stale/archive handling: Task 3. -- Production adapter cutover to unified reader/stager/chunker: Task 5. -- Old skill and legacy code deletion: Task 6. -- PGSS baseline cleanup: Task 5 via `legacyPostgresBaselineRootDir` removal. -- CLI setup `minCalls` to `minExecutions` alias: Task 7. -- Search surfaces: already implemented by `2026-05-11-historic-sql-search-enrichment.md`; final verification keeps them covered. - -Placeholder scan: - -- No unresolved placeholder markers are present. -- Every code-changing task includes exact paths, test commands, and expected pass/fail outcomes. -- Complex projection internals are described as concrete behavior with named fields and deterministic matching rules. - -Type consistency: - -- `HistoricSqlEvidenceEnvelope`, `HistoricSqlProjectionResult`, `HistoricSqlProjectionPostProcessor`, and `HistoricSqlSourceAdapterDeps` names are introduced before use. -- Skill names match the new adapter metadata and runtime asset tests: `historic_sql_table_digest`, `historic_sql_patterns`. -- `PostgresPgssReader` remains the single public PGSS reader after legacy deletion. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md b/docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md deleted file mode 100644 index c3228e8a..00000000 --- a/docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md +++ /dev/null @@ -1,1698 +0,0 @@ -# Historic SQL Unified Hot Path Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build the deterministic historic-SQL hot path that reads warehouse-aggregated query templates, batch-parses them once, and writes stable table-bucket and pattern-input staged artifacts. - -**Architecture:** This slice adds the unified reader/stager contracts from the historic-SQL redesign without doing the LLM cold path or projection work. Dialect-specific SQL lives in reader classes; shared TypeScript code filters, batch-parses, bucketizes, and writes `manifest.json`, `tables/*.json`, and `patterns-input.json`. The existing production adapter remains on the legacy path until the follow-up skills/projection cutover can switch it without loading missing skills. - -**Tech Stack:** TypeScript ESM/NodeNext, zod 4, Vitest, `SqlAnalysisPort.analyzeBatch()`, warehouse query clients. - ---- - -## Starting Point - -Spec: `docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md` - -Plans found that are based on this spec: - -- `docs/superpowers/plans/2026-05-11-historic-sql-foundations.md` -- `docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md` - -Implemented status from this worktree: - -- `2026-05-11-historic-sql-foundations.md` is implemented. Evidence: `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`, `SemanticLayerSource.usage` in `packages/context/src/sl/types.ts`, `mergeUsagePreservingExternal()` in `packages/context/src/ingest/adapters/live-database/manifest.ts`, `SqlAnalysisPort.analyzeBatch()` in `packages/context/src/sql-analysis/ports.ts`, and `/sql/analyze-batch` in `python/ktx-daemon/src/ktx_daemon/app.py`. -- `2026-05-11-historic-sql-search-enrichment.md` is implemented. Evidence: `buildSemanticLayerSourceSearchText()` indexes `source.usage` in `packages/context/src/sl/sl-search.service.ts`, SQLite FTS returns `snippet()` in `packages/context/src/sl/sqlite-sl-sources-index.ts`, and local/MCP list results expose `frequencyTier` and `snippet` in `packages/context/src/sl/local-sl.ts` and `packages/context/src/mcp/local-project-ports.ts`. - -Still not implemented: - -- `packages/context/src/ingest/adapters/historic-sql/stage.ts` still calls `SqlAnalysisPort.analyzeForFingerprint()` per raw query and emits `templates/*/{metadata.json,page.md,usage.json}`. -- `packages/context/src/ingest/adapters/historic-sql/stage-pgss.ts` still owns Postgres baseline-diff state and writes `.ktx/cache/historic-sql/*/pgss-baseline.json`. -- `packages/context/src/ingest/adapters/historic-sql/chunk.ts` still emits one WorkUnit per template page for `historic_sql_ingest`. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` still advertises `historic_sql_ingest` and `historic_sql_curator`. -- Old code strings still exist: `stagePgStatStatementsTemplates`, `expandCategoricalTemplates`, `classifySlot`, and `pgss-baseline`. - -This plan covers the deterministic hot path from the spec: unified aggregate contracts, aggregate readers, batch parsing, table bucketing, pattern input staging, and a new chunker for the new staged shape. It does not switch `HistoricSqlSourceAdapter` to the new WorkUnits; the cutover plan must create `historic_sql_table_digest`, `historic_sql_patterns`, and projection before changing production `skillNames`. - -## File Structure - -Create: - -- `packages/context/src/ingest/adapters/historic-sql/types.test.ts` - Locks the new public zod contracts and the one-release `minCalls` to `minExecutions` config alias. -- `packages/context/src/ingest/adapters/historic-sql/buckets.ts` - Owns deterministic bucket labels and frequency-tier helpers used by staging. -- `packages/context/src/ingest/adapters/historic-sql/buckets.test.ts` - Locks stable bucket boundaries so small numeric drift does not churn staged files. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` - Implements the new deterministic stager behind `stageHistoricSqlAggregatedSnapshot()`. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` - Tests batch parsing, parse failures, service-account filtering, per-table bucketing, and `patterns-input.json`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` - Implements the new Postgres aggregate reader over `pg_stat_statements`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` - Tests the aggregate PGSS query shape, probe warnings, and row mapping. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` - Implements the new chunker for `tables/*.json` plus `patterns-input.json`. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` - Tests table WorkUnits, the patterns WorkUnit, diff filtering, eviction, and scope detection. - -Modify: - -- `packages/context/src/ingest/adapters/historic-sql/types.ts` - Adds aggregate input, staged artifact, reader, and manifest schemas. Keeps legacy exported types until adapter cutover, but marks the new contracts as the target API for the next slice. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` - Adds `fetchAggregated()` while retaining the existing `fetch()` until the adapter cutover. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` - Adds aggregate-query tests. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` - Adds `fetchAggregated()` while retaining the existing `fetch()` until the adapter cutover. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` - Adds aggregate-query tests. -- `packages/context/src/ingest/index.ts` - Exports the new hot-path contracts and helpers. -- `packages/context/src/package-exports.test.ts` - Asserts the new exports exist without removing old exports in this slice. - -Do not modify in this plan: - -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` -- `packages/context/skills/historic_sql_ingest/SKILL.md` -- `packages/context/skills/historic_sql_curator/SKILL.md` -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -Those files change in the cutover/projection plan after the replacement skills exist. - -## Task 1: Add Unified Contracts - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/types.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/types.ts` -- Modify: `packages/context/src/ingest/index.ts` -- Modify: `packages/context/src/package-exports.test.ts` - -- [ ] **Step 1: Write failing contract tests** - -Create `packages/context/src/ingest/adapters/historic-sql/types.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { - aggregatedTemplateSchema, - historicSqlUnifiedPullConfigSchema, - stagedManifestSchema, - stagedPatternsInputSchema, - stagedTableInputSchema, -} from './types.js'; - -describe('historic-sql unified contracts', () => { - it('parses minExecutions and accepts minCalls as a one-release alias', () => { - expect(historicSqlUnifiedPullConfigSchema.parse({ dialect: 'postgres', minExecutions: 9 })).toMatchObject({ - dialect: 'postgres', - minExecutions: 9, - windowDays: 90, - concurrency: 12, - redactionPatterns: [], - staleArchiveAfterDays: 90, - }); - - expect(historicSqlUnifiedPullConfigSchema.parse({ dialect: 'postgres', minCalls: 7 }).minExecutions).toBe(7); - }); - - it('validates aggregate templates from warehouse readers', () => { - const parsed = aggregatedTemplateSchema.parse({ - templateId: 'pg:123', - canonicalSql: 'select status, count(*) from public.orders group by status', - dialect: 'postgres', - stats: { - executions: 42, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 12.5, - p95RuntimeMs: 40, - errorRate: 0, - rowsProduced: 100, - }, - topUsers: [{ user: 'analyst', executions: 40 }], - }); - - expect(parsed.templateId).toBe('pg:123'); - expect(parsed.topUsers).toEqual([{ user: 'analyst', executions: 40 }]); - }); - - it('validates staged table, patterns, and manifest artifacts', () => { - expect( - stagedTableInputSchema.parse({ - table: 'public.orders', - stats: { - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - errorRateBucket: 'none', - p95RuntimeBucket: '<100ms', - recencyBucket: 'current', - }, - columnsByClause: { - select: [['status', 'high']], - where: [['created_at', 'mid']], - }, - observedJoins: [{ withTable: 'public.customers', on: ['customer_id'], freq: 'high' }], - topTemplates: [{ id: 'pg:123', canonicalSql: 'select * from public.orders', topUsers: [{ user: 'analyst' }] }], - }).table, - ).toBe('public.orders'); - - expect( - stagedPatternsInputSchema.parse({ - templates: [ - { - id: 'pg:123', - canonicalSql: 'select * from public.orders', - tablesTouched: ['public.orders'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ], - }).templates, - ).toHaveLength(1); - - expect( - stagedManifestSchema.parse({ - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 2, - touchedTableCount: 1, - parseFailures: 1, - warnings: ['parse_failed:bad'], - probeWarnings: [], - }).parseFailures, - ).toBe(1); - }); -}); -``` - -Add these assertions near the historic-SQL export assertions in `packages/context/src/package-exports.test.ts`: - -```typescript - expect(ingest.historicSqlUnifiedPullConfigSchema).toBeDefined(); - expect(ingest.aggregatedTemplateSchema).toBeDefined(); - expect(ingest.stagedTableInputSchema).toBeDefined(); -``` - -- [ ] **Step 2: Run the contract tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/types.test.ts src/package-exports.test.ts -``` - -Expected: FAIL with missing exports for `historicSqlUnifiedPullConfigSchema`, `aggregatedTemplateSchema`, and `stagedTableInputSchema`. - -- [ ] **Step 3: Add the new schemas and reader contracts** - -Insert this block immediately after the existing `historicSqlPullConfigSchema` definition in `packages/context/src/ingest/adapters/historic-sql/types.ts`. Keep `historicSqlPullConfigSchema` and `HistoricSqlPullConfig` unchanged in this plan because the current production adapter still reads `lastSuccessfulCursor`, `maxTemplatesPerRun`, and `minCalls`. - -```typescript -const filterModeSchema = z.enum(['exclude', 'include', 'mark-only']); - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -export const historicSqlUnifiedPullConfigSchema = z.preprocess((value) => { - if (!isRecord(value)) { - return value; - } - if (value.minExecutions === undefined && typeof value.minCalls === 'number') { - return { ...value, minExecutions: value.minCalls }; - } - return value; -}, z.object({ - dialect: historicSqlDialectSchema, - windowDays: z.number().int().positive().default(90), - minExecutions: z.number().int().nonnegative().default(5), - concurrency: z.number().int().positive().default(12), - filters: z.object({ - serviceAccounts: z.object({ - patterns: z.array(z.string()).default([]), - mode: filterModeSchema.default('exclude'), - }).optional(), - orchestrators: z.object({ - mode: filterModeSchema.default('mark-only'), - }).optional(), - dropTrivialProbes: z.boolean().default(true), - dropFailedBelow: z.object({ - errorRate: z.number().min(0).max(1), - executions: z.number().int().nonnegative(), - }).optional(), - }).default({}), - redactionPatterns: z.array(z.string()).default([]), - staleArchiveAfterDays: z.number().int().positive().default(90), -})); - -export type HistoricSqlUnifiedPullConfig = z.infer; - -export const aggregatedTemplateSchema = z.object({ - templateId: z.string().min(1), - canonicalSql: z.string().min(1), - dialect: historicSqlDialectSchema, - stats: z.object({ - executions: z.number().int().nonnegative(), - distinctUsers: z.number().int().nonnegative(), - firstSeen: z.iso.datetime(), - lastSeen: z.iso.datetime(), - p50RuntimeMs: z.number().nonnegative().nullable(), - p95RuntimeMs: z.number().nonnegative().nullable(), - errorRate: z.number().min(0).max(1), - rowsProduced: z.number().int().nonnegative().nullable(), - }), - topUsers: z.array(z.object({ - user: z.string().nullable(), - executions: z.number().int().nonnegative(), - })).default([]), -}); -export type AggregatedTemplate = z.infer; - -export const stagedTableInputSchema = z.object({ - table: z.string().min(1), - stats: z.object({ - executionsBucket: z.string(), - distinctUsersBucket: z.string(), - errorRateBucket: z.string(), - p95RuntimeBucket: z.string(), - recencyBucket: z.string(), - }), - columnsByClause: z.record(z.string(), z.array(z.tuple([z.string(), z.string()]))), - observedJoins: z.array(z.object({ - withTable: z.string(), - on: z.array(z.string()), - freq: z.string(), - })), - topTemplates: z.array(z.object({ - id: z.string(), - canonicalSql: z.string(), - topUsers: z.array(z.object({ user: z.string().nullable() })), - })), -}); -export type StagedTableInput = z.infer; - -export const stagedPatternsInputSchema = z.object({ - templates: z.array(z.object({ - id: z.string(), - canonicalSql: z.string(), - tablesTouched: z.array(z.string()), - executionsBucket: z.string(), - distinctUsersBucket: z.string(), - dialect: historicSqlDialectSchema, - })), -}); -export type StagedPatternsInput = z.infer; - -export const stagedManifestSchema = z.object({ - source: z.literal(HISTORIC_SQL_SOURCE_KEY), - connectionId: z.string().min(1), - dialect: historicSqlDialectSchema, - fetchedAt: z.iso.datetime(), - windowStart: z.iso.datetime(), - windowEnd: z.iso.datetime(), - snapshotRowCount: z.number().int().nonnegative(), - touchedTableCount: z.number().int().nonnegative(), - parseFailures: z.number().int().nonnegative(), - warnings: z.array(z.string()), - probeWarnings: z.array(z.string()), -}); -export type StagedManifest = z.infer; - -export interface HistoricSqlProbeResult { - warnings: string[]; -} - -export interface HistoricSqlReader { - probe(client: unknown): Promise; - fetchAggregated( - client: unknown, - window: HistoricSqlTimeWindow, - config: HistoricSqlUnifiedPullConfig, - ): AsyncIterable; -} -``` - -- [ ] **Step 4: Export the new contracts** - -In `packages/context/src/ingest/index.ts`, add exports for the new types and schemas: - -```typescript -export type { - AggregatedTemplate, - HistoricSqlProbeResult, - HistoricSqlReader, - HistoricSqlUnifiedPullConfig, - StagedManifest, - StagedPatternsInput, - StagedTableInput, -} from './adapters/historic-sql/types.js'; -export { - aggregatedTemplateSchema, - historicSqlUnifiedPullConfigSchema, - stagedManifestSchema, - stagedPatternsInputSchema, - stagedTableInputSchema, -} from './adapters/historic-sql/types.js'; -``` - -- [ ] **Step 5: Run the contract tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/types.test.ts src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/types.ts packages/context/src/ingest/adapters/historic-sql/types.test.ts packages/context/src/ingest/index.ts packages/context/src/package-exports.test.ts -git commit -m "feat: add historic sql unified contracts" -``` - -## Task 2: Add Stable Bucket Helpers - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/buckets.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/buckets.test.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Write failing bucket tests** - -Create `packages/context/src/ingest/adapters/historic-sql/buckets.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { - bucketDistinctUsers, - bucketErrorRate, - bucketExecutions, - bucketFrequency, - bucketP95Runtime, - bucketRecency, -} from './buckets.js'; - -describe('historic-sql bucket helpers', () => { - it('uses stable execution buckets', () => { - expect([0, 9, 10, 99, 100, 999, 1000, 4999, 5000, 49999, 50000].map(bucketExecutions)).toEqual([ - '<10', - '<10', - '10-100', - '10-100', - '100-1k', - '100-1k', - '1k-5k', - '1k-5k', - '5k-50k', - '5k-50k', - '>50k', - ]); - }); - - it('uses stable distinct-user, error-rate, runtime, and recency buckets', () => { - expect([0, 1, 2, 5, 6, 10, 11].map(bucketDistinctUsers)).toEqual([ - '0', - '1', - '2-5', - '2-5', - '5-10', - '5-10', - '>10', - ]); - expect([0, 0.01, 0.05, 0.2].map(bucketErrorRate)).toEqual(['none', 'low', 'low', 'high']); - expect([null, 99, 100, 999, 1000, 9999, 10000].map(bucketP95Runtime)).toEqual([ - 'unknown', - '<100ms', - '100ms-1s', - '100ms-1s', - '1s-10s', - '1s-10s', - '>10s', - ]); - expect(bucketRecency('2026-05-11T00:00:00.000Z', new Date('2026-05-11T12:00:00.000Z'))).toBe('current'); - expect(bucketRecency('2026-04-20T00:00:00.000Z', new Date('2026-05-11T12:00:00.000Z'))).toBe('recent'); - expect(bucketRecency('2026-01-01T00:00:00.000Z', new Date('2026-05-11T12:00:00.000Z'))).toBe('stale'); - }); - - it('maps frequency counts to high, mid, and low labels', () => { - expect(bucketFrequency(80, 100)).toBe('high'); - expect(bucketFrequency(20, 100)).toBe('mid'); - expect(bucketFrequency(1, 100)).toBe('low'); - expect(bucketFrequency(0, 0)).toBe('low'); - }); -}); -``` - -- [ ] **Step 2: Run the bucket test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/buckets.test.ts -``` - -Expected: FAIL because `buckets.js` does not exist. - -- [ ] **Step 3: Add the bucket helper implementation** - -Create `packages/context/src/ingest/adapters/historic-sql/buckets.ts`: - -```typescript -export function bucketExecutions(value: number): string { - if (value < 10) return '<10'; - if (value < 100) return '10-100'; - if (value < 1000) return '100-1k'; - if (value < 5000) return '1k-5k'; - if (value < 50000) return '5k-50k'; - return '>50k'; -} - -export function bucketDistinctUsers(value: number): string { - if (value <= 0) return '0'; - if (value === 1) return '1'; - if (value <= 5) return '2-5'; - if (value <= 10) return '5-10'; - return '>10'; -} - -export function bucketErrorRate(value: number): string { - if (value <= 0) return 'none'; - if (value < 0.1) return 'low'; - return 'high'; -} - -export function bucketP95Runtime(value: number | null): string { - if (value === null) return 'unknown'; - if (value < 100) return '<100ms'; - if (value < 1000) return '100ms-1s'; - if (value < 10000) return '1s-10s'; - return '>10s'; -} - -export function bucketRecency(lastSeen: string, now: Date): string { - const parsed = new Date(lastSeen); - if (Number.isNaN(parsed.getTime())) { - return 'unknown'; - } - const ageDays = (now.getTime() - parsed.getTime()) / (24 * 60 * 60 * 1000); - if (ageDays <= 7) return 'current'; - if (ageDays <= 45) return 'recent'; - return 'stale'; -} - -export function bucketFrequency(count: number, total: number): 'high' | 'mid' | 'low' { - if (total <= 0 || count <= 0) return 'low'; - const ratio = count / total; - if (ratio >= 0.5) return 'high'; - if (ratio >= 0.1) return 'mid'; - return 'low'; -} -``` - -- [ ] **Step 4: Run the bucket test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/buckets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Export bucket helpers** - -In `packages/context/src/ingest/index.ts`, add: - -```typescript -export { bucketDistinctUsers, bucketErrorRate, bucketExecutions, bucketP95Runtime, bucketRecency } from './adapters/historic-sql/buckets.js'; -``` - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/buckets.ts packages/context/src/ingest/adapters/historic-sql/buckets.test.ts packages/context/src/ingest/index.ts -git commit -m "feat: add historic sql bucket helpers" -``` - -## Task 3: Stage Aggregated Snapshots - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Write failing staged-artifact tests** - -Create `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`: - -```typescript -import { mkdtemp, readFile, readdir } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; -import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js'; -import type { AggregatedTemplate, HistoricSqlReader } from './types.js'; - -async function tempDir(): Promise { - return mkdtemp(join(tmpdir(), 'historic-sql-unified-stage-')); -} - -async function readJson(root: string, relPath: string): Promise { - return JSON.parse(await readFile(join(root, relPath), 'utf-8')) as T; -} - -function aggregate(overrides: Partial & { templateId: string; canonicalSql: string }): AggregatedTemplate { - return { - templateId: overrides.templateId, - canonicalSql: overrides.canonicalSql, - dialect: overrides.dialect ?? 'postgres', - stats: overrides.stats ?? { - executions: 42, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 20, - p95RuntimeMs: 80, - errorRate: 0, - rowsProduced: 100, - }, - topUsers: overrides.topUsers ?? [{ user: 'analyst', executions: 40 }], - }; -} - -describe('stageHistoricSqlAggregatedSnapshot', () => { - it('batch parses templates and writes stable table and patterns artifacts', async () => { - const stagedDir = await tempDir(); - const reader: HistoricSqlReader = { - async probe() { - return { warnings: ['pg_stat_statements.max is low; aggregation still proceeds'] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'orders-by-status', - canonicalSql: 'select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.created_at >= $1 group by o.status', - }); - yield aggregate({ - templateId: 'service-account-only', - canonicalSql: 'select * from public.orders where id = $1', - stats: { - executions: 20, - distinctUsers: 1, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 5, - p95RuntimeMs: 10, - errorRate: 0, - rowsProduced: 1, - }, - topUsers: [{ user: 'svc_loader', executions: 20 }], - }); - yield aggregate({ - templateId: 'bad-parse', - canonicalSql: 'select broken from', - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'orders-by-status', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: ['status'], - where: ['created_at'], - join: ['customer_id'], - groupBy: ['status'], - }, - }, - ], - ['bad-parse', { tablesTouched: [], columnsByClause: {}, error: 'parse failed' }], - ])), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { - dialect: 'postgres', - filters: { - serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, - }, - }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledTimes(1); - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( - [ - { - id: 'orders-by-status', - sql: 'select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.created_at >= $1 group by o.status', - }, - { id: 'bad-parse', sql: 'select broken from' }, - ], - 'postgres', - ); - - expect(await readdir(join(stagedDir, 'tables'))).toEqual(['public.customers.json', 'public.orders.json']); - - const manifest = await readJson>(stagedDir, 'manifest.json'); - expect(manifest).toMatchObject({ - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - snapshotRowCount: 3, - touchedTableCount: 2, - parseFailures: 1, - warnings: ['parse_failed:bad-parse'], - probeWarnings: ['pg_stat_statements.max is low; aggregation still proceeds'], - }); - - const orders = await readJson>(stagedDir, 'tables/public.orders.json'); - expect(orders).toMatchObject({ - table: 'public.orders', - stats: { - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - errorRateBucket: 'none', - p95RuntimeBucket: '<100ms', - recencyBucket: 'current', - }, - columnsByClause: { - select: [['status', 'high']], - where: [['created_at', 'high']], - join: [['customer_id', 'high']], - groupBy: [['status', 'high']], - }, - observedJoins: [{ withTable: 'public.customers', on: ['customer_id'], freq: 'high' }], - topTemplates: [ - { - id: 'orders-by-status', - topUsers: [{ user: 'analyst' }], - }, - ], - }); - expect(orders.topTemplates[0].canonicalSql).toContain('group by o.status'); - - const patterns = await readJson>(stagedDir, 'patterns-input.json'); - expect(patterns.templates).toEqual([ - { - id: 'orders-by-status', - canonicalSql: expect.stringContaining('public.orders'), - tablesTouched: ['public.customers', 'public.orders'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ]); - }); -}); -``` - -- [ ] **Step 2: Run the stage test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: FAIL because `stage-unified.js` does not exist. - -- [ ] **Step 3: Add the unified stager** - -Create `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` with these exported shapes and helpers: - -```typescript -import { mkdir, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; -import { - bucketDistinctUsers, - bucketErrorRate, - bucketExecutions, - bucketFrequency, - bucketP95Runtime, - bucketRecency, -} from './buckets.js'; -import { - HISTORIC_SQL_SOURCE_KEY, - aggregatedTemplateSchema, - historicSqlUnifiedPullConfigSchema, - type AggregatedTemplate, - type HistoricSqlReader, - type HistoricSqlUnifiedPullConfig, - type StagedPatternsInput, - type StagedTableInput, -} from './types.js'; - -interface StageHistoricSqlAggregatedSnapshotInput { - stagedDir: string; - connectionId: string; - queryClient: unknown; - reader: HistoricSqlReader; - sqlAnalysis: SqlAnalysisPort; - pullConfig: unknown; - now?: Date; -} - -interface ParsedTemplate { - template: AggregatedTemplate; - tablesTouched: string[]; - columnsByClause: Record; -} - -interface TableAccumulator { - table: string; - executions: number; - distinctUsers: number; - errorRateNumerator: number; - p95RuntimeMs: number | null; - lastSeen: string; - columnsByClause: Map>; - observedJoins: Map>; - topTemplates: AggregatedTemplate[]; -} - -const TRIVIAL_SQL_RE = /^\s*SELECT\s+(1|NOW\(\)|CURRENT_TIMESTAMP|VERSION\(\))\s*;?\s*$/i; -const NOISE_PREFIX_RE = /^\s*(SHOW|DESCRIBE|DESC|EXPLAIN|USE|SET)\b/i; -const SYSTEM_TABLE_RE = /\b(INFORMATION_SCHEMA|SNOWFLAKE\.ACCOUNT_USAGE|pg_|system\.)/i; -const ORCHESTRATOR_RE = /\b(dbt|looker|metabase)\b/i; - -function writeJson(root: string, relPath: string, value: unknown): Promise { - const target = join(root, relPath); - return mkdir(dirname(target), { recursive: true }).then(() => - writeFile(target, `${JSON.stringify(value, null, 2)}\n`, 'utf-8'), - ); -} - -function compilePatterns(patterns: string[]): RegExp[] { - return patterns.map((pattern) => new RegExp(pattern)); -} - -function matchesAny(value: string | null, patterns: RegExp[]): boolean { - return !!value && patterns.some((pattern) => pattern.test(value)); -} - -function shouldDropBySql(sql: string, config: HistoricSqlUnifiedPullConfig): boolean { - if (NOISE_PREFIX_RE.test(sql) || SYSTEM_TABLE_RE.test(sql)) return true; - if (config.filters.dropTrivialProbes !== false && TRIVIAL_SQL_RE.test(sql)) return true; - return false; -} - -function shouldDropByUsers(template: AggregatedTemplate, config: HistoricSqlUnifiedPullConfig): boolean { - const service = config.filters.serviceAccounts; - if (!service || service.mode === 'mark-only' || service.patterns.length === 0) return false; - const patterns = compilePatterns(service.patterns); - const matchingExecutions = template.topUsers - .filter((entry) => matchesAny(entry.user, patterns)) - .reduce((sum, entry) => sum + entry.executions, 0); - const allExecutions = template.topUsers.reduce((sum, entry) => sum + entry.executions, 0); - const serviceOnly = allExecutions > 0 && matchingExecutions >= allExecutions; - return service.mode === 'exclude' ? serviceOnly : !serviceOnly; -} - -function shouldDropByFailure(template: AggregatedTemplate, config: HistoricSqlUnifiedPullConfig): boolean { - const failed = config.filters.dropFailedBelow; - return !!failed && template.stats.errorRate > failed.errorRate && template.stats.executions < failed.executions; -} - -function shouldDropTemplate(template: AggregatedTemplate, config: HistoricSqlUnifiedPullConfig): boolean { - if (shouldDropBySql(template.canonicalSql, config)) return true; - if (shouldDropByUsers(template, config)) return true; - if (shouldDropByFailure(template, config)) return true; - return false; -} - -function recordColumn(acc: TableAccumulator, clause: string, column: string, executions: number): void { - const byColumn = acc.columnsByClause.get(clause) ?? new Map(); - byColumn.set(column, (byColumn.get(column) ?? 0) + executions); - acc.columnsByClause.set(clause, byColumn); -} - -function recordJoin(acc: TableAccumulator, otherTable: string, columns: string[], executions: number): void { - const byColumns = acc.observedJoins.get(otherTable) ?? new Map(); - const key = [...new Set(columns)].sort().join(','); - if (key.length > 0) { - byColumns.set(key, (byColumns.get(key) ?? 0) + executions); - acc.observedJoins.set(otherTable, byColumns); - } -} - -function accumulatorFor(table: string): TableAccumulator { - return { - table, - executions: 0, - distinctUsers: 0, - errorRateNumerator: 0, - p95RuntimeMs: null, - lastSeen: '1970-01-01T00:00:00.000Z', - columnsByClause: new Map(), - observedJoins: new Map(), - topTemplates: [], - }; -} - -function addTemplate(acc: TableAccumulator, parsed: ParsedTemplate): void { - const executions = parsed.template.stats.executions; - acc.executions += executions; - acc.distinctUsers = Math.max(acc.distinctUsers, parsed.template.stats.distinctUsers); - acc.errorRateNumerator += parsed.template.stats.errorRate * executions; - acc.p95RuntimeMs = - acc.p95RuntimeMs === null - ? parsed.template.stats.p95RuntimeMs - : parsed.template.stats.p95RuntimeMs === null - ? acc.p95RuntimeMs - : Math.max(acc.p95RuntimeMs, parsed.template.stats.p95RuntimeMs); - acc.lastSeen = parsed.template.stats.lastSeen > acc.lastSeen ? parsed.template.stats.lastSeen : acc.lastSeen; - for (const [clause, columns] of Object.entries(parsed.columnsByClause)) { - for (const column of columns) { - recordColumn(acc, clause, column, executions); - } - } - const joinColumns = parsed.columnsByClause.join ?? []; - for (const otherTable of parsed.tablesTouched.filter((table) => table !== acc.table)) { - recordJoin(acc, otherTable, joinColumns, executions); - } - acc.topTemplates.push(parsed.template); -} -``` - -In the same file, add the staging function: - -```typescript -function toStagedTable(acc: TableAccumulator, now: Date): StagedTableInput { - const errorRate = acc.executions > 0 ? acc.errorRateNumerator / acc.executions : 0; - const columnsByClause = Object.fromEntries( - [...acc.columnsByClause.entries()] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([clause, counts]) => [ - clause, - [...counts.entries()] - .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])) - .map(([column, count]) => [column, bucketFrequency(count, acc.executions)]), - ]), - ); - const observedJoins = [...acc.observedJoins.entries()] - .flatMap(([withTable, byColumns]) => - [...byColumns.entries()].map(([columns, count]) => ({ - withTable, - on: columns.split(',').filter(Boolean), - freq: bucketFrequency(count, acc.executions), - })), - ) - .sort((left, right) => left.withTable.localeCompare(right.withTable) || left.on.join(',').localeCompare(right.on.join(','))); - const topTemplates = [...acc.topTemplates] - .sort((left, right) => right.stats.executions - left.stats.executions || left.templateId.localeCompare(right.templateId)) - .slice(0, 5) - .map((template) => ({ - id: template.templateId, - canonicalSql: template.canonicalSql, - topUsers: template.topUsers.slice(0, 5).map((entry) => ({ user: entry.user })), - })); - - return { - table: acc.table, - stats: { - executionsBucket: bucketExecutions(acc.executions), - distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers), - errorRateBucket: bucketErrorRate(errorRate), - p95RuntimeBucket: bucketP95Runtime(acc.p95RuntimeMs), - recencyBucket: bucketRecency(acc.lastSeen, now), - }, - columnsByClause, - observedJoins, - topTemplates, - }; -} - -function toPatternsInput(parsedTemplates: ParsedTemplate[]): StagedPatternsInput { - return { - templates: parsedTemplates - .map(({ template, tablesTouched }) => ({ - id: template.templateId, - canonicalSql: template.canonicalSql, - tablesTouched: [...tablesTouched].sort(), - executionsBucket: bucketExecutions(template.stats.executions), - distinctUsersBucket: bucketDistinctUsers(template.stats.distinctUsers), - dialect: template.dialect, - })) - .sort((left, right) => left.id.localeCompare(right.id)), - }; -} - -export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSqlAggregatedSnapshotInput): Promise { - const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig); - const now = input.now ?? new Date(); - const windowStart = new Date(now.getTime() - config.windowDays * 24 * 60 * 60 * 1000); - const probe = await input.reader.probe(input.queryClient); - const snapshot: AggregatedTemplate[] = []; - - for await (const row of input.reader.fetchAggregated(input.queryClient, { start: windowStart, end: now }, config)) { - const parsed = aggregatedTemplateSchema.parse(row); - if (!shouldDropTemplate(parsed, config)) { - snapshot.push(parsed); - } - } - - const analysis = await input.sqlAnalysis.analyzeBatch( - snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })), - config.dialect, - ); - const warnings: string[] = []; - const parsedTemplates: ParsedTemplate[] = []; - for (const template of snapshot) { - const parsed = analysis.get(template.templateId); - if (!parsed || parsed.error) { - warnings.push(`parse_failed:${template.templateId}`); - continue; - } - const tablesTouched = [...new Set(parsed.tablesTouched)].filter((table) => table.length > 0).sort(); - if (tablesTouched.length === 0) { - continue; - } - parsedTemplates.push({ - template, - tablesTouched, - columnsByClause: Object.fromEntries( - Object.entries(parsed.columnsByClause).map(([clause, columns]) => [clause, [...new Set(columns)].sort()]), - ), - }); - } - - const byTable = new Map(); - for (const parsed of parsedTemplates) { - for (const table of parsed.tablesTouched) { - const acc = byTable.get(table) ?? accumulatorFor(table); - addTemplate(acc, parsed); - byTable.set(table, acc); - } - } - - await mkdir(input.stagedDir, { recursive: true }); - for (const [table, acc] of [...byTable.entries()].sort(([left], [right]) => left.localeCompare(right))) { - await writeJson(input.stagedDir, `tables/${table}.json`, toStagedTable(acc, now)); - } - await writeJson(input.stagedDir, 'patterns-input.json', toPatternsInput(parsedTemplates)); - await writeJson(input.stagedDir, 'manifest.json', { - source: HISTORIC_SQL_SOURCE_KEY, - connectionId: input.connectionId, - dialect: config.dialect, - fetchedAt: now.toISOString(), - windowStart: windowStart.toISOString(), - windowEnd: now.toISOString(), - snapshotRowCount: snapshot.length, - touchedTableCount: byTable.size, - parseFailures: warnings.filter((warning) => warning.startsWith('parse_failed:')).length, - warnings, - probeWarnings: probe.warnings, - }); -} -``` - -- [ ] **Step 4: Run the staged-artifact test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/stage-unified.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Export the unified stager** - -In `packages/context/src/ingest/index.ts`, add: - -```typescript -export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js'; -``` - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/stage-unified.ts packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts packages/context/src/ingest/index.ts -git commit -m "feat: stage historic sql aggregate snapshots" -``` - -## Task 4: Add Aggregate Readers - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` -- Modify: `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Write failing Postgres aggregate reader tests** - -Create `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts`: - -```typescript -import { describe, expect, it, vi } from 'vitest'; -import { PostgresPgssReader } from './postgres-pgss-reader.js'; - -describe('PostgresPgssReader aggregate path', () => { - it('aggregates pg_stat_statements rows by queryid and query', async () => { - const executeQuery = vi.fn(async (sql: string, params?: unknown[]) => { - if (sql.includes('pg_stat_statements_info')) { - return { headers: ['stats_reset', 'dealloc'], rows: [['2026-05-01T00:00:00.000Z', 1]] }; - } - expect(sql).toContain('GROUP BY queryid, query'); - expect(sql).toContain('HAVING SUM(calls) >= $1'); - expect(params).toEqual([5]); - return { - headers: ['template_id', 'canonical_sql', 'executions', 'distinct_users', 'mean_ms', 'rows_produced', 'top_users'], - rows: [ - [ - '123', - 'select status from public.orders', - '42', - '3', - '11.5', - '100', - JSON.stringify([{ user: 'analyst', executions: 40 }]), - ], - ], - }; - }); - - const reader = new PostgresPgssReader(); - const rows = []; - for await (const row of reader.fetchAggregated( - { executeQuery }, - { start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') }, - { dialect: 'postgres', minExecutions: 5, windowDays: 90, concurrency: 12, filters: {}, redactionPatterns: [], staleArchiveAfterDays: 90 }, - )) { - rows.push(row); - } - - expect(rows).toEqual([ - { - templateId: '123', - canonicalSql: 'select status from public.orders', - dialect: 'postgres', - stats: { - executions: 42, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 11.5, - p95RuntimeMs: 11.5, - errorRate: 0, - rowsProduced: 100, - }, - topUsers: [{ user: 'analyst', executions: 40 }], - }, - ]); - }); -}); -``` - -- [ ] **Step 2: Add failing BigQuery and Snowflake aggregate assertions** - -In `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts`, add a test that constructs `new BigQueryHistoricSqlQueryHistoryReader({ projectId: 'demo', region: 'us' })`, calls `fetchAggregated()`, and asserts the SQL contains: - -```typescript -expect(sql).toContain('COUNT(*) AS executions'); -expect(sql).toContain('COUNT(DISTINCT user_email) AS distinct_users'); -expect(sql).toContain('GROUP BY query_hash'); -expect(sql).toContain('HAVING COUNT(*) >= 5'); -``` - -Map one returned row with headers: - -```typescript -[ - 'template_id', - 'canonical_sql', - 'executions', - 'distinct_users', - 'first_seen', - 'last_seen', - 'p50_ms', - 'p95_ms', - 'error_rate', - 'rows_produced', - 'top_users', -] -``` - -and assert `templateId`, `stats.executions`, `stats.errorRate`, and `topUsers` match the row. - -In `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts`, add the same shape but assert the SQL contains: - -```typescript -expect(sql).toContain('SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'); -expect(sql).toContain('COUNT(*) AS executions'); -expect(sql).toContain('GROUP BY query_hash'); -expect(sql).toContain('HAVING COUNT(*) >= 5'); -``` - -- [ ] **Step 3: Run aggregate reader tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts -``` - -Expected: FAIL because `fetchAggregated()` and `postgres-pgss-reader.js` do not exist. - -- [ ] **Step 4: Implement the aggregate reader methods** - -Create `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` with the same probe behavior currently implemented in `postgres-pgss-query-history-reader.ts`: `queryClient`, `execute`, `indexByHeader`, `value`, `nullableString`, `requiredString`, `requiredFiniteNumber`, `nullableInteger`, `nullableIsoTimestamp`, `firstRow`, `extensionMissingError`, and `grantsMissingError` keep their current behavior. Add this aggregate query and row mapper: - -```typescript -const AGGREGATE_SQL = ` -SELECT queryid::text AS template_id, - query AS canonical_sql, - SUM(calls)::bigint AS executions, - COUNT(DISTINCT userid) AS distinct_users, - SUM(total_exec_time) / NULLIF(SUM(calls), 0) AS mean_ms, - SUM(rows)::bigint AS rows_produced, - COALESCE( - json_agg(json_build_object('user', rolname, 'executions', calls) ORDER BY calls DESC) - FILTER (WHERE userid IS NOT NULL), - '[]'::json - )::text AS top_users -FROM pg_stat_statements -LEFT JOIN pg_roles ON pg_roles.oid = pg_stat_statements.userid -WHERE toplevel = true -GROUP BY queryid, query -HAVING SUM(calls) >= $1 -ORDER BY SUM(total_exec_time) DESC -`.trim(); -``` - -The `fetchAggregated()` method must: - -```typescript - async *fetchAggregated( - client: unknown, - window: HistoricSqlTimeWindow, - config: HistoricSqlUnifiedPullConfig, - ): AsyncIterable { - const pgClient = queryClient(client); - const statsResult = await execute(pgClient, STATS_INFO_SQL); - const { row: statsRow, headers: statsHeaders } = firstRow(statsResult, 'stats-info'); - const firstSeen = nullableIsoTimestamp(value(statsRow, statsHeaders, 'stats_reset')) ?? window.start.toISOString(); - const result = await execute(pgClient, AGGREGATE_SQL, [config.minExecutions]); - const indexes = indexByHeader(result.headers); - for (const row of result.rows) { - yield aggregatedTemplateSchema.parse({ - templateId: requiredString(value(row, indexes, 'template_id'), 'template_id'), - canonicalSql: requiredString(value(row, indexes, 'canonical_sql'), 'canonical_sql'), - dialect: 'postgres', - stats: { - executions: requiredInteger(value(row, indexes, 'executions'), 'executions'), - distinctUsers: requiredInteger(value(row, indexes, 'distinct_users'), 'distinct_users'), - firstSeen, - lastSeen: window.end.toISOString(), - p50RuntimeMs: nullableNumber(value(row, indexes, 'mean_ms')), - p95RuntimeMs: nullableNumber(value(row, indexes, 'mean_ms')), - errorRate: 0, - rowsProduced: nullableInteger(value(row, indexes, 'rows_produced')), - }, - topUsers: parseTopUsers(value(row, indexes, 'top_users')), - }); - } - } -``` - -In `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts`, add this aggregate query inside `fetchAggregated()`: - -```typescript -const sql = ` -SELECT - query_hash AS template_id, - MIN(query) AS canonical_sql, - COUNT(*) AS executions, - COUNT(DISTINCT user_email) AS distinct_users, - MIN(creation_time) AS first_seen, - MAX(creation_time) AS last_seen, - APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(50)] AS p50_ms, - APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(95)] AS p95_ms, - SAFE_DIVIDE(COUNTIF(error_result IS NOT NULL), COUNT(*)) AS error_rate, - CAST(NULL AS INT64) AS rows_produced, - TO_JSON_STRING(ARRAY_AGG(STRUCT(user_email AS user, 1 AS executions) ORDER BY creation_time DESC LIMIT 5)) AS top_users -FROM ${this.viewPath} -WHERE job_type = 'QUERY' - AND statement_type IN ('SELECT', 'MERGE') - AND creation_time >= ${timestampExpression(window.start)} - AND creation_time < ${timestampExpression(window.end)} - AND query IS NOT NULL -GROUP BY query_hash -HAVING COUNT(*) >= ${config.minExecutions} -ORDER BY executions DESC`.trim(); -``` - -Map each result row into `aggregatedTemplateSchema.parse({ templateId, canonicalSql, dialect: 'bigquery', stats: { executions, distinctUsers, firstSeen, lastSeen, p50RuntimeMs, p95RuntimeMs, errorRate, rowsProduced }, topUsers })`, where `topUsers` is parsed from the `top_users` JSON string and invalid JSON becomes `[]`. - -In `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts`, add this aggregate query inside `fetchAggregated()`: - -```typescript -const sql = ` -SELECT - query_hash AS template_id, - MIN(query_text) AS canonical_sql, - COUNT(*) AS executions, - COUNT(DISTINCT user_name) AS distinct_users, - MIN(start_time) AS first_seen, - MAX(start_time) AS last_seen, - APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms, - APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms, - DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate, - SUM(rows_produced) AS rows_produced, - ARRAY_AGG(OBJECT_CONSTRUCT('user', user_name, 'executions', 1)) WITHIN GROUP (ORDER BY start_time DESC)::string AS top_users -FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY -WHERE query_text IS NOT NULL - AND query_type IN ('SELECT', 'MERGE') - AND start_time >= ${timestampLiteral(window.start)} - AND start_time < ${timestampLiteral(window.end)} -GROUP BY query_hash -HAVING COUNT(*) >= ${config.minExecutions} -ORDER BY executions DESC`.trim(); -``` - -Map each result row into `aggregatedTemplateSchema.parse({ templateId, canonicalSql, dialect: 'snowflake', stats: { executions, distinctUsers, firstSeen, lastSeen, p50RuntimeMs, p95RuntimeMs, errorRate, rowsProduced }, topUsers })`, where `topUsers` is parsed from the `top_users` JSON string and invalid JSON becomes `[]`. Keep the existing `fetch()` methods unchanged in this plan so current adapter behavior does not move before the skill/projection cutover. - -- [ ] **Step 5: Export the new Postgres reader** - -In `packages/context/src/ingest/index.ts`, add: - -```typescript -export { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js'; -``` - -- [ ] **Step 6: Run aggregate reader tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts packages/context/src/ingest/index.ts -git commit -m "feat: add historic sql aggregate readers" -``` - -## Task 5: Add Unified Chunking - -**Files:** -- Create: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` -- Create: `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Write failing unified chunk tests** - -Create `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts`: - -```typescript -import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; -import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js'; - -async function tempDir(): Promise { - return mkdtemp(join(tmpdir(), 'historic-sql-unified-chunk-')); -} - -async function writeJson(root: string, relPath: string, value: unknown): Promise { - const target = join(root, relPath); - await mkdir(join(target, '..'), { recursive: true }); - await writeFile(target, `${JSON.stringify(value, null, 2)}\n`, 'utf-8'); -} - -async function writeUnifiedStagedDir(root: string): Promise { - await writeJson(root, 'manifest.json', { - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - fetchedAt: '2026-05-11T00:00:00.000Z', - windowStart: '2026-02-10T00:00:00.000Z', - windowEnd: '2026-05-11T00:00:00.000Z', - snapshotRowCount: 1, - touchedTableCount: 1, - parseFailures: 0, - warnings: [], - probeWarnings: [], - }); - await writeJson(root, 'tables/public.orders.json', { - table: 'public.orders', - stats: { - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - errorRateBucket: 'none', - p95RuntimeBucket: '<100ms', - recencyBucket: 'current', - }, - columnsByClause: { select: [['status', 'high']] }, - observedJoins: [], - topTemplates: [{ id: 'orders', canonicalSql: 'select * from public.orders', topUsers: [{ user: 'analyst' }] }], - }); - await writeJson(root, 'patterns-input.json', { - templates: [ - { - id: 'orders', - canonicalSql: 'select * from public.orders', - tablesTouched: ['public.orders'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ], - }); -} - -describe('chunkHistoricSqlUnifiedStagedDir', () => { - it('emits one table WorkUnit plus one patterns WorkUnit', async () => { - const stagedDir = await tempDir(); - await writeUnifiedStagedDir(stagedDir); - - const result = await chunkHistoricSqlUnifiedStagedDir(stagedDir); - - expect(result.workUnits).toEqual([ - expect.objectContaining({ - unitKey: 'historic-sql-table-public-orders', - displayLabel: 'Historic SQL usage: public.orders', - rawFiles: ['tables/public.orders.json'], - dependencyPaths: ['manifest.json'], - notes: expect.stringContaining('historic_sql_table_digest'), - }), - expect.objectContaining({ - unitKey: 'historic-sql-patterns', - displayLabel: 'Historic SQL cross-table patterns', - rawFiles: ['patterns-input.json'], - dependencyPaths: ['manifest.json'], - notes: expect.stringContaining('historic_sql_patterns'), - }), - ]); - expect(result.reconcileNotes).toEqual(['Historic-SQL touched tables=1 parseFailures=0']); - }); - - it('respects diff sets for unchanged table and patterns files', async () => { - const stagedDir = await tempDir(); - await writeUnifiedStagedDir(stagedDir); - - await expect( - chunkHistoricSqlUnifiedStagedDir(stagedDir, { - added: [], - modified: ['tables/public.orders.json'], - deleted: [], - unchanged: ['manifest.json', 'patterns-input.json'], - }), - ).resolves.toMatchObject({ - workUnits: [expect.objectContaining({ unitKey: 'historic-sql-table-public-orders' })], - }); - - await expect( - chunkHistoricSqlUnifiedStagedDir(stagedDir, { - added: [], - modified: ['patterns-input.json'], - deleted: [], - unchanged: ['manifest.json', 'tables/public.orders.json'], - }), - ).resolves.toMatchObject({ - workUnits: [expect.objectContaining({ unitKey: 'historic-sql-patterns' })], - }); - }); - - it('describes unified staged scope', async () => { - const stagedDir = await tempDir(); - await writeUnifiedStagedDir(stagedDir); - - const scope = await describeHistoricSqlUnifiedScope(stagedDir); - - expect(scope.isPathInScope('manifest.json')).toBe(true); - expect(scope.isPathInScope('patterns-input.json')).toBe(true); - expect(scope.isPathInScope('tables/public.orders.json')).toBe(true); - expect(scope.isPathInScope('templates/old/page.md')).toBe(false); - }); -}); -``` - -- [ ] **Step 2: Run the unified chunk tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/chunk-unified.test.ts -``` - -Expected: FAIL because `chunk-unified.js` does not exist. - -- [ ] **Step 3: Add the unified chunker** - -Create `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`: - -```typescript -import { createHash } from 'node:crypto'; -import { readFile, readdir } from 'node:fs/promises'; -import { join, relative } from 'node:path'; -import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../types.js'; -import { stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema } from './types.js'; - -async function walk(root: string): Promise { - const entries = await readdir(root, { withFileTypes: true, recursive: true }); - return entries - .filter((entry) => entry.isFile()) - .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/')) - .sort(); -} - -async function readJson(stagedDir: string, relPath: string): Promise { - return JSON.parse(await readFile(join(stagedDir, relPath), 'utf-8')) as T; -} - -function safeUnitKey(value: string): string { - return value.replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, ''); -} - -function touchedPath(path: string, touched: Set | null): boolean { - return !touched || touched.has(path); -} - -export async function chunkHistoricSqlUnifiedStagedDir(stagedDir: string, diffSet?: DiffSet): Promise { - const files = await walk(stagedDir); - const manifest = stagedManifestSchema.parse(await readJson(stagedDir, 'manifest.json')); - const touched = diffSet ? new Set([...diffSet.added, ...diffSet.modified]) : null; - const workUnits: WorkUnit[] = []; - - for (const path of files.filter((file) => /^tables\/.+\.json$/.test(file))) { - if (!touchedPath(path, touched)) { - continue; - } - const table = stagedTableInputSchema.parse(await readJson(stagedDir, path)); - workUnits.push({ - unitKey: `historic-sql-table-${safeUnitKey(table.table)}`, - displayLabel: `Historic SQL usage: ${table.table}`, - rawFiles: [path], - dependencyPaths: ['manifest.json'], - peerFileIndex: files.filter((file) => file !== path && file !== 'manifest.json').sort(), - notes: - 'Use historic_sql_table_digest. Read this table usage JSON and the existing semantic-layer source for the table; output only table usage evidence shaped like tableUsageOutputSchema.', - }); - } - - if (files.includes('patterns-input.json') && touchedPath('patterns-input.json', touched)) { - stagedPatternsInputSchema.parse(await readJson(stagedDir, 'patterns-input.json')); - workUnits.push({ - unitKey: 'historic-sql-patterns', - displayLabel: 'Historic SQL cross-table patterns', - rawFiles: ['patterns-input.json'], - dependencyPaths: ['manifest.json'], - peerFileIndex: files.filter((file) => file !== 'patterns-input.json' && file !== 'manifest.json').sort(), - notes: - 'Use historic_sql_patterns. Read patterns-input.json and produce cross-table pattern evidence shaped like patternsArraySchema.', - }); - } - - const deleted = diffSet?.deleted.filter((path) => path === 'patterns-input.json' || /^tables\/.+\.json$/.test(path)).sort(); - return { - workUnits, - eviction: deleted && deleted.length > 0 ? { deletedRawPaths: deleted } : undefined, - reconcileNotes: [`Historic-SQL touched tables=${manifest.touchedTableCount} parseFailures=${manifest.parseFailures}`], - contextReport: { - capped: false, - warnings: [...manifest.probeWarnings, ...manifest.warnings], - }, - }; -} - -export async function describeHistoricSqlUnifiedScope(stagedDir: string): Promise { - const manifest = stagedManifestSchema.parse(await readJson(stagedDir, 'manifest.json')); - const fingerprint = createHash('sha256') - .update(JSON.stringify({ - connectionId: manifest.connectionId, - dialect: manifest.dialect, - windowStart: manifest.windowStart, - windowEnd: manifest.windowEnd, - })) - .digest('hex'); - return { - fingerprint, - isPathInScope: (rawPath) => - rawPath === 'manifest.json' || rawPath === 'patterns-input.json' || /^tables\/.+\.json$/.test(rawPath), - }; -} -``` - -- [ ] **Step 4: Run the unified chunk tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/historic-sql/chunk-unified.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Export the unified chunker** - -In `packages/context/src/ingest/index.ts`, add: - -```typescript -export { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './adapters/historic-sql/chunk-unified.js'; -``` - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts packages/context/src/ingest/index.ts -git commit -m "feat: chunk historic sql unified staging" -``` - -## Task 6: Verify the Hot Path Slice - -**Files:** -- Modify: files changed in Tasks 1-5 - -- [ ] **Step 1: Run focused historic-SQL tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/historic-sql/types.test.ts \ - src/ingest/adapters/historic-sql/buckets.test.ts \ - src/ingest/adapters/historic-sql/stage-unified.test.ts \ - src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts \ - src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts \ - src/ingest/adapters/historic-sql/chunk-unified.test.ts \ - src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run type-check for the context package** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Confirm legacy production adapter was not switched** - -Run: - -```bash -rg -n "historic_sql_ingest|historic_sql_curator|stagePgStatStatementsTemplates" packages/context/src/ingest/adapters/historic-sql packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: Results still include `historic-sql.adapter.ts`, the old skill files, and runtime-asset tests. This is correct for this plan because the replacement skills and projection are not present yet. - -- [ ] **Step 4: Confirm new hot-path exports exist** - -Run: - -```bash -rg -n "stageHistoricSqlAggregatedSnapshot|chunkHistoricSqlUnifiedStagedDir|PostgresPgssReader|aggregatedTemplateSchema" packages/context/src/ingest/index.ts packages/context/src/ingest/adapters/historic-sql -``` - -Expected: Results include the new stager, chunker, reader, and schemas. - -- [ ] **Step 5: Commit verification fixes only when verification changed files** - -```bash -git status --short -``` - -Expected: no output. If verification forced a fix, run: - -```bash -git add packages/context/src/ingest/adapters/historic-sql packages/context/src/ingest/index.ts packages/context/src/package-exports.test.ts -git commit -m "test: verify historic sql unified hot path" -``` - -## Follow-Up Plan Boundary - -The next plan after this one should switch the production adapter only after it also creates the cold-path pieces: - -- `packages/context/skills/historic_sql_table_digest/SKILL.md` -- `packages/context/skills/historic_sql_patterns/SKILL.md` -- adapter `skillNames` change from `historic_sql_ingest` to the two new skills -- `onPullSucceeded()` projection of table usage into `_schema/{shard}.yaml` -- pattern wiki page projection and slug stability -- one-time cleanup of legacy template wiki pages and PGSS baselines -- deletion of `stage-pgss.ts`, old template staging exports, and old historic-SQL skill assets - -## Self-Review - -Spec coverage: - -- Unified aggregate reader contracts: Task 1 and Task 4. -- Trailing-window aggregate fetch shape: Task 4. -- Batch SQL parse through `SqlAnalysisPort.analyzeBatch()`: Task 3. -- Service-account, trivial query, failed-template, parse-failure, and zero-table filtering: Task 3. -- Bucketed `tables/*.json`, `patterns-input.json`, and `manifest.json`: Task 2 and Task 3. -- WorkUnits for one table file plus patterns input: Task 5. -- Hard production cutover, LLM skills, projection, wiki pages, stale handling, and legacy deletion: explicitly excluded from this plan and listed as the next plan boundary. - -Placeholder scan: - -- No unresolved placeholders are left in task steps. -- Every code-changing task includes concrete test code, implementation code, commands, and expected results. - -Type consistency: - -- `HistoricSqlUnifiedPullConfig`, `AggregatedTemplate`, `StagedTableInput`, `StagedPatternsInput`, and `StagedManifest` are defined in Task 1 and reused consistently by Tasks 3-5. -- `PostgresPgssReader`, `fetchAggregated()`, `stageHistoricSqlAggregatedSnapshot()`, and `chunkHistoricSqlUnifiedStagedDir()` names match exports and test imports. - -Plan complete and saved to `docs/superpowers/plans/2026-05-11-historic-sql-unified-hot-path.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md b/docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md deleted file mode 100644 index a73f163f..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md +++ /dev/null @@ -1,1109 +0,0 @@ -# Managed Agent and MCP Semantic Runtime Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make hidden agent semantic queries and MCP semantic compute use the -KTX-managed core Python runtime instead of relying on a user-provided -`python -m ktx_daemon`. - -**Architecture:** Reuse the existing managed runtime command helper so every -CLI semantic compute surface resolves the same bundled `ktx-daemon` executable. -Keep explicit HTTP daemon URLs working for `ktx serve --semantic-compute-url`, -and add runtime install policy flags where commands can lazily install the core -runtime. - -**Tech Stack:** TypeScript, Commander, Vitest, KTX CLI managed Python runtime, -`@ktx/context/daemon`. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plans are based on that spec and are already implemented in this -worktree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` - -Implementation evidence found before writing this plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and - `packages/cli/assets/python/manifest.json`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, and - `packages/cli/src/commands/runtime-commands.ts`. -- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime - install policy flags. -- `packages/cli/src/managed-python-daemon.ts`, daemon state files, and - `ktx runtime start` / `ktx runtime stop`. -- `packages/cli/src/managed-local-embeddings.ts` and setup embedding wiring. -- `scripts/build-public-npm-package.mjs`, `release-policy.json`, and release - smoke coverage for `@kaelio/ktx`. -- `scripts/package-artifacts.mjs` release smoke coverage for lazy core runtime - install, `ktx sl query`, runtime status, doctor, daemon start, daemon reuse, - and daemon stop. -- `scripts/local-embeddings-runtime-smoke.mjs` opt-in release smoke coverage - for `local-embeddings`. - -The next remaining semantic compute gap is that these CLI paths still create a -raw Python semantic-layer compute port: - -- `packages/cli/src/agent-runtime.ts` -- `packages/cli/src/serve.ts` - -Those paths can call `semantic-query`, `semantic-validate`, and -`semantic-generate-sources` through `@ktx/context/daemon`, so they must resolve -the managed runtime just like `ktx sl query`. - -This plan intentionally does not change live-database introspection or Looker -table-identifier parsing. Those use daemon HTTP endpoints through local ingest -adapters and fit a separate managed-daemon adapter plan. - -## File structure - -- Modify `packages/cli/src/managed-python-command.ts`: export a shared - `runtimeInstallPolicyFromFlags()` helper so CLI commands do not duplicate - `--yes` / `--no-input` behavior. -- Modify `packages/cli/src/managed-python-command.test.ts`: cover the shared - policy helper. -- Modify `packages/cli/src/commands/sl-commands.ts`: replace its private - runtime policy helper with the shared helper. -- Modify `packages/cli/src/agent-runtime.ts`: create managed semantic compute - when agent SL query needs Python and no dependency override is injected. -- Modify `packages/cli/src/agent-runtime.test.ts`: cover the managed agent - runtime path. -- Modify `packages/cli/src/agent.ts`: pass CLI version, install policy, and - CLI IO into default agent runtime creation for `sl-query`. -- Modify `packages/cli/src/agent.test.ts`: cover runtime options passed through - agent SL query execution. -- Modify `packages/cli/src/commands/agent-commands.ts`: add `--yes` and - `--no-input` to hidden `ktx agent sl query`. -- Modify `packages/cli/src/serve.ts`: create managed semantic compute for - `ktx serve --semantic-compute` when no explicit HTTP URL is provided. -- Modify `packages/cli/src/serve.test.ts`: cover the managed MCP semantic - compute path. -- Modify `packages/cli/src/commands/serve-commands.ts`: add `--yes` and - `--no-input` to `ktx serve`. -- Modify `packages/cli/src/index.test.ts`: update CLI argument routing for the - new managed runtime policy fields. - -### Task 1: Share managed runtime install policy parsing - -**Files:** - -- Modify: `packages/cli/src/managed-python-command.test.ts` -- Modify: `packages/cli/src/managed-python-command.ts` -- Modify: `packages/cli/src/commands/sl-commands.ts` -- Test: `packages/cli/src/managed-python-command.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing policy helper tests** - -In `packages/cli/src/managed-python-command.test.ts`, update the import from -`./managed-python-command.js` to include `runtimeInstallPolicyFromFlags`: - -```typescript -import { - createManagedPythonSemanticLayerComputePort, - managedRuntimeInstallCommand, - runtimeInstallPolicyFromFlags, -} from './managed-python-command.js'; -``` - -Add this block after the existing `describe('managedRuntimeInstallCommand', ...)` -block: - -```typescript -describe('runtimeInstallPolicyFromFlags', () => { - it('maps command flags to managed runtime install policies', () => { - expect(runtimeInstallPolicyFromFlags({})).toBe('prompt'); - expect(runtimeInstallPolicyFromFlags({ yes: false })).toBe('prompt'); - expect(runtimeInstallPolicyFromFlags({ yes: true })).toBe('auto'); - expect(runtimeInstallPolicyFromFlags({ input: false })).toBe('never'); - }); - - it('rejects conflicting runtime install flags', () => { - expect(() => runtimeInstallPolicyFromFlags({ yes: true, input: false })).toThrow( - 'Choose only one runtime install mode: --yes or --no-input', - ); - }); -}); -``` - -- [ ] **Step 2: Run the failing helper tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts -``` - -Expected: FAIL with an import error for `runtimeInstallPolicyFromFlags`. - -- [ ] **Step 3: Export the shared policy helper** - -In `packages/cli/src/managed-python-command.ts`, add this function immediately -after the `KtxManagedPythonInstallPolicy` type: - -```typescript -export function runtimeInstallPolicyFromFlags(options: { - yes?: boolean; - input?: boolean; -}): KtxManagedPythonInstallPolicy { - if (options.yes === true && options.input === false) { - throw new Error('Choose only one runtime install mode: --yes or --no-input'); - } - if (options.yes === true) { - return 'auto'; - } - return options.input === false ? 'never' : 'prompt'; -} -``` - -- [ ] **Step 4: Replace the private SL policy helper** - -In `packages/cli/src/commands/sl-commands.ts`, replace this import: - -```typescript -import type { KtxManagedPythonInstallPolicy } from '../managed-python-command.js'; -``` - -with this import: - -```typescript -import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; -``` - -Delete this private function from `packages/cli/src/commands/sl-commands.ts`: - -```typescript -function runtimeInstallPolicy(options: { yes?: boolean; input?: boolean }): KtxManagedPythonInstallPolicy { - if (options.yes === true && options.input === false) { - throw new Error('Choose only one runtime install mode: --yes or --no-input'); - } - if (options.yes === true) { - return 'auto'; - } - return options.input === false ? 'never' : 'prompt'; -} -``` - -In the `sl.command('query')` action, replace: - -```typescript -runtimeInstallPolicy: runtimeInstallPolicy(options), -``` - -with: - -```typescript -runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), -``` - -- [ ] **Step 5: Run focused helper and routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the shared helper** - -```bash -git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/commands/sl-commands.ts -git commit -m "refactor: share managed runtime install policy parsing" -``` - -### Task 2: Use managed semantic compute for hidden agent SL query - -**Files:** - -- Modify: `packages/cli/src/agent-runtime.test.ts` -- Modify: `packages/cli/src/agent-runtime.ts` -- Modify: `packages/cli/src/agent.test.ts` -- Modify: `packages/cli/src/agent.ts` -- Modify: `packages/cli/src/commands/agent-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Test: `packages/cli/src/agent-runtime.test.ts` -- Test: `packages/cli/src/agent.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Add failing agent runtime tests** - -In `packages/cli/src/agent-runtime.test.ts`, add this test after -`constructs local context ports with semantic compute and query executor`: - -```typescript - it('creates managed semantic compute when no test override is injected', async () => { - const project = { - projectDir: tempDir, - configPath: join(tempDir, 'ktx.yaml'), - config: { project: 'revenue', connections: {} }, - coreConfig: {}, - git: {}, - fileStore: {}, - } as never; - const ports = { semanticLayer: {} } as never; - const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; - const loadProject = vi.fn(async () => project); - const createContextTools = vi.fn(() => ports); - const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); - const { io } = makeIo(); - - await expect( - createKtxAgentRuntime( - { - projectDir: tempDir, - enableSemanticCompute: true, - enableQueryExecution: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - io, - }, - { - loadProject, - createContextTools, - createManagedSemanticLayerCompute, - }, - ), - ).resolves.toMatchObject({ project, ports, semanticLayerCompute }); - - expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io, - }); - expect(createContextTools).toHaveBeenCalledWith(project, { - semanticLayerCompute, - }); - }); -``` - -- [ ] **Step 2: Add failing agent command/runtime tests** - -In `packages/cli/src/agent.test.ts`, update the existing -`executes SL queries from a JSON query file` test so the `sl-query` args include -the managed runtime fields: - -```typescript - { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile, - execute: true, - maxRows: 100, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'never', - }, -``` - -Add this test immediately after `executes SL queries from a JSON query file`: - -```typescript - it('passes managed runtime options into default SL query runtime creation', async () => { - const queryFile = join(tempDir, 'sl-query.json'); - const io = makeIo(); - const createRuntime = vi.fn(async () => runtime()); - await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8'); - - await expect( - runKtxAgent( - { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile, - execute: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - }, - io.io, - { createRuntime }, - ), - ).resolves.toBe(0); - - expect(createRuntime).toHaveBeenCalledWith({ - projectDir: tempDir, - enableSemanticCompute: true, - enableQueryExecution: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - io: io.io, - }); - }); -``` - -- [ ] **Step 3: Add failing CLI routing tests** - -In `packages/cli/src/index.test.ts`, update the existing -`dispatches full hidden agent commands without exposing agent in root help` -case for `agent sl query` so its expected args include: - -```typescript - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'prompt', -``` - -Add this test after that existing full hidden agent command test: - -```typescript - it('routes hidden agent SL query managed runtime policies', async () => { - const autoIo = makeIo(); - const neverIo = makeIo(); - const conflictIo = makeIo(); - const agent = vi.fn(async () => 0); - - await expect( - runKtxCli( - [ - '--project-dir', - tempDir, - 'agent', - 'sl', - 'query', - '--json', - '--connection-id', - 'warehouse', - '--query-file', - '/tmp/query.json', - '--yes', - ], - autoIo.io, - { agent }, - ), - ).resolves.toBe(0); - - await expect( - runKtxCli( - [ - '--project-dir', - tempDir, - 'agent', - 'sl', - 'query', - '--json', - '--connection-id', - 'warehouse', - '--query-file', - '/tmp/query.json', - '--no-input', - ], - neverIo.io, - { agent }, - ), - ).resolves.toBe(0); - - await expect( - runKtxCli( - [ - '--project-dir', - tempDir, - 'agent', - 'sl', - 'query', - '--json', - '--connection-id', - 'warehouse', - '--query-file', - '/tmp/query.json', - '--yes', - '--no-input', - ], - conflictIo.io, - { agent }, - ), - ).resolves.toBe(1); - - expect(agent).toHaveBeenNthCalledWith( - 1, - { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile: '/tmp/query.json', - execute: false, - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'auto', - }, - autoIo.io, - ); - expect(agent).toHaveBeenNthCalledWith( - 2, - { - command: 'sl-query', - projectDir: tempDir, - json: true, - connectionId: 'warehouse', - queryFile: '/tmp/query.json', - execute: false, - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'never', - }, - neverIo.io, - ); - expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); - }); -``` - -- [ ] **Step 4: Run the failing agent tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/agent-runtime.test.ts src/agent.test.ts src/index.test.ts -``` - -Expected: FAIL with TypeScript or runtime errors for -`createManagedSemanticLayerCompute`, missing `cliVersion`, missing -`runtimeInstallPolicy`, or unsupported hidden agent `--yes` / `--no-input`. - -- [ ] **Step 5: Implement managed agent runtime creation** - -In `packages/cli/src/agent-runtime.ts`, replace the direct -`@ktx/context/daemon` import: - -```typescript -import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; -``` - -with: - -```typescript -import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; -import { - createManagedPythonSemanticLayerComputePort, - type KtxManagedPythonInstallPolicy, -} from './managed-python-command.js'; -``` - -Update `KtxAgentRuntimeOptions` to: - -```typescript -export interface KtxAgentRuntimeOptions { - projectDir: string; - enableSemanticCompute: boolean; - enableQueryExecution: boolean; - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; - io?: KtxCliIo; -} -``` - -Update `KtxAgentRuntimeDeps` to: - -```typescript -export interface KtxAgentRuntimeDeps { - loadProject?: typeof loadKtxProject; - createContextTools?: typeof createLocalProjectMcpContextPorts; - createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; - createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; - createQueryExecutor?: () => KtxSqlQueryExecutorPort; -} -``` - -Add this helper before `createKtxAgentRuntime`: - -```typescript -async function createAgentSemanticLayerCompute( - options: KtxAgentRuntimeOptions, - deps: KtxAgentRuntimeDeps, -): Promise { - if (!options.enableSemanticCompute) { - return undefined; - } - if (deps.createSemanticLayerCompute) { - return deps.createSemanticLayerCompute(); - } - if (!options.cliVersion || !options.runtimeInstallPolicy || !options.io) { - throw new Error('Managed Python semantic compute requires cliVersion, runtimeInstallPolicy, and io.'); - } - const createManagedSemanticLayerCompute = - deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; - return createManagedSemanticLayerCompute({ - cliVersion: options.cliVersion, - installPolicy: options.runtimeInstallPolicy, - io: options.io, - }); -} -``` - -In `createKtxAgentRuntime`, replace: - -```typescript - const semanticLayerCompute = options.enableSemanticCompute - ? (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)() - : undefined; -``` - -with: - -```typescript - const semanticLayerCompute = await createAgentSemanticLayerCompute(options, deps); -``` - -- [ ] **Step 6: Pass runtime options through agent execution** - -In `packages/cli/src/agent.ts`, add this import: - -```typescript -import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -``` - -Update the `sl-query` variant in `KtxAgentArgs` to: - -```typescript - | { - command: 'sl-query'; - projectDir: string; - json: true; - connectionId: string; - queryFile: string; - execute: boolean; - maxRows?: number; - cliVersion: string; - runtimeInstallPolicy: KtxManagedPythonInstallPolicy; - } -``` - -Update `KtxAgentDeps.createRuntime` to use the shared runtime options type: - -```typescript - createRuntime?: (options: { - projectDir: string; - enableSemanticCompute: boolean; - enableQueryExecution: boolean; - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; - io?: KtxCliIo; - }) => Promise; -``` - -Change `runtimeFor` from: - -```typescript -async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps): Promise { - const needsSemanticCompute = args.command === 'sl-query'; - const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute); - return deps.createRuntime - ? deps.createRuntime({ - projectDir: args.projectDir, - enableSemanticCompute: needsSemanticCompute, - enableQueryExecution: needsQueryExecution, - }) - : createKtxAgentRuntime( - { - projectDir: args.projectDir, - enableSemanticCompute: needsSemanticCompute, - enableQueryExecution: needsQueryExecution, - }, - deps, - ); -} -``` - -to: - -```typescript -async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps, io: KtxCliIo): Promise { - const needsSemanticCompute = args.command === 'sl-query'; - const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute); - const runtimeOptions = { - projectDir: args.projectDir, - enableSemanticCompute: needsSemanticCompute, - enableQueryExecution: needsQueryExecution, - ...(args.command === 'sl-query' - ? { - cliVersion: args.cliVersion, - runtimeInstallPolicy: args.runtimeInstallPolicy, - io, - } - : {}), - }; - return deps.createRuntime ? deps.createRuntime(runtimeOptions) : createKtxAgentRuntime(runtimeOptions, deps); -} -``` - -In `runKtxAgent`, replace: - -```typescript - const runtime = await runtimeFor(args, deps); -``` - -with: - -```typescript - const runtime = await runtimeFor(args, deps, io); -``` - -- [ ] **Step 7: Add hidden agent runtime policy flags** - -In `packages/cli/src/commands/agent-commands.ts`, add this import: - -```typescript -import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; -``` - -In the `agent sl query` command chain, add these options after -`.option('--execute', ...)`: - -```typescript - .option('--yes', 'Install the managed Python runtime without prompting when required', false) - .option('--no-input', 'Disable interactive managed runtime installation') -``` - -Update the action options type from: - -```typescript - options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number }, -``` - -to: - -```typescript - options: { - connectionId: string; - queryFile: string; - execute: boolean; - maxRows?: number; - yes?: boolean; - input?: boolean; - }, -``` - -Add these fields to the `runAgent` argument object: - -```typescript - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), -``` - -- [ ] **Step 8: Run focused agent tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/agent-runtime.test.ts src/agent.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Commit the agent integration** - -```bash -git add packages/cli/src/agent-runtime.ts packages/cli/src/agent-runtime.test.ts packages/cli/src/agent.ts packages/cli/src/agent.test.ts packages/cli/src/commands/agent-commands.ts packages/cli/src/index.test.ts -git commit -m "feat: use managed runtime for agent semantic queries" -``` - -### Task 3: Use managed semantic compute for MCP serve - -**Files:** - -- Modify: `packages/cli/src/serve.test.ts` -- Modify: `packages/cli/src/serve.ts` -- Modify: `packages/cli/src/commands/serve-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Test: `packages/cli/src/serve.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Add a failing serve managed runtime test** - -In `packages/cli/src/serve.test.ts`, add this helper after the imports: - -```typescript -function makeManagedRuntimeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { write: (chunk: string) => (stdout += chunk) }, - stderr: { write: (chunk: string) => (stderr += chunk) }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} -``` - -Add this test before `uses the HTTP semantic compute port when a daemon URL is -provided`: - -```typescript - it('uses managed semantic compute when MCP semantic compute has no explicit HTTP URL', async () => { - const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; - const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; - const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute); - const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } })); - const managedRuntimeIo = makeManagedRuntimeIo(); - - await expect( - runKtxServeStdio( - { - mcp: 'stdio', - projectDir: '/tmp/ktx-project', - userId: 'agent', - semanticCompute: true, - semanticComputeUrl: undefined, - databaseIntrospectionUrl: undefined, - executeQueries: false, - memoryCapture: false, - memoryModel: undefined, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - }, - { - loadProject: async () => project, - createContextTools, - createManagedSemanticLayerCompute, - managedRuntimeIo: managedRuntimeIo.io, - createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never), - createTransport: vi.fn(() => ({}) as never), - stderr: { write: vi.fn() }, - }, - ), - ).resolves.toBe(0); - - expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: managedRuntimeIo.io, - }); - expect(createContextTools).toHaveBeenCalledWith( - project, - expect.objectContaining({ - semanticLayerCompute, - }), - ); - }); -``` - -- [ ] **Step 2: Add failing serve routing tests** - -In `packages/cli/src/index.test.ts`, update both existing `serveStdio` -expectations so the expected args include: - -```typescript - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'prompt', -``` - -Add this test after `dispatches serve public command options through Commander`: - -```typescript - it('routes serve managed runtime install policies', async () => { - const autoIo = makeIo(); - const neverIo = makeIo(); - const conflictIo = makeIo(); - const serveStdio = vi.fn(async () => 0); - - await expect( - runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes'], autoIo.io, { - serveStdio, - }), - ).resolves.toBe(0); - await expect( - runKtxCli(['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--no-input'], neverIo.io, { - serveStdio, - }), - ).resolves.toBe(0); - await expect( - runKtxCli( - ['serve', '--mcp', 'stdio', '--project-dir', tempDir, '--semantic-compute', '--yes', '--no-input'], - conflictIo.io, - { serveStdio }, - ), - ).resolves.toBe(1); - - expect(serveStdio).toHaveBeenNthCalledWith(1, { - mcp: 'stdio', - projectDir: tempDir, - userId: 'local', - semanticCompute: true, - semanticComputeUrl: undefined, - databaseIntrospectionUrl: undefined, - executeQueries: false, - memoryCapture: false, - memoryModel: undefined, - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'auto', - }); - expect(serveStdio).toHaveBeenNthCalledWith(2, { - mcp: 'stdio', - projectDir: tempDir, - userId: 'local', - semanticCompute: true, - semanticComputeUrl: undefined, - databaseIntrospectionUrl: undefined, - executeQueries: false, - memoryCapture: false, - memoryModel: undefined, - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'never', - }); - expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); - }); -``` - -- [ ] **Step 3: Run the failing serve tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/serve.test.ts src/index.test.ts -``` - -Expected: FAIL with missing `createManagedSemanticLayerCompute` support and -missing `cliVersion` / `runtimeInstallPolicy` fields in command routing. - -- [ ] **Step 4: Implement managed serve semantic compute** - -In `packages/cli/src/serve.ts`, add this import: - -```typescript -import type { KtxCliIo } from './cli-runtime.js'; -import { - createManagedPythonSemanticLayerComputePort, - type KtxManagedPythonInstallPolicy, -} from './managed-python-command.js'; -``` - -Update `KtxServeArgs` to: - -```typescript -export interface KtxServeArgs { - mcp: 'stdio'; - projectDir: string; - userId: string; - semanticCompute: boolean; - semanticComputeUrl?: string; - databaseIntrospectionUrl?: string; - executeQueries: boolean; - memoryCapture: boolean; - memoryModel?: string; - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; -} -``` - -Update `KtxServeDeps` to include: - -```typescript - createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort; - managedRuntimeIo?: KtxCliIo; -``` - -Add these helpers before `runKtxServeStdio`: - -```typescript -function requiredManagedRuntimeCliVersion(args: KtxServeArgs): string { - if (!args.cliVersion) { - throw new Error('Managed Python semantic compute requires a CLI version.'); - } - return args.cliVersion; -} - -async function createServeSemanticLayerCompute( - args: KtxServeArgs, - deps: KtxServeDeps, -): Promise { - if (!args.semanticCompute) { - return undefined; - } - if (args.semanticComputeUrl) { - return (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))( - args.semanticComputeUrl, - ); - } - if (deps.createSemanticLayerCompute) { - return deps.createSemanticLayerCompute(); - } - const createManagedSemanticLayerCompute = - deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort; - return createManagedSemanticLayerCompute({ - cliVersion: requiredManagedRuntimeCliVersion(args), - installPolicy: args.runtimeInstallPolicy ?? 'prompt', - io: deps.managedRuntimeIo ?? process, - }); -} -``` - -In `runKtxServeStdio`, replace: - -```typescript - const semanticLayerCompute = args.semanticCompute - ? args.semanticComputeUrl - ? (deps.createHttpSemanticLayerCompute ?? ((baseUrl) => createHttpSemanticLayerComputePort({ baseUrl })))( - args.semanticComputeUrl, - ) - : (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)() - : undefined; -``` - -with: - -```typescript - const semanticLayerCompute = await createServeSemanticLayerCompute(args, deps); -``` - -Remove `createPythonSemanticLayerComputePort` from the -`@ktx/context/daemon` import list. - -- [ ] **Step 5: Add serve runtime policy flags** - -In `packages/cli/src/commands/serve-commands.ts`, add this import: - -```typescript -import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; -``` - -Add these command options after `.option('--semantic-compute-url ', ...)`: - -```typescript - .option('--yes', 'Install the managed Python runtime without prompting when required', false) - .option('--no-input', 'Disable interactive managed runtime installation') -``` - -Add these fields to the `KtxServeArgs` object: - -```typescript - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), -``` - -- [ ] **Step 6: Run focused serve tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/serve.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit the serve integration** - -```bash -git add packages/cli/src/serve.ts packages/cli/src/serve.test.ts packages/cli/src/commands/serve-commands.ts packages/cli/src/index.test.ts -git commit -m "feat: use managed runtime for MCP semantic compute" -``` - -### Task 4: Verify managed semantic runtime integration - -**Files:** - -- Verify: `packages/cli/src/managed-python-command.ts` -- Verify: `packages/cli/src/agent-runtime.ts` -- Verify: `packages/cli/src/agent.ts` -- Verify: `packages/cli/src/commands/agent-commands.ts` -- Verify: `packages/cli/src/serve.ts` -- Verify: `packages/cli/src/commands/serve-commands.ts` -- Verify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Run all focused CLI tests touched by this plan** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/agent-runtime.test.ts src/agent.test.ts src/serve.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run package build** - -Run: - -```bash -pnpm --filter @ktx/cli run build -``` - -Expected: PASS. - -- [ ] **Step 5: Commit any verification fixes** - -If verification required code edits, run: - -```bash -git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/agent-runtime.ts packages/cli/src/agent-runtime.test.ts packages/cli/src/agent.ts packages/cli/src/agent.test.ts packages/cli/src/commands/agent-commands.ts packages/cli/src/serve.ts packages/cli/src/serve.test.ts packages/cli/src/commands/serve-commands.ts packages/cli/src/index.test.ts -git commit -m "fix: verify managed semantic runtime surfaces" -``` - -If no files changed after Step 1 through Step 4, do not create an empty commit. - -## Acceptance criteria - -- `ktx agent sl query` has `--yes` and `--no-input` managed runtime policy - flags. -- `ktx agent sl query --yes` passes `runtimeInstallPolicy: 'auto'` and the - current CLI package version into default runtime creation. -- `ktx agent sl query --no-input` passes `runtimeInstallPolicy: 'never'`. -- `ktx agent sl query --yes --no-input` exits with - `Choose only one runtime install mode: --yes or --no-input`. -- Default agent SL query runtime creation uses - `createManagedPythonSemanticLayerComputePort()` and therefore invokes the - bundled managed `ktx-daemon` executable. -- `ktx serve --mcp stdio --semantic-compute` has `--yes` and `--no-input` - managed runtime policy flags. -- `ktx serve --mcp stdio --semantic-compute --yes` passes - `runtimeInstallPolicy: 'auto'` and the current CLI package version into - serve runtime creation. -- `ktx serve --mcp stdio --semantic-compute --no-input` passes - `runtimeInstallPolicy: 'never'`. -- `ktx serve --mcp stdio --semantic-compute-url ` continues to use the - explicit HTTP semantic compute port and does not install or start a managed - runtime. -- Focused CLI tests, full CLI tests, CLI type-check, and CLI build pass. - -## Self-review - -- Spec coverage: this plan extends managed Python one-shot semantic compute to - hidden agent SL query and MCP `serve --semantic-compute`, covering additional - semantic query, validation, and source-generation paths that use - `@ktx/context/daemon`. -- Remaining intentional gap: local ingest daemon-backed database introspection - and Looker SQL table-identifier parsing still need a managed daemon adapter - plan because they use HTTP daemon endpoints rather than the one-shot semantic - compute port. -- Placeholder scan: all steps contain concrete edits, commands, and expected - results. -- Type consistency: runtime policy values stay `prompt`, `auto`, and `never`; - runtime feature values stay `core` and `local-embeddings`; package version - fields are named `cliVersion`. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md deleted file mode 100644 index 907c3ba9..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md +++ /dev/null @@ -1,856 +0,0 @@ -# Managed Local Embeddings Release Smoke Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add an opt-in release smoke that proves the public `@kaelio/ktx` -package can install `local-embeddings`, start the managed daemon, compute a real -local embedding, and persist the managed embedding marker through setup. - -**Architecture:** Keep the default `artifacts:verify` path lightweight. Add a -separate Node smoke script with an explicit opt-in gate, source-level tests, and -a package script that a release job can run only when large Python and model -downloads are acceptable. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv, KTX managed Python -runtime assets, FastAPI embedding endpoint, sentence-transformers. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plans are based on that spec and are already implemented in this -worktree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` - -Implementation evidence found before writing this plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and matching tests. -- `packages/cli/src/managed-python-runtime.ts`, `runtime.ts`, and - `commands/runtime-commands.ts`. -- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime - install policy flags. -- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` / - `ktx runtime stop`. -- `packages/cli/src/managed-local-embeddings.ts`, - `packages/context/src/llm/local-config.ts`, and setup embedding wiring. -- `scripts/build-public-npm-package.mjs`, `release-policy.json` listing - `@kaelio/ktx`, and public-package smoke command construction. -- `scripts/package-artifacts.mjs` installed CLI smoke that isolates - `KTX_RUNTIME_ROOT`, lazily installs the core runtime, runs `ktx sl query`, - checks runtime status and doctor output, and starts, reuses, and stops the - core daemon. - -The remaining spec gap is the release-check item that permits local embeddings -coverage in a separate job or opt-in check. The default release artifact smoke -must not download `sentence-transformers`, `torch`, or the -`all-MiniLM-L6-v2` model. - -## File structure - -- Create `scripts/local-embeddings-runtime-smoke.mjs`: an opt-in smoke script - that consumes the built public npm tarball, installs it in a temporary pnpm - project, isolates all runtime and model caches, installs the - `local-embeddings` feature, starts the managed daemon, computes one real - embedding, runs setup with local embeddings, verifies the managed config - marker, and stops the daemon. -- Create `scripts/local-embeddings-runtime-smoke.test.mjs`: fast source-level - tests for opt-in gating, public tarball selection, cache isolation, command - construction, daemon URL parsing, embedding response validation, and package - script registration. -- Modify `package.json`: add `release:local-embeddings-smoke` without adding - it to default `check`, `test`, `artifacts:verify`, or release readiness. - -### Task 1: Add failing local embeddings smoke tests - -**Files:** - -- Create: `scripts/local-embeddings-runtime-smoke.test.mjs` -- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` - -- [ ] **Step 1: Write the failing test file** - -Create `scripts/local-embeddings-runtime-smoke.test.mjs` with this content: - -```javascript -import assert from 'node:assert/strict'; -import { readFile } from 'node:fs/promises'; -import { describe, it } from 'node:test'; - -import { - buildLocalEmbeddingsSmokeEnv, - localEmbeddingsSmokeCommands, - localEmbeddingsSmokeOptIn, - parseDaemonBaseUrl, - publicKtxTarballName, - validateEmbeddingResponse, -} from './local-embeddings-runtime-smoke.mjs'; - -describe('localEmbeddingsSmokeOptIn', () => { - it('skips unless the smoke is explicitly enabled', () => { - assert.deepEqual(localEmbeddingsSmokeOptIn({}, []), { - run: false, - message: 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.', - }); - }); - - it('runs when the environment opt-in is set', () => { - assert.deepEqual(localEmbeddingsSmokeOptIn({ KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1' }, []), { - run: true, - }); - }); - - it('runs when --force is present', () => { - assert.deepEqual(localEmbeddingsSmokeOptIn({}, ['--force']), { - run: true, - }); - }); -}); - -describe('publicKtxTarballName', () => { - it('selects the public @kaelio/ktx tarball name', () => { - assert.equal( - publicKtxTarballName(['kaelio-ktx-0.0.0-private.tgz', 'ignore-me.tgz']), - 'kaelio-ktx-0.0.0-private.tgz', - ); - }); - - it('fails when the public package tarball is missing', () => { - assert.throws( - () => publicKtxTarballName(['ktx-cli-0.0.0-private.tgz']), - /Expected exactly one @kaelio\/ktx tarball/, - ); - }); - - it('fails when multiple public package tarballs are present', () => { - assert.throws( - () => publicKtxTarballName(['kaelio-ktx-0.1.0.tgz', 'kaelio-ktx-0.2.0.tgz']), - /Expected exactly one @kaelio\/ktx tarball/, - ); - }); -}); - -describe('buildLocalEmbeddingsSmokeEnv', () => { - it('isolates the runtime root and model caches inside the smoke root', () => { - const env = buildLocalEmbeddingsSmokeEnv('/tmp/ktx-local-embedding-smoke', { - PATH: '/usr/bin', - }); - - assert.equal(env.PATH, '/usr/bin'); - assert.equal(env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE, '1'); - assert.equal(env.KTX_RUNTIME_ROOT, '/tmp/ktx-local-embedding-smoke/managed-runtime'); - assert.equal(env.HF_HOME, '/tmp/ktx-local-embedding-smoke/hf-home'); - assert.equal(env.TRANSFORMERS_CACHE, '/tmp/ktx-local-embedding-smoke/transformers-cache'); - assert.equal(env.SENTENCE_TRANSFORMERS_HOME, '/tmp/ktx-local-embedding-smoke/sentence-transformers-home'); - assert.equal(env.TORCH_HOME, '/tmp/ktx-local-embedding-smoke/torch-home'); - }); -}); - -describe('localEmbeddingsSmokeCommands', () => { - it('describes the installed-package commands needed for the smoke', () => { - const commands = localEmbeddingsSmokeCommands({ - projectDir: '/tmp/ktx-local-embedding-smoke/project', - }); - - assert.deepEqual(commands.map((command) => command.label), [ - 'ktx public package version', - 'ktx runtime status missing', - 'ktx runtime install local embeddings', - 'ktx runtime status local embeddings ready', - 'ktx runtime start local embeddings', - 'ktx setup local embeddings', - 'ktx runtime stop local embeddings', - ]); - assert.deepEqual(commands[2], { - label: 'ktx runtime install local embeddings', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], - timeoutMs: 1_200_000, - }); - assert.deepEqual(commands[4], { - label: 'ktx runtime start local embeddings', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'], - timeoutMs: 300_000, - }); - assert.deepEqual(commands[5].args, [ - 'exec', - 'ktx', - 'setup', - '--project-dir', - '/tmp/ktx-local-embedding-smoke/project', - '--new', - '--no-input', - '--yes', - '--skip-llm', - '--embedding-backend', - 'sentence-transformers', - '--skip-databases', - '--skip-sources', - '--skip-agents', - ]); - }); -}); - -describe('parseDaemonBaseUrl', () => { - it('extracts the daemon URL from runtime start output', () => { - assert.equal( - parseDaemonBaseUrl('Started KTX Python daemon\nurl: http://127.0.0.1:61234\nfeatures: local-embeddings\n'), - 'http://127.0.0.1:61234', - ); - }); - - it('rejects output without a daemon URL', () => { - assert.throws(() => parseDaemonBaseUrl('Started KTX Python daemon\n'), /Daemon URL was not printed/); - }); -}); - -describe('validateEmbeddingResponse', () => { - it('accepts a finite embedding vector with the expected dimensions', () => { - validateEmbeddingResponse({ embedding: [0.1, -0.2, 0.3] }, 3); - }); - - it('rejects a vector with the wrong dimensions', () => { - assert.throws( - () => validateEmbeddingResponse({ embedding: [0.1, 0.2] }, 3), - /Expected embedding dimension 3, got 2/, - ); - }); - - it('rejects non-finite embedding values', () => { - assert.throws( - () => validateEmbeddingResponse({ embedding: [0.1, Number.NaN, 0.3] }, 3), - /Embedding value at index 1 is not a finite number/, - ); - }); -}); - -describe('package script', () => { - it('registers the opt-in local embeddings smoke command', async () => { - const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); - - assert.equal( - packageJson.scripts['release:local-embeddings-smoke'], - 'node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in', - ); - }); -}); -``` - -- [ ] **Step 2: Run the failing test** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: FAIL with an import error for -`./local-embeddings-runtime-smoke.mjs`. - -- [ ] **Step 3: Commit the failing tests** - -Run: - -```bash -git add scripts/local-embeddings-runtime-smoke.test.mjs -git commit -m "test: specify local embeddings release smoke" -``` - -### Task 2: Implement the opt-in smoke script - -**Files:** - -- Create: `scripts/local-embeddings-runtime-smoke.mjs` -- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` - -- [ ] **Step 1: Create the smoke script** - -Create `scripts/local-embeddings-runtime-smoke.mjs` with this content: - -```javascript -import { execFile } from 'node:child_process'; -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); -const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); -const DEFAULT_ROOT_DIR = resolve(SCRIPT_DIR, '..'); -const PUBLIC_NPM_ARTIFACT_DIR = join('dist', 'artifacts', 'npm'); -const OPT_IN_MESSAGE = - 'Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke.'; - -export function localEmbeddingsSmokeOptIn(env = process.env, args = process.argv.slice(2)) { - if (env.KTX_RUN_LOCAL_EMBEDDINGS_SMOKE === '1' || args.includes('--force')) { - return { run: true }; - } - return { run: false, message: OPT_IN_MESSAGE }; -} - -export function publicKtxTarballName(files) { - const matches = files.filter((file) => /^kaelio-ktx-.+\.tgz$/.test(file)).sort(); - if (matches.length !== 1) { - throw new Error( - `Expected exactly one @kaelio/ktx tarball in ${PUBLIC_NPM_ARTIFACT_DIR}, found ${matches.length}: ${ - matches.join(', ') || 'none' - }. Run pnpm run artifacts:build first.`, - ); - } - return matches[0]; -} - -export async function selectPublicKtxTarball(rootDir = DEFAULT_ROOT_DIR) { - const npmArtifactDir = join(rootDir, PUBLIC_NPM_ARTIFACT_DIR); - const files = await readdir(npmArtifactDir); - return join(npmArtifactDir, publicKtxTarballName(files)); -} - -export function buildLocalEmbeddingsSmokeEnv(root, baseEnv = process.env) { - return { - ...baseEnv, - KTX_RUN_LOCAL_EMBEDDINGS_SMOKE: '1', - KTX_RUNTIME_ROOT: join(root, 'managed-runtime'), - HF_HOME: join(root, 'hf-home'), - TRANSFORMERS_CACHE: join(root, 'transformers-cache'), - SENTENCE_TRANSFORMERS_HOME: join(root, 'sentence-transformers-home'), - TORCH_HOME: join(root, 'torch-home'), - }; -} - -export function localEmbeddingsSmokeCommands(input) { - return [ - { - label: 'ktx public package version', - command: 'pnpm', - args: ['exec', 'ktx', '--version'], - timeoutMs: 60_000, - }, - { - label: 'ktx runtime status missing', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'status', '--json'], - timeoutMs: 60_000, - }, - { - label: 'ktx runtime install local embeddings', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], - timeoutMs: 1_200_000, - }, - { - label: 'ktx runtime status local embeddings ready', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'status', '--json'], - timeoutMs: 60_000, - }, - { - label: 'ktx runtime start local embeddings', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'start', '--feature', 'local-embeddings'], - timeoutMs: 300_000, - }, - { - label: 'ktx setup local embeddings', - command: 'pnpm', - args: [ - 'exec', - 'ktx', - 'setup', - '--project-dir', - input.projectDir, - '--new', - '--no-input', - '--yes', - '--skip-llm', - '--embedding-backend', - 'sentence-transformers', - '--skip-databases', - '--skip-sources', - '--skip-agents', - ], - timeoutMs: 900_000, - }, - { - label: 'ktx runtime stop local embeddings', - command: 'pnpm', - args: ['exec', 'ktx', 'runtime', 'stop'], - timeoutMs: 60_000, - }, - ]; -} - -export function parseDaemonBaseUrl(stdout) { - const match = stdout.match(/^url: (http:\/\/127\.0\.0\.1:\d+)$/m); - if (!match) { - throw new Error(`Daemon URL was not printed by runtime start:\n${stdout}`); - } - return match[1]; -} - -export function validateEmbeddingResponse(raw, expectedDimensions) { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { - throw new Error('Embedding response must be a JSON object'); - } - const embedding = raw.embedding; - if (!Array.isArray(embedding)) { - throw new Error('Embedding response must include an embedding array'); - } - if (embedding.length !== expectedDimensions) { - throw new Error(`Expected embedding dimension ${expectedDimensions}, got ${embedding.length}`); - } - for (const [index, value] of embedding.entries()) { - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new Error(`Embedding value at index ${index} is not a finite number`); - } - } -} - -async function run(command, args, options = {}) { - process.stdout.write(`$ ${command} ${args.join(' ')}\n`); - try { - const result = await execFileAsync(command, args, { - cwd: options.cwd, - env: { ...process.env, ...options.env }, - encoding: 'utf8', - maxBuffer: 1024 * 1024 * 20, - timeout: options.timeoutMs ?? 120_000, - }); - if (result.stdout) { - process.stdout.write(result.stdout); - } - if (result.stderr) { - process.stderr.write(result.stderr); - } - return { code: 0, stdout: result.stdout, stderr: result.stderr }; - } catch (error) { - const stdout = typeof error.stdout === 'string' ? error.stdout : ''; - const stderr = typeof error.stderr === 'string' ? error.stderr : error.message; - if (stdout) { - process.stdout.write(stdout); - } - if (stderr) { - process.stderr.write(stderr); - } - return { - code: typeof error.code === 'number' ? error.code : 1, - stdout, - stderr, - }; - } -} - -function requireSuccess(label, result, options = {}) { - if (result.code !== 0) { - throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); - } - if (options.stderrPattern && !options.stderrPattern.test(result.stderr)) { - throw new Error(`${label} stderr did not match ${options.stderrPattern}\nstderr:\n${result.stderr}`); - } -} - -function parseJsonStdout(label, result) { - requireSuccess(label, result); - try { - return JSON.parse(result.stdout); - } catch (error) { - throw new Error(`${label} did not write JSON stdout: ${error.message}\nstdout:\n${result.stdout}`); - } -} - -function requireOutput(label, result, pattern) { - if (!pattern.test(result.stdout)) { - throw new Error(`${label} stdout did not match ${pattern}\nstdout:\n${result.stdout}`); - } -} - -async function postJson(baseUrl, path, payload, timeoutMs) { - const response = await fetch(new URL(path, baseUrl), { - method: 'POST', - headers: { - accept: 'application/json', - 'content-type': 'application/json', - }, - body: JSON.stringify(payload), - signal: AbortSignal.timeout(timeoutMs), - }); - const text = await response.text(); - if (!response.ok) { - throw new Error(`POST ${path} failed with ${response.status}: ${text}`); - } - try { - return JSON.parse(text); - } catch (error) { - throw new Error(`POST ${path} returned non-JSON response: ${error.message}\n${text}`); - } -} - -async function writeSmokePackage(projectDir, tarballPath) { - await mkdir(projectDir, { recursive: true }); - await writeFile( - join(projectDir, 'package.json'), - `${JSON.stringify( - { - name: 'ktx-local-embeddings-runtime-smoke', - version: '0.0.0', - private: true, - type: 'module', - dependencies: { - '@kaelio/ktx': `file:${tarballPath}`, - }, - }, - null, - 2, - )}\n`, - ); -} - -export async function runLocalEmbeddingsRuntimeSmoke(options = {}) { - const rootDir = options.rootDir ?? DEFAULT_ROOT_DIR; - const tarballPath = options.tarballPath ?? (await selectPublicKtxTarball(rootDir)); - const root = await mkdtemp(join(tmpdir(), 'ktx-local-embeddings-smoke-')); - const keepTemp = options.keepTemp ?? process.env.KTX_KEEP_LOCAL_EMBEDDINGS_SMOKE === '1'; - const installDir = join(root, 'installed-package'); - const projectDir = join(root, 'project'); - const smokeEnv = buildLocalEmbeddingsSmokeEnv(root); - const commands = localEmbeddingsSmokeCommands({ projectDir }); - let daemonStarted = false; - - try { - await writeSmokePackage(installDir, tarballPath); - requireSuccess( - 'pnpm install public package', - await run('pnpm', ['install', '--ignore-scripts=false'], { - cwd: installDir, - env: smokeEnv, - timeoutMs: 300_000, - }), - ); - - const version = await run(commands[0].command, commands[0].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[0].timeoutMs, - }); - requireSuccess(commands[0].label, version); - requireOutput(commands[0].label, version, /@kaelio\/ktx 0\.0\.0-private/); - - const missingStatus = parseJsonStdout( - commands[1].label, - await run(commands[1].command, commands[1].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[1].timeoutMs, - }), - ); - if (missingStatus.kind !== 'missing') { - throw new Error(`Expected missing runtime before install, got ${JSON.stringify(missingStatus)}`); - } - - const install = await run(commands[2].command, commands[2].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[2].timeoutMs, - }); - requireSuccess(commands[2].label, install); - requireOutput(commands[2].label, install, /Installed KTX Python runtime/); - requireOutput(commands[2].label, install, /features: core, local-embeddings/); - - const readyStatus = parseJsonStdout( - commands[3].label, - await run(commands[3].command, commands[3].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[3].timeoutMs, - }), - ); - if (readyStatus.kind !== 'ready') { - throw new Error(`Expected ready runtime after install, got ${JSON.stringify(readyStatus)}`); - } - if (!readyStatus.manifest?.features?.includes('local-embeddings')) { - throw new Error(`Runtime manifest did not include local-embeddings: ${JSON.stringify(readyStatus.manifest)}`); - } - - const start = await run(commands[4].command, commands[4].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[4].timeoutMs, - }); - requireSuccess(commands[4].label, start); - daemonStarted = true; - const baseUrl = parseDaemonBaseUrl(start.stdout); - - const embeddingResponse = await postJson( - baseUrl, - '/embeddings/compute', - { text: 'KTX local embeddings release smoke' }, - 900_000, - ); - validateEmbeddingResponse(embeddingResponse, 384); - process.stdout.write('KTX local embeddings daemon computed a 384-dimensional embedding\n'); - - const setup = await run(commands[5].command, commands[5].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[5].timeoutMs, - }); - requireSuccess(commands[5].label, setup); - requireOutput(commands[5].label, setup, /Embeddings ready: yes \(all-MiniLM-L6-v2\)/); - - const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf8'); - if (!config.includes('base_url: managed:local-embeddings')) { - throw new Error(`ktx.yaml did not contain managed local embeddings marker:\n${config}`); - } - process.stdout.write('KTX setup persisted managed local embeddings marker\n'); - - const stop = await run(commands[6].command, commands[6].args, { - cwd: installDir, - env: smokeEnv, - timeoutMs: commands[6].timeoutMs, - }); - requireSuccess(commands[6].label, stop); - daemonStarted = false; - requireOutput(commands[6].label, stop, /Stopped KTX Python daemon/); - - process.stdout.write('KTX local embeddings runtime smoke verified\n'); - } finally { - if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'runtime', 'stop'], { - cwd: installDir, - env: smokeEnv, - timeoutMs: 60_000, - }); - } - if (!keepTemp) { - await rm(root, { recursive: true, force: true }); - } else { - process.stdout.write(`Kept local embeddings smoke root: ${root}\n`); - } - } -} - -async function main() { - const args = process.argv.slice(2); - const optIn = localEmbeddingsSmokeOptIn(process.env, args); - if (!optIn.run) { - process.stdout.write(`Skipping KTX local embeddings runtime smoke. ${optIn.message}\n`); - if (args.includes('--require-opt-in')) { - process.exitCode = 1; - } - return; - } - - await runLocalEmbeddingsRuntimeSmoke(); -} - -if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { - main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); - process.exitCode = 1; - }); -} -``` - -- [ ] **Step 2: Run the smoke test** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: FAIL only in the package script test because -`release:local-embeddings-smoke` is not registered yet. - -- [ ] **Step 3: Commit the smoke script** - -Run: - -```bash -git add scripts/local-embeddings-runtime-smoke.mjs -git commit -m "feat: add local embeddings runtime smoke" -``` - -### Task 3: Register the opt-in package script - -**Files:** - -- Modify: `package.json` -- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` - -- [ ] **Step 1: Add the package script** - -In `package.json`, add this script immediately after -`"release:published-smoke"`: - -```json -"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", -``` - -The surrounding `scripts` section must contain this sequence after the edit: - -```json -"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config", -"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", -"release:readiness": "node scripts/release-readiness.mjs", -``` - -- [ ] **Step 2: Run the focused test** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 3: Verify the script stays opt-in** - -Run: - -```bash -pnpm run release:local-embeddings-smoke -``` - -Expected: FAIL with: - -```text -Skipping KTX local embeddings runtime smoke. Set KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 or pass --force to run the local embeddings smoke. -``` - -The command must exit non-zero because `--require-opt-in` is present. This -protects local and CI runs from downloading large dependencies by accident. - -- [ ] **Step 4: Commit the package script** - -Run: - -```bash -git add package.json -git commit -m "chore: register local embeddings smoke" -``` - -### Task 4: Verify the opt-in smoke path - -**Files:** - -- Verify: `scripts/local-embeddings-runtime-smoke.mjs` -- Verify: `scripts/local-embeddings-runtime-smoke.test.mjs` -- Verify: `package.json` - -- [ ] **Step 1: Run fast script tests** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/package-artifacts.test.mjs -``` - -Expected: PASS. Existing package artifact tests must still prove that the -default npm artifact smoke does not prepare an external Python environment or -run local embeddings downloads. - -- [ ] **Step 2: Build release artifacts for the smoke** - -Run: - -```bash -pnpm run artifacts:build -``` - -Expected: PASS and `dist/artifacts/npm/` contains exactly one -`kaelio-ktx-*.tgz` tarball. - -- [ ] **Step 3: Run the opt-in local embeddings smoke** - -Run this only in an environment where downloading `sentence-transformers`, -`torch`, and `all-MiniLM-L6-v2` is acceptable: - -```bash -KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 pnpm run release:local-embeddings-smoke -``` - -Expected: PASS with output containing: - -```text -KTX local embeddings daemon computed a 384-dimensional embedding -KTX setup persisted managed local embeddings marker -KTX local embeddings runtime smoke verified -``` - -- [ ] **Step 4: Run release readiness** - -Run: - -```bash -pnpm run release:readiness -``` - -Expected: PASS. The readiness report must not require -`release:local-embeddings-smoke`; that smoke remains a separately triggered -release job. - -- [ ] **Step 5: Run pre-commit for changed files when configured** - -Run: - -```bash -uv run pre-commit run --files scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs package.json -``` - -Expected: PASS. If pre-commit is unavailable in the environment, record the -tooling failure and keep the previous verification output. - -- [ ] **Step 6: Commit verification fixes if needed** - -If verification required edits, run: - -```bash -git add scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs package.json -git commit -m "fix: verify local embeddings smoke" -``` - -Skip this commit when no files changed after the previous commits. - -## Acceptance criteria - -- `node --test scripts/local-embeddings-runtime-smoke.test.mjs` passes. -- `pnpm run release:local-embeddings-smoke` fails fast without the opt-in - environment variable and prints the exact opt-in guidance. -- `KTX_RUN_LOCAL_EMBEDDINGS_SMOKE=1 pnpm run release:local-embeddings-smoke` - installs the public `@kaelio/ktx` tarball into a clean project, isolates - `KTX_RUNTIME_ROOT` and model caches, installs `local-embeddings`, starts the - managed daemon, computes a 384-dimensional embedding through - `/embeddings/compute`, runs setup with `--embedding-backend - sentence-transformers`, verifies `base_url: managed:local-embeddings` in - `ktx.yaml`, and stops the daemon. -- The default `pnpm run artifacts:verify`, `pnpm run release:readiness`, and - `pnpm run check` paths do not run the local embeddings smoke. - -## Self-review - -- Spec coverage: this plan covers the remaining release-check item for local - embeddings in a separate job or opt-in check. Earlier implemented plans cover - the bundled wheel, managed runtime installer, `sl query` command integration, - daemon lifecycle, managed local embeddings runtime behavior, public npm - package assembly, and default core runtime release smoke. -- Placeholder scan: no steps contain placeholder implementation language. -- Type consistency: runtime feature names are consistently `core` and - `local-embeddings`; the public npm package name is `@kaelio/ktx`; the opt-in - environment variable is `KTX_RUN_LOCAL_EMBEDDINGS_SMOKE`; the managed local - embedding marker remains `managed:local-embeddings`. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md deleted file mode 100644 index c2023c96..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md +++ /dev/null @@ -1,1122 +0,0 @@ -# Managed Local Embeddings Runtime Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make local `sentence-transformers` embedding setup use the -KTX-managed Python runtime and daemon instead of requiring users to start a -manual `ktx-daemon` process. - -**Architecture:** Add one managed local-embedding helper in the CLI that -prompts or fails according to the existing runtime install policy, starts the -managed daemon with the `local-embeddings` feature, and returns the daemon URL -for health checks. Store a stable managed-runtime marker in `ktx.yaml`, and -teach context embedding config resolution to turn that marker into a daemon URL -only when the CLI has provided one through the environment. - -**Tech Stack:** TypeScript, Vitest, Commander, `@clack/prompts`, KTX managed -Python runtime commands, `@ktx/llm` embedding health checks. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -Existing plans based on the spec: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is - implemented. The worktree contains the runtime wheel builder, runtime wheel - packaging, the `kaelio-ktx` Python artifact policy entry, and matching - artifact tests. -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is - implemented. The worktree contains `managed-python-runtime.ts`, the runtime - command runner, `runtime install`, `status`, `doctor`, and `prune` command - registration, and matching CLI tests. -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` - is implemented. The worktree contains `managed-python-command.ts`, `ktx sl - query` runtime policy flags, schema validation, and matching `sl` tests. -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` - is implemented. The worktree contains `managed-python-daemon.ts`, daemon - state paths in the runtime layout, `runtime start`, `runtime stop`, Python - `/health` version metadata, and matching TypeScript and Python tests. - -Spec requirements still outside this plan: - -- Public npm package surface rename from `@ktx/cli` to `@kaelio/ktx`. -- Managed runtime usage for non-embedding Python-backed command paths beyond - `ktx sl query`. -- Release smoke coverage for `npx @kaelio/ktx ...` invocation modes. - -This plan implements the next local-embedding runtime slice: - -- Selecting local embeddings installs only the `local-embeddings` runtime - feature. -- Local embedding setup starts or reuses the managed HTTP daemon. -- `--yes` installs and starts without prompting. -- `--no-input` fails with an exact preparation command when the managed local - embedding runtime is missing. -- Project config records a managed local embedding marker instead of a random - daemon port. -- Context embedding resolution only resolves the marker when the CLI provides - the active daemon URL. - -## File structure - -- Modify `packages/context/src/llm/local-config.ts`: define the managed local - embeddings marker and environment variable, and resolve that marker to a - runtime daemon URL. -- Modify `packages/context/src/llm/local-config.test.ts`: cover marker - resolution, missing daemon URL behavior, and provider construction. -- Modify `packages/context/src/llm/index.ts`: export the marker constants. -- Modify `packages/context/src/package-exports.test.ts`: assert root exports - expose the marker constants. -- Create `packages/cli/src/managed-local-embeddings.ts`: start or reuse the - managed daemon with `local-embeddings` and build health/project configs. -- Create `packages/cli/src/managed-local-embeddings.test.ts`: cover ready, - `--yes`, prompt, and `--no-input` behavior. -- Modify `packages/cli/src/setup-embeddings.ts`: use the managed helper for - local embeddings and persist the managed marker. -- Modify `packages/cli/src/setup-embeddings.test.ts`: update local embedding - setup expectations and no-input failure behavior. -- Modify `packages/cli/src/setup.ts`: pass CLI version and runtime install - policy into the embeddings step. -- Modify `packages/cli/src/commands/setup-commands.ts`: attach package version - to setup runs. -- Modify `packages/cli/src/cli-program.ts`: attach package version to the bare - interactive setup path. -- Modify `packages/cli/src/index.ts`: export the managed local embedding helper - for tests and programmatic use. -- Modify `packages/cli/src/index.test.ts` and `packages/cli/src/setup.test.ts`: - update setup argument expectations for `cliVersion`. - -### Task 1: Add managed embedding marker resolution in context - -**Files:** - -- Modify: `packages/context/src/llm/local-config.test.ts` -- Modify: `packages/context/src/llm/local-config.ts` -- Modify: `packages/context/src/llm/index.ts` -- Modify: `packages/context/src/package-exports.test.ts` - -- [ ] **Step 1: Write failing marker resolution tests** - -In `packages/context/src/llm/local-config.test.ts`, extend the import from -`./local-config.js` so it includes the new constants: - -```typescript -import { - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, - createLocalKtxEmbeddingProviderFromConfig, - createLocalKtxLlmProviderFromConfig, - resolveLocalKtxEmbeddingConfig, - resolveLocalKtxLlmConfig, -} from './local-config.js'; -``` - -Add these tests inside `describe('local KTX embedding config', () => { ... })` -after the existing `resolves sentence-transformers config` test: - -```typescript - it('resolves managed sentence-transformers config from the CLI-provided daemon URL', () => { - const config: KtxProjectEmbeddingConfig = { - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { - base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - pathPrefix: '', - }, - batchSize: 32, - }; - - expect( - resolveLocalKtxEmbeddingConfig(config, { - [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', - }), - ).toEqual({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, - batchSize: 32, - }); - }); - - it('returns null for managed sentence-transformers when no daemon URL is available', () => { - const config: KtxProjectEmbeddingConfig = { - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { - base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - pathPrefix: '', - }, - }; - - expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull(); - }); -``` - -In `packages/context/src/package-exports.test.ts`, add these assertions after -the existing `root.createLocalKtxEmbeddingProviderFromConfig` assertion: - -```typescript - expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBe('managed:local-embeddings'); - expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV).toBe( - 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL', - ); -``` - -- [ ] **Step 2: Run the failing context tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -- src/llm/local-config.test.ts src/package-exports.test.ts -``` - -Expected: FAIL with missing exports for -`MANAGED_SENTENCE_TRANSFORMERS_BASE_URL` and -`MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV`. - -- [ ] **Step 3: Implement marker resolution** - -In `packages/context/src/llm/local-config.ts`, add these exports after the -`LocalConfigDeps` interface: - -```typescript -export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = 'managed:local-embeddings'; -export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV = 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL'; -``` - -Add this helper before `resolveLocalKtxEmbeddingConfig`: - -```typescript -function resolveSentenceTransformersBaseUrl(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined { - if (!value) { - return undefined; - } - if (value === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL) { - return resolveOptional(`env:${MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV}`, env); - } - return value; -} -``` - -Replace `resolveLocalKtxEmbeddingConfig` with this implementation: - -```typescript -export function resolveLocalKtxEmbeddingConfig( - config: KtxProjectEmbeddingConfig, - env: NodeJS.ProcessEnv, -): KtxEmbeddingConfig | null { - if (config.backend === 'none') { - return null; - } - if (config.backend === 'sentence-transformers') { - const baseURL = resolveSentenceTransformersBaseUrl(config.sentenceTransformers?.base_url, env); - if (!baseURL) { - return null; - } - return { - backend: config.backend, - model: config.model ?? 'all-MiniLM-L6-v2', - dimensions: config.dimensions, - sentenceTransformers: { - baseURL, - pathPrefix: config.sentenceTransformers?.pathPrefix, - }, - batchSize: config.batchSize, - }; - } - return { - backend: config.backend, - model: config.model ?? 'deterministic', - dimensions: config.dimensions, - ...(resolvedProviderConfig(config.openai, env) ? { openai: resolvedProviderConfig(config.openai, env) } : {}), - batchSize: config.batchSize, - }; -} -``` - -In `packages/context/src/llm/index.ts`, add the new constants to the existing -export from `./local-config.js`: - -```typescript -export { - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, - createLocalKtxEmbeddingProviderFromConfig, - createLocalKtxLlmProviderFromConfig, - resolveLocalKtxEmbeddingConfig, - resolveLocalKtxLlmConfig, -} from './local-config.js'; -``` - -- [ ] **Step 4: Verify the context marker tests pass** - -Run: - -```bash -pnpm --filter @ktx/context run test -- src/llm/local-config.test.ts src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/llm/local-config.ts packages/context/src/llm/local-config.test.ts packages/context/src/llm/index.ts packages/context/src/package-exports.test.ts -git commit -m "feat: add managed local embeddings config marker" -``` - -### Task 2: Add the managed local embeddings CLI helper - -**Files:** - -- Create: `packages/cli/src/managed-local-embeddings.test.ts` -- Create: `packages/cli/src/managed-local-embeddings.ts` -- Modify: `packages/cli/src/index.ts` - -- [ ] **Step 1: Write the failing helper tests** - -Create `packages/cli/src/managed-local-embeddings.test.ts` with this content: - -```typescript -import { describe, expect, it, vi } from 'vitest'; -import { - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, -} from '@ktx/context'; -import { - ensureManagedLocalEmbeddingsDaemon, - managedLocalEmbeddingHealthConfig, - managedLocalEmbeddingProjectConfig, -} from './managed-local-embeddings.js'; -import type { ManagedPythonCommandRuntime } from './managed-python-command.js'; -import type { ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -function runtime(): ManagedPythonCommandRuntime { - return { - layout: { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - daemonStatePath: '/runtime/0.2.0/daemon.json', - daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', - daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', - }, - manifest: { - schemaVersion: 1, - cliVersion: '0.2.0', - installedAt: '2026-05-11T00:00:00.000Z', - asset: { - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.2.0', - wheel: { - file: 'kaelio_ktx-0.2.0-py3-none-any.whl', - sha256: 'a'.repeat(64), - bytes: 123, - }, - }, - features: ['core', 'local-embeddings'], - python: { - executable: '/runtime/0.2.0/.venv/bin/python', - daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }, - installLog: '/runtime/0.2.0/install.log', - }, - }; -} - -function daemonResult(status: 'started' | 'reused' = 'reused'): ManagedPythonDaemonStartResult { - return { - status, - layout: runtime().layout, - baseUrl: 'http://127.0.0.1:61234', - state: { - schemaVersion: 1, - pid: 12345, - host: '127.0.0.1', - port: 61234, - version: '0.2.0', - features: ['core', 'local-embeddings'], - startedAt: '2026-05-11T00:00:00.000Z', - stdoutLog: '/runtime/0.2.0/daemon.stdout.log', - stderrLog: '/runtime/0.2.0/daemon.stderr.log', - }, - }; -} - -describe('managedLocalEmbeddingProjectConfig', () => { - it('uses a stable managed runtime marker instead of a random daemon port', () => { - expect( - managedLocalEmbeddingProjectConfig({ - model: 'all-MiniLM-L6-v2', - dimensions: 384, - }), - ).toEqual({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { - base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - pathPrefix: '', - }, - }); - }); -}); - -describe('managedLocalEmbeddingHealthConfig', () => { - it('uses the active managed daemon URL for the immediate health check', () => { - expect( - managedLocalEmbeddingHealthConfig({ - baseUrl: 'http://127.0.0.1:61234', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - }), - ).toEqual({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, - }); - }); -}); - -describe('ensureManagedLocalEmbeddingsDaemon', () => { - it('ensures the local-embeddings feature and starts the managed daemon', async () => { - const io = makeIo(); - const ensureRuntime = vi.fn(async () => runtime()); - const startDaemon = vi.fn(async () => daemonResult('started')); - - await expect( - ensureManagedLocalEmbeddingsDaemon({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: io.io, - ensureRuntime, - startDaemon, - }), - ).resolves.toEqual({ - baseUrl: 'http://127.0.0.1:61234', - env: { - [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', - }, - }); - - expect(ensureRuntime).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: io.io, - feature: 'local-embeddings', - }); - expect(startDaemon).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - features: ['local-embeddings'], - force: false, - }); - expect(io.stderr()).toContain('Started KTX local embeddings daemon: http://127.0.0.1:61234'); - }); - - it('reuses an already running daemon without reporting a new start', async () => { - const io = makeIo(); - - await ensureManagedLocalEmbeddingsDaemon({ - cliVersion: '0.2.0', - installPolicy: 'prompt', - io: io.io, - ensureRuntime: vi.fn(async () => runtime()), - startDaemon: vi.fn(async () => daemonResult('reused')), - }); - - expect(io.stderr()).toContain('Using KTX local embeddings daemon: http://127.0.0.1:61234'); - }); -}); -``` - -- [ ] **Step 2: Run the failing helper tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-local-embeddings.test.ts -``` - -Expected: FAIL with an import error for -`./managed-local-embeddings.js`. - -- [ ] **Step 3: Implement the helper** - -Create `packages/cli/src/managed-local-embeddings.ts` with this content: - -```typescript -import { - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, -} from '@ktx/context'; -import type { KtxProjectEmbeddingConfig } from '@ktx/context/project'; -import type { KtxEmbeddingConfig } from '@ktx/llm'; -import type { KtxCliIo } from './cli-runtime.js'; -import { - ensureManagedPythonCommandRuntime, - type KtxManagedPythonInstallPolicy, - type ManagedPythonCommandRuntime, -} from './managed-python-command.js'; -import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; - -export interface ManagedLocalEmbeddingsDaemon { - baseUrl: string; - env: Record; -} - -export interface ManagedLocalEmbeddingsOptions { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxCliIo; - ensureRuntime?: (options: { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxCliIo; - feature: 'local-embeddings'; - }) => Promise; - startDaemon?: (options: { - cliVersion: string; - features: ['local-embeddings']; - force: boolean; - }) => Promise; -} - -export function managedLocalEmbeddingProjectConfig(input: { - model: string; - dimensions: number; -}): KtxProjectEmbeddingConfig { - return { - backend: 'sentence-transformers', - model: input.model, - dimensions: input.dimensions, - sentenceTransformers: { - base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, - pathPrefix: '', - }, - }; -} - -export function managedLocalEmbeddingHealthConfig(input: { - baseUrl: string; - model: string; - dimensions: number; -}): KtxEmbeddingConfig { - return { - backend: 'sentence-transformers', - model: input.model, - dimensions: input.dimensions, - sentenceTransformers: { - baseURL: input.baseUrl, - pathPrefix: '', - }, - }; -} - -export async function ensureManagedLocalEmbeddingsDaemon( - options: ManagedLocalEmbeddingsOptions, -): Promise { - const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime; - const startDaemon = options.startDaemon ?? startManagedPythonDaemon; - - await ensureRuntime({ - cliVersion: options.cliVersion, - installPolicy: options.installPolicy, - io: options.io, - feature: 'local-embeddings', - }); - const daemon = await startDaemon({ - cliVersion: options.cliVersion, - features: ['local-embeddings'], - force: false, - }); - - const verb = daemon.status === 'started' ? 'Started' : 'Using'; - options.io.stderr.write(`${verb} KTX local embeddings daemon: ${daemon.baseUrl}\n`); - - return { - baseUrl: daemon.baseUrl, - env: { - [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: daemon.baseUrl, - }, - }; -} -``` - -In `packages/cli/src/index.ts`, add this export after the existing -`managed-python-daemon.js` exports: - -```typescript -export { - ensureManagedLocalEmbeddingsDaemon, - managedLocalEmbeddingHealthConfig, - managedLocalEmbeddingProjectConfig, - type ManagedLocalEmbeddingsDaemon, - type ManagedLocalEmbeddingsOptions, -} from './managed-local-embeddings.js'; -``` - -- [ ] **Step 4: Verify helper tests pass** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-local-embeddings.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/cli/src/managed-local-embeddings.ts packages/cli/src/managed-local-embeddings.test.ts packages/cli/src/index.ts -git commit -m "feat: add managed local embeddings daemon helper" -``` - -### Task 3: Wire setup embeddings to the managed runtime - -**Files:** - -- Modify: `packages/cli/src/setup-embeddings.ts` -- Modify: `packages/cli/src/setup-embeddings.test.ts` - -- [ ] **Step 1: Write failing setup tests for managed local embeddings** - -In `packages/cli/src/setup-embeddings.test.ts`, update the import from -`./setup-embeddings.js` so it also imports the managed install policy type: - -```typescript -import { - type KtxSetupEmbeddingsPromptAdapter, - runKtxSetupEmbeddingsStep, -} from './setup-embeddings.js'; -``` - -Add this helper near `makePromptAdapter`: - -```typescript -function managedDaemon(baseUrl = 'http://127.0.0.1:61234') { - return { - baseUrl, - env: { - KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl, - }, - }; -} -``` - -In every `runKtxSetupEmbeddingsStep` call that does not inject an `embeddingBackend: -'openai'`, add these arguments: - -```typescript - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', -``` - -In the test named `configures local sentence-transformers embeddings after -interactive selection`, add this dependency: - -```typescript - const ensureLocalEmbeddings = vi.fn(async () => managedDaemon()); -``` - -Pass it in the deps object: - -```typescript - { prompts, env: {}, healthCheck, ensureLocalEmbeddings }, -``` - -Replace the expected health check config in that test with: - -```typescript - expect(ensureLocalEmbeddings).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: io.io, - }); - expect(healthCheck).toHaveBeenCalledWith({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' }, - }); -``` - -Replace the persisted local embedding expectation in that test with: - -```typescript - expect(config.ingest.embeddings).toMatchObject({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, - }); -``` - -Add this new test after the existing non-interactive local embeddings test: - -```typescript - it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => { - const io = makeIo(); - const ensureLocalEmbeddings = vi.fn(async () => { - throw new Error( - 'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes', - ); - }); - - const result = await runKtxSetupEmbeddingsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - cliVersion: '0.2.0', - runtimeInstallPolicy: 'never', - skipEmbeddings: false, - }, - io.io, - { env: {}, ensureLocalEmbeddings }, - ); - - expect(result.status).toBe('failed'); - expect(io.stderr()).toContain( - 'KTX Python runtime is required for this command. Run: ktx runtime install --feature local-embeddings --yes', - ); - }); -``` - -- [ ] **Step 2: Run the failing setup tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/setup-embeddings.test.ts -``` - -Expected: FAIL because `KtxSetupEmbeddingsArgs` has no `cliVersion` or -`runtimeInstallPolicy`, and `KtxSetupEmbeddingsDeps` has no -`ensureLocalEmbeddings`. - -- [ ] **Step 3: Update setup embeddings types and imports** - -In `packages/cli/src/setup-embeddings.ts`, add these imports: - -```typescript -import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -import { - ensureManagedLocalEmbeddingsDaemon, - managedLocalEmbeddingHealthConfig, - managedLocalEmbeddingProjectConfig, - type ManagedLocalEmbeddingsDaemon, -} from './managed-local-embeddings.js'; -``` - -Add these fields to `KtxSetupEmbeddingsArgs` after `inputMode`: - -```typescript - cliVersion: string; - runtimeInstallPolicy: KtxManagedPythonInstallPolicy; -``` - -Add this dependency to `KtxSetupEmbeddingsDeps`: - -```typescript - ensureLocalEmbeddings?: (options: { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxCliIo; - }) => Promise; -``` - -- [ ] **Step 4: Replace manual local daemon messaging and config** - -In `packages/cli/src/setup-embeddings.ts`, remove these constants: - -```typescript -const LOCAL_EMBEDDING_DAEMON_COMMAND = 'ktx-daemon serve-http --host 127.0.0.1 --port 8765'; -const LOCAL_EMBEDDING_DAEMON_DEV_COMMAND = - 'cd ktx && source .venv/bin/activate && uv run ktx-daemon serve-http --host 127.0.0.1 --port 8765'; -``` - -Replace `localEmbeddingSetupMessage` with: - -```typescript -function localEmbeddingSetupMessage(message: string): string { - return [ - `Local embedding health check failed: ${message}`, - 'Local embeddings use the KTX-managed Python runtime.', - 'Prepare the runtime with: ktx runtime start --feature local-embeddings', - 'Use --yes with setup to install and start the runtime without prompting.', - 'The first run may download Python packages and the all-MiniLM-L6-v2 model.', - ].join('\n'); -} -``` - -Inside `runKtxSetupEmbeddingsStep`, before building `healthConfig`, add this -block after the OpenAI credential block: - -```typescript - let managedLocalEmbeddings: ManagedLocalEmbeddingsDaemon | undefined; - if (selectedBackend === LOCAL_EMBEDDING_BACKEND) { - const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon; - try { - managedLocalEmbeddings = await ensureLocalEmbeddings({ - cliVersion: args.cliVersion, - installPolicy: args.runtimeInstallPolicy, - io, - }); - } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - return { status: 'failed', projectDir: args.projectDir }; - } - } -``` - -Replace the `healthConfig` assignment with: - -```typescript - const healthConfig = - selectedBackend === LOCAL_EMBEDDING_BACKEND && managedLocalEmbeddings - ? managedLocalEmbeddingHealthConfig({ - baseUrl: managedLocalEmbeddings.baseUrl, - model, - dimensions, - }) - : buildHealthConfig({ - backend: selectedBackend, - model, - dimensions, - credentialValue, - }); -``` - -Replace the successful local persistence call inside `if (health.ok) { ... }` -with: - -```typescript - await persistEmbeddingConfig( - args.projectDir, - selectedBackend === LOCAL_EMBEDDING_BACKEND - ? managedLocalEmbeddingProjectConfig({ model, dimensions }) - : buildProjectEmbeddingConfig({ - backend: selectedBackend, - model, - dimensions, - credentialRef, - }), - ); -``` - -- [ ] **Step 5: Verify setup embeddings tests pass** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/setup-embeddings.test.ts src/managed-local-embeddings.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -Run: - -```bash -git add packages/cli/src/setup-embeddings.ts packages/cli/src/setup-embeddings.test.ts -git commit -m "feat: use managed runtime for local embedding setup" -``` - -### Task 4: Pass runtime policy and CLI version through setup commands - -**Files:** - -- Modify: `packages/cli/src/setup.ts` -- Modify: `packages/cli/src/commands/setup-commands.ts` -- Modify: `packages/cli/src/cli-program.ts` -- Modify: `packages/cli/src/setup.test.ts` -- Modify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing setup argument expectations** - -In `packages/cli/src/index.test.ts`, find the test that routes the main setup -command and add `cliVersion: '0.0.0-private'` to the expected setup run -argument object. - -Add this assertion to the same test when `--yes` is present: - -```typescript - yes: true, - cliVersion: '0.0.0-private', -``` - -In `packages/cli/src/setup.test.ts`, find the setup test that asserts the -embeddings runner arguments. Add these expected fields to the embeddings step -argument object: - -```typescript - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', -``` - -Add one focused unit test near the other setup flow tests: - -```typescript - it('passes no-input runtime policy to the embeddings step', async () => { - const io = makeIo(); - const embeddings = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir })); - - await expect( - runKtxSetup( - { - command: 'run', - projectDir: tempDir, - mode: 'existing', - agents: false, - agentScope: 'project', - agentInstallMode: 'cli', - skipAgents: true, - inputMode: 'disabled', - yes: false, - cliVersion: '0.2.0', - skipLlm: true, - skipEmbeddings: false, - databaseSchemas: [], - skipDatabases: true, - skipSources: true, - }, - io.io, - { - project: { - run: vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir })), - }, - embeddings, - }, - ), - ).resolves.toBe(1); - - expect(embeddings).toHaveBeenCalledWith( - expect.objectContaining({ - cliVersion: '0.2.0', - runtimeInstallPolicy: 'never', - }), - io.io, - ); - }); -``` - -- [ ] **Step 2: Run the failing setup routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/setup.test.ts src/index.test.ts -``` - -Expected: FAIL because setup args do not carry `cliVersion` yet and embeddings -args do not derive `runtimeInstallPolicy`. - -- [ ] **Step 3: Add `cliVersion` to setup run args** - -In `packages/cli/src/setup.ts`, add this field to the run variant of -`KtxSetupArgs` immediately after `yes`: - -```typescript - cliVersion: string; -``` - -Add this helper near the other setup helpers: - -```typescript -function setupRuntimeInstallPolicy(args: Extract): 'prompt' | 'auto' | 'never' { - if (args.yes) { - return 'auto'; - } - return args.inputMode === 'disabled' ? 'never' : 'prompt'; -} -``` - -In the embeddings step call inside `runKtxSetupInner`, add: - -```typescript - cliVersion: args.cliVersion, - runtimeInstallPolicy: setupRuntimeInstallPolicy(args), -``` - -- [ ] **Step 4: Pass package version from Commander and bare setup** - -In `packages/cli/src/commands/setup-commands.ts`, add this field to the setup -run argument object: - -```typescript - cliVersion: context.packageInfo.version, -``` - -Place it immediately after `yes: options.yes === true`. - -In `packages/cli/src/cli-program.ts`, add this field to the bare interactive -setup argument object inside `runBareInteractiveCommand`: - -```typescript - cliVersion: context.packageInfo.version, -``` - -Place it immediately after `yes: false`. - -- [ ] **Step 5: Verify setup routing tests pass** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/setup.test.ts src/index.test.ts src/setup-embeddings.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -Run: - -```bash -git add packages/cli/src/setup.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/cli-program.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts -git commit -m "feat: pass managed runtime policy through setup" -``` - -### Task 5: Final verification - -**Files:** - -- Verify: `packages/context/src/llm/local-config.ts` -- Verify: `packages/cli/src/managed-local-embeddings.ts` -- Verify: `packages/cli/src/setup-embeddings.ts` -- Verify: `packages/cli/src/setup.ts` - -- [ ] **Step 1: Run focused context tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -- src/llm/local-config.test.ts src/package-exports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-local-embeddings.test.ts src/setup-embeddings.test.ts src/setup.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run TypeScript checks for changed packages** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run package-level tests if type-check changed public exports** - -Run: - -```bash -pnpm --filter @ktx/context run test -pnpm --filter @ktx/cli run test -``` - -Expected: PASS. - -- [ ] **Step 5: Run pre-commit for changed files** - -Run: - -```bash -uv run pre-commit run --files packages/context/src/llm/local-config.ts packages/context/src/llm/local-config.test.ts packages/context/src/llm/index.ts packages/context/src/package-exports.test.ts packages/cli/src/managed-local-embeddings.ts packages/cli/src/managed-local-embeddings.test.ts packages/cli/src/setup-embeddings.ts packages/cli/src/setup-embeddings.test.ts packages/cli/src/setup.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/cli-program.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts packages/cli/src/index.ts -``` - -Expected: PASS. If pre-commit is unavailable because local hook versions are -missing, run the focused tests and type-check commands from steps 1 through 3 -and record the pre-commit error. - -- [ ] **Step 6: Commit final verification adjustments** - -Run this only if final verification required small fixes: - -```bash -git add packages/context/src/llm/local-config.ts packages/context/src/llm/local-config.test.ts packages/context/src/llm/index.ts packages/context/src/package-exports.test.ts packages/cli/src/managed-local-embeddings.ts packages/cli/src/managed-local-embeddings.test.ts packages/cli/src/setup-embeddings.ts packages/cli/src/setup-embeddings.test.ts packages/cli/src/setup.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/cli-program.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts packages/cli/src/index.ts -git commit -m "test: verify managed local embeddings runtime setup" -``` - -## Acceptance criteria - -- `ktx setup --embedding-backend sentence-transformers --yes` installs the - `local-embeddings` runtime feature when needed, starts or reuses the managed - daemon, probes the active daemon URL, and writes `managed:local-embeddings` - to `ktx.yaml`. -- `ktx setup --embedding-backend sentence-transformers --no-input` fails with - the exact runtime preparation command when the runtime is missing. -- Existing OpenAI embedding setup behavior is unchanged. -- The project config no longer stores the daemon's random port. -- `resolveLocalKtxEmbeddingConfig` returns a usable `KtxEmbeddingConfig` for - managed local embeddings only when - `KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL` is present. -- Focused CLI and context tests pass. - -## Self-review - -- Spec coverage: This plan covers lazy `local-embeddings` installation after - local embeddings are selected, separate prompt/no-input behavior, and managed - daemon reuse for local embedding setup health checks. -- Placeholder scan: This plan contains concrete file paths, code snippets, - commands, expected outcomes, and commit commands. -- Type consistency: The new `ManagedLocalEmbeddingsDaemon` type, managed marker - constants, setup argument fields, and helper function names are used - consistently across tasks. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md b/docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md deleted file mode 100644 index 0495ab97..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-local-embeddings-smoke-public-version.md +++ /dev/null @@ -1,239 +0,0 @@ -# Managed Local Embeddings Smoke Public Version Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the opt-in local embeddings release smoke validate the public -`@kaelio/ktx` package version instead of the private workspace version. - -**Architecture:** Reuse the public package constants from -`scripts/build-public-npm-package.mjs` inside the local embeddings smoke. Add a -small exported RegExp helper so the unit test can lock the version expectation -without running the expensive model-download smoke. - -**Tech Stack:** Node.js ESM scripts, `node:test`, pnpm release scripts. - ---- - -## Current State - -The npm-managed Python runtime spec is -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. -The current branch already contains implementation commits for each existing -plan derived from that spec. - -Implemented spec-derived plans: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` -- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` -- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md` -- `docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md` - -The remaining gap is in -`scripts/local-embeddings-runtime-smoke.mjs`. The script selects and installs a -public tarball named `kaelio-ktx-*.tgz` and writes a smoke package dependency on -`@kaelio/ktx`, but line 267 still expects `@kaelio/ktx 0.0.0-private`. The -public package builder defines `PUBLIC_NPM_PACKAGE_VERSION = '0.1.0'`, and the -main packed-package smoke already expects `@kaelio/ktx 0.1.0`. - -## File Structure - -This change keeps the release version source of truth in one script and reuses -it from the opt-in smoke. - -- Modify `scripts/local-embeddings-runtime-smoke.mjs`: import the public package - constants, export `expectedPublicKtxVersionPattern()`, and use that pattern - for the smoke version assertion. -- Modify `scripts/local-embeddings-runtime-smoke.test.mjs`: import - `expectedPublicKtxVersionPattern()` and assert that it accepts - `@kaelio/ktx 0.1.0` and rejects `@kaelio/ktx 0.0.0-private`. - -### Task 1: Align the local embeddings smoke version assertion - -**Files:** -- Modify: `scripts/local-embeddings-runtime-smoke.mjs:1-267` -- Modify: `scripts/local-embeddings-runtime-smoke.test.mjs:5-118` -- Test: `scripts/local-embeddings-runtime-smoke.test.mjs` - -- [ ] **Step 1: Write the failing version-pattern test** - -In `scripts/local-embeddings-runtime-smoke.test.mjs`, update the import block -to include `expectedPublicKtxVersionPattern`: - -```js -import { - buildLocalEmbeddingsSmokeEnv, - expectedPublicKtxVersionPattern, - localEmbeddingsSmokeCommands, - localEmbeddingsSmokeOptIn, - parseDaemonBaseUrl, - publicKtxTarballName, - validateEmbeddingResponse, -} from './local-embeddings-runtime-smoke.mjs'; -``` - -Then add this test after the `publicKtxTarballName` describe block: - -```js -describe('expectedPublicKtxVersionPattern', () => { - it('matches the public package version and rejects the private workspace version', () => { - const pattern = expectedPublicKtxVersionPattern(); - - assert.match('@kaelio/ktx 0.1.0\n', pattern); - assert.doesNotMatch('@kaelio/ktx 0.0.0-private\n', pattern); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: FAIL with an ESM export error that says -`expectedPublicKtxVersionPattern` is not exported from -`./local-embeddings-runtime-smoke.mjs`. - -- [ ] **Step 3: Import the public package constants** - -In `scripts/local-embeddings-runtime-smoke.mjs`, add this import after the -existing Node imports: - -```js -import { - PUBLIC_NPM_PACKAGE_NAME, - PUBLIC_NPM_PACKAGE_VERSION, -} from './build-public-npm-package.mjs'; -``` - -The top of the file becomes: - -```js -import { execFile } from 'node:child_process'; -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { promisify } from 'node:util'; - -import { - PUBLIC_NPM_PACKAGE_NAME, - PUBLIC_NPM_PACKAGE_VERSION, -} from './build-public-npm-package.mjs'; -``` - -- [ ] **Step 4: Add the version-pattern helper** - -In `scripts/local-embeddings-runtime-smoke.mjs`, add these functions after the -`OPT_IN_MESSAGE` constant: - -```js -function escapeRegExp(value) { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -export function expectedPublicKtxVersionPattern() { - return new RegExp( - `${escapeRegExp(PUBLIC_NPM_PACKAGE_NAME)} ${escapeRegExp(PUBLIC_NPM_PACKAGE_VERSION)}`, - ); -} -``` - -- [ ] **Step 5: Use the helper in the smoke** - -In `scripts/local-embeddings-runtime-smoke.mjs`, replace this line: - -```js -requireOutput(commands[0].label, version, /@kaelio\/ktx 0\.0\.0-private/); -``` - -with: - -```js -requireOutput(commands[0].label, version, expectedPublicKtxVersionPattern()); -``` - -- [ ] **Step 6: Run the focused test** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: PASS. The new test proves the smoke accepts `@kaelio/ktx 0.1.0` and -rejects `@kaelio/ktx 0.0.0-private`. - -- [ ] **Step 7: Run related release-script tests** - -Run: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs -``` - -Expected: PASS. These tests cover the public package constants, tarball name, -artifact smoke source, and local embeddings smoke helpers. - -- [ ] **Step 8: Run a stale-expectation search** - -Run: - -```bash -rg -n "@kaelio/ktx 0\\.0\\.0-private|0\\\\\\.0\\\\\\.0-private" scripts/local-embeddings-runtime-smoke.mjs -``` - -Expected: no output. The opt-in local embeddings smoke no longer contains the -private package version expectation. The test file still uses -`@kaelio/ktx 0.0.0-private` as a negative fixture. - -- [ ] **Step 9: Commit** - -Run: - -```bash -git add scripts/local-embeddings-runtime-smoke.mjs scripts/local-embeddings-runtime-smoke.test.mjs -git commit -m "fix: align local embeddings smoke with public version" -``` - -## Verification - -Run these checks before marking the plan complete: - -```bash -node --test scripts/local-embeddings-runtime-smoke.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs -rg -n "@kaelio/ktx 0\\.0\\.0-private|0\\\\\\.0\\\\\\.0-private" scripts/local-embeddings-runtime-smoke.mjs -``` - -Expected results: - -- `node --test ...` exits with code 0. -- `rg ...` prints no matches. -- No Python files changed, so the repository Python pre-commit requirement does - not apply. - -## Self-Review - -- Spec coverage: this plan fixes the opt-in local embeddings release smoke from - the npm-managed runtime spec so it validates the public npm package produced - by the current release artifact flow. -- Placeholder scan: the plan contains concrete file paths, code blocks, - commands, and expected outcomes. -- Type consistency: the helper name is consistently - `expectedPublicKtxVersionPattern`, and it uses - `PUBLIC_NPM_PACKAGE_NAME` plus `PUBLIC_NPM_PACKAGE_VERSION` from the public - package builder. diff --git a/docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md b/docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md deleted file mode 100644 index fbba6289..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md +++ /dev/null @@ -1,1650 +0,0 @@ -# Managed Local Ingest Daemon Runtime Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make local ingest, scan, and MCP daemon-backed helper paths use the -KTX-managed core Python daemon instead of requiring `KTX_DAEMON_URL` or a -manually started daemon on `127.0.0.1:8765`. - -**Architecture:** Add lazy managed-daemon HTTP ports in the CLI package. Thread -those ports through CLI local ingest adapter creation and pull-config options so -Looker table identifier parsing, historic SQL analysis, and live-database daemon -fallbacks resolve the managed core daemon only when a request is made. - -**Tech Stack:** TypeScript, Vitest, Commander, KTX CLI managed Python runtime, -KTX context local ingest adapters, MCP local project ports. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plans are based on that spec and are already implemented in this -worktree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` - -Implementation evidence found before writing this plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and - `packages/cli/assets/python/manifest.json`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, and - `packages/cli/src/commands/runtime-commands.ts`. -- `packages/cli/src/managed-python-command.ts` and managed `ktx sl query`, - hidden agent SL query, and MCP semantic compute paths. -- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` / - `ktx runtime stop`. -- `packages/cli/src/managed-local-embeddings.ts` and local embeddings setup - wiring. -- `scripts/build-public-npm-package.mjs`, release policy updates, release - smoke coverage, and opt-in local embeddings smoke coverage. -- `packages/cli/src/agent-runtime.ts` and `packages/cli/src/serve.ts` now - create managed semantic-layer compute when no explicit semantic HTTP URL is - provided. - -The remaining spec gap is local ingest daemon-backed helper behavior: - -- `packages/context/src/ingest/local-adapters.ts` still creates the Looker - table identifier parser from `options.looker.daemonBaseUrl`, - `KTX_DAEMON_URL`, or `http://127.0.0.1:8765`. -- `packages/cli/src/local-adapters.ts` still creates historic SQL analysis from - `options.sqlAnalysisUrl`, `KTX_SQL_ANALYSIS_URL`, `KTX_DAEMON_URL`, or - `http://127.0.0.1:8765`. -- `packages/cli/src/serve.ts` passes adapters to MCP local ingest, but - `LocalIngestMcpOptions` has no `pullConfigOptions`, so Looker pull-config - generation cannot receive CLI-managed daemon options. - -This plan closes that gap without changing explicit daemon URL behavior. -Explicit `--database-introspection-url`, explicit test dependency injection, -`KTX_SQL_ANALYSIS_URL`, and `KTX_DAEMON_URL` continue to win over the managed -daemon. - -## File structure - -- Create `packages/cli/src/managed-python-http.ts`: lazy managed core daemon - resolver, generic HTTP JSON runner, managed Looker table identifier parser, - managed SQL analysis port, and managed live-database daemon request options. -- Create `packages/cli/src/managed-python-http.test.ts`: verifies lazy daemon - resolution, install policy propagation, daemon reuse caching, and HTTP runner - delegation. -- Modify `packages/cli/src/local-adapters.ts`: accepts managed daemon options - and wires them into daemon-backed local ingest helpers only when no explicit - daemon URL is configured. -- Modify `packages/cli/src/ingest.ts`: adds runtime install policy fields to - run args and passes managed daemon options to both adapter creation and - local pull-config resolution. -- Modify `packages/cli/src/ingest.test.ts`: covers managed daemon option - threading and preserves explicit daemon URL behavior. -- Modify `packages/cli/src/commands/ingest-commands.ts`: adds `--yes` to - `ktx ingest run` and uses existing `--no-input` as the runtime noninteractive - mode. -- Modify `packages/cli/src/scan.ts`: adds runtime install policy fields and - passes managed daemon options to local ingest adapters used during scan. -- Modify `packages/cli/src/scan.test.ts`: covers managed daemon option - threading and explicit daemon URL behavior. -- Modify `packages/cli/src/commands/scan-commands.ts`: adds `--yes` and - `--no-input` to `ktx scan`. -- Modify `packages/context/src/ingest/local-ingest.ts`: adds - `pullConfigOptions` to `LocalIngestMcpOptions`. -- Modify `packages/context/src/mcp/local-project-ports.ts`: passes MCP local - ingest pull-config options into `runLocalIngest()`. -- Modify `packages/context/src/mcp/local-project-ports.test.ts`: covers MCP - pull-config option forwarding. -- Modify `packages/cli/src/serve.ts`: passes managed daemon options and - pull-config options to MCP local ingest. -- Modify `packages/cli/src/serve.test.ts`: covers MCP local ingest managed - daemon option wiring. -- Modify `packages/cli/src/index.test.ts`: updates Commander routing - expectations for ingest and scan runtime install policy flags. - -### Task 1: Add managed daemon HTTP helpers - -**Files:** - -- Create: `packages/cli/src/managed-python-http.test.ts` -- Create: `packages/cli/src/managed-python-http.ts` -- Test: `packages/cli/src/managed-python-http.test.ts` - -- [ ] **Step 1: Write failing tests for lazy daemon HTTP helpers** - -Create `packages/cli/src/managed-python-http.test.ts` with this content: - -```typescript -import { describe, expect, it, vi } from 'vitest'; -import { - createManagedDaemonHttpJsonRunner, - createManagedDaemonLookerTableIdentifierParser, - createManagedDaemonSqlAnalysisPort, - createManagedPythonDaemonBaseUrlResolver, - managedDaemonDatabaseIntrospectionOptions, -} from './managed-python-http.js'; - -function io() { - let stderr = ''; - return { - io: { - stdout: { write: vi.fn() }, - stderr: { write: (chunk: string) => (stderr += chunk) }, - }, - stderr: () => stderr, - }; -} - -describe('createManagedPythonDaemonBaseUrlResolver', () => { - it('ensures the core runtime, starts the daemon, reports the URL, and caches the result', async () => { - const testIo = io(); - const ensureRuntime = vi.fn(async () => ({ - layout: {} as never, - manifest: {} as never, - })); - const startDaemon = vi.fn(async () => ({ - status: 'started' as const, - layout: {} as never, - state: { pid: 1234 } as never, - baseUrl: 'http://127.0.0.1:61234', - })); - const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: testIo.io, - ensureRuntime, - startDaemon, - }); - - await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); - await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); - - expect(ensureRuntime).toHaveBeenCalledTimes(1); - expect(ensureRuntime).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: testIo.io, - feature: 'core', - }); - expect(startDaemon).toHaveBeenCalledTimes(1); - expect(startDaemon).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - features: ['core'], - force: false, - }); - expect(testIo.stderr()).toContain('Started KTX Python daemon: http://127.0.0.1:61234'); - }); - - it('reports daemon reuse without reinstalling after the first resolved URL', async () => { - const testIo = io(); - const ensureRuntime = vi.fn(async () => ({ - layout: {} as never, - manifest: {} as never, - })); - const startDaemon = vi.fn(async () => ({ - status: 'reused' as const, - layout: {} as never, - state: { pid: 1234 } as never, - baseUrl: 'http://127.0.0.1:61234', - })); - const resolveBaseUrl = createManagedPythonDaemonBaseUrlResolver({ - cliVersion: '0.2.0', - installPolicy: 'never', - io: testIo.io, - ensureRuntime, - startDaemon, - }); - - await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); - await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234'); - - expect(ensureRuntime).toHaveBeenCalledTimes(1); - expect(startDaemon).toHaveBeenCalledTimes(1); - expect(testIo.stderr()).toContain('Using existing KTX Python daemon: http://127.0.0.1:61234'); - }); -}); - -describe('createManagedDaemonHttpJsonRunner', () => { - it('resolves the managed base URL lazily for each HTTP JSON request', async () => { - const postJson = vi.fn(async () => ({ ok: true })); - const runner = createManagedDaemonHttpJsonRunner({ - resolveBaseUrl: async () => 'http://127.0.0.1:61234', - postJson, - }); - - await expect(runner('/sql/parse-table-identifier', { items: [] })).resolves.toEqual({ ok: true }); - - expect(postJson).toHaveBeenCalledWith('http://127.0.0.1:61234', '/sql/parse-table-identifier', { items: [] }); - }); -}); - -describe('managed daemon ingest ports', () => { - it('creates a Looker table parser backed by the managed daemon runner', async () => { - const requestJson = vi.fn(async () => ({ - results: { - 'model.explore': { - ok: true, - catalog: 'warehouse', - schema: 'public', - name: 'orders', - canonical_table: 'public.orders', - }, - }, - })); - const parser = createManagedDaemonLookerTableIdentifierParser({ requestJson }); - - await expect( - parser.parse([{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }]), - ).resolves.toEqual({ - 'model.explore': { - ok: true, - catalog: 'warehouse', - schema: 'public', - name: 'orders', - canonical_table: 'public.orders', - }, - }); - expect(requestJson).toHaveBeenCalledWith('/sql/parse-table-identifier', { - items: [{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }], - }); - }); - - it('creates a SQL analysis port backed by the managed daemon runner', async () => { - const requestJson = vi.fn(async () => ({ - fingerprint: 'select-orders', - normalized_sql: 'SELECT * FROM public.orders WHERE id = ?', - tables_touched: ['public.orders'], - literal_slots: [{ position: 1, type: 'number', example_value: '42' }], - })); - const sqlAnalysis = createManagedDaemonSqlAnalysisPort({ requestJson }); - - await expect(sqlAnalysis.analyzeForFingerprint('SELECT * FROM public.orders WHERE id = 42', 'postgres')).resolves - .toEqual({ - fingerprint: 'select-orders', - normalizedSql: 'SELECT * FROM public.orders WHERE id = ?', - tablesTouched: ['public.orders'], - literalSlots: [{ position: 1, type: 'number', exampleValue: '42' }], - }); - expect(requestJson).toHaveBeenCalledWith('/api/sql/analyze-for-fingerprint', { - sql: 'SELECT * FROM public.orders WHERE id = 42', - dialect: 'postgres', - }); - }); - - it('returns live-database daemon request options backed by the managed runner', async () => { - const requestJson = vi.fn(async () => ({ - connection_id: 'warehouse', - tables: [], - })); - const options = managedDaemonDatabaseIntrospectionOptions({ requestJson }); - - await expect(options.requestJson('/database/introspect', { connection_id: 'warehouse' })).resolves.toEqual({ - connection_id: 'warehouse', - tables: [], - }); - expect(requestJson).toHaveBeenCalledWith('/database/introspect', { connection_id: 'warehouse' }); - }); -}); -``` - -- [ ] **Step 2: Run the failing helper tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-http.test.ts -``` - -Expected: FAIL with an import error for `./managed-python-http.js`. - -- [ ] **Step 3: Implement managed daemon HTTP helpers** - -Create `packages/cli/src/managed-python-http.ts` with this content: - -```typescript -import { request as httpRequest } from 'node:http'; -import { request as httpsRequest } from 'node:https'; -import { URL } from 'node:url'; -import { - createDaemonLookerTableIdentifierParser, - type DaemonLiveDatabaseIntrospectionOptions, - type KtxDaemonDatabaseHttpJsonRunner, - type KtxDaemonTableIdentifierHttpJsonRunner, - type LookerTableIdentifierParser, -} from '@ktx/context/ingest'; -import { - createHttpSqlAnalysisPort, - type KtxSqlAnalysisHttpJsonRunner, - type SqlAnalysisPort, -} from '@ktx/context/sql-analysis'; -import type { KtxCliIo } from './cli-runtime.js'; -import { - ensureManagedPythonCommandRuntime, - type KtxManagedPythonInstallPolicy, - type ManagedPythonCommandRuntime, -} from './managed-python-command.js'; -import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; - -export type ManagedPythonHttpJsonRunner = ( - path: string, - payload: Record, -) => Promise>; - -export type ManagedPythonHttpPostJson = ( - baseUrl: string, - path: string, - payload: Record, -) => Promise>; - -export interface ManagedPythonCoreDaemonOptions { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxCliIo; - ensureRuntime?: (options: { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxCliIo; - feature: 'core'; - }) => Promise; - startDaemon?: (options: { - cliVersion: string; - features: ['core']; - force: false; - }) => Promise; -} - -export type ManagedPythonDaemonHttpOptions = - | { - requestJson: ManagedPythonHttpJsonRunner; - } - | { - resolveBaseUrl: () => Promise; - postJson?: ManagedPythonHttpPostJson; - } - | (ManagedPythonCoreDaemonOptions & { - postJson?: ManagedPythonHttpPostJson; - }); - -function normalizedBaseUrl(baseUrl: string): string { - return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`; -} - -function parseJsonObject(raw: string, path: string): Record { - const parsed = JSON.parse(raw) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error(`KTX managed daemon HTTP ${path} returned non-object JSON`); - } - return parsed as Record; -} - -export async function postManagedDaemonJson( - baseUrl: string, - path: string, - payload: Record, -): Promise> { - return await new Promise((resolve, reject) => { - const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl)); - const body = JSON.stringify(payload); - const client = target.protocol === 'https:' ? httpsRequest : httpRequest; - const request = client( - target, - { - method: 'POST', - headers: { - accept: 'application/json', - 'content-type': 'application/json', - 'content-length': Buffer.byteLength(body), - }, - }, - (response) => { - const chunks: Buffer[] = []; - response.on('data', (chunk: Buffer) => chunks.push(chunk)); - response.on('end', () => { - const text = Buffer.concat(chunks).toString('utf8'); - const statusCode = response.statusCode ?? 0; - if (statusCode < 200 || statusCode >= 300) { - reject(new Error(`KTX managed daemon HTTP ${path} failed with ${statusCode}: ${text}`)); - return; - } - try { - resolve(parseJsonObject(text, path)); - } catch (error) { - reject(error); - } - }); - }, - ); - request.on('error', reject); - request.end(body); - }); -} - -export function createManagedPythonDaemonBaseUrlResolver( - options: ManagedPythonCoreDaemonOptions, -): () => Promise { - let cachedBaseUrl: string | undefined; - - return async () => { - if (cachedBaseUrl) { - return cachedBaseUrl; - } - - const ensureRuntime = options.ensureRuntime ?? ensureManagedPythonCommandRuntime; - const startDaemon = options.startDaemon ?? startManagedPythonDaemon; - await ensureRuntime({ - cliVersion: options.cliVersion, - installPolicy: options.installPolicy, - io: options.io, - feature: 'core', - }); - const daemon = await startDaemon({ - cliVersion: options.cliVersion, - features: ['core'], - force: false, - }); - const verb = daemon.status === 'started' ? 'Started' : 'Using existing'; - options.io.stderr.write(`${verb} KTX Python daemon: ${daemon.baseUrl}\n`); - cachedBaseUrl = daemon.baseUrl; - return cachedBaseUrl; - }; -} - -function isRequestJsonOnly(options: ManagedPythonDaemonHttpOptions): options is { requestJson: ManagedPythonHttpJsonRunner } { - return 'requestJson' in options; -} - -function isResolveBaseUrlOnly( - options: ManagedPythonDaemonHttpOptions, -): options is { resolveBaseUrl: () => Promise; postJson?: ManagedPythonHttpPostJson } { - return 'resolveBaseUrl' in options; -} - -export function createManagedDaemonHttpJsonRunner( - options: ManagedPythonDaemonHttpOptions, -): ManagedPythonHttpJsonRunner { - if (isRequestJsonOnly(options)) { - return options.requestJson; - } - const resolveBaseUrl = isResolveBaseUrlOnly(options) - ? options.resolveBaseUrl - : createManagedPythonDaemonBaseUrlResolver(options); - const postJson = options.postJson ?? postManagedDaemonJson; - - return async (path, payload) => postJson(await resolveBaseUrl(), path, payload); -} - -export function createManagedDaemonLookerTableIdentifierParser( - options: ManagedPythonDaemonHttpOptions, -): LookerTableIdentifierParser { - return createDaemonLookerTableIdentifierParser({ - baseUrl: 'http://127.0.0.1:0', - requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonTableIdentifierHttpJsonRunner, - }); -} - -export function createManagedDaemonSqlAnalysisPort(options: ManagedPythonDaemonHttpOptions): SqlAnalysisPort { - return createHttpSqlAnalysisPort({ - baseUrl: 'http://127.0.0.1:0', - requestJson: createManagedDaemonHttpJsonRunner(options) as KtxSqlAnalysisHttpJsonRunner, - }); -} - -export function managedDaemonDatabaseIntrospectionOptions( - options: ManagedPythonDaemonHttpOptions, -): Pick { - return { - requestJson: createManagedDaemonHttpJsonRunner(options) as KtxDaemonDatabaseHttpJsonRunner, - }; -} -``` - -- [ ] **Step 4: Verify the helper tests pass** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-http.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the helper** - -Run: - -```bash -git add packages/cli/src/managed-python-http.ts packages/cli/src/managed-python-http.test.ts -git commit -m "feat(cli): add managed daemon HTTP helpers" -``` - -Expected: commit succeeds. - -### Task 2: Wire managed daemon options into CLI local adapters - -**Files:** - -- Modify: `packages/cli/src/local-adapters.ts` -- Test: `packages/cli/src/managed-python-http.test.ts` - -- [ ] **Step 1: Update local adapter imports** - -In `packages/cli/src/local-adapters.ts`, add this import after the -`createHttpSqlAnalysisPort` import: - -```typescript -import { - createManagedDaemonLookerTableIdentifierParser, - createManagedDaemonSqlAnalysisPort, - managedDaemonDatabaseIntrospectionOptions, - type ManagedPythonCoreDaemonOptions, -} from './managed-python-http.js'; -``` - -- [ ] **Step 2: Add managed daemon options to the local adapter option type** - -Replace this interface: - -```typescript -interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { - historicSqlConnectionId?: string; - sqlAnalysisUrl?: string; -} -``` - -with this interface: - -```typescript -export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions { - historicSqlConnectionId?: string; - sqlAnalysisUrl?: string; - managedDaemon?: ManagedPythonCoreDaemonOptions; -} -``` - -- [ ] **Step 3: Add helper functions for managed daemon adapter options** - -Add these helpers immediately after `hasSnowflakeDriver()`: - -```typescript -function ktxCliDaemonDatabaseIntrospectionOptions( - options: KtxCliLocalIngestAdaptersOptions, -): DefaultLocalIngestAdaptersOptions['databaseIntrospection'] { - if (options.databaseIntrospectionUrl || options.databaseIntrospection?.requestJson || !options.managedDaemon) { - return options.databaseIntrospection; - } - return { - ...(options.databaseIntrospection ?? {}), - ...managedDaemonDatabaseIntrospectionOptions(options.managedDaemon), - }; -} - -function ktxCliLookerOptions( - options: KtxCliLocalIngestAdaptersOptions, -): DefaultLocalIngestAdaptersOptions['looker'] { - const looker = options.looker; - if (looker?.parser || looker?.daemonBaseUrl || process.env.KTX_DAEMON_URL || !options.managedDaemon) { - return looker; - } - return { - ...(looker ?? {}), - parser: createManagedDaemonLookerTableIdentifierParser(options.managedDaemon), - }; -} - -function ktxCliHistoricSqlAnalysis(options: KtxCliLocalIngestAdaptersOptions) { - if (options.sqlAnalysisUrl) { - return createHttpSqlAnalysisPort({ baseUrl: options.sqlAnalysisUrl }); - } - if (process.env.KTX_SQL_ANALYSIS_URL) { - return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_SQL_ANALYSIS_URL }); - } - if (process.env.KTX_DAEMON_URL) { - return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_DAEMON_URL }); - } - if (options.managedDaemon) { - return createManagedDaemonSqlAnalysisPort(options.managedDaemon); - } - return createHttpSqlAnalysisPort({ baseUrl: 'http://127.0.0.1:8765' }); -} -``` - -- [ ] **Step 4: Use managed daemon request options for daemon live-database fallback** - -In `createKtxCliLiveDatabaseIntrospection()`, insert this line before the -`const daemon = createDaemonLiveDatabaseIntrospection({` statement: - -```typescript - const databaseIntrospection = ktxCliDaemonDatabaseIntrospectionOptions(options); -``` - -Then replace the daemon creation block: - -```typescript - const daemon = createDaemonLiveDatabaseIntrospection({ - connections: project.config.connections, - ...options.databaseIntrospection, - ...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}), - }); -``` - -with this block: - -```typescript - const daemon = createDaemonLiveDatabaseIntrospection({ - connections: project.config.connections, - ...databaseIntrospection, - ...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}), - }); -``` - -- [ ] **Step 5: Use managed daemon SQL analysis for historic SQL** - -In `historicSqlOptionsForLocalRun()`, replace this block: - -```typescript - return { - sqlAnalysis: createHttpSqlAnalysisPort({ - baseUrl: - options.sqlAnalysisUrl ?? - process.env.KTX_SQL_ANALYSIS_URL ?? - process.env.KTX_DAEMON_URL ?? - 'http://127.0.0.1:8765', - }), - postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), - postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), - }; -``` - -with this block: - -```typescript - return { - sqlAnalysis: ktxCliHistoricSqlAnalysis(options), - postgresQueryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), - postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), - }; -``` - -- [ ] **Step 6: Pass managed Looker options into default local adapters** - -In `createKtxCliLocalIngestAdapters()`, replace: - -```typescript - const base = createDefaultLocalIngestAdapters(project, { - ...options, - ...(historicSql ? { historicSql } : {}), - }); -``` - -with: - -```typescript - const base = createDefaultLocalIngestAdapters(project, { - ...options, - databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options), - looker: ktxCliLookerOptions(options), - ...(historicSql ? { historicSql } : {}), - }); -``` - -- [ ] **Step 7: Run the CLI type check for local adapter changes** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 8: Commit local adapter wiring** - -Run: - -```bash -git add packages/cli/src/local-adapters.ts -git commit -m "feat(cli): route local adapters through managed daemon" -``` - -Expected: commit succeeds. - -### Task 3: Thread managed daemon options through ingest commands - -**Files:** - -- Modify: `packages/cli/src/ingest.ts` -- Modify: `packages/cli/src/ingest.test.ts` -- Modify: `packages/cli/src/commands/ingest-commands.ts` -- Test: `packages/cli/src/ingest.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing ingest option-threading tests** - -In `packages/cli/src/ingest.test.ts`, add this test after -`passes daemon database introspection URL to default local ingest adapters`: - -```typescript - it('passes managed daemon options to adapters and pull-config options when no explicit daemon URL is set', async () => { - const projectDir = join(tempDir, 'managed-daemon-ingest-project'); - await initKtxProject({ projectDir, projectName: 'managed-daemon-ingest-project' }); - await writeWarehouseConfig(projectDir); - const createdAdapters: SourceAdapter[] = [ - { source: 'fake', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, - ]; - const createAdapters = vi.fn(() => createdAdapters as never); - const runLocal = vi.fn(async (input: RunLocalIngestOptions) => - completedLocalBundleRun(input, input.jobId ?? 'local-job-1'), - ); - const io = makeIo(); - - await expect( - runKtxIngest( - { - command: 'run', - projectDir, - connectionId: 'warehouse', - adapter: 'fake', - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - outputMode: 'plain', - } satisfies KtxIngestArgs, - io.io, - { - createAdapters, - runLocalIngest: runLocal, - jobIdFactory: () => 'local-job-1', - }, - ), - ).resolves.toBe(0); - - const expectedManagedDaemon = { - cliVersion: '0.2.0', - installPolicy: 'auto', - io: io.io, - }; - expect(createAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir }), { - managedDaemon: expectedManagedDaemon, - }); - expect(runLocal).toHaveBeenCalledWith( - expect.objectContaining({ - pullConfigOptions: { - managedDaemon: expectedManagedDaemon, - }, - }), - ); - }); -``` - -In the existing `passes daemon database introspection URL to default local ingest -adapters` test, add this assertion inside the existing `expect(runLocal)` block: - -```typescript - pullConfigOptions: { - databaseIntrospectionUrl: 'http://127.0.0.1:8765', - }, -``` - -- [ ] **Step 2: Run the failing ingest tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/ingest.test.ts -``` - -Expected: FAIL because `KtxIngestArgs` has no `cliVersion` or -`runtimeInstallPolicy`, and `runKtxIngest()` does not pass managed daemon -options into `createAdapters()` or `pullConfigOptions`. - -- [ ] **Step 3: Add runtime install policy fields to ingest args** - -In `packages/cli/src/ingest.ts`, add this import after the local adapters -import: - -```typescript -import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -``` - -In the `KtxIngestArgs` `command: 'run'` branch, add these fields after -`databaseIntrospectionUrl?: string;`: - -```typescript - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; -``` - -- [ ] **Step 4: Add a managed daemon option helper to ingest** - -In `packages/cli/src/ingest.ts`, add this helper after -`initialRunMemoryFlowInput()`: - -```typescript -function managedDaemonOptionsForIngestRun( - args: Extract, - io: KtxIngestIo, -) { - if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) { - return undefined; - } - return { - cliVersion: args.cliVersion, - installPolicy: args.runtimeInstallPolicy, - io, - }; -} -``` - -- [ ] **Step 5: Pass managed daemon options to adapters and pull-config resolution** - -In the `args.command === 'run'` branch of `runKtxIngest()`, replace the -`adapterOptions` block: - -```typescript - const adapterOptions = { - ...(localIngestOptions.pullConfigOptions ?? {}), - ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), - ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}), - }; -``` - -with: - -```typescript - const managedDaemon = managedDaemonOptionsForIngestRun(args, io); - const adapterOptions = { - ...(localIngestOptions.pullConfigOptions ?? {}), - ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), - ...(managedDaemon ? { managedDaemon } : {}), - ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}), - }; -``` - -In the non-Metabase `executeLocalIngest()` call, move `...localIngestOptions` -before `pullConfigOptions` and add `pullConfigOptions: adapterOptions`. -The call must contain this sequence after the edit: - -```typescript - const result = await executeLocalIngest({ - project, - adapters: createAdapters(project, adapterOptions), - adapter: args.adapter, - connectionId: args.connectionId, - sourceDir: args.sourceDir, - trigger: 'manual_resync', - jobId, - ...localIngestOptions, - pullConfigOptions: adapterOptions, - ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}), - ...(memoryFlow ? { memoryFlow } : {}), - }); -``` - -- [ ] **Step 6: Add runtime flags to `ktx ingest run` routing** - -In `packages/cli/src/commands/ingest-commands.ts`, add this import after the -`KtxCliDeps` import: - -```typescript -import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; -``` - -In the `ingest run` command options, add this option immediately before -`.option('--no-input', ...)`: - -```typescript - .option('--yes', 'Install the managed Python runtime without prompting when required', false) -``` - -In the `KtxIngestArgs` object built for `ingest run`, add these fields after -`databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined,`: - -```typescript - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), -``` - -- [ ] **Step 7: Update Commander ingest routing expectations** - -In `packages/cli/src/index.test.ts`, in the test that routes -`dev ingest run`, add these expected fields after -`databaseIntrospectionUrl: undefined,`: - -```typescript - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'never', -``` - -Add this test after that existing routing test: - -```typescript - it('routes ingest managed runtime install policies', async () => { - const autoIo = makeIo(); - const conflictIo = makeIo(); - const ingest = vi.fn(async () => 0); - - await expect( - runKtxCli( - [ - 'dev', - 'ingest', - 'run', - '--project-dir', - tempDir, - '--connection-id', - 'warehouse', - '--adapter', - 'looker', - '--yes', - ], - autoIo.io, - { ingest }, - ), - ).resolves.toBe(0); - await expect( - runKtxCli( - [ - 'dev', - 'ingest', - 'run', - '--project-dir', - tempDir, - '--connection-id', - 'warehouse', - '--adapter', - 'looker', - '--yes', - '--no-input', - ], - conflictIo.io, - { ingest }, - ), - ).resolves.toBe(1); - - expect(ingest).toHaveBeenCalledWith( - expect.objectContaining({ - command: 'run', - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'auto', - }), - autoIo.io, - ); - expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); - }); -``` - -- [ ] **Step 8: Run focused ingest and routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/ingest.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Commit ingest runtime policy wiring** - -Run: - -```bash -git add packages/cli/src/ingest.ts packages/cli/src/ingest.test.ts packages/cli/src/commands/ingest-commands.ts packages/cli/src/index.test.ts -git commit -m "feat(cli): use managed daemon for ingest helpers" -``` - -Expected: commit succeeds. - -### Task 4: Thread managed daemon options through scan commands - -**Files:** - -- Modify: `packages/cli/src/scan.ts` -- Modify: `packages/cli/src/scan.test.ts` -- Modify: `packages/cli/src/commands/scan-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Test: `packages/cli/src/scan.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing scan option-threading test** - -In `packages/cli/src/scan.test.ts`, add this test after the test that passes -`databaseIntrospectionUrl`: - -```typescript - it('passes managed daemon options to local ingest adapters when no explicit daemon URL is set', async () => { - const report = minimalScanReport(); - const createLocalIngestAdapters = vi.fn(() => []); - const runLocalScan = vi.fn( - async (_input: RunLocalScanOptions): Promise => ({ - runId: 'scan-run-1', - status: 'done', - done: true, - connectionId: 'warehouse', - mode: 'structural', - dryRun: false, - syncId: 'sync-1', - report, - }), - ); - const io = makeIo(); - - await expect( - runKtxScan( - { - command: 'run', - projectDir: tempDir, - connectionId: 'warehouse', - mode: 'structural', - detectRelationships: false, - dryRun: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - }, - io.io, - { runLocalScan, createLocalIngestAdapters }, - ), - ).resolves.toBe(0); - - expect(createLocalIngestAdapters).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir }), { - managedDaemon: { - cliVersion: '0.2.0', - installPolicy: 'auto', - io: io.io, - }, - }); - }); -``` - -- [ ] **Step 2: Run the failing scan tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/scan.test.ts -``` - -Expected: FAIL because `KtxScanArgs` has no `cliVersion` or -`runtimeInstallPolicy`, and `runKtxScan()` does not pass managed daemon options -to adapter creation. - -- [ ] **Step 3: Add runtime install policy fields to scan args** - -In `packages/cli/src/scan.ts`, add this import after the local adapter import: - -```typescript -import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -``` - -In the `KtxScanArgs` `command: 'run'` branch, add these fields after -`databaseIntrospectionUrl?: string;`: - -```typescript - cliVersion?: string; - runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; -``` - -- [ ] **Step 4: Add managed daemon option construction to scan** - -In `packages/cli/src/scan.ts`, add this helper after `warningLine()`: - -```typescript -function managedDaemonOptionsForScanRun(args: Extract, io: KtxCliIo) { - if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) { - return undefined; - } - return { - cliVersion: args.cliVersion, - installPolicy: args.runtimeInstallPolicy, - io, - }; -} -``` - -In the `runLocalScan()` call, replace this adapter creation block: - -```typescript - adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { - databaseIntrospectionUrl: args.databaseIntrospectionUrl, - }), -``` - -with: - -```typescript - adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { - ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), - ...(managedDaemonOptionsForScanRun(args, io) - ? { managedDaemon: managedDaemonOptionsForScanRun(args, io) } - : {}), - }), -``` - -Then replace the repeated helper call with a local constant to keep the code -single-pass. The final block must be: - -```typescript - const managedDaemon = managedDaemonOptionsForScanRun(args, io); - const connector = - args.mode !== 'structural' || args.detectRelationships - ? await createKtxCliScanConnector(project, args.connectionId) - : undefined; - const progress = createCliScanProgress(io); - try { - const result = await (deps.runLocalScan ?? runLocalScan)({ - project, - connectionId: args.connectionId, - mode: args.mode, - detectRelationships: args.detectRelationships, - dryRun: args.dryRun, - trigger: 'cli', - databaseIntrospectionUrl: args.databaseIntrospectionUrl, - connector, - adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, { - ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), - ...(managedDaemon ? { managedDaemon } : {}), - }), - progress, - }); -``` - -- [ ] **Step 5: Add runtime flags to scan routing** - -In `packages/cli/src/commands/scan-commands.ts`, add this import after the -`cli-program.js` import: - -```typescript -import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; -``` - -In the top-level `scan` command options, add these options after -`--database-introspection-url`: - -```typescript - .option('--yes', 'Install the managed Python runtime without prompting when required', false) - .option('--no-input', 'Disable interactive managed runtime installation') -``` - -In the scan run action, add these fields after -`databaseIntrospectionUrl: options.databaseIntrospectionUrl,`: - -```typescript - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options), -``` - -- [ ] **Step 6: Update Commander scan routing expectations** - -In `packages/cli/src/index.test.ts`, update the `routes low-level scan through -ktx dev with top-level project-dir` expected args by adding: - -```typescript - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'prompt', -``` - -Add this test after that routing test: - -```typescript - it('routes scan managed runtime install policies', async () => { - const autoIo = makeIo(); - const neverIo = makeIo(); - const conflictIo = makeIo(); - const scan = vi.fn().mockResolvedValue(0); - - await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes'], autoIo.io, { scan })) - .resolves.toBe(0); - await expect(runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--no-input'], neverIo.io, { scan })) - .resolves.toBe(0); - await expect( - runKtxCli(['--project-dir', tempDir, 'dev', 'scan', 'warehouse', '--yes', '--no-input'], conflictIo.io, { - scan, - }), - ).resolves.toBe(1); - - expect(scan).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - command: 'run', - runtimeInstallPolicy: 'auto', - }), - autoIo.io, - ); - expect(scan).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - command: 'run', - runtimeInstallPolicy: 'never', - }), - neverIo.io, - ); - expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); - }); -``` - -- [ ] **Step 7: Run focused scan and routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/scan.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit scan runtime policy wiring** - -Run: - -```bash -git add packages/cli/src/scan.ts packages/cli/src/scan.test.ts packages/cli/src/commands/scan-commands.ts packages/cli/src/index.test.ts -git commit -m "feat(cli): pass managed daemon options to scan" -``` - -Expected: commit succeeds. - -### Task 5: Pass pull-config options through MCP local ingest - -**Files:** - -- Modify: `packages/context/src/ingest/local-ingest.ts` -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` -- Test: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write failing MCP pull-config forwarding test** - -In `packages/context/src/mcp/local-project-ports.test.ts`, add this test in -the local ingest tool describe block, next to the existing local ingest tests: - -```typescript - it('passes local ingest pull-config options into runLocalIngest', async () => { - const runLocalIngest = vi.fn(async () => ({ - result: { ok: true }, - report: { - id: 'report-1', - runId: 'run-1', - jobId: 'job-1', - sourceKey: 'looker', - connectionId: 'warehouse', - body: { - syncId: 'sync-1', - workUnits: [], - failedWorkUnits: [], - diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 }, - provenanceRows: [], - }, - }, - } as never)); - const ports = createLocalProjectMcpContextPorts(project, { - localIngest: { - adapters: [{ source: 'looker', skillNames: [] }], - pullConfigOptions: { - looker: { - daemonBaseUrl: 'http://127.0.0.1:61234', - }, - }, - runLocalIngest, - }, - }); - - await expect( - ports.ingest.run({ - adapter: 'looker', - connectionId: 'warehouse', - trigger: 'manual_resync', - config: {}, - }), - ).resolves.toMatchObject({ - runId: 'run-1', - jobId: 'job-1', - reportId: 'report-1', - }); - - expect(runLocalIngest).toHaveBeenCalledWith( - expect.objectContaining({ - pullConfigOptions: { - looker: { - daemonBaseUrl: 'http://127.0.0.1:61234', - }, - }, - }), - ); - }); -``` - -- [ ] **Step 2: Run the failing MCP test** - -Run: - -```bash -pnpm --filter @ktx/context run test -- src/mcp/local-project-ports.test.ts -``` - -Expected: FAIL because `LocalIngestMcpOptions` does not accept -`pullConfigOptions`, and MCP local ingest does not pass it to -`runLocalIngest()`. - -- [ ] **Step 3: Add pull-config options to MCP local ingest options** - -In `packages/context/src/ingest/local-ingest.ts`, update -`LocalIngestMcpOptions` so the `Pick` includes -`'pullConfigOptions'`. The interface must contain this sequence after the edit: - -```typescript -export interface LocalIngestMcpOptions - extends Pick< - RunLocalIngestOptions, - | 'agentRunner' - | 'llmProvider' - | 'memoryModel' - | 'semanticLayerCompute' - | 'queryExecutor' - | 'logger' - | 'pullConfigOptions' - > { - adapters?: SourceAdapter[]; - jobIdFactory?: () => string; - runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise; -} -``` - -- [ ] **Step 4: Pass pull-config options in MCP local ingest execution** - -In `packages/context/src/mcp/local-project-ports.ts`, in the -`runLocalIngest({ ... })` call, add this field after `sourceDir,`: - -```typescript - pullConfigOptions: options.localIngest?.pullConfigOptions, -``` - -- [ ] **Step 5: Run MCP tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -- src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit MCP pull-config forwarding** - -Run: - -```bash -git add packages/context/src/ingest/local-ingest.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat(context): pass MCP ingest pull config options" -``` - -Expected: commit succeeds. - -### Task 6: Wire managed daemon options through MCP serve - -**Files:** - -- Modify: `packages/cli/src/serve.ts` -- Modify: `packages/cli/src/serve.test.ts` -- Test: `packages/cli/src/serve.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing serve managed daemon wiring test** - -In `packages/cli/src/serve.test.ts`, add this test after -`uses managed semantic compute when MCP semantic compute has no explicit HTTP -URL`: - -```typescript - it('passes managed daemon options to MCP local ingest adapters and pull-config options', async () => { - const project = { projectDir: '/tmp/ktx-project', config: { connections: {} } } as never; - const adapters = [{ source: 'looker', skillNames: [] }]; - const createIngestAdapters = vi.fn(() => adapters); - const createContextTools = vi.fn(() => ({ connections: { list: async () => [] } })); - const managedRuntimeIo = makeManagedRuntimeIo(); - - await expect( - runKtxServeStdio( - { - mcp: 'stdio', - projectDir: '/tmp/ktx-project', - userId: 'agent', - semanticCompute: false, - semanticComputeUrl: undefined, - databaseIntrospectionUrl: undefined, - executeQueries: false, - memoryCapture: false, - memoryModel: undefined, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - }, - { - loadProject: async () => project, - createContextTools, - createIngestAdapters, - managedRuntimeIo: managedRuntimeIo.io, - createServer: vi.fn(() => ({ connect: vi.fn(async () => undefined) }) as never), - createTransport: vi.fn(() => ({}) as never), - stderr: { write: vi.fn() }, - }, - ), - ).resolves.toBe(0); - - const expectedManagedDaemon = { - cliVersion: '0.2.0', - installPolicy: 'auto', - io: managedRuntimeIo.io, - }; - expect(createIngestAdapters).toHaveBeenCalledWith(project, { - managedDaemon: expectedManagedDaemon, - }); - expect(createContextTools).toHaveBeenCalledWith( - project, - expect.objectContaining({ - localIngest: expect.objectContaining({ - adapters, - pullConfigOptions: { - managedDaemon: expectedManagedDaemon, - }, - }), - }), - ); - }); -``` - -Add this assertion to the existing test that passes -`databaseIntrospectionUrl: 'http://127.0.0.1:8765'`: - -```typescript - localIngest: expect.objectContaining({ - pullConfigOptions: { - databaseIntrospectionUrl: 'http://127.0.0.1:8765', - }, - }), -``` - -- [ ] **Step 2: Run the failing serve tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/serve.test.ts -``` - -Expected: FAIL because `runKtxServeStdio()` does not pass managed daemon -options or pull-config options into local ingest. - -- [ ] **Step 3: Add serve managed daemon option helper** - -In `packages/cli/src/serve.ts`, add this import after the managed command -import: - -```typescript -import type { ManagedPythonCoreDaemonOptions } from './managed-python-http.js'; -``` - -Add this helper after `requiredManagedRuntimeCliVersion()`: - -```typescript -function managedDaemonOptionsForServe( - args: KtxServeArgs, - deps: KtxServeDeps, -): ManagedPythonCoreDaemonOptions | undefined { - if (args.databaseIntrospectionUrl || !args.cliVersion) { - return undefined; - } - return { - cliVersion: args.cliVersion, - installPolicy: args.runtimeInstallPolicy ?? 'prompt', - io: deps.managedRuntimeIo ?? process, - }; -} -``` - -- [ ] **Step 4: Pass managed daemon options to serve local ingest** - -In `runKtxServeStdio()`, replace this block: - -```typescript - const createIngestAdapters = deps.createIngestAdapters ?? createKtxCliLocalIngestAdapters; - const localAdapters = createIngestAdapters(project, { - databaseIntrospectionUrl: args.databaseIntrospectionUrl, - }); -``` - -with: - -```typescript - const createIngestAdapters = deps.createIngestAdapters ?? createKtxCliLocalIngestAdapters; - const managedDaemon = managedDaemonOptionsForServe(args, deps); - const localAdapterOptions = { - ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}), - ...(managedDaemon ? { managedDaemon } : {}), - }; - const localAdapters = createIngestAdapters(project, localAdapterOptions); -``` - -In the `localIngest` object, add this field after `adapters: localAdapters,`: - -```typescript - pullConfigOptions: localAdapterOptions, -``` - -- [ ] **Step 5: Run serve and routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/serve.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit serve managed daemon wiring** - -Run: - -```bash -git add packages/cli/src/serve.ts packages/cli/src/serve.test.ts -git commit -m "feat(cli): pass managed daemon options to serve ingest" -``` - -Expected: commit succeeds. - -### Task 7: Verify managed local ingest daemon integration - -**Files:** - -- Verify: `packages/cli/src/managed-python-http.ts` -- Verify: `packages/cli/src/local-adapters.ts` -- Verify: `packages/cli/src/ingest.ts` -- Verify: `packages/cli/src/scan.ts` -- Verify: `packages/cli/src/serve.ts` -- Verify: `packages/context/src/ingest/local-ingest.ts` -- Verify: `packages/context/src/mcp/local-project-ports.ts` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-http.test.ts src/ingest.test.ts src/scan.test.ts src/serve.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused context tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -- src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run affected package type checks** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -pnpm --filter @ktx/context run type-check -``` - -Expected: both commands PASS. - -- [ ] **Step 4: Run the broader TypeScript test surface** - -Run: - -```bash -pnpm --filter @ktx/cli run test -pnpm --filter @ktx/context run test -``` - -Expected: both commands PASS. - -- [ ] **Step 5: Commit verification-only fixes if needed** - -If Step 1 through Step 4 require mechanical test expectation or type fixes, run: - -```bash -git add packages/cli/src packages/context/src -git commit -m "test: verify managed local ingest daemon runtime" -``` - -Expected: commit succeeds only when files changed during verification. If no -files changed, skip this commit. - -## Self-review - -Spec coverage: - -- The plan uses the managed core runtime and daemon for Python-backed local - ingest helper behavior. -- The plan preserves explicit daemon URLs and environment-variable override - behavior. -- The plan keeps the first-use installation policy aligned with existing - `--yes`, `--no-input`, and prompt semantics. -- The plan avoids local embedding dependency installation by requesting only - the `core` runtime feature. - -Placeholder scan: - -- No placeholder markers remain in the task steps. -- Every code-changing step includes the exact code block or replacement to use. - -Type consistency: - -- The new managed daemon option type is named `ManagedPythonCoreDaemonOptions`. -- CLI runtime policy fields use the existing - `KtxManagedPythonInstallPolicy` type. -- MCP local ingest reuses the existing `DefaultLocalIngestAdaptersOptions` - through `RunLocalIngestOptions['pullConfigOptions']`. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md deleted file mode 100644 index 733b6915..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md +++ /dev/null @@ -1,935 +0,0 @@ -# Managed Python Runtime Command Integration Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `ktx sl query` use the KTX-managed bundled Python runtime -instead of relying on a user-provided `python -m ktx_daemon`. - -**Architecture:** Add a small CLI helper that resolves the managed runtime, -installs the `core` feature when policy permits it, and creates the existing -`@ktx/context/daemon` one-shot semantic-layer compute port with the managed -`ktx-daemon` executable. Wire `ktx sl query` to pass an explicit runtime -install policy from `--yes`, `--no-input`, or the default interactive mode. - -**Tech Stack:** TypeScript, Commander, Vitest, `@clack/prompts`, -`@ktx/context/daemon`, existing KTX managed runtime installer. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -Existing plans based on the spec: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is - implemented. The worktree contains - `scripts/build-python-runtime-wheel.mjs`, - `scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel packaging in - `scripts/package-artifacts.mjs`, release-policy coverage, and matching - artifact tests. The targeted verification passes: - `node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs`. -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is - implemented. The worktree contains - `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, - `packages/cli/src/commands/runtime-commands.ts`, CLI registration, and - matching Vitest coverage. The targeted CLI verification passes: - `pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts`. - -Spec requirements still outside this plan: - -- `ktx runtime start` and `ktx runtime stop`. -- Managed HTTP daemon state, health checks, reuse, and stale daemon repair. -- Lazy `local-embeddings` installation and local embedding daemon reuse. -- Public npm package rename from `@ktx/cli` to `@kaelio/ktx`. - -This plan implements the next runnable user path: `ktx sl query` installs or -uses the managed `core` Python runtime according to the command's input policy. - -## File structure - -- Create `packages/cli/src/managed-python-command.ts`: CLI helper for managed - runtime policy, optional prompt, runtime install, and managed semantic-layer - compute port creation. -- Create `packages/cli/src/managed-python-command.test.ts`: unit tests for - ready runtime reuse, `--no-input` failure, `--yes` installation, and - interactive prompt acceptance. -- Modify `packages/cli/src/sl.ts`: extend `KtxSlArgs` with CLI version and - runtime install policy for `query`, and use the managed helper when no test - compute port is injected. -- Modify `packages/cli/src/sl.test.ts`: update existing `query` arguments and - assert `runKtxSl` delegates default compute creation to the managed helper. -- Modify `packages/cli/src/commands/sl-commands.ts`: add `--yes` and - `--no-input` to `sl query`, derive the runtime install policy, and pass the - CLI package version. -- Modify `packages/cli/src/command-schemas.ts`: validate `cliVersion` and - `runtimeInstallPolicy` on parsed `sl query` arguments. -- Modify `packages/cli/src/index.test.ts`: assert Commander routes the new - `sl query` runtime policy flags. - -### Task 1: Add failing managed Python command helper tests - -**Files:** - -- Create: `packages/cli/src/managed-python-command.test.ts` -- Test: `packages/cli/src/managed-python-command.test.ts` - -- [ ] **Step 1: Write the failing test file** - -Create `packages/cli/src/managed-python-command.test.ts` with this content: - -```typescript -import { describe, expect, it, vi } from 'vitest'; -import { - createManagedPythonSemanticLayerComputePort, - managedRuntimeInstallCommand, -} from './managed-python-command.js'; -import type { - InstalledKtxRuntimeManifest, - KtxRuntimeFeature, - ManagedPythonRuntimeInstallResult, - ManagedPythonRuntimeLayout, - ManagedPythonRuntimeStatus, -} from './managed-python-runtime.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -function layout(): ManagedPythonRuntimeLayout { - return { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }; -} - -function manifest(features: KtxRuntimeFeature[] = ['core']): InstalledKtxRuntimeManifest { - return { - schemaVersion: 1, - cliVersion: '0.2.0', - installedAt: '2026-05-11T00:00:00.000Z', - asset: { - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.2.0', - wheel: { - file: 'kaelio_ktx-0.2.0-py3-none-any.whl', - sha256: 'a'.repeat(64), - bytes: 123, - }, - }, - features, - python: { - executable: '/runtime/0.2.0/.venv/bin/python', - daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }, - installLog: '/runtime/0.2.0/install.log', - }; -} - -function readyStatus(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeStatus { - return { - kind: 'ready', - detail: 'Runtime ready at /runtime/0.2.0', - layout: layout(), - manifest: manifest(features), - }; -} - -function missingStatus(): ManagedPythonRuntimeStatus { - return { - kind: 'missing', - detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', - layout: layout(), - }; -} - -function installResult(features: KtxRuntimeFeature[] = ['core']): ManagedPythonRuntimeInstallResult { - const installedManifest = manifest(features); - return { - status: 'installed', - layout: layout(), - asset: { - manifest: installedManifest.asset, - wheelPath: '/assets/python/kaelio_ktx-0.2.0-py3-none-any.whl', - }, - manifest: installedManifest, - }; -} - -describe('managedRuntimeInstallCommand', () => { - it('prints the exact command for each managed runtime feature', () => { - expect(managedRuntimeInstallCommand('core')).toBe('ktx runtime install --yes'); - expect(managedRuntimeInstallCommand('local-embeddings')).toBe( - 'ktx runtime install --feature local-embeddings --yes', - ); - }); -}); - -describe('createManagedPythonSemanticLayerComputePort', () => { - it('uses the managed ktx-daemon executable when the runtime is ready', async () => { - const io = makeIo(); - const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; - const createPythonCompute = vi.fn(() => compute); - - await expect( - createManagedPythonSemanticLayerComputePort({ - cliVersion: '0.2.0', - installPolicy: 'never', - io: io.io, - readStatus: vi.fn(async () => readyStatus()), - installRuntime: vi.fn(), - createPythonCompute, - }), - ).resolves.toBe(compute); - - expect(createPythonCompute).toHaveBeenCalledWith({ - command: '/runtime/0.2.0/.venv/bin/ktx-daemon', - args: [], - }); - expect(io.stderr()).toBe(''); - }); - - it('fails with a preparation command when input is disabled and the runtime is missing', async () => { - const io = makeIo(); - const installRuntime = vi.fn(); - - await expect( - createManagedPythonSemanticLayerComputePort({ - cliVersion: '0.2.0', - installPolicy: 'never', - io: io.io, - readStatus: vi.fn(async () => missingStatus()), - installRuntime, - }), - ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx runtime install --yes'); - - expect(installRuntime).not.toHaveBeenCalled(); - }); - - it('installs the core runtime without prompting when policy is auto', async () => { - const io = makeIo(); - const compute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() }; - const createPythonCompute = vi.fn(() => compute); - const installRuntime = vi.fn(async () => installResult()); - - await expect( - createManagedPythonSemanticLayerComputePort({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: io.io, - readStatus: vi.fn(async () => missingStatus()), - installRuntime, - createPythonCompute, - }), - ).resolves.toBe(compute); - - expect(installRuntime).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - features: ['core'], - force: false, - }); - expect(io.stderr()).toContain('Installing KTX Python runtime (core) with uv'); - expect(io.stderr()).toContain('KTX Python runtime ready: /runtime/0.2.0'); - }); - - it('prompts before installing when policy is prompt', async () => { - const io = makeIo(); - const confirmInstall = vi.fn(async () => true); - const installRuntime = vi.fn(async () => installResult()); - - await createManagedPythonSemanticLayerComputePort({ - cliVersion: '0.2.0', - installPolicy: 'prompt', - io: io.io, - readStatus: vi.fn(async () => missingStatus()), - installRuntime, - createPythonCompute: vi.fn(() => ({ query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() })), - confirmInstall, - }); - - expect(confirmInstall).toHaveBeenCalledWith( - 'KTX needs to install the core Python runtime. This downloads Python dependencies with uv. Continue?', - ); - expect(installRuntime).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - features: ['core'], - force: false, - }); - }); -}); -``` - -- [ ] **Step 2: Run the failing test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts -``` - -Expected: FAIL with an import error for -`./managed-python-command.js`. - -### Task 2: Implement the managed Python command helper - -**Files:** - -- Create: `packages/cli/src/managed-python-command.ts` -- Test: `packages/cli/src/managed-python-command.test.ts` - -- [ ] **Step 1: Create the helper** - -Create `packages/cli/src/managed-python-command.ts` with this content: - -```typescript -import { cancel, confirm, isCancel } from '@clack/prompts'; -import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; -import type { KtxCliIo } from './cli-runtime.js'; -import { - installManagedPythonRuntime, - readManagedPythonRuntimeStatus, - type InstalledKtxRuntimeManifest, - type KtxRuntimeFeature, - type ManagedPythonRuntimeInstallOptions, - type ManagedPythonRuntimeInstallResult, - type ManagedPythonRuntimeLayout, - type ManagedPythonRuntimeLayoutOptions, - type ManagedPythonRuntimeStatus, -} from './managed-python-runtime.js'; - -export type KtxManagedPythonInstallPolicy = 'prompt' | 'auto' | 'never'; - -export interface ManagedPythonCommandRuntime { - layout: ManagedPythonRuntimeLayout; - manifest: InstalledKtxRuntimeManifest; -} - -export interface ManagedPythonCommandDeps { - readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; - installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; - confirmInstall?: (message: string) => Promise; -} - -export interface ManagedPythonCommandOptions extends ManagedPythonCommandDeps { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxCliIo; - feature?: KtxRuntimeFeature; -} - -export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonCommandOptions { - createPythonCompute?: typeof createPythonSemanticLayerComputePort; -} - -export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string { - return feature === 'local-embeddings' - ? 'ktx runtime install --feature local-embeddings --yes' - : 'ktx runtime install --yes'; -} - -function installPrompt(feature: KtxRuntimeFeature): string { - const label = feature === 'local-embeddings' ? 'local embeddings Python runtime' : 'core Python runtime'; - return `KTX needs to install the ${label}. This downloads Python dependencies with uv. Continue?`; -} - -function runtimeRequiredMessage(feature: KtxRuntimeFeature): string { - return `KTX Python runtime is required for this command. Run: ${managedRuntimeInstallCommand(feature)}`; -} - -function hasFeature(manifest: InstalledKtxRuntimeManifest, feature: KtxRuntimeFeature): boolean { - return manifest.features.includes(feature); -} - -async function defaultConfirmInstall(message: string): Promise { - if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) { - return false; - } - const response = await confirm({ message, initialValue: true }); - if (isCancel(response)) { - cancel('Runtime installation cancelled.'); - return false; - } - return response === true; -} - -export async function ensureManagedPythonCommandRuntime( - options: ManagedPythonCommandOptions, -): Promise { - const feature = options.feature ?? 'core'; - const readStatus = options.readStatus ?? readManagedPythonRuntimeStatus; - const installRuntime = options.installRuntime ?? installManagedPythonRuntime; - const status = await readStatus({ cliVersion: options.cliVersion }); - - if (status.kind === 'ready' && status.manifest && hasFeature(status.manifest, feature)) { - return { layout: status.layout, manifest: status.manifest }; - } - - if (options.installPolicy === 'never') { - throw new Error(runtimeRequiredMessage(feature)); - } - - if (options.installPolicy === 'prompt') { - const confirmInstall = options.confirmInstall ?? defaultConfirmInstall; - const confirmed = await confirmInstall(installPrompt(feature)); - if (!confirmed) { - throw new Error(`KTX Python runtime installation was cancelled. Run: ${managedRuntimeInstallCommand(feature)}`); - } - } - - options.io.stderr.write(`Installing KTX Python runtime (${feature}) with uv...\n`); - const installed = await installRuntime({ - cliVersion: options.cliVersion, - features: [feature], - force: false, - }); - options.io.stderr.write(`KTX Python runtime ready: ${installed.layout.versionDir}\n`); - return { layout: installed.layout, manifest: installed.manifest }; -} - -export async function createManagedPythonSemanticLayerComputePort( - options: ManagedPythonSemanticLayerComputeOptions, -): Promise { - const runtime = await ensureManagedPythonCommandRuntime({ - cliVersion: options.cliVersion, - installPolicy: options.installPolicy, - io: options.io, - feature: 'core', - ...(options.readStatus ? { readStatus: options.readStatus } : {}), - ...(options.installRuntime ? { installRuntime: options.installRuntime } : {}), - ...(options.confirmInstall ? { confirmInstall: options.confirmInstall } : {}), - }); - const createPythonCompute = options.createPythonCompute ?? createPythonSemanticLayerComputePort; - return createPythonCompute({ - command: runtime.manifest.python.daemonExecutable, - args: [], - }); -} -``` - -- [ ] **Step 2: Run the helper test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Commit** - -Run: - -```bash -git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts -git commit -m "feat: add managed python command helper" -``` - -Expected: commit succeeds. - -### Task 3: Add failing `runKtxSl` managed runtime tests - -**Files:** - -- Modify: `packages/cli/src/sl.test.ts` -- Test: `packages/cli/src/sl.test.ts` - -- [ ] **Step 1: Add runtime fields to existing `query` test args** - -In each existing `runKtxSl` call whose argument object has -`command: 'query'`, add these properties: - -```typescript -cliVersion: '0.2.0', -runtimeInstallPolicy: 'auto', -``` - -For example, the first `query` argument object becomes: - -```typescript -{ - command: 'query', - projectDir: '/tmp/project', - connectionId: 'warehouse', - query: { measures: ['orders.order_count'], dimensions: [] }, - format: 'sql', - execute: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', -} -``` - -- [ ] **Step 2: Add the managed helper delegation test** - -In `packages/cli/src/sl.test.ts`, add this test inside -`describe('runKtxSl', () => { ... })` after the existing -`runs sl query and prints SQL output` test: - -```typescript - it('creates default sl query compute through the managed runtime helper', async () => { - const projectDir = join(tempDir, 'project'); - const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; - await project.fileStore.writeFile( - 'semantic-layer/warehouse/orders.yaml', - `name: orders -table: public.orders -grain: [id] -columns: - - name: id - type: number -measures: - - name: order_count - expr: count(*) -joins: [] -`, - 'ktx', - 'ktx@example.com', - 'Add orders source', - ); - - const stdout = { write: vi.fn() }; - const stderr = { write: vi.fn() }; - const compute = { - query: vi.fn(async () => ({ - sql: 'select count(*) as order_count from public.orders', - dialect: 'postgres', - columns: [{ name: 'orders.order_count' }], - plan: {}, - })), - validateSources: vi.fn(), - generateSources: vi.fn(), - }; - const createManagedSemanticLayerCompute = vi.fn(async () => compute); - - await expect( - runKtxSl( - { - command: 'query', - projectDir, - connectionId: 'warehouse', - query: { measures: ['orders.order_count'], dimensions: [] }, - format: 'sql', - execute: false, - cliVersion: '0.2.0', - runtimeInstallPolicy: 'auto', - }, - { stdout, stderr }, - { createManagedSemanticLayerCompute }, - ), - ).resolves.toBe(0); - - expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - installPolicy: 'auto', - io: { stdout, stderr }, - }); - expect(stdout.write).toHaveBeenCalledWith('select count(*) as order_count from public.orders\n'); - }); -``` - -- [ ] **Step 3: Run the failing `sl` test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/sl.test.ts -``` - -Expected: FAIL with a TypeScript/Vitest error because `runKtxSl` does not -accept `createManagedSemanticLayerCompute` yet. - -### Task 4: Wire `runKtxSl` to the managed helper - -**Files:** - -- Modify: `packages/cli/src/sl.ts` -- Test: `packages/cli/src/sl.test.ts` - -- [ ] **Step 1: Add the managed helper imports** - -In `packages/cli/src/sl.ts`, add this import after the existing imports: - -```typescript -import { - createManagedPythonSemanticLayerComputePort, - type KtxManagedPythonInstallPolicy, -} from './managed-python-command.js'; -``` - -- [ ] **Step 2: Extend the `query` args type** - -In the `KtxSlArgs` union, replace the current `query` object type with this -shape: - -```typescript - | { - command: 'query'; - projectDir: string; - connectionId?: string; - query: SemanticLayerQueryInput; - format: SlQueryFormat; - execute: boolean; - maxRows?: number; - cliVersion: string; - runtimeInstallPolicy: KtxManagedPythonInstallPolicy; - }; -``` - -- [ ] **Step 3: Extend `KtxSlDeps`** - -In `packages/cli/src/sl.ts`, replace `KtxSlDeps` with this interface: - -```typescript -interface KtxSlDeps { - loadProject?: typeof loadKtxProject; - createSemanticLayerCompute?: () => KtxSemanticLayerComputePort; - createManagedSemanticLayerCompute?: (options: { - cliVersion: string; - installPolicy: KtxManagedPythonInstallPolicy; - io: KtxSlIo; - }) => Promise; - createQueryExecutor?: () => KtxSqlQueryExecutorPort; -} -``` - -- [ ] **Step 4: Use the managed helper in the `query` branch** - -In the `args.command === 'query'` branch, replace: - -```typescript - const compute = (deps.createSemanticLayerCompute ?? createPythonSemanticLayerComputePort)(); -``` - -with: - -```typescript - const compute = deps.createSemanticLayerCompute - ? deps.createSemanticLayerCompute() - : await (deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort)({ - cliVersion: args.cliVersion, - installPolicy: args.runtimeInstallPolicy, - io, - }); -``` - -- [ ] **Step 5: Run the `sl` test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/sl.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -Run: - -```bash -git add packages/cli/src/sl.ts packages/cli/src/sl.test.ts -git commit -m "feat: use managed runtime for sl query compute" -``` - -Expected: commit succeeds. - -### Task 5: Add failing Commander routing tests for `sl query` - -**Files:** - -- Modify: `packages/cli/src/index.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Add routing tests** - -In `packages/cli/src/index.test.ts`, add this test near the other command -routing tests: - -```typescript - it('routes sl query managed runtime install policies', async () => { - const sl = vi.fn(async () => 0); - - const promptIo = makeIo(); - await expect( - runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count'], promptIo.io, { sl }), - ).resolves.toBe(0); - expect(sl).toHaveBeenLastCalledWith( - expect.objectContaining({ - command: 'query', - projectDir: tempDir, - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'prompt', - query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }), - }), - promptIo.io, - ); - - const autoIo = makeIo(); - await expect( - runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes'], autoIo.io, { - sl, - }), - ).resolves.toBe(0); - expect(sl).toHaveBeenLastCalledWith( - expect.objectContaining({ - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'auto', - }), - autoIo.io, - ); - - const noInputIo = makeIo(); - await expect( - runKtxCli( - ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'], - noInputIo.io, - { sl }, - ), - ).resolves.toBe(0); - expect(sl).toHaveBeenLastCalledWith( - expect.objectContaining({ - cliVersion: '0.0.0-private', - runtimeInstallPolicy: 'never', - }), - noInputIo.io, - ); - }); - - it('rejects conflicting sl query runtime install flags', async () => { - const io = makeIo(); - const sl = vi.fn(async () => 0); - - await expect( - runKtxCli( - ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes', '--no-input'], - io.io, - { sl }, - ), - ).resolves.toBe(1); - - expect(sl).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); - }); -``` - -- [ ] **Step 2: Run the failing routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -Expected: FAIL because `sl query` does not accept `--yes` or `--no-input` -and does not pass runtime policy fields yet. - -### Task 6: Wire `sl query` flags and schema validation - -**Files:** - -- Modify: `packages/cli/src/commands/sl-commands.ts` -- Modify: `packages/cli/src/command-schemas.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Add the runtime policy type import** - -In `packages/cli/src/commands/sl-commands.ts`, add this import: - -```typescript -import type { KtxManagedPythonInstallPolicy } from '../managed-python-command.js'; -``` - -- [ ] **Step 2: Add the runtime policy parser** - -In `packages/cli/src/commands/sl-commands.ts`, add this function near the -other option parsers: - -```typescript -function runtimeInstallPolicy(options: { yes?: boolean; input?: boolean }): KtxManagedPythonInstallPolicy { - if (options.yes === true && options.input === false) { - throw new Error('Choose only one runtime install mode: --yes or --no-input'); - } - if (options.yes === true) { - return 'auto'; - } - return options.input === false ? 'never' : 'prompt'; -} -``` - -- [ ] **Step 3: Add the command options** - -In the `sl.command('query')` option chain, add these options after -`.option('--execute', 'Execute the compiled query', false)`: - -```typescript - .option('--yes', 'Install the managed Python runtime without prompting when required', false) - .option('--no-input', 'Disable interactive managed runtime installation') -``` - -- [ ] **Step 4: Pass runtime fields into `slQueryCommandSchema.parse`** - -In the `sl.command('query')` action, add these properties to the parsed object: - -```typescript - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicy(options), -``` - -The parsed object must include these fields next to `execute` and `format`: - -```typescript - const args = slQueryCommandSchema.parse({ - command: 'query', - projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, - query: { - measures: options.measure, - dimensions: options.dimension, - ...(options.filter.length > 0 ? { filters: options.filter } : {}), - ...(options.segment.length > 0 ? { segments: options.segment } : {}), - ...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}), - ...(options.limit !== undefined ? { limit: options.limit } : {}), - ...(options.includeEmpty === true ? { include_empty: true } : {}), - }, - format: options.format, - execute: options.execute === true, - cliVersion: context.packageInfo.version, - runtimeInstallPolicy: runtimeInstallPolicy(options), - ...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}), - }); -``` - -- [ ] **Step 5: Extend the command schema** - -In `packages/cli/src/command-schemas.ts`, add these fields to -`slQueryCommandSchema` after `execute: z.boolean()`: - -```typescript - cliVersion: z.string().min(1), - runtimeInstallPolicy: z.enum(['prompt', 'auto', 'never']), -``` - -- [ ] **Step 6: Run the routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts -git commit -m "feat: route sl query managed runtime policy" -``` - -Expected: commit succeeds. - -### Task 7: Verify the full changed surface - -**Files:** - -- Verify: `packages/cli/src/managed-python-command.test.ts` -- Verify: `packages/cli/src/sl.test.ts` -- Verify: `packages/cli/src/index.test.ts` -- Verify: `packages/cli/src/managed-python-command.ts` -- Verify: `packages/cli/src/sl.ts` -- Verify: `packages/cli/src/commands/sl-commands.ts` -- Verify: `packages/cli/src/command-schemas.ts` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-command.test.ts src/sl.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type checking** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run pre-commit for changed TypeScript files** - -Run: - -```bash -uv run pre-commit run --files packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/sl.ts packages/cli/src/sl.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts -``` - -Expected: PASS. If pre-commit is unavailable because the local `uv` version -does not satisfy `pyproject.toml`, record the version mismatch and run the -focused CLI tests plus type checking from Steps 1 and 2. - -- [ ] **Step 4: Commit verification fixes when needed** - -If Step 1, Step 2, or Step 3 changes files through formatting hooks, run: - -```bash -git add packages/cli/src/managed-python-command.ts packages/cli/src/managed-python-command.test.ts packages/cli/src/sl.ts packages/cli/src/sl.test.ts packages/cli/src/commands/sl-commands.ts packages/cli/src/command-schemas.ts packages/cli/src/index.test.ts -git commit -m "test: verify managed runtime sl query integration" -``` - -Expected: commit succeeds only when verification changed files. If no files -changed, leave the branch with the commits from Tasks 2, 4, and 6. - -## Acceptance criteria - -When this plan is complete: - -- `ktx sl query` uses the managed runtime's installed `ktx-daemon` executable - for semantic-layer compilation when no test compute dependency is injected. -- `ktx sl query --yes` installs the `core` runtime feature without prompting - when the managed runtime is missing. -- `ktx sl query --no-input` fails with - `KTX Python runtime is required for this command. Run: ktx runtime install --yes` - when the managed runtime is missing. -- `ktx sl query` prompts before first managed runtime installation in an - interactive terminal. -- Existing injected-compute tests still bypass runtime installation. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md deleted file mode 100644 index 280ec728..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md +++ /dev/null @@ -1,1546 +0,0 @@ -# Managed Python Runtime Daemon Lifecycle Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `ktx runtime start` and `ktx runtime stop` for the -KTX-managed Python HTTP daemon, including state files, health checks, reuse, -and stale daemon repair. - -**Architecture:** Keep daemon process management in a new CLI-owned module that -depends on the existing managed runtime installer. The module starts -`ktx-daemon serve-http` from the installed runtime on `127.0.0.1`, writes an -adjacent daemon state file, verifies `/health` before reuse, and removes stale -state when the process, port, version, or requested feature set no longer -matches. - -**Tech Stack:** TypeScript, Node 22 ESM, Commander, Vitest, `zod`, FastAPI, -`uvicorn`, `uv`, KTX managed runtime assets. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -Existing plans based on the spec: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` is - implemented. The worktree contains - `scripts/build-python-runtime-wheel.mjs`, - `scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel packaging in - `scripts/package-artifacts.mjs`, release-policy coverage, and matching - artifact tests. -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` is - implemented. The worktree contains - `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, - `packages/cli/src/commands/runtime-commands.ts`, CLI registration, and - matching Vitest coverage. -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` - is implemented. The worktree contains - `packages/cli/src/managed-python-command.ts`, `ktx sl query` runtime policy - flags, schema validation, and matching CLI tests. - -Implementation evidence collected before writing this plan: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs -``` - -Expected current result: - -```text -# pass 38 -# fail 0 -``` - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts src/managed-python-command.test.ts src/sl.test.ts -``` - -Expected current result: - -```text -Test Files 58 passed (58) -Tests 699 passed (699) -``` - -Spec requirements still outside this plan: - -- Lazy `local-embeddings` installation and daemon reuse from embedding setup, - embedding health checks, and ingest paths. -- Managed runtime usage for Python-backed operations beyond `ktx sl query`. -- Public npm package rename from `@ktx/cli` to `@kaelio/ktx`. - -This plan implements the daemon lifecycle requirement: - -- `ktx runtime start` -- `ktx runtime stop` -- A versioned daemon state file adjacent to the installed runtime manifest. -- Random localhost port allocation. -- Captured daemon stdout and stderr logs. -- `/health` validation before daemon reuse. -- Stale daemon cleanup when process, health, version, or features don't match. - -## File structure - -- Modify `python/ktx-daemon/src/ktx_daemon/app.py`: include a daemon version in - `/health`, supplied by `KTX_DAEMON_VERSION` for managed runtime starts. -- Modify `python/ktx-daemon/tests/test_app.py`: assert the health endpoint - returns the managed version when the environment variable is set. -- Modify `packages/cli/src/managed-python-runtime.ts`: add daemon state and log - paths to `ManagedPythonRuntimeLayout`. -- Modify `packages/cli/src/managed-python-runtime.test.ts`: assert the new - layout paths. -- Modify `packages/cli/src/runtime.test.ts` and - `packages/cli/src/managed-python-command.test.ts`: add daemon paths to - layout fixtures after the layout type changes. -- Create `packages/cli/src/managed-python-daemon.ts`: start, stop, status, - health-check, stale-state, and state-file logic for the managed HTTP daemon. -- Create `packages/cli/src/managed-python-daemon.test.ts`: unit tests for - stopped status, start, reuse, stale repair, and stop. -- Modify `packages/cli/src/runtime.ts`: route `runtime start` and - `runtime stop` through the daemon lifecycle module and print concise output. -- Modify `packages/cli/src/runtime.test.ts`: assert command runner behavior for - start and stop. -- Modify `packages/cli/src/commands/runtime-commands.ts`: register - `ktx runtime start` and `ktx runtime stop`, and accept `--yes` on - `ktx runtime install` so the preparation command printed by - `ktx sl query --no-input` is valid. -- Modify `packages/cli/src/index.test.ts`: assert Commander routes the new - runtime subcommands with the CLI package version. -- Modify `packages/cli/src/index.ts`: export the daemon lifecycle helpers for - tests and programmatic use. - -### Task 1: Add daemon metadata to runtime layout and Python health - -**Files:** - -- Modify: `packages/cli/src/managed-python-runtime.ts` -- Modify: `packages/cli/src/managed-python-runtime.test.ts` -- Modify: `packages/cli/src/runtime.test.ts` -- Modify: `packages/cli/src/managed-python-command.test.ts` -- Modify: `python/ktx-daemon/src/ktx_daemon/app.py` -- Modify: `python/ktx-daemon/tests/test_app.py` - -- [ ] **Step 1: Write failing TypeScript layout assertions** - -In `packages/cli/src/managed-python-runtime.test.ts`, update the first -`managedPythonRuntimeLayout` test so it includes these expectations after the -existing `daemonPath` assertion: - -```typescript - expect(layout.daemonStatePath).toBe( - '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.json', - ); - expect(layout.daemonStdoutPath).toBe( - '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stdout.log', - ); - expect(layout.daemonStderrPath).toBe( - '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/daemon.stderr.log', - ); -``` - -- [ ] **Step 2: Run the failing layout test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts -``` - -Expected: FAIL with TypeScript or assertion errors for missing -`daemonStatePath`, `daemonStdoutPath`, and `daemonStderrPath`. - -- [ ] **Step 3: Add daemon paths to the runtime layout type** - -In `packages/cli/src/managed-python-runtime.ts`, add these fields to -`ManagedPythonRuntimeLayout` immediately after `daemonPath`: - -```typescript - daemonStatePath: string; - daemonStdoutPath: string; - daemonStderrPath: string; -``` - -In `managedPythonRuntimeLayout`, add these properties to the returned object -immediately after `daemonPath`: - -```typescript - daemonStatePath: join(versionDir, 'daemon.json'), - daemonStdoutPath: join(versionDir, 'daemon.stdout.log'), - daemonStderrPath: join(versionDir, 'daemon.stderr.log'), -``` - -- [ ] **Step 4: Update layout fixtures used by existing tests** - -In `packages/cli/src/runtime.test.ts`, every object literal that represents a -`ManagedPythonRuntimeLayout` must include these fields: - -```typescript - daemonStatePath: '/runtime/0.2.0/daemon.json', - daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', - daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', -``` - -In `packages/cli/src/managed-python-command.test.ts`, update the `layout()` -helper to return these fields: - -```typescript - daemonStatePath: '/runtime/0.2.0/daemon.json', - daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', - daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', -``` - -- [ ] **Step 5: Verify the TypeScript layout change** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/runtime.test.ts src/managed-python-command.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Write the failing Python health-version test** - -In `python/ktx-daemon/tests/test_app.py`, add this test after -`test_health_endpoint_returns_healthy`: - -```python -def test_health_endpoint_returns_managed_runtime_version(monkeypatch) -> None: - monkeypatch.setenv("KTX_DAEMON_VERSION", "0.2.0") - client = TestClient(create_app()) - - response = client.get("/health") - - assert response.status_code == 200 - assert response.json() == {"status": "healthy", "version": "0.2.0"} -``` - -- [ ] **Step 7: Run the failing Python health test** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py::test_health_endpoint_returns_managed_runtime_version -q -``` - -Expected: FAIL because `/health` does not include `version`. - -- [ ] **Step 8: Include version metadata in daemon health** - -In `python/ktx-daemon/src/ktx_daemon/app.py`, add this import with the existing -imports: - -```python -import os -``` - -Replace the `health` endpoint with: - -```python - @app.get("/health") - async def health() -> dict[str, str]: - response = {"status": "healthy"} - version = os.environ.get("KTX_DAEMON_VERSION") - if version: - response["version"] = version - return response -``` - -- [ ] **Step 9: Verify Python health tests** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py -q -``` - -Expected: PASS. - -- [ ] **Step 10: Run Python pre-commit for modified Python files** - -Run: - -```bash -source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py -``` - -Expected: PASS. If pre-commit cannot run because hooks or tool versions are -missing, capture the error and run: - -```bash -source .venv/bin/activate && uv run ruff check python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py -``` - -- [ ] **Step 11: Commit** - -Run: - -```bash -git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/runtime.test.ts packages/cli/src/managed-python-command.test.ts python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py -git commit -m "feat: add managed runtime daemon metadata" -``` - -### Task 2: Implement managed daemon lifecycle library - -**Files:** - -- Create: `packages/cli/src/managed-python-daemon.test.ts` -- Create: `packages/cli/src/managed-python-daemon.ts` -- Test: `packages/cli/src/managed-python-daemon.test.ts` - -- [ ] **Step 1: Write the failing daemon lifecycle tests** - -Create `packages/cli/src/managed-python-daemon.test.ts` with this content: - -```typescript -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - readManagedPythonDaemonStatus, - startManagedPythonDaemon, - stopManagedPythonDaemon, - type ManagedPythonDaemonChild, - type ManagedPythonDaemonFetch, - type ManagedPythonDaemonSpawn, - type ManagedPythonDaemonState, -} from './managed-python-daemon.js'; -import type { - InstalledKtxRuntimeManifest, - ManagedPythonRuntimeInstallResult, - ManagedPythonRuntimeLayout, -} from './managed-python-runtime.js'; - -function layout(root: string): ManagedPythonRuntimeLayout { - return { - cliVersion: '0.2.0', - runtimeRoot: join(root, 'runtime'), - versionDir: join(root, 'runtime', '0.2.0'), - venvDir: join(root, 'runtime', '0.2.0', '.venv'), - manifestPath: join(root, 'runtime', '0.2.0', 'manifest.json'), - installLogPath: join(root, 'runtime', '0.2.0', 'install.log'), - assetDir: join(root, 'assets', 'python'), - assetManifestPath: join(root, 'assets', 'python', 'manifest.json'), - pythonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'python'), - daemonPath: join(root, 'runtime', '0.2.0', '.venv', 'bin', 'ktx-daemon'), - daemonStatePath: join(root, 'runtime', '0.2.0', 'daemon.json'), - daemonStdoutPath: join(root, 'runtime', '0.2.0', 'daemon.stdout.log'), - daemonStderrPath: join(root, 'runtime', '0.2.0', 'daemon.stderr.log'), - }; -} - -function manifest(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): InstalledKtxRuntimeManifest { - const runtimeLayout = layout(root); - return { - schemaVersion: 1, - cliVersion: '0.2.0', - installedAt: '2026-05-11T00:00:00.000Z', - asset: { - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.2.0', - wheel: { - file: 'kaelio_ktx-0.2.0-py3-none-any.whl', - sha256: 'a'.repeat(64), - bytes: 123, - }, - }, - features, - python: { - executable: runtimeLayout.pythonPath, - daemonExecutable: runtimeLayout.daemonPath, - }, - installLog: runtimeLayout.installLogPath, - }; -} - -function installResult(root: string, features: Array<'core' | 'local-embeddings'> = ['core']): ManagedPythonRuntimeInstallResult { - return { - status: 'ready', - layout: layout(root), - asset: { - manifest: manifest(root, features).asset, - wheelPath: join(root, 'assets', 'python', 'kaelio_ktx-0.2.0-py3-none-any.whl'), - }, - manifest: manifest(root, features), - }; -} - -function makeFetch(version = '0.2.0'): ManagedPythonDaemonFetch { - return vi.fn(async () => ({ - ok: true, - status: 200, - json: async () => ({ status: 'healthy', version }), - text: async () => '', - })); -} - -function makeSpawn(pid = 4242): ManagedPythonDaemonSpawn { - return vi.fn((_command, _args, _options): ManagedPythonDaemonChild => ({ - pid, - unref: vi.fn(), - })); -} - -function runningState(root: string, overrides: Partial = {}): ManagedPythonDaemonState { - const runtimeLayout = layout(root); - return { - schemaVersion: 1, - pid: 4242, - host: '127.0.0.1', - port: 58731, - version: '0.2.0', - features: ['core'], - startedAt: '2026-05-11T00:00:00.000Z', - stdoutLog: runtimeLayout.daemonStdoutPath, - stderrLog: runtimeLayout.daemonStderrPath, - ...overrides, - }; -} - -describe('managed Python daemon lifecycle', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-managed-daemon-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('reports stopped when no daemon state exists', async () => { - const status = await readManagedPythonDaemonStatus({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - processAlive: vi.fn(() => false), - fetch: makeFetch(), - }); - - expect(status.kind).toBe('stopped'); - expect(status.detail).toContain('No daemon state'); - }); - - it('starts ktx-daemon serve-http, waits for health, and writes state', async () => { - const spawnDaemon = makeSpawn(5555); - const installRuntime = vi.fn(async () => installResult(tempDir)); - - const result = await startManagedPythonDaemon({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - features: ['core'], - installRuntime, - spawnDaemon, - fetch: makeFetch(), - allocatePort: vi.fn(async () => 61234), - now: () => new Date('2026-05-11T00:00:00.000Z'), - pollIntervalMs: 1, - }); - - expect(result.status).toBe('started'); - expect(result.baseUrl).toBe('http://127.0.0.1:61234'); - expect(installRuntime).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - features: ['core'], - force: false, - }); - expect(spawnDaemon).toHaveBeenCalledWith( - layout(tempDir).daemonPath, - ['serve-http', '--host', '127.0.0.1', '--port', '61234'], - expect.objectContaining({ - detached: true, - env: expect.objectContaining({ KTX_DAEMON_VERSION: '0.2.0' }), - }), - ); - expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({ - pid: 5555, - port: 61234, - version: '0.2.0', - features: ['core'], - stdoutLog: layout(tempDir).daemonStdoutPath, - stderrLog: layout(tempDir).daemonStderrPath, - }); - }); - - it('reuses a healthy daemon with the requested feature set', async () => { - await mkdir(layout(tempDir).versionDir, { recursive: true }); - await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); - const spawnDaemon = makeSpawn(9999); - - const result = await startManagedPythonDaemon({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - features: ['core'], - installRuntime: vi.fn(async () => installResult(tempDir)), - spawnDaemon, - fetch: makeFetch(), - processAlive: vi.fn(() => true), - pollIntervalMs: 1, - }); - - expect(result.status).toBe('reused'); - expect(result.baseUrl).toBe('http://127.0.0.1:58731'); - expect(spawnDaemon).not.toHaveBeenCalled(); - }); - - it('starts a fresh daemon when the previous state is stale', async () => { - await mkdir(layout(tempDir).versionDir, { recursive: true }); - await writeFile( - layout(tempDir).daemonStatePath, - `${JSON.stringify(runningState(tempDir, { version: '0.1.0' }), null, 2)}\n`, - ); - - const result = await startManagedPythonDaemon({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - features: ['core'], - installRuntime: vi.fn(async () => installResult(tempDir)), - spawnDaemon: makeSpawn(6666), - fetch: makeFetch(), - processAlive: vi.fn(() => true), - killProcess: vi.fn(), - allocatePort: vi.fn(async () => 61235), - now: () => new Date('2026-05-11T00:00:00.000Z'), - pollIntervalMs: 1, - }); - - expect(result.status).toBe('started'); - expect(JSON.parse(await readFile(layout(tempDir).daemonStatePath, 'utf8'))).toMatchObject({ - pid: 6666, - port: 61235, - version: '0.2.0', - }); - }); - - it('stops a recorded daemon and removes the state file', async () => { - await mkdir(layout(tempDir).versionDir, { recursive: true }); - await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); - const killProcess = vi.fn(); - - const result = await stopManagedPythonDaemon({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - processAlive: vi.fn(() => true), - killProcess, - }); - - expect(result.status).toBe('stopped'); - expect(killProcess).toHaveBeenCalledWith(4242); - await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toThrow(); - }); -}); -``` - -- [ ] **Step 2: Run the failing daemon lifecycle tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-daemon.test.ts -``` - -Expected: FAIL with an import error for `./managed-python-daemon.js`. - -- [ ] **Step 3: Implement the daemon lifecycle module** - -Create `packages/cli/src/managed-python-daemon.ts` with this content: - -```typescript -import { spawn } from 'node:child_process'; -import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises'; -import { createServer } from 'node:net'; -import { setTimeout as delay } from 'node:timers/promises'; -import { z } from 'zod'; -import { - installManagedPythonRuntime, - managedPythonRuntimeLayout, - runtimeFeatureSchema, - type KtxRuntimeFeature, - type ManagedPythonRuntimeInstallOptions, - type ManagedPythonRuntimeInstallResult, - type ManagedPythonRuntimeLayout, - type ManagedPythonRuntimeLayoutOptions, -} from './managed-python-runtime.js'; - -export interface ManagedPythonDaemonState { - schemaVersion: 1; - pid: number; - host: '127.0.0.1'; - port: number; - version: string; - features: KtxRuntimeFeature[]; - startedAt: string; - stdoutLog: string; - stderrLog: string; -} - -export type ManagedPythonDaemonStatus = - | { kind: 'stopped'; detail: string; layout: ManagedPythonRuntimeLayout } - | { kind: 'running'; detail: string; layout: ManagedPythonRuntimeLayout; state: ManagedPythonDaemonState; baseUrl: string } - | { kind: 'stale'; detail: string; layout: ManagedPythonRuntimeLayout; state?: ManagedPythonDaemonState }; - -export interface ManagedPythonDaemonStartResult { - status: 'started' | 'reused'; - layout: ManagedPythonRuntimeLayout; - state: ManagedPythonDaemonState; - baseUrl: string; -} - -export interface ManagedPythonDaemonStopResult { - status: 'stopped' | 'already-stopped'; - layout: ManagedPythonRuntimeLayout; - state?: ManagedPythonDaemonState; -} - -export interface ManagedPythonDaemonChild { - pid?: number; - unref(): void; -} - -export type ManagedPythonDaemonSpawn = ( - command: string, - args: string[], - options: { - detached: boolean; - stdio: ['ignore', number, number]; - env: NodeJS.ProcessEnv; - }, -) => ManagedPythonDaemonChild; - -export type ManagedPythonDaemonFetch = ( - url: string, -) => Promise<{ - ok: boolean; - status: number; - json(): Promise; - text(): Promise; -}>; - -export interface ManagedPythonDaemonStartOptions extends ManagedPythonRuntimeLayoutOptions { - features: KtxRuntimeFeature[]; - force?: boolean; - installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; - spawnDaemon?: ManagedPythonDaemonSpawn; - fetch?: ManagedPythonDaemonFetch; - allocatePort?: () => Promise; - processAlive?: (pid: number) => boolean; - killProcess?: (pid: number) => void; - now?: () => Date; - startupTimeoutMs?: number; - pollIntervalMs?: number; -} - -export interface ManagedPythonDaemonStatusOptions extends ManagedPythonRuntimeLayoutOptions { - fetch?: ManagedPythonDaemonFetch; - processAlive?: (pid: number) => boolean; -} - -export interface ManagedPythonDaemonStopOptions extends ManagedPythonRuntimeLayoutOptions { - processAlive?: (pid: number) => boolean; - killProcess?: (pid: number) => void; -} - -const daemonStateSchema = z.object({ - schemaVersion: z.literal(1), - pid: z.number().int().positive(), - host: z.literal('127.0.0.1'), - port: z.number().int().min(1).max(65535), - version: z.string().min(1), - features: z.array(runtimeFeatureSchema).min(1), - startedAt: z.string().min(1), - stdoutLog: z.string().min(1), - stderrLog: z.string().min(1), -}); - -function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] { - const requested = new Set(['core', ...features]); - return runtimeFeatureSchema.options.filter((feature) => requested.has(feature)); -} - -function hasFeatures(state: ManagedPythonDaemonState, features: KtxRuntimeFeature[]): boolean { - return normalizeFeatures(features).every((feature) => state.features.includes(feature)); -} - -function defaultFetch(url: string): ReturnType { - return fetch(url) as ReturnType; -} - -function defaultProcessAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -function defaultKillProcess(pid: number): void { - try { - process.kill(pid, 'SIGTERM'); - } catch (error) { - const code = (error as { code?: unknown }).code; - if (code !== 'ESRCH') { - throw error; - } - } -} - -function defaultSpawnDaemon( - command: string, - args: string[], - options: Parameters[2], -): ManagedPythonDaemonChild { - return spawn(command, args, options); -} - -function baseUrl(state: Pick): string { - return `http://${state.host}:${state.port}`; -} - -async function readState(path: string): Promise { - try { - return daemonStateSchema.parse(JSON.parse(await readFile(path, 'utf8')) as unknown); - } catch (error) { - const code = (error as { code?: unknown }).code; - if (code === 'ENOENT') { - return undefined; - } - throw error; - } -} - -async function writeState(path: string, state: ManagedPythonDaemonState): Promise { - await writeFile(path, `${JSON.stringify(state, null, 2)}\n`); -} - -async function healthOk(input: { - state: ManagedPythonDaemonState; - cliVersion: string; - fetch: ManagedPythonDaemonFetch; -}): Promise<{ ok: true } | { ok: false; detail: string }> { - try { - const response = await input.fetch(`${baseUrl(input.state)}/health`); - if (!response.ok) { - return { ok: false, detail: `Health check returned HTTP ${response.status}: ${await response.text()}` }; - } - const body = (await response.json()) as unknown; - if (!body || typeof body !== 'object' || Array.isArray(body)) { - return { ok: false, detail: 'Health check returned non-object JSON' }; - } - const record = body as Record; - if (record.status !== 'healthy') { - return { ok: false, detail: `Health check returned status ${String(record.status)}` }; - } - if (record.version !== input.cliVersion) { - return { - ok: false, - detail: `Daemon version ${String(record.version)} does not match CLI ${input.cliVersion}`, - }; - } - return { ok: true }; - } catch (error) { - return { ok: false, detail: error instanceof Error ? error.message : String(error) }; - } -} - -export async function readManagedPythonDaemonStatus( - options: ManagedPythonDaemonStatusOptions, -): Promise { - const layout = managedPythonRuntimeLayout(options); - let state: ManagedPythonDaemonState | undefined; - try { - state = await readState(layout.daemonStatePath); - } catch (error) { - return { - kind: 'stale', - detail: `Daemon state is invalid: ${error instanceof Error ? error.message : String(error)}`, - layout, - }; - } - if (!state) { - return { kind: 'stopped', detail: `No daemon state at ${layout.daemonStatePath}`, layout }; - } - if (state.version !== options.cliVersion) { - return { - kind: 'stale', - detail: `Daemon is for CLI ${state.version}, current CLI is ${options.cliVersion}`, - layout, - state, - }; - } - const processAlive = options.processAlive ?? defaultProcessAlive; - if (!processAlive(state.pid)) { - return { kind: 'stale', detail: `Daemon process ${state.pid} is not running`, layout, state }; - } - const health = await healthOk({ - state, - cliVersion: options.cliVersion, - fetch: options.fetch ?? defaultFetch, - }); - if (!health.ok) { - return { kind: 'stale', detail: health.detail, layout, state }; - } - return { kind: 'running', detail: `Daemon running at ${baseUrl(state)}`, layout, state, baseUrl: baseUrl(state) }; -} - -export async function allocateDaemonPort(): Promise { - return await new Promise((resolve, reject) => { - const server = createServer(); - server.on('error', reject); - server.listen(0, '127.0.0.1', () => { - const address = server.address(); - server.close(() => { - if (address && typeof address === 'object') { - resolve(address.port); - return; - } - reject(new Error('Failed to allocate a daemon port')); - }); - }); - }); -} - -async function waitForHealth(input: { - state: ManagedPythonDaemonState; - cliVersion: string; - fetch: ManagedPythonDaemonFetch; - timeoutMs: number; - pollIntervalMs: number; -}): Promise { - const deadline = Date.now() + input.timeoutMs; - let lastDetail = 'daemon did not answer health checks'; - while (Date.now() <= deadline) { - const health = await healthOk({ - state: input.state, - cliVersion: input.cliVersion, - fetch: input.fetch, - }); - if (health.ok) { - return; - } - lastDetail = health.detail; - await delay(input.pollIntervalMs); - } - throw new Error(`KTX Python daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`); -} - -async function removeState(layout: ManagedPythonRuntimeLayout): Promise { - await rm(layout.daemonStatePath, { force: true }); -} - -async function stopRecordedDaemon(input: { - layout: ManagedPythonRuntimeLayout; - state: ManagedPythonDaemonState; - processAlive: (pid: number) => boolean; - killProcess: (pid: number) => void; -}): Promise { - if (input.processAlive(input.state.pid)) { - input.killProcess(input.state.pid); - } - await removeState(input.layout); -} - -export async function startManagedPythonDaemon( - options: ManagedPythonDaemonStartOptions, -): Promise { - const features = normalizeFeatures(options.features); - const installRuntime = options.installRuntime ?? installManagedPythonRuntime; - const layoutOverrides = { - ...(options.runtimeRoot !== undefined ? { runtimeRoot: options.runtimeRoot } : {}), - ...(options.assetDir !== undefined ? { assetDir: options.assetDir } : {}), - ...(options.platform !== undefined ? { platform: options.platform } : {}), - ...(options.env !== undefined ? { env: options.env } : {}), - ...(options.homeDir !== undefined ? { homeDir: options.homeDir } : {}), - }; - const layout = managedPythonRuntimeLayout({ cliVersion: options.cliVersion, ...layoutOverrides }); - const processAlive = options.processAlive ?? defaultProcessAlive; - const killProcess = options.killProcess ?? defaultKillProcess; - const fetchImpl = options.fetch ?? defaultFetch; - - const status = await readManagedPythonDaemonStatus({ - cliVersion: options.cliVersion, - ...layoutOverrides, - fetch: fetchImpl, - processAlive, - }); - if (options.force !== true && status.kind === 'running' && hasFeatures(status.state, features)) { - return { status: 'reused', layout, state: status.state, baseUrl: status.baseUrl }; - } - if (status.state) { - await stopRecordedDaemon({ layout, state: status.state, processAlive, killProcess }); - } else { - await removeState(layout); - } - - const installed = await installRuntime({ - cliVersion: options.cliVersion, - ...layoutOverrides, - features, - force: false, - }); - - await mkdir(layout.versionDir, { recursive: true }); - const stdout = await open(layout.daemonStdoutPath, 'a'); - const stderr = await open(layout.daemonStderrPath, 'a'); - try { - const port = await (options.allocatePort ?? allocateDaemonPort)(); - const spawnDaemon = options.spawnDaemon ?? defaultSpawnDaemon; - const child = spawnDaemon( - installed.manifest.python.daemonExecutable, - ['serve-http', '--host', '127.0.0.1', '--port', String(port)], - { - detached: true, - stdio: ['ignore', stdout.fd, stderr.fd], - env: { - ...process.env, - KTX_DAEMON_VERSION: options.cliVersion, - }, - }, - ); - child.unref(); - if (!child.pid) { - throw new Error(`KTX Python daemon did not report a pid. stderr: ${layout.daemonStderrPath}`); - } - const state: ManagedPythonDaemonState = { - schemaVersion: 1, - pid: child.pid, - host: '127.0.0.1', - port, - version: options.cliVersion, - features: installed.manifest.features, - startedAt: (options.now ?? (() => new Date()))().toISOString(), - stdoutLog: layout.daemonStdoutPath, - stderrLog: layout.daemonStderrPath, - }; - await waitForHealth({ - state, - cliVersion: options.cliVersion, - fetch: fetchImpl, - timeoutMs: options.startupTimeoutMs ?? 10_000, - pollIntervalMs: options.pollIntervalMs ?? 100, - }); - await writeState(layout.daemonStatePath, state); - return { status: 'started', layout, state, baseUrl: baseUrl(state) }; - } finally { - await stdout.close(); - await stderr.close(); - } -} - -export async function stopManagedPythonDaemon( - options: ManagedPythonDaemonStopOptions, -): Promise { - const layout = managedPythonRuntimeLayout(options); - const state = await readState(layout.daemonStatePath); - if (!state) { - return { status: 'already-stopped', layout }; - } - await stopRecordedDaemon({ - layout, - state, - processAlive: options.processAlive ?? defaultProcessAlive, - killProcess: options.killProcess ?? defaultKillProcess, - }); - return { status: 'stopped', layout, state }; -} -``` - -- [ ] **Step 4: Run daemon lifecycle tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-daemon.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/cli/src/managed-python-daemon.ts packages/cli/src/managed-python-daemon.test.ts -git commit -m "feat: manage python daemon lifecycle" -``` - -### Task 3: Wire runtime start and stop commands - -**Files:** - -- Modify: `packages/cli/src/runtime.ts` -- Modify: `packages/cli/src/runtime.test.ts` -- Modify: `packages/cli/src/commands/runtime-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Modify: `packages/cli/src/index.ts` - -- [ ] **Step 1: Write failing runtime command runner tests** - -In `packages/cli/src/runtime.test.ts`, add these imports: - -```typescript -import type { - ManagedPythonDaemonStartResult, - ManagedPythonDaemonStopResult, -} from './managed-python-daemon.js'; -``` - -Add these tests inside `describe('runKtxRuntime', () => { ... })` after the -install test: - -```typescript - it('starts the managed Python daemon and prints the base URL', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - startDaemon: vi.fn(async (): Promise => ({ - status: 'started', - baseUrl: 'http://127.0.0.1:61234', - layout: { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - daemonStatePath: '/runtime/0.2.0/daemon.json', - daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', - daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', - }, - state: { - schemaVersion: 1, - pid: 4242, - host: '127.0.0.1', - port: 61234, - version: '0.2.0', - features: ['core', 'local-embeddings'], - startedAt: '2026-05-11T00:00:00.000Z', - stdoutLog: '/runtime/0.2.0/daemon.stdout.log', - stderrLog: '/runtime/0.2.0/daemon.stderr.log', - }, - })), - }; - - await expect( - runKtxRuntime( - { command: 'start', cliVersion: '0.2.0', feature: 'local-embeddings', force: true }, - io.io, - deps, - ), - ).resolves.toBe(0); - - expect(deps.startDaemon).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - features: ['local-embeddings'], - force: true, - }); - expect(io.stdout()).toContain('Started KTX Python daemon'); - expect(io.stdout()).toContain('url: http://127.0.0.1:61234'); - expect(io.stdout()).toContain('pid: 4242'); - expect(io.stdout()).toContain('features: core, local-embeddings'); - expect(io.stdout()).toContain('stderr: /runtime/0.2.0/daemon.stderr.log'); - }); - - it('stops the managed Python daemon', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - stopDaemon: vi.fn(async (): Promise => ({ - status: 'stopped', - layout: { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - daemonStatePath: '/runtime/0.2.0/daemon.json', - daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log', - daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log', - }, - state: { - schemaVersion: 1, - pid: 4242, - host: '127.0.0.1', - port: 61234, - version: '0.2.0', - features: ['core'], - startedAt: '2026-05-11T00:00:00.000Z', - stdoutLog: '/runtime/0.2.0/daemon.stdout.log', - stderrLog: '/runtime/0.2.0/daemon.stderr.log', - }, - })), - }; - - await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0' }, io.io, deps)).resolves.toBe(0); - - expect(deps.stopDaemon).toHaveBeenCalledWith({ cliVersion: '0.2.0' }); - expect(io.stdout()).toContain('Stopped KTX Python daemon'); - expect(io.stdout()).toContain('pid: 4242'); - }); -``` - -- [ ] **Step 2: Run the failing command runner tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/runtime.test.ts -``` - -Expected: FAIL because `KtxRuntimeArgs` and `KtxRuntimeDeps` do not include -`start`, `stop`, `startDaemon`, or `stopDaemon`. - -- [ ] **Step 3: Update the runtime command runner** - -In `packages/cli/src/runtime.ts`, add these imports: - -```typescript -import { - startManagedPythonDaemon, - stopManagedPythonDaemon, - type ManagedPythonDaemonStartResult, - type ManagedPythonDaemonStopResult, -} from './managed-python-daemon.js'; -``` - -Extend `KtxRuntimeArgs` with: - -```typescript - | { command: 'start'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } - | { command: 'stop'; cliVersion: string } -``` - -Extend `KtxRuntimeDeps` with: - -```typescript - startDaemon?: (options: { - cliVersion: string; - features: KtxRuntimeFeature[]; - force?: boolean; - }) => Promise; - stopDaemon?: (options: { cliVersion: string }) => Promise; -``` - -Add these writer helpers after `writeInstallResult`: - -```typescript -function writeDaemonStart(io: KtxCliIo, result: ManagedPythonDaemonStartResult): void { - const verb = result.status === 'reused' ? 'Using existing' : 'Started'; - io.stdout.write(`${verb} KTX Python daemon\n`); - io.stdout.write(`url: ${result.baseUrl}\n`); - io.stdout.write(`pid: ${result.state.pid}\n`); - io.stdout.write(`version: ${result.state.version}\n`); - io.stdout.write(`features: ${result.state.features.join(', ')}\n`); - io.stdout.write(`state: ${result.layout.daemonStatePath}\n`); - io.stdout.write(`stdout: ${result.state.stdoutLog}\n`); - io.stdout.write(`stderr: ${result.state.stderrLog}\n`); -} - -function writeDaemonStop(io: KtxCliIo, result: ManagedPythonDaemonStopResult): void { - if (result.status === 'already-stopped') { - io.stdout.write('KTX Python daemon already stopped\n'); - return; - } - io.stdout.write('Stopped KTX Python daemon\n'); - io.stdout.write(`pid: ${result.state?.pid ?? 'unknown'}\n`); - io.stdout.write(`state: ${result.layout.daemonStatePath}\n`); -} -``` - -Inside `runKtxRuntime`, add these branches after the install branch: - -```typescript - if (args.command === 'start') { - const startDaemon = deps.startDaemon ?? startManagedPythonDaemon; - const result = await startDaemon({ - cliVersion: args.cliVersion, - features: [args.feature], - force: args.force, - }); - writeDaemonStart(io, result); - return 0; - } - if (args.command === 'stop') { - const stopDaemon = deps.stopDaemon ?? stopManagedPythonDaemon; - const result = await stopDaemon({ cliVersion: args.cliVersion }); - writeDaemonStop(io, result); - return 0; - } -``` - -- [ ] **Step 4: Verify runtime command runner tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing Commander routing tests** - -In `packages/cli/src/index.test.ts`, inside -`it('routes runtime management commands with the CLI package version', ...)`, -add two new IO handles after `installIo`: - -```typescript - const startIo = makeIo(); - const stopIo = makeIo(); -``` - -Replace the existing `runtime install` invocation with this version that also -passes `--yes`, then add the new `runtime start` and `runtime stop` -invocations immediately after it: - -```typescript - await expect( - runKtxCli(['runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { - runtime, - }), - ).resolves.toBe(0); - await expect( - runKtxCli(['runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), - ).resolves.toBe(0); - await expect(runKtxCli(['runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); -``` - -Update the `expect(runtime).toHaveBeenNthCalledWith(...)` assertions so the -runtime calls are: - -```typescript - expect(runtime).toHaveBeenNthCalledWith( - 1, - { - command: 'install', - cliVersion: '0.0.0-private', - feature: 'local-embeddings', - force: true, - }, - installIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 2, - { - command: 'start', - cliVersion: '0.0.0-private', - feature: 'local-embeddings', - force: true, - }, - startIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 3, - { - command: 'stop', - cliVersion: '0.0.0-private', - }, - stopIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 4, - { - command: 'status', - cliVersion: '0.0.0-private', - json: true, - }, - statusIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 5, - { - command: 'doctor', - cliVersion: '0.0.0-private', - json: false, - }, - doctorIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 6, - { - command: 'prune', - cliVersion: '0.0.0-private', - dryRun: true, - yes: false, - }, - pruneIo.io, - ); -``` - -- [ ] **Step 6: Run the failing Commander routing test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -Expected: FAIL because `runtime install --yes` is not accepted and -`runtime start` and `runtime stop` are not registered. - -- [ ] **Step 7: Register start and stop subcommands** - -In `packages/cli/src/commands/runtime-commands.ts`, update the existing -runtime feature option to return a fresh Commander option per command: - -```typescript -function createRuntimeFeatureOption() { - return new Option('--feature ', 'Runtime feature level') - .choices(['core', 'local-embeddings']) - .default('core'); -} -``` - -Then update the existing `install` command so it accepts `--yes` without -changing behavior: - -```typescript - runtime - .command('install') - .description('Install the bundled Python runtime wheel into the managed runtime') - .addOption(createRuntimeFeatureOption()) - .option('--yes', 'Accept runtime installation without prompting', false) - .option('--force', 'Reinstall even when the runtime already looks ready', false) - .action(async (options: { feature: RuntimeFeature; yes?: boolean; force?: boolean }) => { - await runRuntimeArgs(context, { - command: 'install', - cliVersion: context.packageInfo.version, - feature: options.feature, - force: options.force === true, - }); - }); -``` - -Add this `start` command after the `install` command: - -```typescript - runtime - .command('start') - .description('Start the KTX-managed Python HTTP daemon') - .addOption(createRuntimeFeatureOption()) - .option('--force', 'Restart even when a matching daemon is already running', false) - .action(async (options: { feature: RuntimeFeature; force?: boolean }) => { - await runRuntimeArgs(context, { - command: 'start', - cliVersion: context.packageInfo.version, - feature: options.feature, - force: options.force === true, - }); - }); -``` - -Add this `stop` command after the `start` command: - -```typescript - runtime - .command('stop') - .description('Stop the KTX-managed Python HTTP daemon') - .action(async () => { - await runRuntimeArgs(context, { - command: 'stop', - cliVersion: context.packageInfo.version, - }); - }); -``` - -- [ ] **Step 8: Export daemon lifecycle helpers** - -In `packages/cli/src/index.ts`, add this export near the other public test and -programmatic exports: - -```typescript -export { - allocateDaemonPort, - readManagedPythonDaemonStatus, - startManagedPythonDaemon, - stopManagedPythonDaemon, -} from './managed-python-daemon.js'; -export type { - ManagedPythonDaemonStartResult, - ManagedPythonDaemonState, - ManagedPythonDaemonStatus, - ManagedPythonDaemonStopResult, -} from './managed-python-daemon.js'; -``` - -- [ ] **Step 9: Verify CLI routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts src/runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 10: Commit** - -Run: - -```bash -git add packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/index.test.ts packages/cli/src/index.ts -git commit -m "feat: add runtime daemon start stop commands" -``` - -### Task 4: Verify daemon lifecycle end to end - -**Files:** - -- Verify: `packages/cli/src/managed-python-daemon.ts` -- Verify: `packages/cli/src/runtime.ts` -- Verify: `python/ktx-daemon/src/ktx_daemon/app.py` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts src/managed-python-daemon.test.ts src/runtime.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused Python tests** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py python/ktx-daemon/tests/test_cli.py -q -``` - -Expected: PASS. - -- [ ] **Step 3: Run TypeScript type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run Python pre-commit for modified files** - -Run: - -```bash -source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/managed-python-daemon.ts packages/cli/src/managed-python-daemon.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/index.test.ts packages/cli/src/index.ts -``` - -Expected: PASS. If pre-commit rejects TypeScript file arguments because a hook -only handles Python, run the Python-only pre-commit command from Task 1 and -then run: - -```bash -pnpm --filter @ktx/cli run check -``` - -- [ ] **Step 5: Build the CLI package** - -Run: - -```bash -pnpm --filter @ktx/cli run build -``` - -Expected: PASS. - -- [ ] **Step 6: Build runtime wheel assets** - -Run: - -```bash -pnpm run artifacts:verify -``` - -Expected: PASS and `packages/cli/assets/python/manifest.json` exists with a -matching `kaelio_ktx-0.1.0-py3-none-any.whl`. - -- [ ] **Step 7: Smoke runtime install, start, reuse, and stop** - -Run: - -```bash -KTX_RUNTIME_ROOT="$(mktemp -d)" -KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime install --yes -KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime start -KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime start -KTX_RUNTIME_ROOT="$KTX_RUNTIME_ROOT" node packages/cli/dist/bin.js runtime stop -rm -rf "$KTX_RUNTIME_ROOT" -``` - -Expected: - -```text -Installed KTX Python runtime -Started KTX Python daemon -Using existing KTX Python daemon -Stopped KTX Python daemon -``` - -If the existing runtime layout does not honor `KTX_RUNTIME_ROOT`, run the same -commands without that environment variable and clean up with: - -```bash -node packages/cli/dist/bin.js runtime stop -node packages/cli/dist/bin.js runtime prune --dry-run -``` - -- [ ] **Step 8: Commit verification-only fixes if needed** - -If verification exposed a small defect inside this plan's files, fix it and -commit only the touched files: - -```bash -git add packages/cli/src/managed-python-daemon.ts packages/cli/src/managed-python-daemon.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/index.test.ts packages/cli/src/index.ts python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_app.py packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/managed-python-command.test.ts -git commit -m "fix: verify managed runtime daemon lifecycle" -``` - -Skip this step when there are no verification fixes. - -## Acceptance criteria - -- `ktx runtime start` installs or reuses the requested runtime feature level and - starts `ktx-daemon serve-http` on `127.0.0.1` with a random available port. -- `ktx runtime start` reuses a healthy matching daemon and starts a fresh daemon - when the recorded process, health response, version, or feature set is stale. -- `ktx runtime stop` terminates the recorded daemon process and removes the - daemon state file. -- The daemon state file records `pid`, `port`, `version`, `features`, - `startedAt`, stdout log path, and stderr log path. -- The daemon health endpoint returns `{"status": "healthy"}` by default and - includes `version` when `KTX_DAEMON_VERSION` is set. -- Daemon stdout and stderr are preserved under the versioned runtime directory. -- Focused TypeScript tests, focused Python tests, CLI type-check, and - Python-file pre-commit pass or have explicitly recorded environment blockers. - -## Self-review checklist - -- Spec coverage: this plan covers `ktx runtime start`, `ktx runtime stop`, - daemon state, random localhost port binding, health validation, version - matching, stale repair, and captured daemon logs. It leaves lazy embedding - command integration and public npm renaming for later plans. -- Placeholder scan: this plan contains no placeholder steps, deferred code - blocks, or undefined function names. -- Type consistency: runtime feature values are consistently `core` and - `local-embeddings`; daemon state uses `schemaVersion`, `pid`, `host`, `port`, - `version`, `features`, `startedAt`, `stdoutLog`, and `stderrLog`; command - runner types use `startDaemon` and `stopDaemon`. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md deleted file mode 100644 index c6271cb1..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md +++ /dev/null @@ -1,1750 +0,0 @@ -# Managed Python Runtime Installer Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Install and inspect the bundled `kaelio-ktx` Python wheel in a -versioned KTX-managed runtime directory. - -**Architecture:** Add a CLI-owned managed-runtime module that knows where the -bundled wheel asset lives, verifies its checksum, creates a versioned virtual -environment with `uv`, installs the requested feature set, and writes an -installed-runtime manifest. Add `ktx runtime install`, `status`, `doctor`, and -`prune` commands that expose this behavior without changing normal -Python-backed commands yet. - -**Tech Stack:** TypeScript, Node 22 ESM, Commander, Vitest, `zod`, `uv`, npm -package assets. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -Plan 1, `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md`, -is implemented in this worktree. The implemented source includes -`scripts/build-python-runtime-wheel.mjs`, -`scripts/build-python-runtime-wheel.test.mjs`, runtime-wheel handling in -`scripts/package-artifacts.mjs`, test coverage in -`scripts/package-artifacts.test.mjs`, and the `kaelio-ktx` release-policy -entry. The targeted verification command passes: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs -``` - -Expected current result: - -```text -# pass 38 -# fail 0 -``` - -No other plan files currently reference the npm-managed Python runtime spec. - -This plan implements the next prerequisite: - -- Platform-specific managed runtime roots. -- Versioned runtime directories keyed by the CLI package version. -- Runtime asset manifest reading and wheel checksum verification. -- `uv` virtual environment creation. -- Core and `local-embeddings` feature installation levels. -- Installed-runtime manifest writing. -- `ktx runtime install`, `ktx runtime status`, `ktx runtime doctor`, and - `ktx runtime prune`. - -This plan intentionally leaves the following spec requirements for later -plans: - -- Lazy install from normal commands such as `ktx sl query`. -- `ktx runtime start` and `ktx runtime stop`. -- Daemon state, health checks, reuse, and stale-daemon repair. -- Public npm package renaming from `@ktx/cli` to `@kaelio/ktx`. - -## File structure - -- Create `packages/cli/src/managed-python-runtime.ts`: pure managed-runtime - library for path calculation, asset verification, install/status/doctor, and - pruning. -- Create `packages/cli/src/managed-python-runtime.test.ts`: unit tests for - runtime roots, manifest validation, install command shape, status checks, and - prune safety. -- Create `packages/cli/src/runtime.ts`: command runner that formats - `install`, `status`, `doctor`, and `prune` output. -- Create `packages/cli/src/runtime.test.ts`: command-runner tests with injected - managed-runtime dependencies. -- Create `packages/cli/src/commands/runtime-commands.ts`: Commander - registration for `ktx runtime ...`. -- Modify `packages/cli/src/cli-runtime.ts`: add the runtime command runner to - CLI dependency injection. -- Modify `packages/cli/src/cli-program.ts`: pass package info into command - registration and register the runtime command group. -- Modify `packages/cli/src/index.ts`: export runtime command types and the - runner for tests and programmatic use. -- Modify `packages/cli/src/index.test.ts`: assert root help exposes - `runtime` and Commander routes runtime subcommands correctly. - -### Task 1: Add failing managed-runtime library tests - -**Files:** - -- Create: `packages/cli/src/managed-python-runtime.test.ts` -- Test: `packages/cli/src/managed-python-runtime.test.ts` - -- [ ] **Step 1: Write the failing test file** - -Create `packages/cli/src/managed-python-runtime.test.ts` with this content: - -```typescript -import { createHash } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - doctorManagedPythonRuntime, - installManagedPythonRuntime, - managedPythonRuntimeLayout, - pruneManagedPythonRuntimes, - readManagedPythonRuntimeStatus, - verifyRuntimeAsset, - type ManagedPythonRuntimeExec, -} from './managed-python-runtime.js'; - -async function writeAsset(root: string, contents = 'wheel-bytes') { - const assetDir = join(root, 'assets', 'python'); - await mkdir(assetDir, { recursive: true }); - const wheelPath = join(assetDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'); - await writeFile(wheelPath, contents); - await writeFile( - join(assetDir, 'manifest.json'), - `${JSON.stringify( - { - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.1.0', - wheel: { - file: 'kaelio_ktx-0.1.0-py3-none-any.whl', - sha256: createHash('sha256').update(contents).digest('hex'), - bytes: Buffer.byteLength(contents), - }, - }, - null, - 2, - )}\n`, - ); - return { assetDir, wheelPath }; -} - -describe('managedPythonRuntimeLayout', () => { - it('uses the macOS application-support runtime root', () => { - const layout = managedPythonRuntimeLayout({ - cliVersion: '0.2.0', - platform: 'darwin', - env: {}, - homeDir: '/Users/alex', - assetDir: '/repo/packages/cli/assets/python', - }); - - expect(layout.runtimeRoot).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime'); - expect(layout.versionDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0'); - expect(layout.venvDir).toBe('/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv'); - expect(layout.pythonPath).toBe( - '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/python', - ); - expect(layout.daemonPath).toBe( - '/Users/alex/Library/Application Support/kaelio/ktx/runtime/0.2.0/.venv/bin/ktx-daemon', - ); - expect(layout.assetManifestPath).toBe('/repo/packages/cli/assets/python/manifest.json'); - }); - - it('honors XDG_DATA_HOME on Linux', () => { - const layout = managedPythonRuntimeLayout({ - cliVersion: '0.2.0', - platform: 'linux', - env: { XDG_DATA_HOME: '/var/xdg' }, - homeDir: '/home/alex', - assetDir: '/repo/packages/cli/assets/python', - }); - - expect(layout.runtimeRoot).toBe('/var/xdg/kaelio/ktx/runtime'); - expect(layout.versionDir).toBe('/var/xdg/kaelio/ktx/runtime/0.2.0'); - }); - - it('uses LocalAppData on Windows', () => { - const layout = managedPythonRuntimeLayout({ - cliVersion: '0.2.0', - platform: 'win32', - env: { LOCALAPPDATA: 'C:\\Users\\Alex\\AppData\\Local' }, - homeDir: 'C:\\Users\\Alex', - assetDir: 'C:\\repo\\packages\\cli\\assets\\python', - }); - - expect(layout.runtimeRoot).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime'); - expect(layout.pythonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/python.exe'); - expect(layout.daemonPath).toBe('C:\\Users\\Alex\\AppData\\Local/Kaelio/KTX/runtime/0.2.0/.venv/Scripts/ktx-daemon.exe'); - }); -}); - -describe('verifyRuntimeAsset', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-asset-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('reads the manifest and verifies the wheel checksum', async () => { - const { assetDir, wheelPath } = await writeAsset(tempDir, 'valid-wheel'); - - const asset = await verifyRuntimeAsset({ assetDir }); - - expect(asset.manifest.distributionName).toBe('kaelio-ktx'); - expect(asset.manifest.normalizedName).toBe('kaelio_ktx'); - expect(asset.wheelPath).toBe(wheelPath); - }); - - it('rejects a wheel whose checksum does not match the manifest', async () => { - const { assetDir, wheelPath } = await writeAsset(tempDir, 'original'); - await writeFile(wheelPath, 'tampered'); - - await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow( - /Bundled Python runtime wheel checksum mismatch/, - ); - }); - - it('rejects an unsafe wheel filename in the manifest', async () => { - const { assetDir } = await writeAsset(tempDir, 'valid-wheel'); - await writeFile( - join(assetDir, 'manifest.json'), - `${JSON.stringify({ - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.1.0', - wheel: { - file: '../kaelio_ktx-0.1.0-py3-none-any.whl', - sha256: 'a'.repeat(64), - bytes: 1, - }, - })}\n`, - ); - - await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow(/Unsafe runtime wheel filename/); - }); -}); - -describe('installManagedPythonRuntime', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-install-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('creates a venv, installs the core wheel, and writes a manifest', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const commands: Array<{ command: string; args: string[] }> = []; - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { - commands.push({ command, args }); - return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; - }); - - const result = await installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }); - - expect(result.status).toBe('installed'); - expect(commands).toEqual([ - { command: 'uv', args: ['--version'] }, - { command: 'uv', args: ['venv', result.layout.venvDir] }, - { - command: 'uv', - args: ['pip', 'install', '--python', result.layout.pythonPath, result.asset.wheelPath], - }, - ]); - const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { - cliVersion: string; - features: string[]; - python: { executable: string; daemonExecutable: string }; - }; - expect(manifest.cliVersion).toBe('0.2.0'); - expect(manifest.features).toEqual(['core']); - expect(manifest.python.executable).toBe(result.layout.pythonPath); - expect(manifest.python.daemonExecutable).toBe(result.layout.daemonPath); - }); - - it('installs the local-embeddings extra when requested', async () => { - const { assetDir } = await writeAsset(tempDir, 'embedding-wheel'); - const commands: Array<{ command: string; args: string[] }> = []; - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { - commands.push({ command, args }); - return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; - }); - - const result = await installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['local-embeddings'], - exec, - }); - - expect(commands.at(-1)).toEqual({ - command: 'uv', - args: ['pip', 'install', '--python', result.layout.pythonPath, `${result.asset.wheelPath}[local-embeddings]`], - }); - const manifest = JSON.parse(await readFile(result.layout.manifestPath, 'utf8')) as { features: string[] }; - expect(manifest.features).toEqual(['core', 'local-embeddings']); - }); - - it('reuses an existing compatible runtime when force is false', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ - stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', - stderr: '', - })); - - const first = await installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }); - await mkdir(join(first.layout.venvDir, 'bin'), { recursive: true }); - await writeFile(first.layout.pythonPath, '#!/usr/bin/env python\n'); - await writeFile(first.layout.daemonPath, '#!/usr/bin/env python\n'); - - const second = await installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }); - - expect(second.status).toBe('ready'); - expect(exec).toHaveBeenCalledTimes(3); - }); - - it('keeps failed install logs in the versioned runtime directory', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { - if (command === 'uv' && args[0] === 'venv') { - throw Object.assign(new Error('uv venv failed'), { stdout: 'creating\n', stderr: 'bad python\n' }); - } - return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; - }); - - await expect( - installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }), - ).rejects.toThrow(/Python runtime install failed/); - - const log = await readFile(join(tempDir, 'runtime', '0.2.0', 'install.log'), 'utf8'); - expect(log).toContain('$ uv venv'); - expect(log).toContain('bad python'); - }); -}); - -describe('readManagedPythonRuntimeStatus', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-status-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('reports missing before install', async () => { - const status = await readManagedPythonRuntimeStatus({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir: join(tempDir, 'assets', 'python'), - }); - - expect(status.kind).toBe('missing'); - expect(status.detail).toContain('No runtime manifest'); - }); - - it('reports ready when manifest and executables exist', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ - stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', - stderr: '', - })); - const install = await installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }); - await mkdir(join(install.layout.venvDir, 'bin'), { recursive: true }); - await writeFile(install.layout.pythonPath, '#!/usr/bin/env python\n'); - await writeFile(install.layout.daemonPath, '#!/usr/bin/env python\n'); - - const status = await readManagedPythonRuntimeStatus({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - }); - - expect(status.kind).toBe('ready'); - expect(status.manifest?.features).toEqual(['core']); - }); - - it('reports broken when an executable is missing', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ - stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', - stderr: '', - })); - await installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }); - - const status = await readManagedPythonRuntimeStatus({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - }); - - expect(status.kind).toBe('broken'); - expect(status.detail).toContain('Missing Python executable'); - }); -}); - -describe('doctorManagedPythonRuntime', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-doctor-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('checks uv, bundled assets, and installed runtime status', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ - stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', - stderr: '', - })); - - const checks = await doctorManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - exec, - }); - - expect(checks.map((check) => [check.id, check.status])).toEqual([ - ['uv', 'pass'], - ['asset', 'pass'], - ['runtime', 'fail'], - ]); - expect(checks[2]?.fix).toBe('Run: ktx runtime install --yes'); - }); -}); - -describe('pruneManagedPythonRuntimes', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-prune-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('removes stale version directories and keeps the current version', async () => { - const runtimeRoot = join(tempDir, 'runtime'); - await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); - await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); - await writeFile(join(runtimeRoot, 'README.txt'), 'not a runtime directory\n'); - - const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot }); - - expect(result.removed).toEqual([join(runtimeRoot, '0.1.0')]); - expect(result.kept).toEqual([join(runtimeRoot, '0.2.0')]); - await expect(stat(join(runtimeRoot, '0.1.0'))).rejects.toThrow(); - expect(await readdir(runtimeRoot)).toEqual(['0.2.0', 'README.txt']); - }); - - it('supports dry-run without deleting stale directories', async () => { - const runtimeRoot = join(tempDir, 'runtime'); - await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true }); - await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true }); - - const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot, dryRun: true }); - - expect(result.removed).toEqual([]); - expect(result.stale).toEqual([join(runtimeRoot, '0.1.0')]); - expect(await readdir(runtimeRoot)).toEqual(['0.1.0', '0.2.0']); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/managed-python-runtime.test.ts -``` - -Expected: FAIL with an import error for `./managed-python-runtime.js`. - -- [ ] **Step 3: Commit the failing tests** - -Run: - -```bash -git add packages/cli/src/managed-python-runtime.test.ts -git commit -m "test: cover managed python runtime lifecycle" -``` - -### Task 2: Implement the managed-runtime library - -**Files:** - -- Create: `packages/cli/src/managed-python-runtime.ts` -- Test: `packages/cli/src/managed-python-runtime.test.ts` - -- [ ] **Step 1: Create the managed-runtime implementation** - -Create `packages/cli/src/managed-python-runtime.ts` with this content: - -```typescript -import { createHash } from 'node:crypto'; -import { execFile } from 'node:child_process'; -import { access, appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; -import { homedir } from 'node:os'; -import { basename, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { promisify } from 'node:util'; -import { z } from 'zod'; - -const execFileAsync = promisify(execFile); - -export const runtimeFeatureSchema = z.enum(['core', 'local-embeddings']); -export type KtxRuntimeFeature = z.infer; - -const runtimeAssetManifestSchema = z.object({ - schemaVersion: z.literal(1), - distributionName: z.literal('kaelio-ktx'), - normalizedName: z.literal('kaelio_ktx'), - version: z.string().min(1), - wheel: z.object({ - file: z.string().min(1), - sha256: z.string().regex(/^[a-f0-9]{64}$/), - bytes: z.number().int().nonnegative(), - }), -}); - -export type KtxRuntimeAssetManifest = z.infer; - -const installedRuntimeManifestSchema = z.object({ - schemaVersion: z.literal(1), - cliVersion: z.string().min(1), - installedAt: z.string().min(1), - asset: runtimeAssetManifestSchema, - features: z.array(runtimeFeatureSchema).min(1), - python: z.object({ - executable: z.string().min(1), - daemonExecutable: z.string().min(1), - }), - installLog: z.string().min(1), -}); - -export type InstalledKtxRuntimeManifest = z.infer; - -export interface ManagedPythonRuntimeLayoutOptions { - cliVersion: string; - platform?: NodeJS.Platform; - env?: NodeJS.ProcessEnv; - homeDir?: string; - runtimeRoot?: string; - assetDir?: string; -} - -export interface ManagedPythonRuntimeLayout { - cliVersion: string; - runtimeRoot: string; - versionDir: string; - venvDir: string; - manifestPath: string; - installLogPath: string; - assetDir: string; - assetManifestPath: string; - pythonPath: string; - daemonPath: string; -} - -export interface ManagedRuntimeAsset { - manifest: KtxRuntimeAssetManifest; - wheelPath: string; -} - -export type ManagedPythonRuntimeExec = ( - command: string, - args: string[], - options?: { cwd?: string; env?: NodeJS.ProcessEnv }, -) => Promise<{ stdout: string; stderr: string }>; - -export interface ManagedPythonRuntimeInstallOptions extends ManagedPythonRuntimeLayoutOptions { - features: KtxRuntimeFeature[]; - force?: boolean; - exec?: ManagedPythonRuntimeExec; -} - -export interface ManagedPythonRuntimeInstallResult { - status: 'ready' | 'installed'; - layout: ManagedPythonRuntimeLayout; - asset: ManagedRuntimeAsset; - manifest: InstalledKtxRuntimeManifest; -} - -export type ManagedPythonRuntimeStatusKind = 'missing' | 'ready' | 'mismatched' | 'broken'; - -export interface ManagedPythonRuntimeStatus { - kind: ManagedPythonRuntimeStatusKind; - detail: string; - layout: ManagedPythonRuntimeLayout; - manifest?: InstalledKtxRuntimeManifest; -} - -export interface ManagedPythonRuntimeDoctorCheck { - id: 'uv' | 'asset' | 'runtime'; - label: string; - status: 'pass' | 'fail'; - detail: string; - fix?: string; -} - -export interface ManagedPythonRuntimePruneResult { - runtimeRoot: string; - stale: string[]; - kept: string[]; - removed: string[]; -} - -function defaultAssetDir(): string { - return fileURLToPath(new URL('../assets/python/', import.meta.url)); -} - -function runtimeRootFor(input: Required>): string { - if (input.platform === 'darwin') { - return join(input.homeDir, 'Library', 'Application Support', 'kaelio', 'ktx', 'runtime'); - } - if (input.platform === 'win32') { - return join(input.env.LOCALAPPDATA ?? join(input.homeDir, 'AppData', 'Local'), 'Kaelio', 'KTX', 'runtime'); - } - return join(input.env.XDG_DATA_HOME ?? join(input.homeDir, '.local', 'share'), 'kaelio', 'ktx', 'runtime'); -} - -function executablePath(venvDir: string, platform: NodeJS.Platform, name: string): string { - if (platform === 'win32') { - return join(venvDir, 'Scripts', `${name}.exe`); - } - return join(venvDir, 'bin', name); -} - -export function managedPythonRuntimeLayout(options: ManagedPythonRuntimeLayoutOptions): ManagedPythonRuntimeLayout { - const platform = options.platform ?? process.platform; - const env = options.env ?? process.env; - const homeDir = options.homeDir ?? homedir(); - const runtimeRoot = options.runtimeRoot ?? runtimeRootFor({ platform, env, homeDir }); - const versionDir = join(runtimeRoot, options.cliVersion); - const venvDir = join(versionDir, '.venv'); - const assetDir = options.assetDir ?? defaultAssetDir(); - - return { - cliVersion: options.cliVersion, - runtimeRoot, - versionDir, - venvDir, - manifestPath: join(versionDir, 'manifest.json'), - installLogPath: join(versionDir, 'install.log'), - assetDir, - assetManifestPath: join(assetDir, 'manifest.json'), - pythonPath: executablePath(venvDir, platform, 'python'), - daemonPath: executablePath(venvDir, platform, 'ktx-daemon'), - }; -} - -async function pathExists(path: string): Promise { - try { - await access(path); - return true; - } catch { - return false; - } -} - -function assertSafeWheelFilename(file: string): void { - if (file !== basename(file) || file.includes('/') || file.includes('\\')) { - throw new Error(`Unsafe runtime wheel filename in bundled manifest: ${file}`); - } -} - -async function readJsonFile(path: string): Promise { - return JSON.parse(await readFile(path, 'utf8')) as unknown; -} - -export async function verifyRuntimeAsset(input: { assetDir: string }): Promise { - const manifestPath = join(input.assetDir, 'manifest.json'); - const manifest = runtimeAssetManifestSchema.parse(await readJsonFile(manifestPath)); - assertSafeWheelFilename(manifest.wheel.file); - const wheelPath = join(input.assetDir, manifest.wheel.file); - const wheel = await readFile(wheelPath); - const sha256 = createHash('sha256').update(wheel).digest('hex'); - if (sha256 !== manifest.wheel.sha256 || wheel.byteLength !== manifest.wheel.bytes) { - throw new Error(`Bundled Python runtime wheel checksum mismatch: ${wheelPath}`); - } - return { manifest, wheelPath }; -} - -function normalizeFeatures(features: KtxRuntimeFeature[]): KtxRuntimeFeature[] { - const requested = new Set(['core', ...features]); - return runtimeFeatureSchema.options.filter((feature) => requested.has(feature)); -} - -async function readInstalledManifest(path: string): Promise { - if (!(await pathExists(path))) { - return undefined; - } - return installedRuntimeManifestSchema.parse(await readJsonFile(path)); -} - -function hasFeatures(manifest: InstalledKtxRuntimeManifest, features: KtxRuntimeFeature[]): boolean { - return normalizeFeatures(features).every((feature) => manifest.features.includes(feature)); -} - -async function defaultExec( - command: string, - args: string[], - options: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, -): Promise<{ stdout: string; stderr: string }> { - const result = await execFileAsync(command, args, { - cwd: options.cwd, - env: options.env, - encoding: 'utf8', - maxBuffer: 1024 * 1024 * 20, - }); - return { stdout: result.stdout, stderr: result.stderr }; -} - -function errorOutput(error: unknown): { stdout: string; stderr: string } { - const value = error as { stdout?: unknown; stderr?: unknown }; - return { - stdout: typeof value.stdout === 'string' ? value.stdout : '', - stderr: typeof value.stderr === 'string' ? value.stderr : '', - }; -} - -async function runLogged(input: { - exec: ManagedPythonRuntimeExec; - logPath: string; - command: string; - args: string[]; - cwd?: string; -}): Promise<{ stdout: string; stderr: string }> { - await appendFile(input.logPath, `$ ${input.command} ${input.args.join(' ')}\n`); - try { - const result = await input.exec(input.command, input.args, { cwd: input.cwd }); - if (result.stdout) { - await appendFile(input.logPath, result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`); - } - if (result.stderr) { - await appendFile(input.logPath, result.stderr.endsWith('\n') ? result.stderr : `${result.stderr}\n`); - } - return result; - } catch (error) { - const output = errorOutput(error); - if (output.stdout) { - await appendFile(input.logPath, output.stdout.endsWith('\n') ? output.stdout : `${output.stdout}\n`); - } - if (output.stderr) { - await appendFile(input.logPath, output.stderr.endsWith('\n') ? output.stderr : `${output.stderr}\n`); - } - throw new Error(`Python runtime install failed. Install log: ${input.logPath}`); - } -} - -async function ensureUv(exec: ManagedPythonRuntimeExec): Promise { - try { - const result = await exec('uv', ['--version']); - return result.stdout.trim() || 'uv available'; - } catch { - throw new Error( - 'uv is required to install the KTX Python runtime. Install uv and retry: ktx runtime install --yes', - ); - } -} - -export async function installManagedPythonRuntime( - options: ManagedPythonRuntimeInstallOptions, -): Promise { - const layout = managedPythonRuntimeLayout(options); - const exec = options.exec ?? defaultExec; - const features = normalizeFeatures(options.features); - const asset = await verifyRuntimeAsset({ assetDir: layout.assetDir }); - const existing = await readInstalledManifest(layout.manifestPath); - if ( - options.force !== true && - existing && - existing.cliVersion === options.cliVersion && - existing.asset.wheel.sha256 === asset.manifest.wheel.sha256 && - hasFeatures(existing, features) && - (await pathExists(existing.python.executable)) && - (await pathExists(existing.python.daemonExecutable)) - ) { - return { status: 'ready', layout, asset, manifest: existing }; - } - - await rm(layout.versionDir, { recursive: true, force: true }); - await mkdir(layout.versionDir, { recursive: true }); - await writeFile(layout.installLogPath, ''); - await ensureUv(exec); - await runLogged({ exec, logPath: layout.installLogPath, command: 'uv', args: ['venv', layout.venvDir] }); - const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath; - await runLogged({ - exec, - logPath: layout.installLogPath, - command: 'uv', - args: ['pip', 'install', '--python', layout.pythonPath, wheelSpec], - }); - - const manifest: InstalledKtxRuntimeManifest = { - schemaVersion: 1, - cliVersion: options.cliVersion, - installedAt: new Date().toISOString(), - asset: asset.manifest, - features, - python: { - executable: layout.pythonPath, - daemonExecutable: layout.daemonPath, - }, - installLog: layout.installLogPath, - }; - await writeFile(layout.manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - return { status: 'installed', layout, asset, manifest }; -} - -export async function readManagedPythonRuntimeStatus( - options: ManagedPythonRuntimeLayoutOptions, -): Promise { - const layout = managedPythonRuntimeLayout(options); - let manifest: InstalledKtxRuntimeManifest | undefined; - try { - manifest = await readInstalledManifest(layout.manifestPath); - } catch (error) { - return { - kind: 'broken', - detail: `Runtime manifest is invalid: ${error instanceof Error ? error.message : String(error)}`, - layout, - }; - } - if (!manifest) { - return { kind: 'missing', detail: `No runtime manifest at ${layout.manifestPath}`, layout }; - } - if (manifest.cliVersion !== options.cliVersion) { - return { - kind: 'mismatched', - detail: `Runtime is for CLI ${manifest.cliVersion}, current CLI is ${options.cliVersion}`, - layout, - manifest, - }; - } - if (!(await pathExists(manifest.python.executable))) { - return { kind: 'broken', detail: `Missing Python executable: ${manifest.python.executable}`, layout, manifest }; - } - if (!(await pathExists(manifest.python.daemonExecutable))) { - return { kind: 'broken', detail: `Missing ktx-daemon executable: ${manifest.python.daemonExecutable}`, layout, manifest }; - } - return { kind: 'ready', detail: `Runtime ready at ${layout.versionDir}`, layout, manifest }; -} - -function check(status: ManagedPythonRuntimeDoctorCheck['status'], input: Omit) { - return { status, ...input }; -} - -export async function doctorManagedPythonRuntime( - options: ManagedPythonRuntimeLayoutOptions & { exec?: ManagedPythonRuntimeExec }, -): Promise { - const exec = options.exec ?? defaultExec; - const checks: ManagedPythonRuntimeDoctorCheck[] = []; - try { - const version = await ensureUv(exec); - checks.push(check('pass', { id: 'uv', label: 'uv', detail: version })); - } catch (error) { - checks.push( - check('fail', { - id: 'uv', - label: 'uv', - detail: error instanceof Error ? error.message : String(error), - fix: 'Install uv, then run: ktx runtime install --yes', - }), - ); - } - - try { - const asset = await verifyRuntimeAsset({ assetDir: managedPythonRuntimeLayout(options).assetDir }); - checks.push(check('pass', { id: 'asset', label: 'Bundled Python wheel', detail: asset.wheelPath })); - } catch (error) { - checks.push( - check('fail', { - id: 'asset', - label: 'Bundled Python wheel', - detail: error instanceof Error ? error.message : String(error), - fix: 'Run: pnpm run artifacts:check', - }), - ); - } - - const status = await readManagedPythonRuntimeStatus(options); - checks.push( - check(status.kind === 'ready' ? 'pass' : 'fail', { - id: 'runtime', - label: 'Managed Python runtime', - detail: status.detail, - ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx runtime install --yes' }), - }), - ); - return checks; -} - -export async function pruneManagedPythonRuntimes(options: { - cliVersion: string; - runtimeRoot: string; - dryRun?: boolean; -}): Promise { - if (!(await pathExists(options.runtimeRoot))) { - return { runtimeRoot: options.runtimeRoot, stale: [], kept: [], removed: [] }; - } - const entries = await readdir(options.runtimeRoot); - const stale: string[] = []; - const kept: string[] = []; - for (const entry of entries) { - const path = join(options.runtimeRoot, entry); - const info = await stat(path); - if (!info.isDirectory()) { - continue; - } - if (entry === options.cliVersion) { - kept.push(path); - } else { - stale.push(path); - } - } - const removed: string[] = []; - if (options.dryRun !== true) { - for (const path of stale) { - await rm(path, { recursive: true, force: true }); - removed.push(path); - } - } - return { runtimeRoot: options.runtimeRoot, stale, kept, removed }; -} -``` - -- [ ] **Step 2: Run the managed-runtime tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/managed-python-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run the CLI type checker** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Commit the implementation** - -Run: - -```bash -git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts -git commit -m "feat: add managed python runtime installer" -``` - -### Task 3: Add the runtime command runner - -**Files:** - -- Create: `packages/cli/src/runtime.ts` -- Create: `packages/cli/src/runtime.test.ts` -- Test: `packages/cli/src/runtime.test.ts` - -- [ ] **Step 1: Write the failing command-runner tests** - -Create `packages/cli/src/runtime.test.ts` with this content: - -```typescript -import { describe, expect, it, vi } from 'vitest'; -import { runKtxRuntime, type KtxRuntimeDeps } from './runtime.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -describe('runKtxRuntime', () => { - it('installs the requested runtime feature and prints the manifest path', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - installRuntime: vi.fn(async () => ({ - status: 'installed', - layout: { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }, - asset: { - wheelPath: '/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl', - manifest: { - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.1.0', - wheel: { - file: 'kaelio_ktx-0.1.0-py3-none-any.whl', - sha256: 'a'.repeat(64), - bytes: 10, - }, - }, - }, - manifest: { - schemaVersion: 1, - cliVersion: '0.2.0', - installedAt: '2026-05-11T00:00:00.000Z', - asset: { - schemaVersion: 1, - distributionName: 'kaelio-ktx', - normalizedName: 'kaelio_ktx', - version: '0.1.0', - wheel: { - file: 'kaelio_ktx-0.1.0-py3-none-any.whl', - sha256: 'a'.repeat(64), - bytes: 10, - }, - }, - features: ['core', 'local-embeddings'], - python: { - executable: '/runtime/0.2.0/.venv/bin/python', - daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }, - installLog: '/runtime/0.2.0/install.log', - }, - })), - }; - - await expect( - runKtxRuntime( - { command: 'install', cliVersion: '0.2.0', feature: 'local-embeddings', force: true }, - io.io, - deps, - ), - ).resolves.toBe(0); - - expect(deps.installRuntime).toHaveBeenCalledWith({ - cliVersion: '0.2.0', - features: ['local-embeddings'], - force: true, - }); - expect(io.stdout()).toContain('Installed KTX Python runtime'); - expect(io.stdout()).toContain('features: core, local-embeddings'); - expect(io.stdout()).toContain('manifest: /runtime/0.2.0/manifest.json'); - expect(io.stderr()).toBe(''); - }); - - it('prints runtime status as JSON', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - readStatus: vi.fn(async () => ({ - kind: 'missing', - detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', - layout: { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }, - })), - }; - - await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: true }, io.io, deps)).resolves.toBe(0); - - expect(JSON.parse(io.stdout())).toMatchObject({ - kind: 'missing', - detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', - layout: { runtimeRoot: '/runtime' }, - }); - }); - - it('returns failure for doctor when any check fails', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - doctorRuntime: vi.fn(async () => [ - { id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' }, - { - id: 'runtime', - label: 'Managed Python runtime', - status: 'fail', - detail: 'No runtime manifest', - fix: 'Run: ktx runtime install --yes', - }, - ]), - }; - - await expect(runKtxRuntime({ command: 'doctor', cliVersion: '0.2.0', json: false }, io.io, deps)).resolves.toBe(1); - - expect(io.stdout()).toContain('PASS uv: uv 0.9.5'); - expect(io.stdout()).toContain('FAIL Managed Python runtime: No runtime manifest'); - expect(io.stdout()).toContain('Fix: Run: ktx runtime install --yes'); - }); - - it('requires --yes before pruning stale runtime directories', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - pruneRuntime: vi.fn(async () => { - throw new Error('should not prune without --yes'); - }), - }; - - await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: false, yes: false }, io.io, deps)) - .resolves.toBe(1); - - expect(io.stderr()).toContain('Refusing to prune without --yes'); - expect(deps.pruneRuntime).not.toHaveBeenCalled(); - }); - - it('prints stale directories during prune dry-run', async () => { - const io = makeIo(); - const deps: KtxRuntimeDeps = { - readStatus: vi.fn(async () => ({ - kind: 'missing', - detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', - layout: { - cliVersion: '0.2.0', - runtimeRoot: '/runtime', - versionDir: '/runtime/0.2.0', - venvDir: '/runtime/0.2.0/.venv', - manifestPath: '/runtime/0.2.0/manifest.json', - installLogPath: '/runtime/0.2.0/install.log', - assetDir: '/assets/python', - assetManifestPath: '/assets/python/manifest.json', - pythonPath: '/runtime/0.2.0/.venv/bin/python', - daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon', - }, - })), - pruneRuntime: vi.fn(async () => ({ - runtimeRoot: '/runtime', - stale: ['/runtime/0.1.0'], - kept: ['/runtime/0.2.0'], - removed: [], - })), - }; - - await expect(runKtxRuntime({ command: 'prune', cliVersion: '0.2.0', dryRun: true, yes: false }, io.io, deps)) - .resolves.toBe(0); - - expect(io.stdout()).toContain('Stale KTX Python runtimes'); - expect(io.stdout()).toContain('/runtime/0.1.0'); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/runtime.test.ts -``` - -Expected: FAIL with an import error for `./runtime.js`. - -- [ ] **Step 3: Create the command runner** - -Create `packages/cli/src/runtime.ts` with this content: - -```typescript -import { - doctorManagedPythonRuntime, - installManagedPythonRuntime, - pruneManagedPythonRuntimes, - readManagedPythonRuntimeStatus, - type KtxRuntimeFeature, - type ManagedPythonRuntimeDoctorCheck, - type ManagedPythonRuntimeInstallOptions, - type ManagedPythonRuntimeInstallResult, - type ManagedPythonRuntimeLayoutOptions, - type ManagedPythonRuntimePruneResult, - type ManagedPythonRuntimeStatus, -} from './managed-python-runtime.js'; -import type { KtxCliIo } from './cli-runtime.js'; - -export type KtxRuntimeArgs = - | { command: 'install'; cliVersion: string; feature: KtxRuntimeFeature; force: boolean } - | { command: 'status'; cliVersion: string; json: boolean } - | { command: 'doctor'; cliVersion: string; json: boolean } - | { command: 'prune'; cliVersion: string; dryRun: boolean; yes: boolean }; - -export interface KtxRuntimeDeps { - installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; - readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; - doctorRuntime?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; - pruneRuntime?: (options: { cliVersion: string; runtimeRoot: string; dryRun?: boolean }) => Promise; -} - -function writeJson(io: KtxCliIo, value: unknown): void { - io.stdout.write(`${JSON.stringify(value, null, 2)}\n`); -} - -function writeInstallResult(io: KtxCliIo, result: ManagedPythonRuntimeInstallResult): void { - const verb = result.status === 'ready' ? 'Using existing' : 'Installed'; - io.stdout.write(`${verb} KTX Python runtime\n`); - io.stdout.write(`version: ${result.manifest.cliVersion}\n`); - io.stdout.write(`features: ${result.manifest.features.join(', ')}\n`); - io.stdout.write(`python: ${result.manifest.python.executable}\n`); - io.stdout.write(`daemon: ${result.manifest.python.daemonExecutable}\n`); - io.stdout.write(`manifest: ${result.layout.manifestPath}\n`); - io.stdout.write(`install log: ${result.layout.installLogPath}\n`); -} - -function writeStatus(io: KtxCliIo, status: ManagedPythonRuntimeStatus): void { - io.stdout.write('KTX Python runtime\n'); - io.stdout.write(`status: ${status.kind}\n`); - io.stdout.write(`detail: ${status.detail}\n`); - io.stdout.write(`runtime root: ${status.layout.runtimeRoot}\n`); - io.stdout.write(`version dir: ${status.layout.versionDir}\n`); - if (status.manifest) { - io.stdout.write(`features: ${status.manifest.features.join(', ')}\n`); - io.stdout.write(`python: ${status.manifest.python.executable}\n`); - io.stdout.write(`daemon: ${status.manifest.python.daemonExecutable}\n`); - } -} - -function writeDoctor(io: KtxCliIo, checks: ManagedPythonRuntimeDoctorCheck[]): void { - io.stdout.write('KTX Python runtime doctor\n'); - for (const check of checks) { - io.stdout.write(`${check.status.toUpperCase()} ${check.label}: ${check.detail}\n`); - if (check.fix) { - io.stdout.write(` Fix: ${check.fix}\n`); - } - } -} - -function writePrune(io: KtxCliIo, result: ManagedPythonRuntimePruneResult, dryRun: boolean): void { - if (result.stale.length === 0) { - io.stdout.write(`No stale KTX Python runtimes found under ${result.runtimeRoot}\n`); - return; - } - io.stdout.write(dryRun ? 'Stale KTX Python runtimes\n' : 'Removed stale KTX Python runtimes\n'); - for (const path of dryRun ? result.stale : result.removed) { - io.stdout.write(`${path}\n`); - } -} - -export async function runKtxRuntime( - args: KtxRuntimeArgs, - io: KtxCliIo = process, - deps: KtxRuntimeDeps = {}, -): Promise { - try { - if (args.command === 'install') { - const installRuntime = deps.installRuntime ?? installManagedPythonRuntime; - const result = await installRuntime({ - cliVersion: args.cliVersion, - features: [args.feature], - force: args.force, - }); - writeInstallResult(io, result); - return 0; - } - if (args.command === 'status') { - const readStatus = deps.readStatus ?? readManagedPythonRuntimeStatus; - const status = await readStatus({ cliVersion: args.cliVersion }); - if (args.json) { - writeJson(io, status); - } else { - writeStatus(io, status); - } - return 0; - } - if (args.command === 'doctor') { - const doctorRuntime = deps.doctorRuntime ?? doctorManagedPythonRuntime; - const checks = await doctorRuntime({ cliVersion: args.cliVersion }); - if (args.json) { - writeJson(io, { checks }); - } else { - writeDoctor(io, checks); - } - return checks.some((check) => check.status === 'fail') ? 1 : 0; - } - if (!args.dryRun && !args.yes) { - io.stderr.write('Refusing to prune without --yes. Preview with: ktx runtime prune --dry-run\n'); - return 1; - } - const status = await (deps.readStatus ?? readManagedPythonRuntimeStatus)({ cliVersion: args.cliVersion }); - const pruneRuntime = deps.pruneRuntime ?? pruneManagedPythonRuntimes; - const result = await pruneRuntime({ - cliVersion: args.cliVersion, - runtimeRoot: status.layout.runtimeRoot, - dryRun: args.dryRun, - }); - writePrune(io, result, args.dryRun); - return 0; - } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - return 1; - } -} -``` - -- [ ] **Step 4: Run the command-runner tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the command runner** - -Run: - -```bash -git add packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts -git commit -m "feat: add runtime command runner" -``` - -### Task 4: Register `ktx runtime` commands - -**Files:** - -- Create: `packages/cli/src/commands/runtime-commands.ts` -- Modify: `packages/cli/src/cli-runtime.ts` -- Modify: `packages/cli/src/cli-program.ts` -- Modify: `packages/cli/src/index.ts` -- Modify: `packages/cli/src/index.test.ts` -- Test: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Create the runtime command registration** - -Create `packages/cli/src/commands/runtime-commands.ts` with this content: - -```typescript -import { type Command, Option } from '@commander-js/extra-typings'; -import type { KtxCliCommandContext } from '../cli-program.js'; -import type { KtxRuntimeArgs } from '../runtime.js'; - -type RuntimeFeature = Extract['feature']; - -const runtimeFeatureOption = new Option('--feature ', 'Runtime feature level') - .choices(['core', 'local-embeddings']) - .default('core'); - -async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArgs): Promise { - const runner = context.deps.runtime ?? (await import('../runtime.js')).runKtxRuntime; - context.setExitCode(await runner(args, context.io)); -} - -export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void { - const runtime = program - .command('runtime') - .description('Install, inspect, and prune the KTX-managed Python runtime') - .showHelpAfterError(); - - runtime - .command('install') - .description('Install the bundled Python runtime wheel into the managed runtime') - .addOption(runtimeFeatureOption) - .option('--force', 'Reinstall even when the runtime already looks ready', false) - .action(async (options: { feature: RuntimeFeature; force?: boolean }) => { - await runRuntimeArgs(context, { - command: 'install', - cliVersion: context.packageInfo.version, - feature: options.feature, - force: options.force === true, - }); - }); - - runtime - .command('status') - .description('Show managed Python runtime status') - .option('--json', 'Print JSON output', false) - .action(async (options: { json?: boolean }) => { - await runRuntimeArgs(context, { - command: 'status', - cliVersion: context.packageInfo.version, - json: options.json === true, - }); - }); - - runtime - .command('doctor') - .description('Check managed Python runtime prerequisites and installation') - .option('--json', 'Print JSON output', false) - .action(async (options: { json?: boolean }) => { - await runRuntimeArgs(context, { - command: 'doctor', - cliVersion: context.packageInfo.version, - json: options.json === true, - }); - }); - - runtime - .command('prune') - .description('Remove stale managed Python runtimes for older CLI versions') - .option('--dry-run', 'List stale runtimes without deleting them', false) - .option('--yes', 'Confirm deletion of stale runtime directories', false) - .action(async (options: { dryRun?: boolean; yes?: boolean }) => { - await runRuntimeArgs(context, { - command: 'prune', - cliVersion: context.packageInfo.version, - dryRun: options.dryRun === true, - yes: options.yes === true, - }); - }); -} -``` - -- [ ] **Step 2: Add runtime dependency injection to CLI runtime** - -In `packages/cli/src/cli-runtime.ts`, add this import after the existing -`KtxPublicIngestArgs` import: - -```typescript -import type { KtxRuntimeArgs } from './runtime.js'; -``` - -Then add this property to `KtxCliDeps` after `publicIngest`: - -```typescript - runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; -``` - -- [ ] **Step 3: Add package info to command context and register the command** - -In `packages/cli/src/cli-program.ts`, add this import after the -`registerPublicIngestCommands` import: - -```typescript -import { registerRuntimeCommands } from './commands/runtime-commands.js'; -``` - -Add this property to `KtxCliCommandContext` after `deps`: - -```typescript - packageInfo: KtxCliPackageInfo; -``` - -Add this property to the `context` object inside `runCommanderKtxCli` after -`deps`: - -```typescript - packageInfo: info, -``` - -Register the runtime commands after `registerSlCommands(program, context);`: - -```typescript - registerRuntimeCommands(program, context); - profileMark('commander:register-runtime'); -``` - -- [ ] **Step 4: Export runtime APIs from the CLI package** - -In `packages/cli/src/index.ts`, add this export after the setup exports: - -```typescript -export { runKtxRuntime, type KtxRuntimeArgs, type KtxRuntimeDeps } from './runtime.js'; -``` - -- [ ] **Step 5: Update root help and routing tests** - -In `packages/cli/src/index.test.ts`, update the root help command list in the -test named `prints the May 6 public command surface in root help` from: - -```typescript - for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'serve', 'status']) { -``` - -to: - -```typescript - for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'runtime', 'serve', 'status']) { -``` - -Then add this test after the root help test: - -```typescript - it('routes runtime management commands with the CLI package version', async () => { - const runtime = vi.fn(async () => 0); - const installIo = makeIo(); - const statusIo = makeIo(); - const doctorIo = makeIo(); - const pruneIo = makeIo(); - - await expect( - runKtxCli(['runtime', 'install', '--feature', 'local-embeddings', '--force'], installIo.io, { runtime }), - ).resolves.toBe(0); - await expect(runKtxCli(['runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['runtime', 'doctor'], doctorIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(0); - - expect(runtime).toHaveBeenNthCalledWith( - 1, - { - command: 'install', - cliVersion: '0.0.0-private', - feature: 'local-embeddings', - force: true, - }, - installIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 2, - { - command: 'status', - cliVersion: '0.0.0-private', - json: true, - }, - statusIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 3, - { - command: 'doctor', - cliVersion: '0.0.0-private', - json: false, - }, - doctorIo.io, - ); - expect(runtime).toHaveBeenNthCalledWith( - 4, - { - command: 'prune', - cliVersion: '0.0.0-private', - dryRun: true, - yes: false, - }, - pruneIo.io, - ); - }); -``` - -- [ ] **Step 6: Run the CLI routing tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit the command registration** - -Run: - -```bash -git add packages/cli/src/commands/runtime-commands.ts packages/cli/src/cli-runtime.ts packages/cli/src/cli-program.ts packages/cli/src/index.ts packages/cli/src/index.test.ts -git commit -m "feat: expose runtime management commands" -``` - -### Task 5: Verify the managed runtime installer end to end - -**Files:** - -- Verify: `packages/cli/src/managed-python-runtime.ts` -- Verify: `packages/cli/src/runtime.ts` -- Verify: `packages/cli/src/commands/runtime-commands.ts` -- Verify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Run focused Vitest coverage** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/managed-python-runtime.test.ts src/runtime.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the CLI type checker** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Build CLI artifacts so bundled Python assets exist** - -Run: - -```bash -pnpm run artifacts:check -``` - -Expected: PASS. The command must leave these generated files: - -```text -packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl -packages/cli/assets/python/manifest.json -``` - -- [ ] **Step 4: Smoke the status command without installing** - -Run: - -```bash -pnpm --filter @ktx/cli run build -node packages/cli/dist/bin.js runtime status --json -``` - -Expected: PASS with JSON containing `"kind": "missing"` or `"kind": "ready"`. -Both are valid because a developer machine might already have a runtime for -the current CLI version. - -- [ ] **Step 5: Smoke the doctor command** - -Run: - -```bash -node packages/cli/dist/bin.js runtime doctor -``` - -Expected: command exits `0` if the runtime is ready and exits `1` if the -runtime is missing. In both cases, stdout must include: - -```text -KTX Python runtime doctor -``` - -- [ ] **Step 6: Run pre-commit for changed files** - -Run: - -```bash -uv run pre-commit run --files packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/cli-runtime.ts packages/cli/src/cli-program.ts packages/cli/src/index.ts packages/cli/src/index.test.ts -``` - -Expected: PASS. If pre-commit cannot run because this checkout lacks a -compatible pre-commit environment, record the exact failure and keep the -Vitest, type-check, and build results. - -- [ ] **Step 7: Commit final verification fixes** - -If verification required edits, run: - -```bash -git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts packages/cli/src/runtime.ts packages/cli/src/runtime.test.ts packages/cli/src/commands/runtime-commands.ts packages/cli/src/cli-runtime.ts packages/cli/src/cli-program.ts packages/cli/src/index.ts packages/cli/src/index.test.ts -git commit -m "test: verify managed python runtime commands" -``` - -If no verification edits were needed, do not create an empty commit. - -## Self-review - -Spec coverage: - -- Covers runtime root selection for macOS, Linux, and Windows. -- Covers versioned runtime directories based on the CLI package version. -- Covers locating `uv`, creating a virtual environment, installing the bundled - wheel, and writing a runtime manifest. -- Covers feature levels by installing `core` by default and - `local-embeddings` through the wheel extra when requested. -- Covers focused errors for missing `uv`, failed install logs, status output, - doctor output, and stale runtime pruning. -- Leaves lazy install from normal commands, daemon start/stop/reuse, and - public npm renaming for later plans. - -Placeholder scan: - -- The plan contains no placeholder markers and no unspecified implementation - steps. - -Type and name consistency: - -- Runtime feature strings are consistently `core` and `local-embeddings`. -- Runtime command args use `cliVersion`, `feature`, `force`, `json`, `dryRun`, - and `yes` consistently across command registration, tests, and runner code. -- Asset manifest names are consistently `kaelio-ktx`, `kaelio_ktx`, and - `manifest.json`. diff --git a/docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md b/docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md deleted file mode 100644 index 23959596..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md +++ /dev/null @@ -1,585 +0,0 @@ -# Managed Python Runtime Release Smoke Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the public `@kaelio/ktx` artifact smoke prove that the npm -package installs and uses its own managed Python runtime without an externally -prepared Python environment. - -**Architecture:** Keep the release smoke black-box: install the packed public -npm tarball into a clean project, isolate `KTX_RUNTIME_ROOT`, and exercise the -installed `ktx` binary. The first `ktx sl query --yes` performs the lazy core -runtime install from bundled package assets, then the smoke verifies -`runtime status`, `runtime doctor`, daemon start/reuse, and daemon stop. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv, KTX CLI managed -Python runtime assets. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -Existing plans based on the spec: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` - -All six are implemented in this worktree. Evidence found before writing this -plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and - `scripts/build-python-runtime-wheel.test.mjs`. -- `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and - `packages/cli/assets/python/manifest.json`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, and - `packages/cli/src/commands/runtime-commands.ts`. -- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime - install policy flags. -- `packages/cli/src/managed-python-daemon.ts`, daemon state paths, and - `ktx runtime start` / `ktx runtime stop`. -- `packages/cli/src/managed-local-embeddings.ts`, - `packages/context/src/llm/local-config.ts` managed marker constants, and - setup wiring in `packages/cli/src/setup-embeddings.ts`. -- `scripts/build-public-npm-package.mjs`, - `scripts/build-public-npm-package.test.mjs`, `release-policy.json` listing - `@kaelio/ktx`, and published smoke command construction for the required - `@kaelio/ktx` invocation modes. - -The remaining release-smoke gap is in `scripts/package-artifacts.mjs`: - -- `verifyNpmArtifacts()` creates a smoke `.venv`, installs the built Python - runtime wheel into it, and runs installed CLI smoke scripts with that venv at - the front of `PATH`. -- The installed CLI smoke does run `ktx sl query --yes`, but it does not - isolate `KTX_RUNTIME_ROOT`, does not assert that the first query installed - the managed runtime from bundled npm assets, and does not exercise - `ktx runtime status`, `doctor`, `start`, reuse, and `stop`. - -This plan closes that release-flow gap without changing the separate Python -artifact smoke. `verifyPythonArtifacts()` must continue to install the built -Python wheel directly because it verifies the Python artifact itself. - -## File structure - -- Modify `scripts/package-artifacts.test.mjs`: remove the npm-smoke venv test, - add a source-level guard that npm artifact verification does not prepare an - external Python venv, and assert that the installed CLI smoke exercises the - managed runtime lifecycle. -- Modify `scripts/package-artifacts.mjs`: remove npm-smoke Python venv PATH - setup, isolate `KTX_RUNTIME_ROOT` inside `npmRuntimeSmokeSource()`, assert - first-run lazy install, and add runtime status/doctor/start/reuse/stop smoke - commands. - -### Task 1: Add failing release-smoke tests - -**Files:** - -- Modify: `scripts/package-artifacts.test.mjs` -- Test: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Remove the stale npm-smoke venv import** - -In `scripts/package-artifacts.test.mjs`, delete `npmSmokePythonEnv` from the -import list. The surrounding import block must contain this sequence after the -edit: - -```javascript - npmDemoSmokeSource, - npmRuntimeSmokeSource, - npmSmokePackageJson, - npmVerifySource, -``` - -- [ ] **Step 2: Replace the npm-smoke venv test with a source guard** - -Delete this entire test block: - -```javascript -describe('npmSmokePythonEnv', () => { - it('prepends the npm smoke virtualenv bin directory to PATH', () => { - const env = npmSmokePythonEnv('/tmp/ktx-npm-smoke', { PATH: '/usr/bin' }); - - assert.match(env.PATH, /^\/tmp\/ktx-npm-smoke\/\.venv\/(bin|Scripts)/); - assert.match(env.PATH, /\/usr\/bin$/); - }); -}); -``` - -Insert this block in the same location: - -```javascript -describe('verifyNpmArtifacts', () => { - it('does not prepare an external Python environment for the npm smoke', async () => { - const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); - const start = source.indexOf('async function verifyNpmArtifacts'); - const end = source.indexOf('async function verifyNpmDemoArtifacts'); - assert.ok(start > 0, 'verifyNpmArtifacts function must exist'); - assert.ok(end > start, 'verifyNpmDemoArtifacts must follow verifyNpmArtifacts'); - - const body = source.slice(start, end); - assert.doesNotMatch(body, /uv', \['venv', '\.venv'\]/); - assert.doesNotMatch(body, /pythonArtifactInstallArgs/); - assert.doesNotMatch(body, /npmSmokePythonEnv/); - }); -}); -``` - -- [ ] **Step 3: Extend the installed CLI smoke assertions** - -In the `it('runs installed CLI commands through the public package runtime', -...)` test, add these assertions after the existing -`assert.match(source, /ktx sl query sqlite execute/);` assertion: - -```javascript - assert.match(source, /import Database from 'better-sqlite3'/); - assert.doesNotMatch(source, /run\('python'/); - assert.match(source, /KTX_RUNTIME_ROOT/); - assert.match(source, /managed-runtime/); - assert.match(source, /ktx runtime status missing/); - assert.match(source, /runtimeStatusBefore\.kind, 'missing'/); - assert.match(source, /Installing KTX Python runtime \(core\) with uv/); - assert.match(source, /KTX Python runtime ready:/); - assert.match(source, /ktx runtime status ready/); - assert.match(source, /runtimeStatusAfter\.kind, 'ready'/); - assert.match(source, /runtimeStatusAfter\.manifest\.features/); - assert.match(source, /ktx runtime doctor/); - assert.match(source, /PASS Managed Python runtime/); - assert.match(source, /ktx runtime start/); - assert.match(source, /ktx runtime start reuse/); - assert.match(source, /Using existing KTX Python daemon/); - assert.match(source, /ktx runtime stop/); -``` - -- [ ] **Step 4: Run the failing package artifact tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: FAIL. The guard fails because `verifyNpmArtifacts()` still creates -the npm-smoke `.venv`, and the installed CLI smoke assertions fail because -`npmRuntimeSmokeSource()` does not yet isolate or verify the managed runtime. - -### Task 2: Make the npm smoke use only the managed runtime - -**Files:** - -- Modify: `scripts/package-artifacts.mjs` -- Modify: `scripts/package-artifacts.test.mjs` -- Test: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Remove the npm-smoke PATH helper** - -In `scripts/package-artifacts.mjs`, change the path import from: - -```javascript -import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; -``` - -to: - -```javascript -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; -``` - -Then delete this exported helper: - -```javascript -export function npmSmokePythonEnv(projectDir, baseEnv = process.env) { - const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin'); - const existingPath = baseEnv.PATH ?? ''; - - return Object.assign({}, baseEnv, { - PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir, - }); -} -``` - -- [ ] **Step 2: Add runtime-smoke helpers to `npmRuntimeSmokeSource()`** - -Inside the template string returned by `npmRuntimeSmokeSource()`, add this -helper immediately after `requireSuccess()`: - -```javascript -function requireSuccessWithStderr(label, result, stderrPattern) { - assert.equal( - result.code, - 0, - label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr, - ); - assert.match(result.stderr, stderrPattern, label + ' stderr did not match ' + stderrPattern); -} -``` - -Then replace the smoke root setup: - -```javascript -const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-')); -try { - const projectDir = join(root, 'project'); - const sourceDir = join(root, 'source'); -``` - -with: - -```javascript -const root = await mkdtemp(join(tmpdir(), 'ktx-installed-cli-smoke-')); -const previousRuntimeRoot = process.env.KTX_RUNTIME_ROOT; -process.env.KTX_RUNTIME_ROOT = join(root, 'managed-runtime'); -let daemonStarted = false; -try { - const projectDir = join(root, 'project'); - const sourceDir = join(root, 'source'); -``` - -Finally replace the existing `finally` block at the end of -`npmRuntimeSmokeSource()`: - -```javascript -} finally { - await rm(root, { recursive: true, force: true }); -} -``` - -with: - -```javascript -} finally { - if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']); - } - if (previousRuntimeRoot === undefined) { - delete process.env.KTX_RUNTIME_ROOT; - } else { - process.env.KTX_RUNTIME_ROOT = previousRuntimeRoot; - } - await rm(root, { recursive: true, force: true }); -} -``` - -- [ ] **Step 3: Create the sqlite smoke warehouse without Python** - -Inside the template string returned by `npmRuntimeSmokeSource()`, add this -import after the `assert` import: - -```javascript -import Database from 'better-sqlite3'; -``` - -Then replace the current `writeSqliteWarehouse()` function: - -```javascript -async function writeSqliteWarehouse(projectDir) { - const createDb = await run('python', [ - '-c', - [ - 'import sqlite3', - 'import sys', - 'db_path = sys.argv[1]', - 'conn = sqlite3.connect(db_path)', - 'conn.executescript("""', - 'DROP TABLE IF EXISTS orders;', - 'CREATE TABLE orders (', - ' id INTEGER PRIMARY KEY,', - ' status TEXT NOT NULL,', - ' amount INTEGER NOT NULL', - ');', - "INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10);", - '""")', - 'conn.close()', - ].join('\\n'), - join(projectDir, 'warehouse.db'), - ]); - requireSuccess('create sqlite warehouse', createDb); -} -``` - -with: - -```javascript -async function writeSqliteWarehouse(projectDir) { - const database = new Database(join(projectDir, 'warehouse.db')); - try { - database.exec(` -DROP TABLE IF EXISTS orders; -CREATE TABLE orders ( - id INTEGER PRIMARY KEY, - status TEXT NOT NULL, - amount INTEGER NOT NULL -); -INSERT INTO orders (status, amount) VALUES ('paid', 20), ('paid', 30), ('open', 10); -`); - } finally { - database.close(); - } -} -``` - -- [ ] **Step 4: Assert the isolated runtime is initially missing** - -In `npmRuntimeSmokeSource()`, insert this block immediately after the public -package version assertion: - -```javascript - const runtimeStatusBefore = parseJsonResult( - 'ktx runtime status missing', - await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']), - ); - assert.equal(runtimeStatusBefore.kind, 'missing'); - assert.equal(runtimeStatusBefore.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); - process.stdout.write('ktx managed runtime starts missing in isolated root\\n'); -``` - -- [ ] **Step 5: Assert first `sl query --yes` performs lazy managed install** - -In `npmRuntimeSmokeSource()`, replace the current `slQuery` verification block: - -```javascript - const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', - '--connection-id', - 'warehouse', - '--measure', - 'orders.order_count', - '--format', - 'json', - '--yes', - '--project-dir', - projectDir, - ]); - requireSuccess('ktx sl query', slQuery); - requireOutput('ktx sl query', slQuery, /"mode": "compile_only"/); - requireOutput('ktx sl query', slQuery, /orders/); -``` - -with: - -```javascript - const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', - '--connection-id', - 'warehouse', - '--measure', - 'orders.order_count', - '--format', - 'json', - '--yes', - '--project-dir', - projectDir, - ]); - requireSuccessWithStderr( - 'ktx sl query first managed runtime install', - slQuery, - /Installing KTX Python runtime \(core\) with uv[\s\S]*KTX Python runtime ready:/, - ); - requireOutput('ktx sl query first managed runtime install', slQuery, /"mode": "compile_only"/); - requireOutput('ktx sl query first managed runtime install', slQuery, /orders/); - - const runtimeStatusAfter = parseJsonResult( - 'ktx runtime status ready', - await run('pnpm', ['exec', 'ktx', 'runtime', 'status', '--json']), - ); - assert.equal(runtimeStatusAfter.kind, 'ready'); - assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']); - assert.equal(runtimeStatusAfter.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); - process.stdout.write('ktx managed runtime lazy install verified\\n'); -``` - -- [ ] **Step 6: Add runtime doctor and daemon lifecycle smoke** - -In `npmRuntimeSmokeSource()`, insert this block immediately after the -`sqliteSlQuery` verification block: - -```javascript - const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'runtime', 'doctor']); - requireSuccess('ktx runtime doctor', runtimeDoctor); - requireOutput('ktx runtime doctor', runtimeDoctor, /PASS uv/); - requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Bundled Python wheel/); - requireOutput('ktx runtime doctor', runtimeDoctor, /PASS Managed Python runtime/); - process.stdout.write('ktx runtime doctor verified\\n'); - - const runtimeStart = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']); - requireSuccess('ktx runtime start', runtimeStart); - daemonStarted = true; - requireOutput('ktx runtime start', runtimeStart, /Started KTX Python daemon/); - requireOutput('ktx runtime start', runtimeStart, /url: http:\/\/127\.0\.0\.1:\d+/); - requireOutput('ktx runtime start', runtimeStart, /features: core/); - - const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'runtime', 'start']); - requireSuccess('ktx runtime start reuse', runtimeStartReuse); - requireOutput('ktx runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); - requireOutput('ktx runtime start reuse', runtimeStartReuse, /features: core/); - - const runtimeStop = await run('pnpm', ['exec', 'ktx', 'runtime', 'stop']); - requireSuccess('ktx runtime stop', runtimeStop); - daemonStarted = false; - requireOutput('ktx runtime stop', runtimeStop, /Stopped KTX Python daemon/); - process.stdout.write('ktx runtime daemon lifecycle verified\\n'); -``` - -- [ ] **Step 7: Remove npm-smoke Python preparation from artifact verification** - -In `scripts/package-artifacts.mjs`, replace `verifyNpmArtifacts()` with this -implementation: - -```javascript -async function verifyNpmArtifacts(layout, tmpRoot) { - for (const packageInfo of NPM_ARTIFACT_PACKAGES) { - await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); - } - - const projectDir = join(tmpRoot, 'npm-clean-install'); - await mkdir(projectDir, { recursive: true }); - await writeFile( - join(projectDir, 'package.json'), - `${JSON.stringify(npmSmokePackageJson(layout), null, 2)}\n`, - ); - await writeFile(join(projectDir, 'verify-npm.mjs'), npmVerifySource()); - await writeFile(join(projectDir, 'verify-installed-cli.mjs'), npmRuntimeSmokeSource()); - await writeFile(join(projectDir, 'verify-installed-demo.mjs'), npmDemoSmokeSource()); - - await runCommand('pnpm', ['install'], { cwd: projectDir }); - await runCommand('pnpm', ['rebuild', 'better-sqlite3'], { cwd: projectDir }); - await runCommand('node', ['verify-npm.mjs'], { cwd: projectDir }); - await runCommand('pnpm', ['exec', 'ktx', '--version'], { cwd: projectDir }); - await runCommand('node', ['verify-installed-cli.mjs'], { cwd: projectDir }); - await runCommand('node', ['verify-installed-demo.mjs'], { cwd: projectDir }); -} -``` - -- [ ] **Step 8: Run the focused package artifact tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 9: Commit the release-smoke implementation** - -Run: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -git commit -m "test: verify managed runtime in public package smoke" -``` - -### Task 3: Verify the release-smoke surface - -**Files:** - -- Test: `scripts/package-artifacts.test.mjs` -- Test: `scripts/package-artifacts.mjs` - -- [ ] **Step 1: Run script unit tests that cover artifact packaging** - -Run: - -```bash -node --test scripts/build-python-runtime-wheel.test.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 2: Run the public package artifact smoke** - -Run: - -```bash -pnpm run artifacts:verify -``` - -Expected: PASS. The `verify-installed-cli.mjs` output must include: - -```text -ktx managed runtime starts missing in isolated root -ktx managed runtime lazy install verified -ktx runtime doctor verified -ktx runtime daemon lifecycle verified -``` - -- [ ] **Step 3: Run release readiness** - -Run: - -```bash -pnpm run release:readiness -``` - -Expected: PASS. The report must still list `@kaelio/ktx` as the only npm -package and must still report registry publishing as disabled by -`release-policy.json`. - -- [ ] **Step 4: Run pre-commit for changed files** - -Run: - -```bash -if [ -d .venv ]; then source .venv/bin/activate; fi -uv run pre-commit run --files scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -``` - -Expected: PASS. If pre-commit cannot run because the local environment lacks a -compatible hook version, record the exact failure and keep the passing -`node --test` and artifact smoke results. - -- [ ] **Step 5: Commit verification fixes if needed** - -If Step 1, Step 2, Step 3, or Step 4 required edits, run: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -git commit -m "test: finalize managed runtime release smoke" -``` - -If no files changed after Task 2, do not create an empty commit. - -## Acceptance criteria - -- `verifyNpmArtifacts()` no longer creates a Python `.venv`, no longer calls - `pythonArtifactInstallArgs()`, and no longer runs npm smoke scripts with a - custom Python venv at the front of `PATH`. -- The installed public npm smoke creates its sqlite warehouse with - `better-sqlite3` and does not shell out to `python`. -- The installed public npm smoke sets an isolated `KTX_RUNTIME_ROOT` and - confirms that `ktx runtime status --json` starts as `missing`. -- The first installed `ktx sl query --yes` installs the `core` managed Python - runtime from bundled npm package assets and still returns compile-only SQL. -- A second semantic query executes against sqlite using the installed managed - runtime. -- `ktx runtime doctor` passes after lazy install. -- `ktx runtime start` starts a core daemon, a second `ktx runtime start` reuses - the daemon, and `ktx runtime stop` stops it. -- The separate Python artifact verification still installs and tests the - Python wheel directly. -- Focused script tests, `pnpm run artifacts:verify`, release readiness, and - pre-commit pass or have explicitly recorded environment blockers. - -## Self-review - -- Spec coverage: the previous six plans cover the bundled wheel, runtime - installer, `sl query` command integration, daemon lifecycle, local embeddings, - and public npm package surface. This plan covers release-flow checks for clean - install of the packed npm package, first-run managed runtime install from the - bundled wheel, one-shot semantic-layer query through the managed runtime, - runtime status and doctor output, and daemon start/reuse/stop. -- Remaining intentional gap: optional `local-embeddings` smoke remains outside - the default release artifact smoke because the spec permits it in a separate - job or opt-in check and the dependency downloads are large. -- Placeholder scan: no steps contain placeholder implementation language. -- Type consistency: runtime feature names remain `core` and - `local-embeddings`; the public npm package name remains `@kaelio/ktx`; the - runtime root environment variable is `KTX_RUNTIME_ROOT`. diff --git a/docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md b/docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md deleted file mode 100644 index fe28270d..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md +++ /dev/null @@ -1,657 +0,0 @@ -# Managed Runtime Docs and Postgres Smoke Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the remaining manual Python service guidance from the Postgres -historic SQL smoke and update public docs so the npm-managed Python runtime is -the documented path. - -**Architecture:** Keep the existing managed-runtime code unchanged. Add source -and docs guards first, then make the Postgres historic smoke use the -CLI-managed core daemon through `createKtxCliLocalIngestAdapters()`, and update -the README files that still describe internal package artifacts, manual -`ktx-daemon` startup, or `python-service/`. - -**Tech Stack:** Bash, Node 22 ESM, `node:test`, Markdown, pnpm, uv, KTX CLI -managed Python runtime. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plans are based on that spec and are already implemented in this -worktree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` - -Implementation evidence found before writing this plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and - `packages/cli/assets/python/manifest.json`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, and - `packages/cli/src/commands/runtime-commands.ts`. -- `packages/cli/src/managed-python-command.ts` and managed `ktx sl query` - runtime policy flags. -- `packages/cli/src/managed-python-daemon.ts` and `ktx runtime start` / - `ktx runtime stop`. -- `packages/cli/src/managed-local-embeddings.ts` and local embeddings setup - wiring. -- `scripts/build-public-npm-package.mjs`, release policy updates, release - smoke coverage, and opt-in local embeddings smoke coverage. -- `packages/cli/src/agent-runtime.ts` and `packages/cli/src/serve.ts` now - create managed semantic-layer compute when no explicit semantic HTTP URL is - provided. -- `packages/cli/src/managed-python-http.ts`, - `packages/cli/src/local-adapters.ts`, `packages/cli/src/ingest.ts`, - `packages/cli/src/scan.ts`, and `packages/cli/src/serve.ts` wire local ingest - helper paths to the managed core daemon. - -The remaining drift is documentation and one example smoke script: - -- `examples/postgres-historic/scripts/smoke.sh` still checks for - `python-service/.venv`, starts `uvicorn app.main:app`, and exports - `KTX_SQL_ANALYSIS_URL`. -- `examples/postgres-historic/README.md` still documents - `python-service/.venv` or `KTX_SQL_ANALYSIS_URL` as a prerequisite. -- `examples/package-artifacts/README.md` still says the npm smoke installs - generated `@ktx/context` and `@ktx/cli` tarballs. -- `README.md` still presents source-tree `pnpm run ktx -- ...` commands as the - quick start and tells users to start `ktx-daemon` manually for MCP. - -This plan closes that drift. It does not rename internal workspace packages and -does not remove explicit daemon URL override behavior from production code. - -## File structure - -- Modify `scripts/examples-docs.test.mjs`: add regression coverage for managed - runtime docs, public npm package docs, and the Postgres smoke script. -- Modify `examples/postgres-historic/scripts/smoke.sh`: remove - `python-service/` startup and pass managed daemon options into stage-only - historic SQL ingest. -- Modify `examples/postgres-historic/README.md`: document the managed runtime - and remove old SQL-analysis service instructions. -- Modify `examples/package-artifacts/README.md`: describe the single public - `@kaelio/ktx` npm artifact and managed runtime smoke. -- Modify `README.md`: make public `@kaelio/ktx` invocation modes and managed - runtime commands visible while keeping source-tree development commands in - the development section. - -### Task 1: Add failing docs and smoke guards - -**Files:** - -- Modify: `scripts/examples-docs.test.mjs` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Add public runtime README assertions** - -In `scripts/examples-docs.test.mjs`, insert this test after the existing -`walks through ktx connection list and ktx connection test in the README -quickstart` test: - -```javascript - it('documents public npm and managed runtime usage in the README', async () => { - const rootReadme = await readText('README.md'); - - assert.match(rootReadme, /npx @kaelio\/ktx setup demo --no-input/); - assert.match(rootReadme, /npx @kaelio\/ktx sl query/); - assert.match(rootReadme, /npm install @kaelio\/ktx/); - assert.match(rootReadme, /npm install -g @kaelio\/ktx/); - assert.match(rootReadme, /ktx runtime install/); - assert.match(rootReadme, /ktx runtime status/); - assert.match(rootReadme, /ktx runtime doctor/); - assert.match(rootReadme, /ktx runtime start/); - assert.match(rootReadme, /ktx runtime stop/); - assert.match(rootReadme, /ktx serve --mcp stdio/); - assert.doesNotMatch(rootReadme, /uv run ktx-daemon serve-http/); - assert.doesNotMatch(rootReadme, /--semantic-compute-url http:\/\/127\.0\.0\.1:8765/); - }); -``` - -- [ ] **Step 2: Add package artifact README assertions** - -In `scripts/examples-docs.test.mjs`, insert this test after the new public -runtime README test: - -```javascript - it('documents the public package artifact smoke shape', async () => { - const readme = await readText('examples/package-artifacts/README.md'); - - assert.match(readme, /@kaelio\/ktx/); - assert.match(readme, /managed Python runtime/); - assert.match(readme, /ktx runtime status/); - assert.match(readme, /ktx runtime doctor/); - assert.doesNotMatch(readme, /@ktx\/context/); - assert.doesNotMatch(readme, /@ktx\/cli/); - assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); - }); -``` - -- [ ] **Step 3: Extend Postgres smoke assertions** - -In the existing `documents the Postgres historic SQL smoke example` test in -`scripts/examples-docs.test.mjs`, add these assertions after -`assert.match(smoke, /pg_stat_statements_reset/);`: - -```javascript - assert.match(smoke, /KTX_RUNTIME_ROOT/); - assert.match(smoke, /managedDaemon/); - assert.match(smoke, /installPolicy: 'auto'/); - assert.match(smoke, /getKtxCliPackageInfo/); - assert.doesNotMatch(smoke, /python-service/); - assert.doesNotMatch(smoke, /PYTHON_SERVICE/); - assert.doesNotMatch(smoke, /uvicorn app\.main:app/); - assert.doesNotMatch(smoke, /export KTX_SQL_ANALYSIS_URL/); - assert.doesNotMatch(readme, /python-service/); - assert.doesNotMatch(readme, /KTX_SQL_ANALYSIS_URL/); -``` - -- [ ] **Step 4: Run the docs test to verify it fails** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL. The failure includes missing `@kaelio/ktx` README matches and -the existing `python-service` / `KTX_SQL_ANALYSIS_URL` references in the -Postgres smoke files. - -### Task 2: Move the Postgres historic smoke to the managed runtime - -**Files:** - -- Modify: `examples/postgres-historic/scripts/smoke.sh` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Remove Python service process state** - -In `examples/postgres-historic/scripts/smoke.sh`, replace the variable block: - -```bash -KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" -PYTHON_SERVICE_LOG="$PROJECT_PARENT/python-service.log" -PYTHON_SERVICE_PID="" -``` - -with: - -```bash -KTX_BIN="$KTX_ROOT/packages/cli/dist/bin.js" -export KTX_RUNTIME_ROOT="$PROJECT_PARENT/managed-runtime" -unset KTX_DAEMON_URL -unset KTX_SQL_ANALYSIS_URL -``` - -- [ ] **Step 2: Replace cleanup** - -In `examples/postgres-historic/scripts/smoke.sh`, replace the `cleanup()` -function with: - -```bash -cleanup() { - if [[ -f "$KTX_BIN" ]]; then - node "$KTX_BIN" runtime stop >/dev/null 2>&1 || true - fi - if [[ "${KTX_POSTGRES_HISTORIC_KEEP_DOCKER:-0}" != "1" ]]; then - docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true - fi -} -trap cleanup EXIT -``` - -- [ ] **Step 3: Delete the old SQL analysis service starter** - -Delete the entire `start_sql_analysis_if_needed()` function from -`examples/postgres-historic/scripts/smoke.sh`. The deleted function begins with -this line: - -```bash -start_sql_analysis_if_needed() { -``` - -and ends with this line: - -```bash -} -``` - -immediately before the `latest_manifest()` function. - -- [ ] **Step 4: Pass managed daemon options to stage-only ingest** - -In the Node heredoc inside `run_historic_stage_only()`, replace this block: - -```javascript -const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); - -const project = await loadKtxProject({ projectDir }); -const adapters = createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: 'warehouse' }); -``` - -with: - -```javascript -const { createKtxCliLocalIngestAdapters } = await import(join(ktxRoot, 'packages/cli/dist/local-adapters.js')); -const { getKtxCliPackageInfo } = await import(join(ktxRoot, 'packages/cli/dist/index.js')); - -const project = await loadKtxProject({ projectDir }); -const cliVersion = getKtxCliPackageInfo().version; -const managedRuntimeIo = { stdout: process.stdout, stderr: process.stderr }; -const adapters = createKtxCliLocalIngestAdapters(project, { - historicSqlConnectionId: 'warehouse', - managedDaemon: { - cliVersion, - installPolicy: 'auto', - io: managedRuntimeIo, - }, -}); -``` - -- [ ] **Step 5: Remove the old starter call** - -Delete this line from the bottom half of -`examples/postgres-historic/scripts/smoke.sh`: - -```bash -start_sql_analysis_if_needed -``` - -- [ ] **Step 6: Run the docs test to verify the script guards pass** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL remains because README files have not been updated yet. The -Postgres smoke script assertions now pass. - -### Task 3: Update Postgres historic and artifact docs - -**Files:** - -- Modify: `examples/postgres-historic/README.md` -- Modify: `examples/package-artifacts/README.md` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Replace Postgres prerequisites** - -In `examples/postgres-historic/README.md`, replace the `## Prerequisites` -section with: - -```markdown -## Prerequisites - -- Docker with Compose v2 -- Node and pnpm matching the KTX workspace -- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled - runtime wheel -``` - -- [ ] **Step 2: Replace the smoke run description** - -In `examples/postgres-historic/README.md`, replace the paragraph after the -`examples/postgres-historic/scripts/smoke.sh` command with: - -```markdown -The smoke creates a temporary KTX project, isolates the managed Python runtime -under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and -uses this connection URL: -``` - -- [ ] **Step 3: Update the full ingest command** - -In `examples/postgres-historic/README.md`, replace the manual ingest command: - -```bash -node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic dev ingest run \ - --connection-id warehouse \ - --adapter historic-sql \ - --plain \ - --no-input -``` - -with: - -```bash -pnpm run ktx -- dev ingest run --project-dir /tmp/ktx-postgres-historic \ - --connection-id warehouse \ - --adapter historic-sql \ - --plain \ - --yes \ - --no-input -``` - -- [ ] **Step 4: Replace SQL-analysis troubleshooting** - -In `examples/postgres-historic/README.md`, replace the final troubleshooting -bullet: - -```markdown -- SQL-analysis failures: set `KTX_SQL_ANALYSIS_URL` to the running service URL - or create `python-service/.venv` before running `scripts/smoke.sh`. -``` - -with: - -```markdown -- SQL-analysis failures: run `pnpm run ktx -- runtime doctor` from the KTX - repository root and confirm `uv`, the bundled Python wheel, and the managed - runtime all pass. -``` - -- [ ] **Step 5: Replace package artifact README body** - -Replace the full contents of `examples/package-artifacts/README.md` with: - -````markdown -# Package artifact smoke checks - -The package artifact smoke checks create temporary projects instead of storing -sample projects in this directory. Run the checks from `ktx/`: - -```bash -pnpm run artifacts:check -``` - -The npm smoke project installs the generated public `@kaelio/ktx` tarball, -imports the package entry point, and runs installed `ktx` commands against a -generated local project. - -The managed runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies -`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from -the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed -daemon, and stops it. - -The Python smoke project still installs the Python artifacts directly because -it verifies the standalone Python distributions that feed the bundled runtime -wheel. -```` - -- [ ] **Step 6: Run the docs test to verify these docs pass** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL remains because `README.md` still lacks the public npm managed -runtime documentation. The Postgres and package artifact assertions now pass. - -### Task 4: Update the root README public runtime path - -**Files:** - -- Modify: `README.md` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Replace quick start** - -In `README.md`, replace the `## Quick start` section through the end of the -full-demo paragraph with: - -````markdown -## Quick start - -Run the pre-seeded demo through the public npm package: - -```bash -npx @kaelio/ktx setup demo --no-input -npx @kaelio/ktx setup demo inspect -``` - -The default demo uses packaged sample data and prebuilt context. It does not -require API keys, network access, or an LLM provider. - -To replay the packaged ingest run, use: - -```bash -npx @kaelio/ktx setup demo --mode replay --no-input -``` - -To run the full agentic demo with an LLM provider, set a provider key for the -current process: - -```bash -ANTHROPIC_API_KEY=$YOUR_ANTHROPIC_API_KEY \ - npx @kaelio/ktx setup demo --mode full --no-input -``` - -Interactive full-demo setup can prompt for a provider key without writing the -key to `ktx.yaml`. - -You can also install the CLI in a project or globally: - -```bash -npm install @kaelio/ktx -npx ktx --help -npm install -g @kaelio/ktx -ktx --help -``` -```` - -- [ ] **Step 2: Replace local project setup command** - -In the `## Build a local project` section of `README.md`, replace: - -```bash -uv sync --all-packages -source .venv/bin/activate - -PROJECT_DIR="$(mktemp -d)/ktx-demo" -pnpm run ktx -- init "$PROJECT_DIR" --name ktx-demo -``` - -with: - -```bash -npm install @kaelio/ktx -PROJECT_DIR="$(mktemp -d)/ktx-demo" -npx ktx init "$PROJECT_DIR" --name ktx-demo -``` - -- [ ] **Step 3: Replace README command prefixes** - -In `README.md`, replace the source-tree command prefix `pnpm run ktx --` with -`npx ktx` in all user workflow commands under `## Build a local project`, -`### Scan the demo warehouse`, and `## Serve MCP`. Keep `pnpm run ktx --` in -the `## Development` section. - -For example, this command: - -```bash -pnpm run ktx -- sl query --project-dir "$PROJECT_DIR" \ -``` - -becomes: - -```bash -npx ktx sl query --project-dir "$PROJECT_DIR" \ -``` - -- [ ] **Step 4: Add managed runtime section** - -Insert this section after the scan walkthrough in `README.md`: - -````markdown -## Managed Python runtime - -KTX installs its Python runtime only when a Python-backed command needs it. -The runtime lives outside the npm cache, is versioned by the installed CLI -version, and is managed by `ktx runtime` commands: - -```bash -npx ktx runtime install --yes -npx ktx runtime status -npx ktx runtime doctor -npx ktx runtime start -npx ktx runtime stop -``` - -Commands such as `npx @kaelio/ktx sl query ... --yes` can install the core -runtime lazily from the bundled wheel. Local embeddings remain lazy; prepare -them only when you select local `sentence-transformers` embeddings: - -```bash -npx ktx runtime install --feature local-embeddings --yes -npx ktx runtime start --feature local-embeddings -``` -```` - -- [ ] **Step 5: Replace Serve MCP section** - -In `README.md`, replace the full `## Serve MCP` section with: - -````markdown -## Serve MCP - -Start the stdio MCP server from the project directory: - -```bash -npx ktx serve --mcp stdio --project-dir "$PROJECT_DIR" \ - --user-id local \ - --semantic-compute \ - --execute-queries \ - --yes -``` - -The `--semantic-compute` flag uses the managed Python runtime when no explicit -semantic compute URL is provided. KTX starts or reuses the managed runtime as -needed. - -The MCP server exposes `connection_list`, `knowledge_search`, -`knowledge_read`, `knowledge_write`, `sl_list_sources`, `sl_read_source`, -`sl_write_source`, `sl_validate`, `sl_query`, `ingest_trigger`, -`ingest_status`, `ingest_report`, and `ingest_replay`. -```` - -- [ ] **Step 6: Update release status wording** - -In `README.md`, replace this sentence in `## Release status`: - -```markdown -This repository is prepared for source publication. Package publishing is still -disabled by `release-policy.json`; registry names, public versions, package -visibility, and provenance policy must be chosen before publishing artifacts to -npm or Python package indexes. -``` - -with: - -```markdown -This repository builds a single public npm artifact named `@kaelio/ktx`. -Package publishing is still disabled by `release-policy.json`; registry -credentials, public versions, release tags, and provenance policy must be -chosen before publishing artifacts to npm or Python package indexes. -``` - -- [ ] **Step 7: Run the docs test to verify the README passes** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -### Task 5: Final verification and commit - -**Files:** - -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `examples/postgres-historic/scripts/smoke.sh` -- Verify: `examples/postgres-historic/README.md` -- Verify: `examples/package-artifacts/README.md` -- Verify: `README.md` - -- [ ] **Step 1: Run the script test suite affected by docs** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs scripts/check-boundaries.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 2: Run the boundary check** - -Run: - -```bash -node scripts/check-boundaries.mjs -``` - -Expected: - -```text -ktx boundary check passed -``` - -- [ ] **Step 3: Search for removed external runtime references** - -Run: - -```bash -rg -n "python-service|uvicorn app\\.main:app|export KTX_SQL_ANALYSIS_URL|uv run ktx-daemon serve-http|@ktx/context.*@ktx/cli" README.md examples/postgres-historic/README.md examples/postgres-historic/scripts/smoke.sh examples/package-artifacts/README.md -``` - -Expected: no matches. - -- [ ] **Step 4: Commit** - -```bash -git add scripts/examples-docs.test.mjs \ - examples/postgres-historic/scripts/smoke.sh \ - examples/postgres-historic/README.md \ - examples/package-artifacts/README.md \ - README.md -git commit -m "docs: align managed runtime examples" -``` - -## Acceptance criteria - -- The Postgres historic SQL smoke no longer references `python-service/`, - `uvicorn app.main:app`, or `export KTX_SQL_ANALYSIS_URL`. -- The stage-only Postgres historic smoke uses `createKtxCliLocalIngestAdapters` - with managed daemon options and `installPolicy: 'auto'`. -- The root README documents `npx @kaelio/ktx`, local `npx ktx`, global `ktx`, - `ktx runtime ...`, and MCP `--semantic-compute --yes` managed-runtime usage. -- Package artifact docs describe the single public `@kaelio/ktx` tarball and - the managed runtime smoke. -- `node --test scripts/examples-docs.test.mjs scripts/check-boundaries.test.mjs` - passes. -- `node scripts/check-boundaries.mjs` passes. - -## Self-review - -- Spec coverage: This plan covers the remaining user-facing drift from the - npm-managed runtime spec by removing manual Python service guidance, - documenting public `@kaelio/ktx` invocation modes, and making the Postgres - example smoke use the managed core daemon. -- Placeholder scan: The plan contains exact files, edits, commands, expected - outcomes, and commit instructions. -- Type consistency: The plan uses the existing `managedDaemon` option shape - from `packages/cli/src/local-adapters.ts` and the existing - `installPolicy: 'auto'` value from `packages/cli/src/managed-python-command.ts`. diff --git a/docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md b/docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md deleted file mode 100644 index b6cbd405..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md +++ /dev/null @@ -1,377 +0,0 @@ -# Managed Runtime Prune Smoke and Docs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Prove and document `ktx runtime prune` as part of the npm-managed -Python runtime release contract. - -**Architecture:** The prune command already exists in the CLI runtime layer, so -this plan adds black-box package smoke coverage and public documentation only. -The smoke creates an isolated stale versioned runtime directory, previews it, -verifies confirmation is required, and removes it through the installed -`@kaelio/ktx` package. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, Markdown, KTX CLI -managed Python runtime. - ---- - -## Current state - -This plan follows -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plan files are based on that spec and are implemented in the -current tree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` -- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` -- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` - -Implementation evidence found before writing this plan includes: - -- `packages/cli/assets/python/manifest.json` and - `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl`. -- `packages/cli/src/managed-python-runtime.ts`, including - `installManagedPythonRuntime()`, `doctorManagedPythonRuntime()`, and - `pruneManagedPythonRuntimes()`. -- `packages/cli/src/runtime.ts`, including the `install`, `status`, - `doctor`, `start`, `stop`, and `prune` runtime command runner branches. -- `packages/cli/src/commands/runtime-commands.ts`, including the - `runtime prune --dry-run` and `runtime prune --yes` Commander wiring. -- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, - `scripts/published-package-smoke.mjs`, `scripts/local-embeddings-runtime-smoke.mjs`, - `scripts/publish-public-npm-package.mjs`, `release-policy.json`, and - `.github/workflows/release.yml`. -- `README.md` and `examples/package-artifacts/README.md` document the managed - runtime but do not mention `ktx runtime prune`. - -The remaining gap is narrow: the spec lists `ktx runtime prune` as part of the -runtime management command family, but public docs and installed package smoke -coverage only prove `install`, `status`, `doctor`, `start`, and `stop`. - -## File structure - -- Modify `scripts/package-artifacts.test.mjs`: assert that the generated - installed npm smoke covers `ktx runtime prune --dry-run`, confirmation - failure, and confirmed deletion. -- Modify `scripts/package-artifacts.mjs`: extend `npmRuntimeSmokeSource()` to - create a stale runtime directory and exercise `ktx runtime prune`. -- Modify `scripts/examples-docs.test.mjs`: require public docs to mention - `ktx runtime prune --dry-run` and `ktx runtime prune --yes`. -- Modify `README.md`: add prune commands and one sentence describing preview - and confirmed deletion. -- Modify `examples/package-artifacts/README.md`: describe prune coverage in the - package artifact smoke. - -### Task 1: Add installed package prune smoke coverage - -**Files:** - -- Modify: `scripts/package-artifacts.test.mjs` -- Modify: `scripts/package-artifacts.mjs` - -- [ ] **Step 1: Add failing smoke-source assertions** - -In `scripts/package-artifacts.test.mjs`, inside -`it('runs installed CLI commands through the public package runtime', () => {` -and immediately after the existing assertions for `ktx runtime stop`, add: - -```javascript - assert.match(source, /ktx runtime prune dry run/); - assert.match(source, /0\.0\.0/); - assert.match(source, /ktx runtime prune needs confirmation/); - assert.match(source, /Refusing to prune without --yes/); - assert.match(source, /ktx runtime prune confirmed/); - assert.match(source, /Removed stale KTX Python runtimes/); - assert.match(source, /assert\.rejects\(\(\) => access\(staleRuntimeDir\)\)/); -``` - -- [ ] **Step 2: Run the package artifact test and verify failure** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: FAIL in the installed CLI smoke source test because -`npmRuntimeSmokeSource()` does not yet contain the prune labels, confirmation -guard, or stale runtime removal assertion. - -- [ ] **Step 3: Extend the generated installed CLI smoke** - -In `scripts/package-artifacts.mjs`, inside `npmRuntimeSmokeSource()`, add this -block immediately after: - -```javascript - process.stdout.write('ktx runtime daemon lifecycle verified\n'); -``` - -Add: - -```javascript - const staleRuntimeDir = join(process.env.KTX_RUNTIME_ROOT, '0.0.0'); - await mkdir(staleRuntimeDir, { recursive: true }); - - const runtimePruneDryRun = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--dry-run']); - requireSuccess('ktx runtime prune dry run', runtimePruneDryRun); - requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /Stale KTX Python runtimes/); - requireOutput('ktx runtime prune dry run', runtimePruneDryRun, /0\.0\.0/); - await access(staleRuntimeDir); - - const runtimePruneNeedsConfirmation = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune']); - assert.equal(runtimePruneNeedsConfirmation.code, 1, 'ktx runtime prune without --yes must fail'); - assert.equal(runtimePruneNeedsConfirmation.stdout, '', 'ktx runtime prune confirmation failure wrote stdout'); - assert.match(runtimePruneNeedsConfirmation.stderr, /Refusing to prune without --yes/); - - const runtimePruneConfirmed = await run('pnpm', ['exec', 'ktx', 'runtime', 'prune', '--yes']); - requireSuccess('ktx runtime prune confirmed', runtimePruneConfirmed); - requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /Removed stale KTX Python runtimes/); - requireOutput('ktx runtime prune confirmed', runtimePruneConfirmed, /0\.0\.0/); - await assert.rejects(() => access(staleRuntimeDir)); - process.stdout.write('ktx runtime prune verified\n'); -``` - -No import changes are needed because the generated smoke already imports -`assert`, `access`, `mkdir`, and `join`. - -- [ ] **Step 4: Run the package artifact test and verify pass** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: PASS. The source assertions now find prune dry-run coverage, -confirmation failure coverage, confirmed prune coverage, and stale directory -deletion verification. - -- [ ] **Step 5: Commit the smoke coverage** - -Run: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -git commit -m "test: cover managed runtime prune in package smoke" -``` - -### Task 2: Document runtime prune in public docs - -**Files:** - -- Modify: `scripts/examples-docs.test.mjs` -- Modify: `README.md` -- Modify: `examples/package-artifacts/README.md` - -- [ ] **Step 1: Add failing docs assertions** - -In `scripts/examples-docs.test.mjs`, inside -`it('documents public npm and managed runtime usage in the README', async () => {` -and immediately after: - -```javascript - assert.match(rootReadme, /ktx runtime stop/); -``` - -Add: - -```javascript - assert.match(rootReadme, /ktx runtime prune --dry-run/); - assert.match(rootReadme, /ktx runtime prune --yes/); -``` - -In the same file, inside -`it('documents the public package artifact smoke shape', async () => {` and -immediately after: - -```javascript - assert.match(readme, /ktx runtime doctor/); -``` - -Add: - -```javascript - assert.match(readme, /ktx runtime prune --dry-run/); - assert.match(readme, /ktx runtime prune --yes/); -``` - -- [ ] **Step 2: Run the docs test and verify failure** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL because `README.md` and -`examples/package-artifacts/README.md` do not yet mention `ktx runtime prune`. - -- [ ] **Step 3: Update the root README runtime section** - -In `README.md`, in the `## Managed Python runtime` command block, replace: - -```bash -npx ktx runtime install --yes -npx ktx runtime status -npx ktx runtime doctor -npx ktx runtime start -npx ktx runtime stop -``` - -with: - -```bash -npx ktx runtime install --yes -npx ktx runtime status -npx ktx runtime doctor -npx ktx runtime start -npx ktx runtime stop -npx ktx runtime prune --dry-run -npx ktx runtime prune --yes -``` - -Immediately after that command block, add: - -```markdown -Use `runtime prune --dry-run` to preview stale runtime directories from older -CLI versions. Add `--yes` to remove those stale directories after daemon -processes are stopped. -``` - -- [ ] **Step 4: Update package artifact smoke docs** - -In `examples/package-artifacts/README.md`, replace: - -```markdown -The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies -`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from -the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed -daemon, and stops it. -``` - -with: - -```markdown -The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies -`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from -the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed -daemon, stops it, previews a stale runtime with `ktx runtime prune --dry-run`, -verifies confirmation is required, and removes the stale runtime with -`ktx runtime prune --yes`. -``` - -- [ ] **Step 5: Run the docs test and verify pass** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. The public README and package artifact README now document -runtime prune alongside the other managed runtime commands. - -- [ ] **Step 6: Commit the docs coverage** - -Run: - -```bash -git add scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md -git commit -m "docs: document managed runtime prune" -``` - -### Task 3: Verify the completed prune release surface - -**Files:** - -- Verify: `scripts/package-artifacts.mjs` -- Verify: `scripts/package-artifacts.test.mjs` -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `README.md` -- Verify: `examples/package-artifacts/README.md` - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs scripts/examples-docs.test.mjs -``` - -Expected: PASS. The source-level tests cover generated package smoke behavior -and docs assertions. - -- [ ] **Step 2: Run the installed package artifact smoke** - -Run: - -```bash -pnpm run artifacts:check -``` - -Expected: PASS. The generated installed CLI smoke prints: - -```text -ktx runtime prune verified -``` - -and removes the temporary `0.0.0` directory from the isolated -`KTX_RUNTIME_ROOT`. - -- [ ] **Step 3: Inspect git status** - -Run: - -```bash -git status --short -``` - -Expected: only the five planned files are modified before the final commit, or -no modified files remain after the task commits. - -- [ ] **Step 4: Commit verification fixes if needed** - -If verification required small corrections, commit only those intended files: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md -git commit -m "test: verify managed runtime prune release surface" -``` - -## Acceptance criteria - -- The generated installed npm package smoke creates a stale versioned runtime - directory under the isolated `KTX_RUNTIME_ROOT`. -- `ktx runtime prune --dry-run` lists the stale runtime and leaves it on disk. -- `ktx runtime prune` without `--yes` exits nonzero and prints the existing - confirmation guidance. -- `ktx runtime prune --yes` removes the stale runtime directory. -- `README.md` lists `ktx runtime prune --dry-run` and - `ktx runtime prune --yes` with the other managed runtime commands. -- `examples/package-artifacts/README.md` describes prune coverage in the - package artifact smoke. - -## Self-review - -- Spec coverage: this plan covers the remaining visible gap for the runtime - management command family in the npm-managed Python runtime spec. The prune - implementation already exists, and this plan adds release smoke and public - docs coverage. -- Placeholder scan: no placeholder steps, deferred implementation notes, or - unspecified behavior gaps remain. -- Type consistency: the plan uses existing labels and functions: - `npmRuntimeSmokeSource()`, `requireSuccess()`, `requireOutput()`, - `KTX_RUNTIME_ROOT`, `ktx runtime prune --dry-run`, and - `ktx runtime prune --yes`. diff --git a/docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md b/docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md deleted file mode 100644 index b3e1e0f9..00000000 --- a/docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md +++ /dev/null @@ -1,647 +0,0 @@ -# Managed Runtime uv Prerequisite Contract Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining npm-managed Python runtime open decision by -making `uv` a documented, release-policy-checked prerequisite. - -**Architecture:** Keep the runtime installer behavior simple: the CLI locates -`uv` on `PATH` and prints a focused error when it is missing. Encode that -decision in `release-policy.json`, validate it during release readiness, use one -shared runtime error message, and document the prerequisite in public docs. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, TypeScript, Vitest, JSON -release policy, Markdown. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plan files are based on that spec and are already implemented in -this worktree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` -- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` -- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md` - -Implementation evidence found before writing this plan includes: - -- `packages/cli/assets/python/manifest.json` and the bundled - `kaelio_ktx-0.1.0-py3-none-any.whl`. -- `packages/cli/src/managed-python-runtime.ts`, including runtime roots, - bundled wheel verification, install, status, doctor, and prune behavior. -- `packages/cli/src/managed-python-command.ts`, - `packages/cli/src/managed-python-daemon.ts`, - `packages/cli/src/managed-local-embeddings.ts`, and - `packages/cli/src/managed-python-http.ts`. -- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, - `scripts/published-package-smoke.mjs`, - `scripts/local-embeddings-runtime-smoke.mjs`, and - `scripts/publish-public-npm-package.mjs`. -- `release-policy.json` is already in `npm-public-release-ready` mode for - `@kaelio/ktx` `0.1.0` and keeps Python package publishing disabled. -- `README.md` and `examples/package-artifacts/README.md` document the managed - runtime command family, including `runtime prune`. - -The remaining spec gap is the open decision in -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`: - -```text -KTX still needs a final decision on whether uv is a hard prerequisite or a -bootstrap dependency that KTX downloads automatically. -``` - -This plan chooses the hard-prerequisite path for the first public release. KTX -will not download `uv` automatically in this release. - -## File structure - -- Modify `release-policy.json`: add a `runtimeInstaller` policy section that - records the hard `uv` prerequisite decision. -- Modify `scripts/release-readiness.mjs`: validate the runtime installer - policy, include it in readiness reports, and print it in text output. -- Modify `scripts/release-readiness.test.mjs`: cover the accepted policy and - rejection paths for missing or bootstrap-style `uv` policies. -- Modify `packages/cli/src/managed-python-runtime.ts`: export one shared - missing-`uv` message and use it for install and doctor output. -- Modify `packages/cli/src/managed-python-runtime.test.ts`: cover install and - doctor behavior when `uv` is missing. -- Modify `scripts/examples-docs.test.mjs`: require public docs to state the - hard `uv` prerequisite. -- Modify `README.md`: document that `uv` must be on `PATH` and KTX does not - download it automatically. -- Modify `examples/package-artifacts/README.md`: document the artifact smoke - `uv` prerequisite. - -### Task 1: Encode the runtime installer policy - -**Files:** - -- Modify: `release-policy.json` -- Modify: `scripts/release-readiness.test.mjs` -- Modify: `scripts/release-readiness.mjs` -- Test: `scripts/release-readiness.test.mjs` - -- [ ] **Step 1: Add failing release policy tests** - -In `scripts/release-readiness.test.mjs`, inside the `releasePolicy()` helper -return value, add the `runtimeInstaller` object immediately after -`publishedPackageSmoke`: - -```javascript - runtimeInstaller: { - uvStrategy: 'path-prerequisite', - bootstrapUv: false, - missingUvBehavior: 'focused-error', - }, -``` - -In the three `assert.deepEqual(report, { ... })` expectations, add this field -immediately after `publishedPackageSmokeGate`: - -```javascript - runtimeInstaller: { - uvStrategy: 'path-prerequisite', - bootstrapUv: false, - missingUvBehavior: 'focused-error', - }, -``` - -Add these tests immediately after the -`it('accepts the npm public release ready policy', async () => { ... })` block: - -```javascript - it('rejects npm public release ready mode without a runtime installer policy', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-missing-test-')); - try { - await writeReadyFixture(root, { - policy: releasePolicy({ - releaseMode: 'npm-public-release-ready', - npm: { - publish: true, - registry: null, - access: 'public', - tag: 'latest', - }, - publishedPackageSmoke: { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - registry: null, - }, - runtimeInstaller: undefined, - requiredBeforePublishing: [], - }), - }); - - await assert.rejects( - () => releaseReadinessReport(root), - /Release policy runtimeInstaller must be a JSON object/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - it('rejects uv bootstrap download policy for the first public npm release', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-runtime-policy-bootstrap-test-')); - try { - await writeReadyFixture(root, { - policy: releasePolicy({ - releaseMode: 'npm-public-release-ready', - npm: { - publish: true, - registry: null, - access: 'public', - tag: 'latest', - }, - publishedPackageSmoke: { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - registry: null, - }, - runtimeInstaller: { - uvStrategy: 'bootstrap-download', - bootstrapUv: true, - missingUvBehavior: 'download', - }, - requiredBeforePublishing: [], - }), - }); - - await assert.rejects( - () => releaseReadinessReport(root), - /Release policy runtimeInstaller\.uvStrategy must be path-prerequisite/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Run the release readiness tests and verify failure** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs -``` - -Expected: FAIL because `releaseReadinessReport()` does not include -`runtimeInstaller`, and `validateReleasePolicy()` does not validate the new -policy section. - -- [ ] **Step 3: Validate the runtime installer policy** - -In `scripts/release-readiness.mjs`, add this function immediately after the -`assertRequiredBeforePublishing(policy)` function definition: - -```javascript -function assertRuntimeInstallerPolicy(policy) { - assertPlainObject(policy.runtimeInstaller, 'Release policy runtimeInstaller'); - assertString(policy.runtimeInstaller.uvStrategy, 'Release policy runtimeInstaller.uvStrategy'); - assertBoolean(policy.runtimeInstaller.bootstrapUv, 'Release policy runtimeInstaller.bootstrapUv'); - assertString( - policy.runtimeInstaller.missingUvBehavior, - 'Release policy runtimeInstaller.missingUvBehavior', - ); - - if (policy.runtimeInstaller.uvStrategy !== 'path-prerequisite') { - throw new Error('Release policy runtimeInstaller.uvStrategy must be path-prerequisite'); - } - if (policy.runtimeInstaller.bootstrapUv !== false) { - throw new Error('Release policy runtimeInstaller.bootstrapUv must be false'); - } - if (policy.runtimeInstaller.missingUvBehavior !== 'focused-error') { - throw new Error('Release policy runtimeInstaller.missingUvBehavior must be focused-error'); - } -} -``` - -In `validateReleasePolicy(policy)`, add this call immediately after -`assertRequiredBeforePublishing(policy);`: - -```javascript - assertRuntimeInstallerPolicy(policy); -``` - -In `releaseReadinessReport(rootDir = scriptRootDir())`, add -`runtimeInstaller` to the returned object immediately after -`publishedPackageSmokeGate`: - -```javascript - runtimeInstaller: policy.runtimeInstaller, -``` - -In `main()`, add these lines immediately after the published package smoke -registry line: - -```javascript - process.stdout.write(`Runtime uv strategy: ${report.runtimeInstaller.uvStrategy}\n`); - process.stdout.write( - `Runtime uv bootstrap: ${report.runtimeInstaller.bootstrapUv ? 'enabled' : 'disabled'}\n`, - ); -``` - -- [ ] **Step 4: Encode the policy in `release-policy.json`** - -Replace `release-policy.json` with this exact content: - -```json -{ - "schemaVersion": 1, - "releaseMode": "npm-public-release-ready", - "npm": { - "publish": true, - "registry": null, - "access": "public", - "tag": "latest", - "packages": ["@kaelio/ktx"] - }, - "python": { - "publish": false, - "repository": null, - "packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"] - }, - "publishedPackageSmoke": { - "packageName": "@kaelio/ktx", - "version": "0.1.0", - "registry": null - }, - "runtimeInstaller": { - "uvStrategy": "path-prerequisite", - "bootstrapUv": false, - "missingUvBehavior": "focused-error" - }, - "requiredBeforePublishing": [] -} -``` - -- [ ] **Step 5: Run the release readiness tests and verify success** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the release policy contract** - -```bash -git add release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs -git commit -m "chore: encode uv runtime prerequisite policy" -``` - -### Task 2: Centralize missing-uv runtime output - -**Files:** - -- Modify: `packages/cli/src/managed-python-runtime.test.ts` -- Modify: `packages/cli/src/managed-python-runtime.ts` -- Test: `packages/cli/src/managed-python-runtime.test.ts` - -- [ ] **Step 1: Add failing missing-uv runtime tests** - -In `packages/cli/src/managed-python-runtime.test.ts`, add -`MISSING_UV_RUNTIME_INSTALL_MESSAGE` to the import from -`./managed-python-runtime.js`: - -```typescript -import { - MISSING_UV_RUNTIME_INSTALL_MESSAGE, - doctorManagedPythonRuntime, - installManagedPythonRuntime, - managedPythonRuntimeLayout, - pruneManagedPythonRuntimes, - readManagedPythonRuntimeStatus, - verifyRuntimeAsset, - type ManagedPythonRuntimeExec, -} from './managed-python-runtime.js'; -``` - -Inside `describe('installManagedPythonRuntime', () => { ... })`, add this test -after the local embeddings test: - -```typescript - it('fails with the hard-prerequisite message when uv is missing', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const commands: Array<{ command: string; args: string[] }> = []; - const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { - commands.push({ command, args }); - throw new Error('spawn uv ENOENT'); - }); - - await expect( - installManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - features: ['core'], - exec, - }), - ).rejects.toThrow(MISSING_UV_RUNTIME_INSTALL_MESSAGE); - - expect(commands).toEqual([{ command: 'uv', args: ['--version'] }]); - }); -``` - -Inside `describe('doctorManagedPythonRuntime', () => { ... })`, add this test -after the existing doctor test: - -```typescript - it('reports uv as a hard prerequisite when uv is missing', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); - const exec: ManagedPythonRuntimeExec = vi.fn(async () => { - throw new Error('spawn uv ENOENT'); - }); - - const checks = await doctorManagedPythonRuntime({ - cliVersion: '0.2.0', - runtimeRoot: join(tempDir, 'runtime'), - assetDir, - exec, - }); - - expect(checks[0]).toEqual({ - id: 'uv', - label: 'uv', - status: 'fail', - detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE, - fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes', - }); - }); -``` - -- [ ] **Step 2: Run the runtime tests and verify failure** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts -``` - -Expected: FAIL because the shared message constant does not exist and the -doctor fix text still uses the older message. - -- [ ] **Step 3: Add the shared missing-uv message** - -In `packages/cli/src/managed-python-runtime.ts`, add this export immediately -after the `ManagedPythonRuntimePruneResult` interface: - -```typescript -export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = - 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx runtime install --yes'; -``` - -Replace the body of the `catch` block in `ensureUv()` with: - -```typescript - throw new Error(MISSING_UV_RUNTIME_INSTALL_MESSAGE); -``` - -In `doctorManagedPythonRuntime()`, replace the `fix` value for the `uv` check -with: - -```typescript - fix: 'Install uv, make sure it is on PATH, and run: ktx runtime install --yes', -``` - -- [ ] **Step 4: Run the runtime tests and verify success** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the runtime output contract** - -```bash -git add packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts -git commit -m "fix: clarify missing uv runtime error" -``` - -### Task 3: Document the hard uv prerequisite - -**Files:** - -- Modify: `scripts/examples-docs.test.mjs` -- Modify: `README.md` -- Modify: `examples/package-artifacts/README.md` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Add failing docs assertions** - -In `scripts/examples-docs.test.mjs`, inside -`it('documents public npm and managed runtime usage in the README', ... )`, add -these assertions immediately after the existing `ktx runtime prune --yes` -assertion: - -```javascript - assert.match(rootReadme, /KTX requires `uv` on `PATH`/); - assert.match(rootReadme, /KTX doesn't download `uv` automatically/); -``` - -Inside `it('documents the public package artifact smoke shape', ... )`, add -this assertion immediately after the `managed Python runtime` assertion: - -```javascript - assert.match(readme, /requires `uv` on `PATH`/); -``` - -- [ ] **Step 2: Run the docs test and verify failure** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL because the README files do not state the hard `uv` -prerequisite. - -- [ ] **Step 3: Update the root README runtime section** - -In `README.md`, in the `## Managed Python runtime` section, replace this -paragraph: - -```markdown -KTX installs its Python runtime only when a Python-backed command needs it. -The runtime lives outside the npm cache, is versioned by the installed CLI -version, and is managed by `ktx runtime` commands: -``` - -With: - -```markdown -KTX installs its Python runtime only when a Python-backed command needs it. -The runtime lives outside the npm cache, is versioned by the installed CLI -version, and is managed by `ktx runtime` commands. - -KTX requires `uv` on `PATH` to create the managed runtime. Install `uv` with -your system package manager or the official installer before running Python- -backed KTX commands. KTX doesn't download `uv` automatically; run -`ktx runtime doctor` if runtime installation fails: -``` - -- [ ] **Step 4: Update the package artifact smoke README** - -In `examples/package-artifacts/README.md`, replace this paragraph: - -```markdown -The managed Python runtime smoke isolates `KTX_RUNTIME_ROOT`, verifies -`ktx runtime status`, runs `ktx sl query --yes` to install the core runtime from -the bundled wheel, checks `ktx runtime doctor`, starts and reuses the managed -daemon, stops it, previews a stale runtime with `ktx runtime prune --dry-run`, -verifies confirmation is required, and removes the stale runtime with -`ktx runtime prune --yes`. -``` - -With: - -```markdown -The managed Python runtime smoke requires `uv` on `PATH`, isolates -`KTX_RUNTIME_ROOT`, verifies `ktx runtime status`, runs `ktx sl query --yes` to -install the core runtime from the bundled wheel, checks `ktx runtime doctor`, -starts and reuses the managed daemon, stops it, previews a stale runtime with -`ktx runtime prune --dry-run`, verifies confirmation is required, and removes -the stale runtime with `ktx runtime prune --yes`. -``` - -- [ ] **Step 5: Run the docs test and verify success** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the public docs update** - -```bash -git add README.md examples/package-artifacts/README.md scripts/examples-docs.test.mjs -git commit -m "docs: document uv runtime prerequisite" -``` - -### Task 4: Verify the completed contract - -**Files:** - -- Verify: `release-policy.json` -- Verify: `scripts/release-readiness.mjs` -- Verify: `scripts/release-readiness.test.mjs` -- Verify: `packages/cli/src/managed-python-runtime.ts` -- Verify: `packages/cli/src/managed-python-runtime.test.ts` -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `README.md` -- Verify: `examples/package-artifacts/README.md` - -- [ ] **Step 1: Run focused verification** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs -pnpm --filter @ktx/cli run test -- src/managed-python-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Verify release readiness text output** - -Run: - -```bash -pnpm run release:readiness -``` - -Expected output includes: - -```text -KTX release mode: npm-public-release-ready -Runtime uv strategy: path-prerequisite -Runtime uv bootstrap: disabled -NPM publish target: @kaelio/ktx@0.1.0 (latest) -``` - -- [ ] **Step 3: Verify no pre-commit config is required** - -Run: - -```bash -rg --files -g '.pre-commit-config.yaml' -g 'pre-commit-config.yaml' -``` - -Expected: no output and exit code 1. No Python files changed, so the repository -Python pre-commit requirement does not apply. - -- [ ] **Step 4: Review the final diff** - -Run: - -```bash -git diff --stat -git diff -- release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md -``` - -Expected: only the runtime installer policy, missing-`uv` message/tests, and -public docs changed. - -- [ ] **Step 5: Commit final verification notes if needed** - -If Task 4 produces only verification output and no file changes, skip this -step. If a correction was made during verification, commit it: - -```bash -git add release-policy.json scripts/release-readiness.mjs scripts/release-readiness.test.mjs packages/cli/src/managed-python-runtime.ts packages/cli/src/managed-python-runtime.test.ts scripts/examples-docs.test.mjs README.md examples/package-artifacts/README.md -git commit -m "chore: finish uv prerequisite release contract" -``` - -## Self-review - -Spec coverage: - -- The earlier implemented plans cover the single public npm package, bundled - Python wheel, managed runtime installer, runtime commands, daemon lifecycle, - local embeddings, Python-backed command integration, release smoke, published - smoke, docs cleanup, release handoff, and prune coverage. -- This plan closes the spec's remaining `uv` open decision by choosing - `path-prerequisite`, recording that decision in release policy, validating it - in release readiness, using one CLI error message, and documenting it. -- The plan keeps Python package publication disabled and keeps KTX-owned Python - code bundled in the npm package. - -Placeholder scan: - -- No task contains deferred implementation markers. -- Each code-changing step names exact files and includes the concrete code to - add or replace. - -Type consistency: - -- The release policy field is consistently named `runtimeInstaller`. -- The chosen strategy is consistently `path-prerequisite`. -- The shared CLI message constant is consistently - `MISSING_UV_RUNTIME_INSTALL_MESSAGE`. diff --git a/docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md b/docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md deleted file mode 100644 index 5cbe3cf4..00000000 --- a/docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md +++ /dev/null @@ -1,1904 +0,0 @@ -# Public Kaelio KTX npm Package Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Produce one installable public npm package, `@kaelio/ktx`, whose -`ktx` binary includes the bundled Python runtime wheel and does not require -users to install any `@ktx/*` workspace packages directly. - -**Architecture:** Keep the internal pnpm workspace package names unchanged for -development, then assemble a release package under `dist/public-npm-package`. -The release package copies the CLI `dist/` and assets, vendors built internal -`@ktx/*` packages as bundled dependencies, writes a public `@kaelio/ktx` -`package.json`, and packs exactly one npm tarball. Release and smoke scripts -then treat `@kaelio/ktx` as the only npm artifact while preserving internal -workspace builds. - -**Tech Stack:** Node 22 ESM scripts, pnpm, TypeScript, Vitest, `node:test`, -npm bundled dependencies, KTX managed Python runtime assets. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -Existing plans based on the spec: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` - -All five are implemented in this worktree. Evidence found before writing this -plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and - `scripts/build-python-runtime-wheel.test.mjs`. -- `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl` and - `packages/cli/assets/python/manifest.json`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, and - `packages/cli/src/commands/runtime-commands.ts`. -- `packages/cli/src/managed-python-command.ts` and `ktx sl query` runtime - install policy flags. -- `packages/cli/src/managed-python-daemon.ts`, daemon state paths, and - `ktx runtime start` / `ktx runtime stop`. -- `packages/cli/src/managed-local-embeddings.ts`, - `packages/context/src/llm/local-config.ts` managed marker constants, and - setup wiring in `packages/cli/src/setup-embeddings.ts`. - -Spec requirements still outside those plans: - -- The visible npm package is still `@ktx/cli`, not `@kaelio/ktx`. -- Release artifacts still model multiple npm workspace packages instead of one - public npm package. -- Installed-package smoke coverage still relies on installing internal - `@ktx/*` tarballs. -- Published-package smoke coverage does not yet exercise the required - `@kaelio/ktx` invocation modes: - `npx @kaelio/ktx setup demo`, `npx @kaelio/ktx sl query ...`, local - `npm install @kaelio/ktx` plus `npx ktx ...`, and global - `npm install -g @kaelio/ktx` plus `ktx ...`. - -This plan implements the public npm package surface and local tarball smoke -coverage. It intentionally keeps internal package imports such as -`@ktx/context` in source code so development stays compatible with the existing -workspace. - -## File structure - -- Modify `packages/cli/src/cli-runtime.ts`: read the package name and version - from the installed package root so the same `dist/` reports `@ktx/cli` in the - workspace and `@kaelio/ktx` in the assembled public package. -- Modify `packages/cli/src/index.test.ts`: cover dynamic package metadata. -- Create `scripts/build-public-npm-package.mjs`: assemble and pack the - `@kaelio/ktx` release package with bundled internal `@ktx/*` packages. -- Create `scripts/build-public-npm-package.test.mjs`: test dependency union, - bundled package copying, public `package.json` generation, and pack command - shape. -- Modify `scripts/package-artifacts.mjs`: build internal packages, build Python - artifacts, build the public package, and write a manifest with exactly one - npm artifact named `@kaelio/ktx`. -- Modify `scripts/package-artifacts.test.mjs`: update artifact layout, - release metadata, npm smoke package, and manifest expectations for the - single public npm artifact. -- Modify `scripts/published-package-smoke-config.mjs`: add the public-package - invocation commands needed by the runtime spec. -- Modify `scripts/published-package-smoke.mjs`: validate the new command list. -- Modify `scripts/published-package-smoke.test.mjs`: expect `@kaelio/ktx` and - the supported `npx`, local install, and global install invocation modes. -- Modify `scripts/release-readiness.mjs`: allow the one public npm artifact - while `release-policy.json` still disables publishing. -- Modify `scripts/release-readiness.test.mjs`: expect only `@kaelio/ktx` in - npm release metadata and policy checks. -- Modify `release-policy.json`: list `@kaelio/ktx` as the only npm package and - set the published smoke package to `@kaelio/ktx`. -- Modify `scripts/precommit-check.test.mjs` only if package filter assertions - expect the public package name after artifact-script changes. Keep - `scripts/precommit-check.mjs` using internal workspace package names. - -### Task 1: Read CLI package metadata from the installed package root - -**Files:** - -- Modify: `packages/cli/src/cli-runtime.ts` -- Modify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Add a failing dynamic metadata test** - -In `packages/cli/src/index.test.ts`, extend the import from `./index.js` so it -includes `packageInfoFromJson`: - -```typescript -import { - getKtxCliPackageInfo, - packageInfoFromJson, - rendererUnavailableVizFallback, - renderMemoryFlowTui, - resolveVizFallback, - runKtxCli, - sanitizeMemoryFlowTuiError, - startLiveMemoryFlowTui, - warnVizFallbackOnce, -} from './index.js'; -``` - -Add this test inside `describe('getKtxCliPackageInfo', () => { ... })` after the -existing metadata tests: - -```typescript - it('normalizes public package metadata from package.json contents', () => { - expect( - packageInfoFromJson({ - name: '@kaelio/ktx', - version: '0.1.0', - }), - ).toEqual({ - name: '@kaelio/ktx', - version: '0.1.0', - contextPackageName: '@ktx/context', - }); - }); -``` - -- [ ] **Step 2: Run the failing metadata test** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -Expected: FAIL with a missing export for `packageInfoFromJson`. - -- [ ] **Step 3: Implement dynamic package metadata** - -In `packages/cli/src/cli-runtime.ts`, add this import at the top of the file: - -```typescript -import { createRequire } from 'node:module'; -``` - -Replace the `KtxCliPackageInfo` interface and `getKtxCliPackageInfo()` with -this code: - -```typescript -const requirePackageJson = createRequire(import.meta.url); - -export interface KtxCliPackageInfo { - name: string; - version: string; - contextPackageName: '@ktx/context'; -} - -export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo { - if ( - typeof packageJson !== 'object' || - packageJson === null || - !('name' in packageJson) || - !('version' in packageJson) || - typeof packageJson.name !== 'string' || - typeof packageJson.version !== 'string' - ) { - throw new Error('Invalid KTX CLI package metadata'); - } - - return { - name: packageJson.name, - version: packageJson.version, - contextPackageName: '@ktx/context', - }; -} - -export function getKtxCliPackageInfo(): KtxCliPackageInfo { - return packageInfoFromJson(requirePackageJson('../package.json')); -} -``` - -In `packages/cli/src/index.ts`, add `packageInfoFromJson` to the export from -`./cli-runtime.js`: - -```typescript -export { - getKtxCliPackageInfo, - packageInfoFromJson, - runInitForCommander, - runKtxCli, - type KtxCliDeps, - type KtxCliIo, - type KtxCliPackageInfo, -} from './cli-runtime.js'; -``` - -- [ ] **Step 4: Verify CLI metadata tests pass** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/cli/src/cli-runtime.ts packages/cli/src/index.ts packages/cli/src/index.test.ts -git commit -m "feat: read CLI package metadata dynamically" -``` - -### Task 2: Add the public npm package assembly script - -**Files:** - -- Create: `scripts/build-public-npm-package.test.mjs` -- Create: `scripts/build-public-npm-package.mjs` - -- [ ] **Step 1: Write failing tests for the public package builder** - -Create `scripts/build-public-npm-package.test.mjs` with this content: - -```javascript -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; - -import { - PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - PUBLIC_NPM_PACKAGE_NAME, - collectPublicDependencies, - createPublicNpmPackageTree, - publicNpmPackageJson, - publicNpmPackageLayout, - publicNpmPackCommand, -} from './build-public-npm-package.mjs'; - -async function writeJson(path, value) { - await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); -} - -async function writePackage(root, packageRoot, packageJson, files = {}) { - const absoluteRoot = join(root, packageRoot); - await mkdir(absoluteRoot, { recursive: true }); - await writeJson(join(absoluteRoot, 'package.json'), packageJson); - - for (const [relativePath, contents] of Object.entries(files)) { - const target = join(absoluteRoot, relativePath); - await mkdir(join(target, '..'), { recursive: true }); - await writeFile(target, contents); - } -} - -async function writeWorkspaceFixture(root) { - await writePackage( - root, - 'packages/cli', - { - name: '@ktx/cli', - version: '0.0.0-private', - description: 'CLI wrapper for KTX', - type: 'module', - engines: { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: 'dist/index.js', - types: 'dist/index.d.ts', - exports: { - '.': { - types: './dist/index.d.ts', - import: './dist/index.js', - default: './dist/index.js', - }, - './package.json': './package.json', - }, - files: ['dist', 'assets'], - dependencies: { - '@clack/prompts': '1.3.0', - '@ktx/context': 'workspace:*', - commander: '14.0.3', - }, - license: 'Apache-2.0', - repository: { - type: 'git', - url: 'git+https://github.com/kaelio/ktx.git', - directory: 'packages/cli', - }, - }, - { - 'dist/bin.js': '#!/usr/bin/env node\n', - 'dist/index.js': 'export const cli = true;\n', - 'dist/index.d.ts': 'export declare const cli: true;\n', - 'assets/python/manifest.json': '{"schemaVersion":1}\n', - }, - ); - - await writePackage( - root, - 'packages/context', - { - name: '@ktx/context', - version: '0.0.0-private', - type: 'module', - main: 'dist/index.js', - exports: { '.': './dist/index.js' }, - files: ['dist', 'prompts', 'skills'], - dependencies: { - '@ktx/llm': 'workspace:*', - yaml: '^2.8.2', - }, - }, - { - 'dist/index.js': 'export const context = true;\n', - 'prompts/system.md': 'prompt\n', - 'skills/sl/SKILL.md': 'skill\n', - }, - ); - - await writePackage( - root, - 'packages/llm', - { - name: '@ktx/llm', - version: '0.0.0-private', - type: 'module', - main: 'dist/index.js', - exports: { '.': './dist/index.js' }, - files: ['dist'], - dependencies: { - ai: '^6.0.168', - }, - }, - { - 'dist/index.js': 'export const llm = true;\n', - }, - ); - - for (const packageName of PUBLIC_BUNDLED_WORKSPACE_PACKAGES.filter( - (name) => name.startsWith('@ktx/connector-'), - )) { - const directory = packageName.replace('@ktx/', ''); - await writePackage( - root, - `packages/${directory}`, - { - name: packageName, - version: '0.0.0-private', - type: 'module', - main: 'dist/index.js', - exports: { '.': './dist/index.js' }, - files: ['dist'], - dependencies: { - '@ktx/context': 'workspace:*', - }, - }, - { - 'dist/index.js': `export const name = ${JSON.stringify(packageName)};\n`, - }, - ); - } -} - -describe('publicNpmPackageLayout', () => { - it('uses stable public package build and tarball paths', () => { - const layout = publicNpmPackageLayout('/repo/ktx'); - - assert.equal(layout.rootDir, '/repo/ktx'); - assert.equal(layout.packRoot, '/repo/ktx/dist/public-npm-package'); - assert.equal(layout.npmDir, '/repo/ktx/dist/artifacts/npm'); - assert.equal(layout.tarballPath, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz'); - }); -}); - -describe('collectPublicDependencies', () => { - it('unions external runtime dependencies and omits workspace packages', () => { - assert.deepEqual( - collectPublicDependencies([ - { - name: '@ktx/cli', - dependencies: { - '@ktx/context': 'workspace:*', - commander: '14.0.3', - zod: '^4.4.3', - }, - }, - { - name: '@ktx/context', - dependencies: { - '@ktx/llm': 'workspace:*', - commander: '14.0.3', - yaml: '^2.8.2', - zod: '^4.1.13', - }, - }, - ]), - { - commander: '14.0.3', - yaml: '^2.8.2', - zod: '^4.4.3', - }, - ); - }); - - it('fails on incompatible external dependency ranges', () => { - assert.throws( - () => - collectPublicDependencies([ - { name: '@ktx/cli', dependencies: { zod: '^4.4.3' } }, - { name: '@ktx/context', dependencies: { zod: '^3.25.0' } }, - ]), - /Incompatible dependency versions for zod/, - ); - }); -}); - -describe('publicNpmPackageJson', () => { - it('describes the public @kaelio/ktx binary package', () => { - const packageJson = publicNpmPackageJson( - { - name: '@ktx/cli', - version: '0.0.0-private', - engines: { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: 'dist/index.js', - types: 'dist/index.d.ts', - exports: { '.': './dist/index.js', './package.json': './package.json' }, - license: 'Apache-2.0', - }, - { commander: '14.0.3' }, - ); - - assert.equal(packageJson.name, PUBLIC_NPM_PACKAGE_NAME); - assert.equal(packageJson.private, false); - assert.deepEqual(packageJson.bin, { ktx: './dist/bin.js' }); - assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' }); - assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES); - assert.deepEqual(packageJson.files, ['dist', 'assets']); - }); -}); - -describe('createPublicNpmPackageTree', () => { - it('copies CLI files, assets, and bundled internal workspace packages', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-public-npm-test-')); - try { - await writeWorkspaceFixture(root); - const layout = publicNpmPackageLayout(root); - - const result = await createPublicNpmPackageTree(layout); - - assert.equal(result.packageJson.name, '@kaelio/ktx'); - assert.equal(result.packageJson.dependencies.commander, '14.0.3'); - assert.equal(result.packageJson.dependencies.yaml, '^2.8.2'); - assert.equal(result.packageJson.dependencies.ai, '^6.0.168'); - assert.equal( - await readFile(join(layout.packRoot, 'assets', 'python', 'manifest.json'), 'utf8'), - '{"schemaVersion":1}\n', - ); - assert.equal( - await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'dist', 'index.js'), 'utf8'), - 'export const context = true;\n', - ); - assert.equal( - await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'prompts', 'system.md'), 'utf8'), - 'prompt\n', - ); - - const bundledContextJson = JSON.parse( - await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'package.json'), 'utf8'), - ); - assert.equal(bundledContextJson.private, true); - assert.deepEqual(bundledContextJson.dependencies, { yaml: '^2.8.2' }); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); - -describe('publicNpmPackCommand', () => { - it('packs the assembled public package with pnpm', () => { - const layout = publicNpmPackageLayout('/repo/ktx'); - - assert.deepEqual(publicNpmPackCommand(layout), { - command: 'pnpm', - args: ['pack', '--out', '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz'], - cwd: '/repo/ktx/dist/public-npm-package', - }); - }); -}); -``` - -- [ ] **Step 2: Run the failing builder tests** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs -``` - -Expected: FAIL with an import error for -`./build-public-npm-package.mjs`. - -- [ ] **Step 3: Implement the public package builder** - -Create `scripts/build-public-npm-package.mjs` with this content: - -```javascript -#!/usr/bin/env node - -import { execFile } from 'node:child_process'; -import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); - -export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx'; -export const PUBLIC_NPM_PACKAGE_VERSION = '0.0.0-private'; -export const PUBLIC_NPM_PACKAGE_TARBALL = 'kaelio-ktx-0.0.0-private.tgz'; - -export const PUBLIC_BUNDLED_WORKSPACE_PACKAGES = [ - '@ktx/llm', - '@ktx/context', - '@ktx/connector-bigquery', - '@ktx/connector-clickhouse', - '@ktx/connector-mysql', - '@ktx/connector-postgres', - '@ktx/connector-posthog', - '@ktx/connector-snowflake', - '@ktx/connector-sqlite', - '@ktx/connector-sqlserver', -]; - -export const PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS = { - '@ktx/llm': 'packages/llm', - '@ktx/context': 'packages/context', - '@ktx/connector-bigquery': 'packages/connector-bigquery', - '@ktx/connector-clickhouse': 'packages/connector-clickhouse', - '@ktx/connector-mysql': 'packages/connector-mysql', - '@ktx/connector-postgres': 'packages/connector-postgres', - '@ktx/connector-posthog': 'packages/connector-posthog', - '@ktx/connector-snowflake': 'packages/connector-snowflake', - '@ktx/connector-sqlite': 'packages/connector-sqlite', - '@ktx/connector-sqlserver': 'packages/connector-sqlserver', -}; - -function scriptRootDir() { - return resolve(dirname(fileURLToPath(import.meta.url)), '..'); -} - -export function publicNpmPackageLayout(rootDir = scriptRootDir()) { - return { - rootDir, - cliPackageRoot: join(rootDir, 'packages', 'cli'), - packRoot: join(rootDir, 'dist', 'public-npm-package'), - npmDir: join(rootDir, 'dist', 'artifacts', 'npm'), - tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', PUBLIC_NPM_PACKAGE_TARBALL), - }; -} - -async function readJson(path) { - return JSON.parse(await readFile(path, 'utf8')); -} - -async function writeJson(path, value) { - await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); -} - -function sortedObject(entries) { - return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right))); -} - -function isWorkspacePackageName(name) { - return name.startsWith('@ktx/'); -} - -function parseCaretVersion(value) { - const match = /^\^(\d+)\.(\d+)\.(\d+)$/.exec(value); - if (!match) { - return null; - } - return { - major: Number(match[1]), - minor: Number(match[2]), - patch: Number(match[3]), - }; -} - -function compareParsedVersions(left, right) { - return left.major - right.major || left.minor - right.minor || left.patch - right.patch; -} - -function mergeDependencyVersion(name, previous, next) { - if (previous === next) { - return previous; - } - - const previousCaret = parseCaretVersion(previous); - const nextCaret = parseCaretVersion(next); - if (previousCaret && nextCaret && previousCaret.major === nextCaret.major) { - return compareParsedVersions(previousCaret, nextCaret) >= 0 ? previous : next; - } - - throw new Error(`Incompatible dependency versions for ${name}: ${previous} and ${next}`); -} - -export function collectPublicDependencies(packageJsons) { - const dependencies = new Map(); - - for (const packageJson of packageJsons) { - for (const [name, version] of Object.entries(packageJson.dependencies ?? {})) { - if (isWorkspacePackageName(name)) { - continue; - } - const previous = dependencies.get(name); - dependencies.set(name, previous ? mergeDependencyVersion(name, previous, version) : version); - } - } - - return sortedObject(dependencies); -} - -export function publicNpmPackageJson(cliPackageJson, dependencies) { - return { - name: PUBLIC_NPM_PACKAGE_NAME, - version: cliPackageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, - description: 'Standalone KTX context layer for database agents', - private: false, - type: 'module', - engines: cliPackageJson.engines ?? { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: cliPackageJson.main ?? 'dist/index.js', - types: cliPackageJson.types ?? 'dist/index.d.ts', - exports: cliPackageJson.exports ?? { - '.': { - types: './dist/index.d.ts', - import: './dist/index.js', - default: './dist/index.js', - }, - './package.json': './package.json', - }, - files: ['dist', 'assets'], - dependencies, - bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - license: cliPackageJson.license ?? 'Apache-2.0', - repository: { - type: 'git', - url: 'git+https://github.com/kaelio/ktx.git', - }, - bugs: { - url: 'https://github.com/kaelio/ktx/issues', - }, - homepage: 'https://github.com/kaelio/ktx#readme', - }; -} - -function bundledWorkspacePackageJson(packageJson) { - const dependencies = Object.fromEntries( - Object.entries(packageJson.dependencies ?? {}).filter(([name]) => !isWorkspacePackageName(name)), - ); - - return { - name: packageJson.name, - version: packageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, - private: true, - type: packageJson.type ?? 'module', - main: packageJson.main, - types: packageJson.types, - exports: packageJson.exports, - files: packageJson.files, - dependencies: sortedObject(Object.entries(dependencies)), - license: packageJson.license ?? 'Apache-2.0', - }; -} - -async function copyPackageFileEntries(sourceRoot, targetRoot, packageJson) { - for (const entry of packageJson.files ?? ['dist']) { - await cp(join(sourceRoot, entry), join(targetRoot, entry), { - recursive: true, - force: true, - }); - } -} - -async function copyCliPackage(layout, cliPackageJson, dependencies) { - await copyPackageFileEntries(layout.cliPackageRoot, layout.packRoot, cliPackageJson); - await writeJson(join(layout.packRoot, 'package.json'), publicNpmPackageJson(cliPackageJson, dependencies)); -} - -async function copyBundledWorkspacePackage(rootDir, packageName, packageJson) { - const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; - if (!packageRoot) { - throw new Error(`Missing bundled workspace package root for ${packageName}`); - } - - const sourceRoot = join(rootDir, packageRoot); - const targetRoot = join(rootDir, 'dist', 'public-npm-package', 'node_modules', ...packageName.split('/')); - await mkdir(targetRoot, { recursive: true }); - await copyPackageFileEntries(sourceRoot, targetRoot, packageJson); - await writeJson(join(targetRoot, 'package.json'), bundledWorkspacePackageJson(packageJson)); -} - -export async function createPublicNpmPackageTree(layout = publicNpmPackageLayout()) { - const cliPackageJson = await readJson(join(layout.cliPackageRoot, 'package.json')); - const bundledPackageJsons = await Promise.all( - PUBLIC_BUNDLED_WORKSPACE_PACKAGES.map(async (packageName) => { - const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; - const packageJson = await readJson(join(layout.rootDir, packageRoot, 'package.json')); - if (packageJson.name !== packageName) { - throw new Error(`Unexpected package name in ${packageRoot}/package.json: ${packageJson.name}`); - } - return packageJson; - }), - ); - const dependencies = collectPublicDependencies([cliPackageJson, ...bundledPackageJsons]); - - await rm(layout.packRoot, { recursive: true, force: true }); - await mkdir(layout.packRoot, { recursive: true }); - await mkdir(layout.npmDir, { recursive: true }); - await copyCliPackage(layout, cliPackageJson, dependencies); - - for (const packageJson of bundledPackageJsons) { - await copyBundledWorkspacePackage(layout.rootDir, packageJson.name, packageJson); - } - - return { - layout, - packageJson: publicNpmPackageJson(cliPackageJson, dependencies), - bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - }; -} - -export function publicNpmPackCommand(layout = publicNpmPackageLayout()) { - return { - command: 'pnpm', - args: ['pack', '--out', layout.tarballPath], - cwd: layout.packRoot, - }; -} - -export async function buildPublicNpmPackage(layout = publicNpmPackageLayout()) { - await createPublicNpmPackageTree(layout); - const pack = publicNpmPackCommand(layout); - await execFileAsync(pack.command, pack.args, { - cwd: pack.cwd, - encoding: 'utf8', - maxBuffer: 10 * 1024 * 1024, - }); - return layout.tarballPath; -} - -async function main() { - const tarball = await buildPublicNpmPackage(); - process.stdout.write(`Built ${PUBLIC_NPM_PACKAGE_NAME} package: ${tarball}\n`); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - try { - await main(); - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); - process.exitCode = 1; - } -} -``` - -- [ ] **Step 4: Verify builder tests pass** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs -git commit -m "feat: assemble public kaelio ktx npm package" -``` - -### Task 3: Make release artifacts use only `@kaelio/ktx` as the npm artifact - -**Files:** - -- Modify: `scripts/package-artifacts.mjs` -- Modify: `scripts/package-artifacts.test.mjs` -- Modify: `scripts/release-readiness.mjs` -- Modify: `scripts/release-readiness.test.mjs` -- Modify: `release-policy.json` - -- [ ] **Step 1: Add failing artifact expectations for the public package** - -In `scripts/package-artifacts.test.mjs`, update the import from -`./package-artifacts.mjs` so it also imports -`INTERNAL_NPM_WORKSPACE_PACKAGES`: - -```javascript -import { - CLI_PYTHON_ASSET_MANIFEST, - INTERNAL_NPM_WORKSPACE_PACKAGES, - RUNTIME_WHEEL_DISTRIBUTION_NAME, - RUNTIME_WHEEL_NORMALIZED_NAME, - RUNTIME_WHEEL_PACKAGE_VERSION, - artifactManifestPath, - buildArtifactCommands, - copyRuntimeWheelAssets, - findPythonArtifacts, - NPM_ARTIFACT_PACKAGES, - npmDemoSmokeSource, - npmRuntimeSmokeSource, - npmSmokePackageJson, - npmSmokePythonEnv, - npmVerifySource, - packageArtifactLayout, - packageReleaseMetadata, - pythonArtifactInstallArgs, - pythonVerifySource, - verifyArtifactManifest, - writeArtifactManifest, -} from './package-artifacts.mjs'; -``` - -Replace the top-level `NPM_BUILD_PACKAGE_ORDER` declaration with: - -```javascript -const INTERNAL_BUILD_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => packageInfo.name); -const CONNECTOR_PACKAGE_NAMES = INTERNAL_BUILD_PACKAGE_NAMES.filter((packageName) => - packageName.startsWith('@ktx/connector-'), -); -const NPM_BUILD_PACKAGE_ORDER = ['@ktx/llm', '@ktx/context', ...CONNECTOR_PACKAGE_NAMES, '@ktx/cli']; -``` - -Replace `expectedNpmArtifactPath` with: - -```javascript -function expectedNpmArtifactPath(packageName) { - if (packageName === '@kaelio/ktx') { - return 'npm/kaelio-ktx-0.0.0-private.tgz'; - } - return `npm/${packageName.replace('@ktx/', 'ktx-')}-0.0.0-private.tgz`; -} -``` - -Replace `writeReleaseMetadataInputs` with: - -```javascript -async function writeReleaseMetadataInputs(root) { - for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { - await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); - await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { - name: packageInfo.name, - version: '0.0.0-private', - private: true, - }); - } - - await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); - await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); - await writeFile( - join(root, 'python', 'ktx-sl', 'pyproject.toml'), - ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'pyproject.toml'), - ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), - ); -} -``` - -Update the `packageArtifactLayout` test so the npm assertions are: - -```javascript - assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz'); - assert.deepEqual(Object.keys(layout.npmTarballs), ['@kaelio/ktx']); -``` - -Update the `buildArtifactCommands` test so it expects one public package build -command instead of per-package `pnpm pack` commands: - -```javascript - assert.deepEqual( - commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), - NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [ - command.command, - command.args, - ]), - [ - [process.execPath, ['scripts/build-python-runtime-wheel.mjs']], - ['uv', ['build', '--package', 'ktx-sl', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ['uv', ['build', '--package', 'ktx-daemon', '--out-dir', '/repo/ktx/dist/artifacts/python']], - ], - ); - assert.deepEqual(commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 3).map((command) => [command.command, command.args]), [ - [process.execPath, ['scripts/build-public-npm-package.mjs']], - ]); -``` - -In the `packageReleaseMetadata` test, replace the expected npm metadata entries -with: - -```javascript - { - ecosystem: 'npm', - packageName: '@kaelio/ktx', - packageRoot: 'packages/cli', - packageVersion: '0.0.0-private', - private: false, - releaseMode: 'ci-artifact-only', - }, -``` - -In the artifact manifest test, replace the npm package expectations with: - -```javascript - [ - { - ecosystem: 'npm', - packageName: '@kaelio/ktx', - packageRoot: 'packages/cli', - packageVersion: '0.0.0-private', - private: false, - releaseMode: 'ci-artifact-only', - }, - ], -``` - -Also replace the expected npm file entries with: - -```javascript - [ - { - artifactKind: 'tarball', - ecosystem: 'npm', - packageName: '@kaelio/ktx', - packageVersion: '0.0.0-private', - path: 'npm/kaelio-ktx-0.0.0-private.tgz', - }, - ], -``` - -- [ ] **Step 2: Run failing artifact tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: FAIL because `INTERNAL_NPM_WORKSPACE_PACKAGES` is missing and the -artifact layout still points at `@ktx/cli`. - -- [ ] **Step 3: Wire `scripts/package-artifacts.mjs` to the public builder** - -In `scripts/package-artifacts.mjs`, add this import after the runtime wheel -import: - -```javascript -import { - PUBLIC_NPM_PACKAGE_NAME, - PUBLIC_NPM_PACKAGE_TARBALL, -} from './build-public-npm-package.mjs'; -``` - -Replace the `NPM_ARTIFACT_PACKAGES` declaration with: - -```javascript -export const INTERNAL_NPM_WORKSPACE_PACKAGES = [ - { name: '@ktx/context', packageRoot: 'packages/context' }, - { name: '@ktx/llm', packageRoot: 'packages/llm' }, - { name: '@ktx/connector-bigquery', packageRoot: 'packages/connector-bigquery' }, - { name: '@ktx/connector-clickhouse', packageRoot: 'packages/connector-clickhouse' }, - { name: '@ktx/connector-mysql', packageRoot: 'packages/connector-mysql' }, - { name: '@ktx/connector-postgres', packageRoot: 'packages/connector-postgres' }, - { name: '@ktx/connector-posthog', packageRoot: 'packages/connector-posthog' }, - { name: '@ktx/connector-snowflake', packageRoot: 'packages/connector-snowflake' }, - { name: '@ktx/connector-sqlite', packageRoot: 'packages/connector-sqlite' }, - { name: '@ktx/connector-sqlserver', packageRoot: 'packages/connector-sqlserver' }, - { name: '@ktx/cli', packageRoot: 'packages/cli' }, -]; - -export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRoot: 'packages/cli' }]; -``` - -Replace the `CONNECTOR_PACKAGE_NAMES` calculation with: - -```javascript -const CONNECTOR_PACKAGE_NAMES = INTERNAL_NPM_WORKSPACE_PACKAGES - .map((packageInfo) => packageInfo.name) - .filter((packageName) => packageName.startsWith('@ktx/connector-')); -``` - -Replace `npmPackageTarballName` with: - -```javascript -function npmPackageTarballName(packageName) { - if (packageName === PUBLIC_NPM_PACKAGE_NAME) { - return PUBLIC_NPM_PACKAGE_TARBALL; - } - return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`; -} -``` - -In `packageArtifactLayout`, keep `contextTarball` for compatibility but make -`cliTarball` point at the public package: - -```javascript - contextTarball: npmTarballs[PUBLIC_NPM_PACKAGE_NAME], - cliTarball: npmTarballs[PUBLIC_NPM_PACKAGE_NAME], -``` - -In `buildArtifactCommands`, replace `packagesByName` and `npmBuildCommands` -with: - -```javascript - const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); - const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { -``` - -Replace `npmPackCommands` and the final returned pack commands with the public -builder command: - -```javascript - const publicPackageCommand = { - command: process.execPath, - args: ['scripts/build-public-npm-package.mjs'], - cwd: layout.rootDir, - }; -``` - -Return: - -```javascript - return [ - ...npmBuildCommands, - { - command: process.execPath, - args: ['scripts/build-python-runtime-wheel.mjs'], - cwd: layout.rootDir, - }, - { - command: 'uv', - args: ['build', '--package', 'ktx-sl', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, - { - command: 'uv', - args: ['build', '--package', 'ktx-daemon', '--out-dir', layout.pythonDir], - cwd: layout.rootDir, - }, - publicPackageCommand, - ]; -``` - -Replace `readNpmPackageMetadata` with: - -```javascript -async function readNpmPackageMetadata(rootDir, packageInfo) { - const packageJson = await readJson(join(rootDir, packageInfo.packageRoot, 'package.json')); - const expectedSourceName = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME ? '@ktx/cli' : packageInfo.name; - if (packageJson.name !== expectedSourceName) { - throw new Error( - `Unexpected package name in ${packageInfo.packageRoot}/package.json: expected ${expectedSourceName}, got ${packageJson.name}`, - ); - } - return releaseMetadataEntry({ - ecosystem: 'npm', - packageName: packageInfo.name, - packageRoot: packageInfo.packageRoot, - packageVersion: packageJson.version, - privatePackage: packageInfo.name === PUBLIC_NPM_PACKAGE_NAME ? false : packageJson.private === true, - }); -} -``` - -In `buildArtifacts`, replace the command-slicing counters with: - -```javascript - const npmBuildCount = NPM_ARTIFACT_BUILD_ORDER.length; - const npmPackStart = commands.length - 1; -``` - -Keep the existing three loops after those counters. This makes the first loop -build all internal workspace packages, the second loop build and copy Python -runtime artifacts, and the final loop run only -`scripts/build-public-npm-package.mjs`. - -- [ ] **Step 4: Update release policy for one npm package** - -Replace the `release-policy.json` `npm.packages` value with: - -```json -["@kaelio/ktx"] -``` - -Replace `publishedPackageSmoke.packageName` with: - -```json -"@kaelio/ktx" -``` - -Replace `requiredBeforePublishing` with: - -```json -[ - "Choose public release version.", - "Configure registry credentials outside source control.", - "Choose release tag and provenance policy." -] -``` - -- [ ] **Step 5: Allow one public npm artifact while publishing remains disabled** - -In `scripts/release-readiness.mjs`, replace the npm portion of -`assertNonPublishingArtifactPolicy` with: - -```javascript - if (entry.ecosystem === 'npm') { - const isPublicKtxPackage = entry.packageName === '@kaelio/ktx'; - if (isPublicKtxPackage) { - if (entry.private !== false) { - throw new Error(`${policyLabel} npm package @kaelio/ktx must be publishable when npm.publish is false`); - } - } else if (entry.private !== true) { - throw new Error(`${policyLabel} npm package ${entry.packageName} must remain private`); - } - if (!entry.packageVersion.endsWith('-private')) { - throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`); - } - } -``` - -In `scripts/release-readiness.test.mjs`, update the import from -`./package-artifacts.mjs` so it includes `INTERNAL_NPM_WORKSPACE_PACKAGES`: - -```javascript -import { - INTERNAL_NPM_WORKSPACE_PACKAGES, - NPM_ARTIFACT_PACKAGES, - packageArtifactLayout, - writeArtifactManifest, -} from './package-artifacts.mjs'; -``` - -Replace `writeReleaseMetadataInputs` with: - -```javascript -async function writeReleaseMetadataInputs(root) { - for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { - await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); - await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { - name: packageInfo.name, - version: '0.0.0-private', - private: true, - }); - } - - await mkdir(join(root, 'python', 'ktx-sl'), { recursive: true }); - await mkdir(join(root, 'python', 'ktx-daemon'), { recursive: true }); - - await writeFile( - join(root, 'python', 'ktx-sl', 'pyproject.toml'), - ['[project]', 'name = "ktx-sl"', 'version = "0.1.0"', ''].join('\n'), - ); - await writeFile( - join(root, 'python', 'ktx-daemon', 'pyproject.toml'), - ['[project]', 'name = "ktx-daemon"', 'version = "0.1.0"', ''].join('\n'), - ); -} -``` - -Update `releasePolicy()` so `npm.packages` defaults to: - -```javascript -packages: ['@kaelio/ktx'], -``` - -Update expected `packageNames` arrays so the npm section is only: - -```javascript -'@kaelio/ktx', -``` - -Update published smoke fixture package names from `@ktx/cli-public` to -`@kaelio/ktx`. - -Replace the stale public-npm rejection test with this policy mismatch test: - -```javascript - it('rejects release policy that still lists internal npm packages', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-release-stale-internal-npm-policy-test-')); - try { - await writeReadyFixture(root, { - policy: releasePolicy({ - npm: { - packages: ['@kaelio/ktx', '@ktx/context'], - }, - }), - }); - - await assert.rejects( - () => releaseReadinessReport(root), - /Release policy npm\.packages mismatch/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 6: Verify artifact and release readiness tests** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json -git commit -m "feat: release one public kaelio ktx npm artifact" -``` - -### Task 4: Add public package invocation smoke coverage - -**Files:** - -- Modify: `scripts/published-package-smoke-config.mjs` -- Modify: `scripts/published-package-smoke.mjs` -- Modify: `scripts/published-package-smoke.test.mjs` -- Modify: `scripts/package-artifacts.mjs` -- Modify: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Add failing published smoke command expectations** - -In `scripts/published-package-smoke.test.mjs`, change all fixture package names -from `@ktx/cli-public`, `@ktx/cli-from-env`, and `@ktx/cli-from-policy` to -`@kaelio/ktx`. - -In the `builds the full hybrid-search smoke command list` test, replace the -expected command list with: - -```javascript - assert.deepEqual(buildPublishedPackageSmokeCommands(config, '/tmp/ktx-smoke/demo', '/tmp/ktx-smoke/empty'), [ - { - label: 'published package version', - command: 'npx', - args: ['--yes', '@kaelio/ktx@latest', '--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package setup demo', - command: 'npx', - args: [ - '--yes', - '@kaelio/ktx@latest', - 'setup', - 'demo', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--no-input', - '--plain', - ], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package sl query', - command: 'npx', - args: [ - '--yes', - '@kaelio/ktx@latest', - 'sl', - 'query', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--connection-id', - 'orbit_demo', - '--measure', - 'contracts.contract_count', - '--format', - 'sql', - '--yes', - ], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package local install', - command: 'pnpm', - args: ['add', '@kaelio/ktx@latest'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package local binary', - command: 'pnpm', - args: ['exec', 'ktx', '--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package global install', - command: 'pnpm', - args: ['add', '--global', '@kaelio/ktx@latest'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package global binary', - command: 'ktx', - args: ['--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - ]); -``` - -- [ ] **Step 2: Run failing published smoke tests** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs -``` - -Expected: FAIL because the command list still contains the old hybrid-search -commands and package names. - -- [ ] **Step 3: Update published package smoke commands** - -In `scripts/published-package-smoke-config.mjs`, replace -`buildPublishedPackageSmokeCommands` with: - -```javascript -export function buildPublishedPackageSmokeCommands(config, projectDir) { - return [ - buildPublishedPackageNpxCommand(config, ['--version'], 'published package version'), - buildPublishedPackageNpxCommand( - config, - ['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'], - 'published package setup demo', - ), - buildPublishedPackageNpxCommand( - config, - [ - 'sl', - 'query', - '--project-dir', - projectDir, - '--connection-id', - 'orbit_demo', - '--measure', - 'contracts.contract_count', - '--format', - 'sql', - '--yes', - ], - 'published package sl query', - ), - { - label: 'published package local install', - command: 'pnpm', - args: ['add', publishedPackageSpec(config)], - env: config.registry ? { npm_config_registry: config.registry } : {}, - }, - { - label: 'published package local binary', - command: 'pnpm', - args: ['exec', 'ktx', '--version'], - env: config.registry ? { npm_config_registry: config.registry } : {}, - }, - { - label: 'published package global install', - command: 'pnpm', - args: ['add', '--global', publishedPackageSpec(config)], - env: config.registry ? { npm_config_registry: config.registry } : {}, - }, - { - label: 'published package global binary', - command: 'ktx', - args: ['--version'], - env: config.registry ? { npm_config_registry: config.registry } : {}, - }, - ]; -} -``` - -In `scripts/published-package-smoke.mjs`, replace the command execution loop in -`runPublishedPackageSmoke` with: - -```javascript - const commands = buildPublishedPackageSmokeCommands(config, projectDir, emptyProjectDir); - const pnpmHome = join(root, 'pnpm-home'); - const globalEnv = { - PNPM_HOME: pnpmHome, - PATH: `${pnpmHome}${process.platform === 'win32' ? ';' : ':'}${process.env.PATH ?? ''}`, - }; - for (const command of commands) { - const isGlobalCommand = command.label.includes('global'); - const result = await runCommand(command.command, command.args, { - cwd: command.label.includes('local') || isGlobalCommand ? root : undefined, - env: isGlobalCommand ? { ...globalEnv, ...command.env } : command.env, - }); - requireSuccess(command.label, result); - if ( - command.label === 'published package version' || - command.label === 'published package local binary' || - command.label === 'published package global binary' - ) { - assert.match(result.stdout, /@kaelio\/ktx /); - } - if (command.label === 'published package sl query') { - assert.match(result.stdout, /SELECT/i); - assert.match(result.stdout, /contracts/i); - } - } - - process.stdout.write('published package invocation smoke verified\n'); -``` - -Remove `assertHybridWikiSearch`, `assertHybridSlSearch`, and -`assertMissingProjectReadiness` if they are no longer used. - -- [ ] **Step 4: Add local tarball public package smoke to artifact verification** - -In `scripts/package-artifacts.mjs`, replace `npmSmokePackageJson(layout)` with: - -```javascript -export function npmSmokePackageJson(layout) { - return { - name: 'ktx-artifact-npm-smoke', - version: '0.0.0', - private: true, - type: 'module', - dependencies: { - '@kaelio/ktx': `file:${layout.cliTarball}`, - }, - pnpm: { - onlyBuiltDependencies: ['better-sqlite3'], - }, - }; -} -``` - -Replace the top of `npmVerifySource()` with this smaller public-package check: - -```javascript -export function npmVerifySource() { - return ` -const cli = await import('@kaelio/ktx'); - -if (cli.getKtxCliPackageInfo().name !== '@kaelio/ktx') { - throw new Error('Unexpected @kaelio/ktx package info'); -} -if (typeof cli.runKtxCli !== 'function') { - throw new Error('Missing runKtxCli export'); -} -`; -} -``` - -In `npmRuntimeSmokeSource()`, add this assertion after `const root = ...`: - -```javascript -const version = await run('pnpm', ['exec', 'ktx', '--version']); -requireSuccess('ktx public package version', version); -requireOutput('ktx public package version', version, /@kaelio\\/ktx 0\\.0\\.0-private/); -``` - -In `npmRuntimeSmokeSource()`, remove these direct imports because the smoke -project no longer installs internal workspace packages directly: - -```javascript -import { spawn, execFile } from 'node:child_process'; -import { once } from 'node:events'; -import { request as httpRequest } from 'node:http'; -import { createServer } from 'node:net'; -import { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; -import { - createDaemonLookerTableIdentifierParser, - LocalLookerRuntimeStore, -} from '@ktx/context/ingest'; -``` - -Replace them with: - -```javascript -import { execFile } from 'node:child_process'; -``` - -Still inside `npmRuntimeSmokeSource()`, delete these helper functions because -the public tarball smoke must exercise the CLI-managed runtime instead of -manually wiring an internal daemon: - -```javascript -function requireToolNames(tools, expectedNames) { - const names = tools.tools.map((tool) => tool.name).sort(); - for (const expectedName of expectedNames) { - assert.ok(names.includes(expectedName), 'MCP tool list did not include ' + expectedName + ': ' + names.join(', ')); - } -} - -function structuredContent(result) { - assert.ok(result.structuredContent, 'MCP result did not include structuredContent'); - return result.structuredContent; -} - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function getAvailablePort() { - const server = createServer(); - server.listen(0, '127.0.0.1'); - await once(server, 'listening'); - const address = server.address(); - if (!address || typeof address === 'string') { - server.close(); - throw new Error('expected TCP server address for daemon smoke'); - } - const port = address.port; - server.close(); - await once(server, 'close'); - return port; -} - -function httpGetOk(url) { - return new Promise((resolve, reject) => { - const request = httpRequest(url, { method: 'GET' }, (response) => { - response.resume(); - response.on('end', () => resolve((response.statusCode ?? 0) >= 200 && (response.statusCode ?? 0) < 300)); - }); - request.on('error', reject); - request.end(); - }); -} - -function spawnLogged(command, args, options = {}) { - const stdout = []; - const stderr = []; - let spawnError; - const child = spawn(command, args, { - cwd: options.cwd, - env: options.env ?? process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }); - child.stdout.on('data', (chunk) => stdout.push(chunk)); - child.stderr.on('data', (chunk) => stderr.push(chunk)); - child.on('error', (error) => { - spawnError = error; - }); - return { - child, - error() { - return spawnError; - }, - output() { - return { - stdout: Buffer.concat(stdout).toString('utf8'), - stderr: Buffer.concat(stderr).toString('utf8'), - }; - }, - }; -} - -async function waitForHttpHealth(url, daemon) { - const deadline = Date.now() + 15_000; - while (Date.now() < deadline) { - if (daemon.error()) { - const output = daemon.output(); - throw new Error( - 'Failed to start ktx-daemon serve-http: ' + - daemon.error().message + - '\nstdout:\n' + - output.stdout + - '\nstderr:\n' + - output.stderr, - ); - } - if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { - const output = daemon.output(); - throw new Error( - 'ktx-daemon serve-http exited before health check passed\nstdout:\n' + - output.stdout + - '\nstderr:\n' + - output.stderr, - ); - } - try { - if (await httpGetOk(url)) { - return; - } - } catch { - await sleep(100); - continue; - } - await sleep(100); - } - const output = daemon.output(); - throw new Error('Timed out waiting for ' + url + '\nstdout:\n' + output.stdout + '\nstderr:\n' + output.stderr); -} - -async function startSemanticDaemon(port) { - const daemon = spawnLogged('ktx-daemon', [ - 'serve-http', - '--host', - '127.0.0.1', - '--port', - String(port), - '--log-level', - 'warning', - ]); - await waitForHttpHealth('http://127.0.0.1:' + port + '/health', daemon); - return daemon; -} - -async function stopSemanticDaemon(daemon) { - if (daemon.child.exitCode !== null || daemon.child.signalCode !== null) { - return; - } - daemon.child.kill('SIGTERM'); - const closed = once(daemon.child, 'close').then(() => true); - const timedOut = sleep(5_000).then(() => false); - if (!(await Promise.race([closed, timedOut]))) { - daemon.child.kill('SIGKILL'); - await once(daemon.child, 'close'); - } -} -``` - -Replace both `ktx agent sl query` smoke commands with top-level `ktx sl query` -commands so the installed public tarball verifies the managed Python runtime -path: - -```javascript - const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', - '--connection-id', - 'warehouse', - '--measure', - 'orders.order_count', - '--format', - 'json', - '--yes', - '--project-dir', - projectDir, - ]); - requireSuccess('ktx sl query', slQuery); - requireOutput('ktx sl query', slQuery, /"mode": "compile_only"/); - requireOutput('ktx sl query', slQuery, /orders/); - - const sqliteSlQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', - '--connection-id', - 'warehouse', - '--measure', - 'orders.order_count', - '--format', - 'json', - '--execute', - '--max-rows', - '100', - '--yes', - '--project-dir', - projectDir, - ]); - requireSuccess('ktx sl query sqlite execute', sqliteSlQuery); - requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"dialect": "sqlite"/); - requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"mode": "executed"/); - requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"driver": "sqlite"/); - requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); - process.stdout.write('ktx sl query sqlite execute verified\n'); -``` - -In `npmRuntimeSmokeSource()`, delete the MCP smoke block that starts with: - -```javascript - const daemonPort = await getAvailablePort(); -``` - -and ends after this cleanup block: - -```javascript - } finally { - await client.close(); - await stopSemanticDaemon(daemon); - } -``` - -In `npmDemoSmokeSource()`, keep the existing `pnpm exec ktx` flow. Add an -assertion that the public package is the only direct dependency: - -```javascript - assert.deepEqual(Object.keys(packageJson.dependencies), ['@kaelio/ktx']); -``` - -- [ ] **Step 5: Update artifact smoke tests** - -In `scripts/package-artifacts.test.mjs`, update the -`npmSmokePackageJson` expectations so they assert: - -```javascript - assert.deepEqual(npmSmokePackageJson(layout).dependencies, { - '@kaelio/ktx': `file:${layout.cliTarball}`, - }); -``` - -Replace installed export assertions that import `@ktx/context`, `@ktx/llm`, or -connector packages with these assertions: - -```javascript - assert.match(verifySource, /const cli = await import\('@kaelio\/ktx'\);/); - assert.match(verifySource, /getKtxCliPackageInfo/); - assert.match(verifySource, /runKtxCli/); - assert.doesNotMatch(verifySource, /@ktx\/context/); - assert.doesNotMatch(verifySource, /@ktx\/llm/); - assert.doesNotMatch(verifySource, /@ktx\/connector-/); -``` - -Add runtime smoke assertions: - -```javascript - assert.match(runtimeSource, /ktx public package version/); - assert.match(runtimeSource, /@kaelio\\\\\/ktx 0\\\\.0\\\\.0-private/); - assert.match(runtimeSource, /'ktx', 'sl', 'query'/); - assert.doesNotMatch(runtimeSource, /@ktx\/context/); - assert.doesNotMatch(runtimeSource, /@modelcontextprotocol/); - assert.doesNotMatch(runtimeSource, /startSemanticDaemon/); -``` - -- [ ] **Step 6: Verify smoke command tests** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs scripts/package-artifacts.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -git commit -m "test: cover public kaelio ktx package invocations" -``` - -### Task 5: Run focused verification and artifact smoke - -**Files:** - -- Verify: `scripts/build-public-npm-package.mjs` -- Verify: `scripts/package-artifacts.mjs` -- Verify: `scripts/published-package-smoke.mjs` -- Verify: `scripts/release-readiness.mjs` -- Verify: `packages/cli/src/cli-runtime.ts` - -- [ ] **Step 1: Run script unit tests** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI package tests touched by metadata changes** - -Run: - -```bash -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Build artifacts from source** - -Run: - -```bash -pnpm run artifacts:build -``` - -Expected: PASS and create: - -```text -dist/artifacts/npm/kaelio-ktx-0.0.0-private.tgz -dist/artifacts/python/kaelio_ktx-0.1.0-py3-none-any.whl -dist/artifacts/manifest.json -``` - -- [ ] **Step 4: Verify artifact manifest** - -Run: - -```bash -pnpm run artifacts:verify-manifest -``` - -Expected: PASS. - -- [ ] **Step 5: Verify installed public tarball smoke** - -Run: - -```bash -pnpm run artifacts:verify -``` - -Expected: PASS. The installed npm smoke must install only -`@kaelio/ktx` directly and must not require direct `@ktx/*` dependencies in the -smoke project. - -- [ ] **Step 6: Run release readiness** - -Run: - -```bash -pnpm run release:readiness -``` - -Expected: PASS. The report must list `@kaelio/ktx` as the only npm package and -must still state that registry publishing remains disabled by -`release-policy.json`. - -- [ ] **Step 7: Run pre-commit for changed files** - -Run: - -```bash -source .venv/bin/activate && uv run pre-commit run --files packages/cli/src/cli-runtime.ts packages/cli/src/index.ts packages/cli/src/index.test.ts scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json -``` - -Expected: PASS. If pre-commit is unavailable because hook tooling is missing, -run these fallback checks: - -```bash -node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs -pnpm --filter @ktx/cli run type-check -pnpm --filter @ktx/cli run test -- src/index.test.ts -``` - -- [ ] **Step 8: Commit verification fixes** - -If verification required fixes, commit only the changed files from this plan: - -```bash -git status --short -git add packages/cli/src/cli-runtime.ts packages/cli/src/index.ts packages/cli/src/index.test.ts scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json -git commit -m "chore: verify public kaelio ktx package artifacts" -``` - -## Self-review notes - -- Spec coverage: this plan implements the `@kaelio/ktx` npm package name, one - visible `ktx` binary, bundled JavaScript CLI output, packaged demo assets, - bundled Python runtime wheel assets, and smoke coverage for the required - public invocation modes. -- Remaining after this plan: managed runtime use in deeper Python-backed - paths, such as MCP `serve` defaults and Looker table identifier parsing, - still needs a separate plan if those paths must stop accepting externally - supplied daemon URLs. -- Placeholder scan: this plan uses exact paths, exact commands, concrete code - blocks, and no deferred implementation markers. -- Type consistency: public npm package names are consistently `@kaelio/ktx`; - internal workspace package names remain `@ktx/*`. diff --git a/docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md b/docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md deleted file mode 100644 index 11100184..00000000 --- a/docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md +++ /dev/null @@ -1,1332 +0,0 @@ -# Public NPM Release Handoff Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Turn the remaining npm-managed Python runtime release gap into a -guarded public `@kaelio/ktx` npm release handoff for version `0.1.0`. - -**Architecture:** Keep one public npm package and keep Python packages -unpublished. The public package builder stamps the assembled `@kaelio/ktx` -package as `0.1.0`, release readiness accepts a publish-ready policy only when -all blocking decisions are encoded, and a new publish script performs a dry-run -by default before any live registry publish. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm 10 publish, JSON release -policy, GitHub Actions workflow validation. - ---- - -## Spec trace and current state - -This plan follows -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The existing plan files that reference that spec are: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` - -All twelve are implemented in the current tree: their referenced source and -test files exist, and the runtime command, daemon, package artifact, -published-package smoke, local-embedding smoke, and README markers are present. - -The remaining release gap is explicit in `release-policy.json`: the repository -still uses `ci-artifact-only`, `npm.publish` is `false`, and the README states -that registry publishing is disabled. This plan changes that to a guarded -handoff for the first public npm release while leaving Python registry -publication disabled because the spec says KTX-owned Python code ships inside -the npm package as a bundled wheel for this release. - -## File structure - -- Modify `scripts/build-public-npm-package.mjs`: make the assembled public npm - package version and tarball name `0.1.0` instead of `0.0.0-private`. -- Modify `scripts/build-public-npm-package.test.mjs`: cover public version - stamping and the versioned tarball path. -- Modify `scripts/package-artifacts.mjs`: make artifact metadata report - `@kaelio/ktx` as version `0.1.0`. -- Modify `scripts/package-artifacts.test.mjs`: update artifact manifest, - metadata, runtime smoke, and demo smoke expectations for the public tarball. -- Modify `scripts/local-embeddings-runtime-smoke.test.mjs`: update public - tarball selection coverage for `kaelio-ktx-0.1.0.tgz`. -- Modify `scripts/release-readiness.mjs`: add the - `npm-public-release-ready` release mode and policy validation. -- Modify `scripts/release-readiness.test.mjs`: cover the publish-ready policy - and validation failures. -- Modify `release-policy.json`: encode the first public npm release handoff. -- Create `scripts/publish-public-npm-package.mjs`: verify readiness and run - `pnpm publish` in dry-run mode by default. -- Create `scripts/publish-public-npm-package.test.mjs`: cover publish command - construction and policy gating. -- Modify `package.json`: add `release:npm-publish`. -- Create `.github/workflows/release.yml`: add a manual dry-run/live publish - workflow for the public npm tarball. -- Create `scripts/release-workflow.test.mjs`: validate that the release - workflow is manual, uses pnpm, runs readiness checks, and gates live publish. -- Modify `README.md`: replace the disabled publishing note with the guarded - handoff commands. - -### Task 1: Stamp public npm artifacts as `0.1.0` - -**Files:** - -- Modify: `scripts/build-public-npm-package.mjs` -- Modify: `scripts/build-public-npm-package.test.mjs` -- Modify: `scripts/package-artifacts.mjs` -- Modify: `scripts/package-artifacts.test.mjs` -- Modify: `scripts/local-embeddings-runtime-smoke.test.mjs` - -- [ ] **Step 1: Write failing public version tests** - -In `scripts/build-public-npm-package.test.mjs`, extend the import from -`./build-public-npm-package.mjs` so it includes `PUBLIC_NPM_PACKAGE_VERSION` -and `publicNpmPackageTarballName`: - -```js -import { - PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - PUBLIC_NPM_PACKAGE_NAME, - PUBLIC_NPM_PACKAGE_VERSION, - collectPublicDependencies, - createPublicNpmPackageTree, - publicNpmPackageJson, - publicNpmPackageLayout, - publicNpmPackageTarballName, - publicNpmPackCommand, -} from './build-public-npm-package.mjs'; -``` - -Replace the `publicNpmPackageLayout` test expectation with: - -```js -describe('publicNpmPackageLayout', () => { - it('uses the first public npm release version for the tarball name', () => { - const layout = publicNpmPackageLayout('/repo/ktx'); - - assert.equal(PUBLIC_NPM_PACKAGE_VERSION, '0.1.0'); - assert.equal(publicNpmPackageTarballName(), 'kaelio-ktx-0.1.0.tgz'); - assert.equal(layout.tarballPath, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz'); - }); -}); -``` - -In the `publicNpmPackageJson` test, add this assertion after the package name -assertion: - -```js -assert.equal(packageJson.version, '0.1.0'); -``` - -In the `publicNpmPackCommand` test, replace the tarball assertion block with: - -```js -assert.deepEqual(publicNpmPackCommand(layout), { - command: 'pnpm', - args: [ - '--config.node-linker=hoisted', - 'pack', - '--out', - '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', - ], - cwd: '/repo/ktx/dist/public-npm-package', -}); -``` - -- [ ] **Step 2: Run public package tests to verify failure** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs -``` - -Expected: FAIL. The failure mentions at least one stale -`kaelio-ktx-0.0.0-private.tgz` or `0.0.0-private` public package version -expectation. - -- [ ] **Step 3: Implement public version stamping** - -In `scripts/build-public-npm-package.mjs`, replace the current public version -constants and layout helper with: - -```js -export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx'; -export const PUBLIC_NPM_PACKAGE_VERSION = '0.1.0'; - -export function publicNpmPackageTarballName(version = PUBLIC_NPM_PACKAGE_VERSION) { - return `kaelio-ktx-${version}.tgz`; -} -``` - -Replace `publicNpmPackageLayout` with: - -```js -export function publicNpmPackageLayout(rootDir = scriptRootDir(), version = PUBLIC_NPM_PACKAGE_VERSION) { - return { - rootDir, - packageVersion: version, - cliPackageRoot: join(rootDir, 'packages', 'cli'), - packRoot: join(rootDir, 'dist', 'public-npm-package'), - npmDir: join(rootDir, 'dist', 'artifacts', 'npm'), - tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', publicNpmPackageTarballName(version)), - }; -} -``` - -Change `publicNpmPackageJson` so it accepts the public version explicitly: - -```js -export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUBLIC_NPM_PACKAGE_VERSION) { - return { - name: PUBLIC_NPM_PACKAGE_NAME, - version, - description: 'Standalone KTX context layer for database agents', - private: false, - type: 'module', - engines: cliPackageJson.engines ?? { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: cliPackageJson.main ?? 'dist/index.js', - types: cliPackageJson.types ?? 'dist/index.d.ts', - exports: cliPackageJson.exports ?? { - '.': { - types: './dist/index.d.ts', - import: './dist/index.js', - default: './dist/index.js', - }, - './package.json': './package.json', - }, - files: ['dist', 'assets'], - dependencies, - bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - license: cliPackageJson.license ?? 'Apache-2.0', - repository: { - type: 'git', - url: 'git+https://github.com/kaelio/ktx.git', - }, - bugs: { - url: 'https://github.com/kaelio/ktx/issues', - }, - homepage: 'https://github.com/kaelio/ktx#readme', - }; -} -``` - -In `copyCliPackage`, pass the layout version: - -```js -await writeJson( - join(layout.packRoot, 'package.json'), - publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), -); -``` - -In `createPublicNpmPackageTree`, return the versioned package JSON: - -```js -return { - layout, - packageJson: publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), - bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, -}; -``` - -- [ ] **Step 4: Run public package tests to verify pass** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing artifact metadata tests** - -In `scripts/package-artifacts.test.mjs`, replace expectations that use the -public npm tarball or package version: - -```js -assert.equal(layout.cliTarball, '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz'); -``` - -```js -{ - ecosystem: 'npm', - packageName: '@kaelio/ktx', - packageRoot: 'packages/cli', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', -} -``` - -```js -{ - ecosystem: 'npm', - packageName: '@kaelio/ktx', - packageVersion: '0.1.0', - path: 'npm/kaelio-ktx-0.1.0.tgz', - bytes: Buffer.byteLength('@kaelio/ktx-tarball'), - sha256: createHash('sha256').update('@kaelio/ktx-tarball').digest('hex'), -} -``` - -In the runtime smoke source expectation, replace: - -```js -requireOutput('ktx public package version', version, /@kaelio\/ktx 0\.1\.0/); -``` - -In `scripts/local-embeddings-runtime-smoke.test.mjs`, replace the public -tarball selection assertion with: - -```js -assert.equal( - publicKtxTarballName(['kaelio-ktx-0.1.0.tgz', 'ignore-me.tgz']), - 'kaelio-ktx-0.1.0.tgz', -); -``` - -- [ ] **Step 6: Run artifact tests to verify failure** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: FAIL. The failure mentions stale artifact metadata or tarball -expectations for `0.0.0-private`. - -- [ ] **Step 7: Implement artifact metadata versioning** - -In `scripts/package-artifacts.mjs`, change the build-public import to: - -```js -import { - PUBLIC_NPM_PACKAGE_NAME, - PUBLIC_NPM_PACKAGE_VERSION, - publicNpmPackageTarballName, -} from './build-public-npm-package.mjs'; -``` - -Replace `npmPackageTarballName` with: - -```js -function npmPackageTarballName(packageName) { - if (packageName === PUBLIC_NPM_PACKAGE_NAME) { - return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); - } - return `${packageName.replace('@ktx/', 'ktx-')}-${PACKAGE_VERSION}.tgz`; -} -``` - -In `readNpmPackageMetadata`, return the public package version for -`@kaelio/ktx`: - -```js - const isPublicKtxPackage = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME; - return releaseMetadataEntry({ - ecosystem: 'npm', - packageName: packageInfo.name, - packageRoot: packageInfo.packageRoot, - packageVersion: isPublicKtxPackage ? PUBLIC_NPM_PACKAGE_VERSION : packageJson.version, - privatePackage: isPublicKtxPackage ? false : packageJson.private === true, - }); -``` - -In `npmRuntimeSmokeSource`, replace the version output regex with: - -```js -requireOutput('ktx public package version', version, /@kaelio\/ktx 0\.1\.0/); -``` - -- [ ] **Step 8: Run artifact tests to verify pass** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 9: Commit public version stamping** - -Run: - -```bash -git add scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs -git commit -m "build: stamp public npm package version" -``` - -Expected: commit created. - -### Task 2: Add publish-ready release policy validation - -**Files:** - -- Modify: `scripts/release-readiness.mjs` -- Modify: `scripts/release-readiness.test.mjs` -- Modify: `release-policy.json` - -- [ ] **Step 1: Write failing release readiness tests** - -In `scripts/release-readiness.test.mjs`, add `PUBLIC_NPM_PACKAGE_VERSION` to -the imports from `./build-public-npm-package.mjs`: - -```js -import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; -``` - -Update `releasePolicy()` so the default npm block includes publish settings: - -```js -npm: { - publish: false, - registry: null, - access: 'public', - tag: 'latest', - packages: ['@kaelio/ktx'], - ...npmOverrides, -}, -``` - -In each existing `releaseReadinessReport` expected object for -`ci-artifact-only` and `published-package-smoke-required`, add: - -```js -npmPublish: null, -``` - -Place it after `publishedPackageSmokeGate` and before -`blockedPublishingDecisions`. - -In `writeReleaseMetadataInputs`, keep internal workspace package versions -private. The public package version comes from artifact metadata: - -```js -version: '0.0.0-private', -private: true, -``` - -Add this test after the existing -`reports required published package smoke when release mode requires it` test: - -```js -it('accepts the npm public release ready policy', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-test-')); - try { - await writeReadyFixture(root, { - policy: releasePolicy({ - releaseMode: 'npm-public-release-ready', - npm: { - publish: true, - registry: null, - access: 'public', - tag: 'latest', - }, - publishedPackageSmoke: { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - registry: null, - }, - requiredBeforePublishing: [], - }), - }); - - const report = await releaseReadinessReport(root); - - assert.deepEqual(report, { - schemaVersion: 1, - releaseMode: 'npm-public-release-ready', - sourceRevision: 'abc123', - npmPublishEnabled: true, - pythonPublishEnabled: false, - packageNames: ['@kaelio/ktx', 'ktx-sl', 'ktx-daemon', 'kaelio-ktx'], - publishedPackageSmokeGate: { - status: 'required', - script: 'pnpm run release:published-smoke', - reason: 'Run the published package smoke after the npm package is published.', - configSource: 'release-policy', - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - registry: null, - }, - npmPublish: { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - access: 'public', - tag: 'latest', - registry: null, - }, - blockedPublishingDecisions: [], - }); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); -``` - -Add this validation test: - -```js -it('rejects npm public release ready mode when npm publish is disabled', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-disabled-test-')); - try { - await writeReadyFixture(root, { - policy: releasePolicy({ - releaseMode: 'npm-public-release-ready', - npm: { - publish: false, - registry: null, - access: 'public', - tag: 'latest', - }, - publishedPackageSmoke: { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - registry: null, - }, - requiredBeforePublishing: [], - }), - }); - - await assert.rejects( - () => releaseReadinessReport(root), - /npm-public-release-ready policy requires npm.publish true/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); -``` - -Add this validation test: - -```js -it('rejects npm public release ready mode when Python publishing is enabled', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-npm-public-ready-python-test-')); - try { - await writeReadyFixture(root, { - policy: releasePolicy({ - releaseMode: 'npm-public-release-ready', - npm: { - publish: true, - registry: null, - access: 'public', - tag: 'latest', - }, - python: { - publish: true, - repository: 'pypi', - }, - publishedPackageSmoke: { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - registry: null, - }, - requiredBeforePublishing: [], - }), - }); - - await assert.rejects( - () => releaseReadinessReport(root), - /npm-public-release-ready policy keeps python.publish false/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); -``` - -- [ ] **Step 2: Run release readiness tests to verify failure** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs -``` - -Expected: FAIL with `Unsupported release policy releaseMode: -npm-public-release-ready` or missing `npm.access` validation. - -- [ ] **Step 3: Implement publish-ready policy validation** - -In `scripts/release-readiness.mjs`, import the public package version: - -```js -import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; -``` - -Add the release mode constant and include it in `SUPPORTED_RELEASE_MODES`: - -```js -const NPM_PUBLIC_RELEASE_READY_MODE = 'npm-public-release-ready'; -const SUPPORTED_RELEASE_MODES = new Set([ - CI_ARTIFACT_ONLY_RELEASE_MODE, - PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE, - NPM_PUBLIC_RELEASE_READY_MODE, -]); -``` - -Add string validators for the npm publish settings: - -```js -function assertNpmAccess(value) { - if (value !== 'public') { - throw new Error('Release policy npm.access must be public'); - } -} - -function assertNpmTag(value) { - assertString(value, 'Release policy npm.tag'); - if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(value)) { - throw new Error(`Invalid Release policy npm.tag: ${value}`); - } -} -``` - -In `validateReleasePolicy`, validate the new npm fields after -`assertNullableString(policy.npm.registry, 'Release policy npm.registry');`: - -```js - assertNpmAccess(policy.npm.access); - assertNpmTag(policy.npm.tag); -``` - -Replace `assertRequiredBeforePublishing` with: - -```js -function assertRequiredBeforePublishing(policy) { - assertStringArray(policy.requiredBeforePublishing, 'Release policy requiredBeforePublishing'); - - if (policy.releaseMode === CI_ARTIFACT_ONLY_RELEASE_MODE && policy.requiredBeforePublishing.length === 0) { - throw new Error('Release policy requiredBeforePublishing must list the remaining publishing decisions'); - } - - if ( - (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE || - policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) && - policy.requiredBeforePublishing.length > 0 - ) { - throw new Error(`${policy.releaseMode} release mode requires requiredBeforePublishing to be empty`); - } -} -``` - -Replace `publishedPackageSmokeGate` with: - -```js -function publishedPackageSmokeGate(policy) { - const config = readPublishedPackageSmokeConfig({}, [], policy.publishedPackageSmoke); - - if ( - (policy.releaseMode === PUBLISHED_PACKAGE_SMOKE_REQUIRED_RELEASE_MODE || - policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) && - !config.enabled - ) { - throw new Error(`${policy.releaseMode} release mode requires release-policy.json publishedPackageSmoke.packageName`); - } - - const base = - policy.releaseMode === CI_ARTIFACT_ONLY_RELEASE_MODE - ? { - status: 'not_required', - reason: 'Published package smoke remains pending until release-policy.json enables npm registry publishing.', - } - : policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE - ? { - status: 'required', - reason: 'Run the published package smoke after the npm package is published.', - } - : { - status: 'required', - reason: 'Run the published package smoke before accepting the hybrid-search release.', - }; - - return { - ...base, - script: 'pnpm run release:published-smoke', - configSource: config.enabled ? config.configSource : null, - packageName: config.enabled ? config.packageName : null, - version: config.enabled ? config.packageVersion : policy.publishedPackageSmoke.version, - registry: config.enabled ? (config.registry ?? null) : policy.publishedPackageSmoke.registry, - }; -} -``` - -Add this function below `assertNonPublishingArtifactPolicy`: - -```js -function assertNpmPublicReleaseReadyPolicy(policy, metadata) { - if (policy.npm.publish !== true) { - throw new Error('npm-public-release-ready policy requires npm.publish true'); - } - if (policy.python.publish !== false) { - throw new Error('npm-public-release-ready policy keeps python.publish false'); - } - if (policy.python.repository !== null) { - throw new Error('npm-public-release-ready policy keeps python.repository null'); - } - - assertSameMembers(policy.npm.packages, ['@kaelio/ktx'], 'Release policy npm.packages'); - assertSameMembers(policy.python.packages, metadataNames(metadata, 'python'), 'Release policy python.packages'); - - const npmMetadata = metadata.find((entry) => entry.ecosystem === 'npm' && entry.packageName === '@kaelio/ktx'); - if (!npmMetadata) { - throw new Error('npm-public-release-ready policy requires @kaelio/ktx artifact metadata'); - } - if (npmMetadata.private !== false) { - throw new Error('npm-public-release-ready policy requires @kaelio/ktx to be publishable'); - } - if (npmMetadata.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) { - throw new Error( - `npm-public-release-ready policy expected @kaelio/ktx ${PUBLIC_NPM_PACKAGE_VERSION}, got ${npmMetadata.packageVersion}`, - ); - } - if (policy.publishedPackageSmoke.packageName !== '@kaelio/ktx') { - throw new Error('npm-public-release-ready policy requires publishedPackageSmoke.packageName @kaelio/ktx'); - } - if (policy.publishedPackageSmoke.version !== PUBLIC_NPM_PACKAGE_VERSION) { - throw new Error( - `npm-public-release-ready policy requires publishedPackageSmoke.version ${PUBLIC_NPM_PACKAGE_VERSION}`, - ); - } -} -``` - -Inside `assertNonPublishingArtifactPolicy`, replace the npm package version -suffix check with public-package-aware validation: - -```js - if (isPublicKtxPackage) { - if (entry.packageVersion !== PUBLIC_NPM_PACKAGE_VERSION) { - throw new Error( - `${policyLabel} npm package @kaelio/ktx must use public version ${PUBLIC_NPM_PACKAGE_VERSION}`, - ); - } - } else if (!entry.packageVersion.endsWith('-private')) { - throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`); - } -``` - -In `releaseReadinessReport`, replace the unconditional -`assertNonPublishingArtifactPolicy(policy, metadata);` call with: - -```js - if (policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE) { - assertNpmPublicReleaseReadyPolicy(policy, metadata); - } else { - assertNonPublishingArtifactPolicy(policy, metadata); - } -``` - -Add `npmPublish` to the returned report: - -```js - npmPublish: - policy.releaseMode === NPM_PUBLIC_RELEASE_READY_MODE - ? { - packageName: '@kaelio/ktx', - version: PUBLIC_NPM_PACKAGE_VERSION, - access: policy.npm.access, - tag: policy.npm.tag, - registry: policy.npm.registry, - } - : null, -``` - -Update the text output so it prints the npm publish target when present: - -```js - if (report.npmPublish) { - process.stdout.write( - `NPM publish target: ${report.npmPublish.packageName}@${report.npmPublish.version} (${report.npmPublish.tag})\n`, - ); - } else { - process.stdout.write('Registry publishing remains disabled by release-policy.json.\n'); - } -``` - -- [ ] **Step 4: Update release policy** - -Replace `release-policy.json` with: - -```json -{ - "schemaVersion": 1, - "releaseMode": "npm-public-release-ready", - "npm": { - "publish": true, - "registry": null, - "access": "public", - "tag": "latest", - "packages": ["@kaelio/ktx"] - }, - "python": { - "publish": false, - "repository": null, - "packages": ["ktx-sl", "ktx-daemon", "kaelio-ktx"] - }, - "publishedPackageSmoke": { - "packageName": "@kaelio/ktx", - "version": "0.1.0", - "registry": null - }, - "requiredBeforePublishing": [] -} -``` - -- [ ] **Step 5: Run release readiness tests to verify pass** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Commit release policy validation** - -Run: - -```bash -git add scripts/release-readiness.mjs scripts/release-readiness.test.mjs release-policy.json -git commit -m "release: add npm public release policy" -``` - -Expected: commit created. - -### Task 3: Add guarded npm publish script - -**Files:** - -- Create: `scripts/publish-public-npm-package.test.mjs` -- Create: `scripts/publish-public-npm-package.mjs` -- Modify: `package.json` - -- [ ] **Step 1: Write failing publish script tests** - -Create `scripts/publish-public-npm-package.test.mjs` with: - -```js -import assert from 'node:assert/strict'; -import { readFile } from 'node:fs/promises'; -import { describe, it } from 'node:test'; - -import { - buildNpmPublishCommand, - requireNpmPublicReleaseReady, - resolvePublishMode, -} from './publish-public-npm-package.mjs'; - -const readyReport = { - releaseMode: 'npm-public-release-ready', - npmPublishEnabled: true, - npmPublish: { - packageName: '@kaelio/ktx', - version: '0.1.0', - access: 'public', - tag: 'latest', - registry: null, - }, -}; - -describe('resolvePublishMode', () => { - it('dry-runs by default', () => { - assert.deepEqual(resolvePublishMode([]), { live: false }); - }); - - it('requires an explicit flag for live publish', () => { - assert.deepEqual(resolvePublishMode(['--publish']), { live: true }); - }); -}); - -describe('requireNpmPublicReleaseReady', () => { - it('accepts the npm public release ready report', () => { - assert.equal(requireNpmPublicReleaseReady(readyReport), readyReport.npmPublish); - }); - - it('rejects artifact-only reports', () => { - assert.throws( - () => - requireNpmPublicReleaseReady({ - releaseMode: 'ci-artifact-only', - npmPublishEnabled: false, - npmPublish: null, - }), - /release-policy.json must use npm-public-release-ready before publishing/, - ); - }); -}); - -describe('buildNpmPublishCommand', () => { - it('builds a dry-run pnpm publish command by default', () => { - assert.deepEqual(buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', readyReport.npmPublish, { live: false }), { - command: 'pnpm', - args: [ - 'publish', - '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', - '--access', - 'public', - '--tag', - 'latest', - '--dry-run', - ], - env: {}, - }); - }); - - it('omits dry-run only for explicit live publish', () => { - assert.deepEqual(buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', readyReport.npmPublish, { live: true }).args, [ - 'publish', - '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', - '--access', - 'public', - '--tag', - 'latest', - ]); - }); - - it('uses npm_config_registry when a registry is configured', () => { - const publish = { - ...readyReport.npmPublish, - registry: 'https://registry.npmjs.org/', - }; - - assert.deepEqual( - buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0.tgz', publish, { live: false }).env, - { npm_config_registry: 'https://registry.npmjs.org/' }, - ); - }); -}); - -describe('package script', () => { - it('registers release:npm-publish', async () => { - const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); - - assert.equal(packageJson.scripts['release:npm-publish'], 'node scripts/publish-public-npm-package.mjs'); - }); -}); -``` - -- [ ] **Step 2: Run publish script tests to verify failure** - -Run: - -```bash -node --test scripts/publish-public-npm-package.test.mjs -``` - -Expected: FAIL with `Cannot find module` for -`scripts/publish-public-npm-package.mjs`. - -- [ ] **Step 3: Implement publish script** - -Create `scripts/publish-public-npm-package.mjs` with: - -```js -#!/usr/bin/env node - -import { execFile } from 'node:child_process'; -import { access } from 'node:fs/promises'; -import { promisify } from 'node:util'; -import { pathToFileURL } from 'node:url'; - -import { packageArtifactLayout } from './package-artifacts.mjs'; -import { releaseReadinessReport } from './release-readiness.mjs'; - -const execFileAsync = promisify(execFile); - -export function resolvePublishMode(args = process.argv.slice(2)) { - return { live: args.includes('--publish') }; -} - -export function requireNpmPublicReleaseReady(report) { - if (report.releaseMode !== 'npm-public-release-ready' || report.npmPublishEnabled !== true || !report.npmPublish) { - throw new Error('release-policy.json must use npm-public-release-ready before publishing'); - } - return report.npmPublish; -} - -export function buildNpmPublishCommand(tarballPath, publish, mode) { - return { - command: 'pnpm', - args: [ - 'publish', - tarballPath, - '--access', - publish.access, - '--tag', - publish.tag, - ...(mode.live ? [] : ['--dry-run']), - ], - env: publish.registry ? { npm_config_registry: publish.registry } : {}, - }; -} - -async function assertFileExists(path) { - try { - await access(path); - } catch { - throw new Error(`Missing npm tarball: ${path}. Run pnpm run artifacts:check first.`); - } -} - -async function runPublishCommand(command) { - process.stdout.write(`$ ${command.command} ${command.args.join(' ')}\n`); - await execFileAsync(command.command, command.args, { - env: { ...process.env, ...command.env }, - encoding: 'utf8', - maxBuffer: 1024 * 1024 * 20, - }); -} - -export async function publishPublicNpmPackage(options = {}) { - const rootDir = options.rootDir; - const mode = options.mode ?? resolvePublishMode(options.args); - const report = await releaseReadinessReport(rootDir); - const publish = requireNpmPublicReleaseReady(report); - const layout = packageArtifactLayout(rootDir); - const tarballPath = layout.cliTarball; - - await assertFileExists(tarballPath); - const command = buildNpmPublishCommand(tarballPath, publish, mode); - await runPublishCommand(command); - - process.stdout.write( - mode.live - ? `Published ${publish.packageName}@${publish.version} with tag ${publish.tag}\n` - : `Dry-run verified ${publish.packageName}@${publish.version} with tag ${publish.tag}\n`, - ); -} - -async function main() { - await publishPublicNpmPackage({ args: process.argv.slice(2) }); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - try { - await main(); - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); - process.exitCode = 1; - } -} -``` - -- [ ] **Step 4: Add the package script** - -In root `package.json`, add this script after `release:local-embeddings-smoke`: - -```json -"release:npm-publish": "node scripts/publish-public-npm-package.mjs", -``` - -- [ ] **Step 5: Run publish script tests to verify pass** - -Run: - -```bash -node --test scripts/publish-public-npm-package.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Run a dry-run publish after artifacts are built** - -Run: - -```bash -pnpm run artifacts:check -pnpm run release:npm-publish -``` - -Expected: PASS. The publish command includes `--dry-run`, and the final line is: - -```text -Dry-run verified @kaelio/ktx@0.1.0 with tag latest -``` - -- [ ] **Step 7: Commit publish script** - -Run: - -```bash -git add scripts/publish-public-npm-package.mjs scripts/publish-public-npm-package.test.mjs package.json -git commit -m "release: add guarded npm publish script" -``` - -Expected: commit created. - -### Task 4: Add manual release workflow and docs - -**Files:** - -- Create: `.github/workflows/release.yml` -- Create: `scripts/release-workflow.test.mjs` -- Modify: `README.md` - -- [ ] **Step 1: Write failing workflow tests** - -Create `scripts/release-workflow.test.mjs` with: - -```js -import assert from 'node:assert/strict'; -import { readFile } from 'node:fs/promises'; -import { describe, it } from 'node:test'; - -describe('release workflow', () => { - it('publishes only from manual dispatch with an explicit live input', async () => { - const workflow = await readFile(new URL('../.github/workflows/release.yml', import.meta.url), 'utf8'); - - assert.match(workflow, /^name: KTX Release$/m); - assert.match(workflow, /^ workflow_dispatch:$/m); - assert.match(workflow, /publish_live:/); - assert.match(workflow, /default: false/); - assert.match(workflow, /pnpm run artifacts:check/); - assert.match(workflow, /pnpm run release:readiness/); - assert.match(workflow, /pnpm run release:npm-publish$/m); - assert.match(workflow, /pnpm run release:npm-publish -- --publish/); - assert.match(workflow, /NODE_AUTH_TOKEN: \$\{\{ secrets.NPM_TOKEN \}\}/); - assert.doesNotMatch(workflow, /^ push:/m); - assert.doesNotMatch(workflow, /^ pull_request:/m); - }); -}); -``` - -- [ ] **Step 2: Run workflow tests to verify failure** - -Run: - -```bash -node --test scripts/release-workflow.test.mjs -``` - -Expected: FAIL because `.github/workflows/release.yml` does not exist. - -- [ ] **Step 3: Add the release workflow** - -Create `.github/workflows/release.yml` with: - -```yaml -name: KTX Release - -on: - workflow_dispatch: - inputs: - publish_live: - description: "Publish @kaelio/ktx to npm instead of running a dry-run" - required: true - type: boolean - default: false - -permissions: - contents: read - -concurrency: - group: ktx-release-${{ github.ref }} - cancel-in-progress: false - -jobs: - npm-public-release: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - with: - run_install: false - - - name: Setup Node.js - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 - with: - node-version: "24" - cache: "pnpm" - cache-dependency-path: "pnpm-lock.yaml" - - - name: Install TypeScript dependencies - run: pnpm install --frozen-lockfile - - - name: Setup Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.13" - - - name: Setup uv - uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 - with: - enable-cache: true - cache-dependency-glob: "uv.lock" - - - name: Install Python dependencies - run: uv sync --all-packages - - - name: Build and verify artifacts - run: pnpm run artifacts:check - - - name: Check release readiness - run: pnpm run release:readiness - - - name: Dry-run npm publish - if: ${{ !inputs.publish_live }} - run: pnpm run release:npm-publish - - - name: Publish npm package - if: ${{ inputs.publish_live }} - run: pnpm run release:npm-publish -- --publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} -``` - -- [ ] **Step 4: Update release docs** - -In `README.md`, replace the current `## Release status` section with: - -```markdown -## Release status - -This repository builds one public npm artifact named `@kaelio/ktx`. The first -public npm handoff is policy-gated through `release-policy.json`, which keeps -Python package publishing disabled because KTX-owned Python code ships inside -the npm package as a bundled wheel. - -Build local package artifacts and verify the guarded dry-run publish path with: - -```bash -source .venv/bin/activate -pnpm run artifacts:check -pnpm run release:readiness -pnpm run release:npm-publish -``` - -Run the live npm publish only from the manual `KTX Release` workflow with the -`publish_live` input enabled after the `NPM_TOKEN` secret is configured. -``` - -- [ ] **Step 5: Run workflow and README checks** - -Run: - -```bash -node --test scripts/release-workflow.test.mjs scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Commit workflow and docs** - -Run: - -```bash -git add .github/workflows/release.yml scripts/release-workflow.test.mjs README.md -git commit -m "release: document public npm release handoff" -``` - -Expected: commit created. - -### Task 5: Final verification - -**Files:** - -- Verify: `scripts/*.test.mjs` -- Verify: `packages/cli/src/*` -- Verify: `README.md` -- Verify: `release-policy.json` - -- [ ] **Step 1: Run focused script tests** - -Run: - -```bash -node --test scripts/build-public-npm-package.test.mjs scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs scripts/release-readiness.test.mjs scripts/publish-public-npm-package.test.mjs scripts/published-package-smoke.test.mjs scripts/release-workflow.test.mjs scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 2: Run workspace type and package checks** - -Run: - -```bash -pnpm run type-check -pnpm run artifacts:check -``` - -Expected: PASS. The artifact build creates -`dist/artifacts/npm/kaelio-ktx-0.1.0.tgz`. - -- [ ] **Step 3: Run release readiness and dry-run publish** - -Run: - -```bash -pnpm run release:readiness -pnpm run release:npm-publish -``` - -Expected: PASS. `release:readiness` prints `KTX release mode: -npm-public-release-ready`, and `release:npm-publish` prints `Dry-run verified -@kaelio/ktx@0.1.0 with tag latest`. - -- [ ] **Step 4: Run pre-commit for changed files** - -Run: - -```bash -uv run pre-commit run --files scripts/build-public-npm-package.mjs scripts/build-public-npm-package.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/local-embeddings-runtime-smoke.test.mjs scripts/release-readiness.mjs scripts/release-readiness.test.mjs scripts/publish-public-npm-package.mjs scripts/publish-public-npm-package.test.mjs scripts/release-workflow.test.mjs release-policy.json package.json README.md .github/workflows/release.yml -``` - -Expected: PASS. If pre-commit is unavailable because the local `uv` version or -pre-commit environment is missing, report that explicitly and keep the script -tests, `pnpm run type-check`, `pnpm run artifacts:check`, `pnpm run -release:readiness`, and `pnpm run release:npm-publish` results as the closest -checks. - -- [ ] **Step 5: Confirm the worktree is clean** - -Run: - -```bash -git status --short -``` - -Expected: no output. If there are uncommitted tracked changes, inspect them and -commit only files from this plan with the exact task commit commands above. - -## Success criteria - -- `@kaelio/ktx` artifact metadata and tarball names use version `0.1.0`. -- `release-policy.json` encodes `npm-public-release-ready`, - `npm.publish: true`, and `python.publish: false`. -- `pnpm run release:npm-publish` performs a dry-run by default. -- Live npm publishing requires `pnpm run release:npm-publish -- --publish` or - the manual `KTX Release` workflow with `publish_live` enabled. -- Published-package smoke remains the post-publication proof for `npx - @kaelio/ktx`, local `npx ktx`, and global `ktx` invocation modes. -- No Python package publication is added for this release. - -## Self-review - -- Spec coverage: this plan covers the remaining public npm handoff gap while - preserving the bundled Python wheel model and single npm package surface. -- Placeholder scan: no open placeholders or deferred implementation notes are - present. -- Type consistency: the release mode name is consistently - `npm-public-release-ready`; the public npm version is consistently `0.1.0`; - the publish script consumes the `npmPublish` report shape produced by - `release-readiness.mjs`. diff --git a/docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md b/docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md deleted file mode 100644 index ad581827..00000000 --- a/docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md +++ /dev/null @@ -1,602 +0,0 @@ -# Published Package Managed Runtime Smoke Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the post-publication smoke prove that the published -`@kaelio/ktx` package uses the same isolated managed Python runtime across -`npx @kaelio/ktx`, local `npx ktx`, and global `ktx` invocation modes. - -**Architecture:** Keep the smoke black-box and network-gated. Strengthen the -command builder so every Python-backed published-package command receives the -same temporary `KTX_RUNTIME_ROOT`, then run a real semantic-layer query through -the direct `npx`, local install, and global install paths instead of checking -only `--version` for local and global binaries. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, npx, KTX managed Python -runtime, published `@kaelio/ktx` package smoke. - ---- - -## Existing status - -This plan is based on -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plans are based on that spec and are implemented in this -worktree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` - -Implementation evidence found before writing this plan includes: - -- `scripts/build-python-runtime-wheel.mjs` and - `packages/cli/assets/python/manifest.json`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/runtime.ts`, - `packages/cli/src/commands/runtime-commands.ts`, - `packages/cli/src/managed-python-command.ts`, - `packages/cli/src/managed-python-daemon.ts`, - `packages/cli/src/managed-local-embeddings.ts`, and - `packages/cli/src/managed-python-http.ts`. -- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, - `scripts/local-embeddings-runtime-smoke.mjs`, and - `scripts/published-package-smoke.mjs`. -- `packages/cli/src/agent-runtime.ts`, `packages/cli/src/serve.ts`, - `packages/cli/src/ingest.ts`, and `packages/cli/src/scan.ts` thread managed - runtime policy through the Python-backed CLI paths. -- `examples/postgres-historic/scripts/smoke.sh`, - `examples/postgres-historic/README.md`, - `examples/package-artifacts/README.md`, and `README.md` now document the - managed runtime instead of a manual `python-service/` process. - -The remaining release-confidence gap is in the post-publication smoke: - -- `scripts/published-package-smoke-config.mjs` runs `npx @kaelio/ktx setup - demo` and `npx @kaelio/ktx sl query ... --yes`, but it does not isolate - `KTX_RUNTIME_ROOT` for those commands. -- The same smoke installs `@kaelio/ktx` locally and globally, but local and - global verification only run `--version`. -- The design spec requires the direct `npx @kaelio/ktx`, local `npx ktx`, and - global `ktx` modes to work for real KTX commands. A semantic-layer query is - the smallest Python-backed command that proves the bundled managed runtime is - usable in each mode. - -## File structure - -- Modify `scripts/published-package-smoke.test.mjs`: expect a shared - `KTX_RUNTIME_ROOT` in the published smoke commands, expect local and global - semantic query commands, and cover label classification used by the runner. -- Modify `scripts/published-package-smoke-config.mjs`: derive a temporary - runtime root from the smoke project directory, merge it with registry - environment settings, and add local and global `sl query` commands. -- Modify `scripts/published-package-smoke.mjs`: validate the renamed version - labels and semantic query labels when the smoke runs. - -### Task 1: Isolate runtime roots and add real local/global command coverage - -**Files:** - -- Modify: `scripts/published-package-smoke.test.mjs` -- Modify: `scripts/published-package-smoke-config.mjs` -- Test: `scripts/published-package-smoke.test.mjs` - -- [ ] **Step 1: Write the failing command-list test** - -In `scripts/published-package-smoke.test.mjs`, replace the existing -`it('builds the full public package smoke command list', ...)` block with this -test: - -```javascript - it('builds the full public package smoke command list', () => { - assert.deepEqual( - buildPublishedPackageSmokeCommands( - config, - '/tmp/ktx-smoke/demo', - '/tmp/ktx-smoke/managed-runtime', - ), - [ - { - label: 'published package npx version', - command: 'npx', - args: ['--yes', '@kaelio/ktx@latest', '--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package setup demo', - command: 'npx', - args: [ - '--yes', - '@kaelio/ktx@latest', - 'setup', - 'demo', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--no-input', - '--plain', - ], - env: { - npm_config_registry: 'https://registry.npmjs.org/', - KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', - }, - }, - { - label: 'published package npx sl query', - command: 'npx', - args: [ - '--yes', - '@kaelio/ktx@latest', - 'sl', - 'query', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--connection-id', - 'orbit_demo', - '--measure', - 'contracts.contract_count', - '--format', - 'sql', - '--yes', - ], - env: { - npm_config_registry: 'https://registry.npmjs.org/', - KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', - }, - }, - { - label: 'published package local install', - command: 'pnpm', - args: ['add', '@kaelio/ktx@latest'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package local version', - command: 'npx', - args: ['ktx', '--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package local sl query', - command: 'npx', - args: [ - 'ktx', - 'sl', - 'query', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--connection-id', - 'orbit_demo', - '--measure', - 'contracts.contract_count', - '--format', - 'sql', - '--yes', - ], - env: { - npm_config_registry: 'https://registry.npmjs.org/', - KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', - }, - }, - { - label: 'published package global install', - command: 'pnpm', - args: ['add', '--global', '@kaelio/ktx@latest'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package global version', - command: 'ktx', - args: ['--version'], - env: { npm_config_registry: 'https://registry.npmjs.org/' }, - }, - { - label: 'published package global sl query', - command: 'ktx', - args: [ - 'sl', - 'query', - '--project-dir', - '/tmp/ktx-smoke/demo', - '--connection-id', - 'orbit_demo', - '--measure', - 'contracts.contract_count', - '--format', - 'sql', - '--yes', - ], - env: { - npm_config_registry: 'https://registry.npmjs.org/', - KTX_RUNTIME_ROOT: '/tmp/ktx-smoke/managed-runtime', - }, - }, - ], - ); - }); -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs -``` - -Expected: FAIL with an `AssertionError` showing that the actual command list -still uses `published package version`, lacks `KTX_RUNTIME_ROOT`, and lacks the -local/global `sl query` commands. - -- [ ] **Step 3: Implement the command builder changes** - -In `scripts/published-package-smoke-config.mjs`, add this import before the -existing `node:assert/strict` import: - -```javascript -import { dirname, join } from 'node:path'; -``` - -In the same file, add these helper functions after -`assertHttpRegistry(registry, label)`: - -```javascript -function registryEnv(config) { - return config.registry ? { npm_config_registry: config.registry } : {}; -} - -function runtimeCommandEnv(config, runtimeRoot) { - return { ...registryEnv(config), KTX_RUNTIME_ROOT: runtimeRoot }; -} - -function semanticQueryArgs(projectDir) { - return [ - 'sl', - 'query', - '--project-dir', - projectDir, - '--connection-id', - 'orbit_demo', - '--measure', - 'contracts.contract_count', - '--format', - 'sql', - '--yes', - ]; -} -``` - -Replace `buildPublishedPackageNpxCommand()` and -`buildPublishedPackageSmokeCommands()` with this implementation: - -```javascript -export function buildPublishedPackageNpxCommand(config, args, label = 'published package command', extraEnv = {}) { - return { - label, - command: 'npx', - args: ['--yes', publishedPackageSpec(config), ...args], - env: { ...registryEnv(config), ...extraEnv }, - }; -} - -export function buildPublishedPackageSmokeCommands( - config, - projectDir, - runtimeRoot = join(dirname(projectDir), 'managed-runtime'), -) { - const runtimeEnv = runtimeCommandEnv(config, runtimeRoot); - const packageEnv = registryEnv(config); - const queryArgs = semanticQueryArgs(projectDir); - - return [ - buildPublishedPackageNpxCommand(config, ['--version'], 'published package npx version'), - buildPublishedPackageNpxCommand( - config, - ['setup', 'demo', '--project-dir', projectDir, '--no-input', '--plain'], - 'published package setup demo', - { KTX_RUNTIME_ROOT: runtimeRoot }, - ), - buildPublishedPackageNpxCommand(config, queryArgs, 'published package npx sl query', { - KTX_RUNTIME_ROOT: runtimeRoot, - }), - { - label: 'published package local install', - command: 'pnpm', - args: ['add', publishedPackageSpec(config)], - env: packageEnv, - }, - { - label: 'published package local version', - command: 'npx', - args: ['ktx', '--version'], - env: packageEnv, - }, - { - label: 'published package local sl query', - command: 'npx', - args: ['ktx', ...queryArgs], - env: runtimeEnv, - }, - { - label: 'published package global install', - command: 'pnpm', - args: ['add', '--global', publishedPackageSpec(config)], - env: packageEnv, - }, - { - label: 'published package global version', - command: 'ktx', - args: ['--version'], - env: packageEnv, - }, - { - label: 'published package global sl query', - command: 'ktx', - args: queryArgs, - env: runtimeEnv, - }, - ]; -} -``` - -- [ ] **Step 4: Run the command-list test and verify it passes** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs -``` - -Expected: PASS for the command construction tests, with remaining failures only -if the runner label validation test from Task 2 has already been added. - -- [ ] **Step 5: Commit the command-builder change** - -Run: - -```bash -git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.test.mjs -git commit -m "test: cover published package runtime smoke commands" -``` - -### Task 2: Validate smoke runner labels for the new command list - -**Files:** - -- Modify: `scripts/published-package-smoke.test.mjs` -- Modify: `scripts/published-package-smoke.mjs` -- Test: `scripts/published-package-smoke.test.mjs` - -- [ ] **Step 1: Write the failing label classification test** - -In `scripts/published-package-smoke.test.mjs`, replace the import from -`./published-package-smoke.mjs` with this import: - -```javascript -import { - buildPublishedPackageNpxCommand, - buildPublishedPackageSmokeCommands, - isPublishedPackageSemanticQueryLabel, - isPublishedPackageVersionLabel, - publishedPackageSpec, - readPublishedPackageSmokeConfig, -} from './published-package-smoke.mjs'; -``` - -Add this test after the `describe('published package smoke command -construction', ...)` block: - -```javascript -describe('published package smoke output validation labels', () => { - it('classifies version and semantic query commands', () => { - assert.equal(isPublishedPackageVersionLabel('published package npx version'), true); - assert.equal(isPublishedPackageVersionLabel('published package local version'), true); - assert.equal(isPublishedPackageVersionLabel('published package global version'), true); - assert.equal(isPublishedPackageVersionLabel('published package setup demo'), false); - - assert.equal(isPublishedPackageSemanticQueryLabel('published package npx sl query'), true); - assert.equal(isPublishedPackageSemanticQueryLabel('published package local sl query'), true); - assert.equal(isPublishedPackageSemanticQueryLabel('published package global sl query'), true); - assert.equal(isPublishedPackageSemanticQueryLabel('published package local install'), false); - }); -}); -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs -``` - -Expected: FAIL with an import error because -`isPublishedPackageSemanticQueryLabel` and `isPublishedPackageVersionLabel` are -not exported yet. - -- [ ] **Step 3: Implement label classification and runner validation** - -In `scripts/published-package-smoke.mjs`, add these constants and exports after -`const SMOKE_TIMEOUT_MS = 180_000;`: - -```javascript -const VERSION_LABELS = new Set([ - 'published package npx version', - 'published package local version', - 'published package global version', -]); - -const SEMANTIC_QUERY_LABELS = new Set([ - 'published package npx sl query', - 'published package local sl query', - 'published package global sl query', -]); - -export function isPublishedPackageVersionLabel(label) { - return VERSION_LABELS.has(label); -} - -export function isPublishedPackageSemanticQueryLabel(label) { - return SEMANTIC_QUERY_LABELS.has(label); -} -``` - -In `runPublishedPackageSmoke(config)`, replace this block: - -```javascript - if ( - command.label === 'published package version' || - command.label === 'published package local binary' || - command.label === 'published package global binary' - ) { - assert.match(result.stdout, /@kaelio\/ktx /); - } - if (command.label === 'published package sl query') { - assert.match(result.stdout, /SELECT/i); - assert.match(result.stdout, /contracts/i); - } -``` - -with this block: - -```javascript - if (isPublishedPackageVersionLabel(command.label)) { - assert.match(result.stdout, /@kaelio\/ktx /); - } - if (isPublishedPackageSemanticQueryLabel(command.label)) { - assert.match(result.stdout, /SELECT/i); - assert.match(result.stdout, /contracts/i); - } -``` - -- [ ] **Step 4: Run the label tests and verify they pass** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the runner-label change** - -Run: - -```bash -git add scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs -git commit -m "test: validate published package smoke outputs" -``` - -### Task 3: Verify release-script compatibility - -**Files:** - -- Verify: `scripts/published-package-smoke-config.mjs` -- Verify: `scripts/published-package-smoke.mjs` -- Verify: `scripts/published-package-smoke.test.mjs` -- Verify: `scripts/release-readiness.test.mjs` -- Verify: `package.json` - -- [ ] **Step 1: Run the focused Node tests** - -Run: - -```bash -node --test scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs -``` - -Expected: PASS. The release-readiness tests must continue to report the -published package smoke gate without executing the network smoke. - -- [ ] **Step 2: Run release readiness** - -Run: - -```bash -pnpm run release:readiness -``` - -Expected: PASS and output containing these lines: - -```text -Release mode: ci-artifact-only -NPM publish enabled: false -Published package smoke: pending -Published package smoke script: pnpm run release:published-smoke -``` - -- [ ] **Step 3: Confirm the network smoke stays explicit** - -Run: - -```bash -rg -n '"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config"' package.json -``` - -Expected: PASS with one match in `package.json`. Do not run -`pnpm run release:published-smoke` in normal CI before the package is published -to the configured registry. - -- [ ] **Step 4: Check pre-commit availability** - -Run: - -```bash -test ! -f .pre-commit-config.yaml -``` - -Expected: PASS in the current worktree. If a pre-commit config exists when this -plan is executed, run this instead after activating `.venv`: - -```bash -source .venv/bin/activate -uv run pre-commit run --files scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs -``` - -- [ ] **Step 5: Commit verification-only fixes if needed** - -If Step 1 or Step 2 required additional source changes, commit them with: - -```bash -git add scripts/published-package-smoke-config.mjs scripts/published-package-smoke.mjs scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs package.json -git commit -m "chore: verify published package runtime smoke" -``` - -If no files changed after Task 2, do not create an empty commit. - -## Acceptance criteria - -- `buildPublishedPackageSmokeCommands()` derives - `/managed-runtime` from the demo project directory by default. -- Direct `npx @kaelio/ktx`, local `npx ktx`, and global `ktx` semantic query - commands all receive the same `KTX_RUNTIME_ROOT`. -- Local and global post-publication smoke coverage runs `sl query ... --yes`, - not only `--version`. -- `runPublishedPackageSmoke()` validates version output for all version labels - and validates generated SQL output for all semantic query labels. -- `node --test scripts/published-package-smoke.test.mjs scripts/release-readiness.test.mjs` - passes. -- `pnpm run release:readiness` still reports the published-package smoke as a - pending explicit release gate while registry publishing is disabled. - -## Self-review notes - -- Spec coverage: this plan covers the remaining invocation-mode confidence gap - from the spec by proving the published package uses an isolated managed - runtime across direct `npx`, local binary, and global binary paths. -- Placeholder scan: the plan contains concrete file paths, exact code blocks, - exact commands, and exact expected outcomes. -- Type consistency: the command label strings are consistent across tests, - command construction, and smoke-runner output validation. diff --git a/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md b/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md deleted file mode 100644 index 52b55ab1..00000000 --- a/docs/superpowers/plans/2026-05-11-single-public-runtime-artifact-cleanup.md +++ /dev/null @@ -1,978 +0,0 @@ -# Single Public Runtime Artifact Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make release artifacts match the npm-managed Python runtime design: -one public `@kaelio/ktx` npm tarball plus one bundled `kaelio-ktx` runtime -wheel, with no standalone `ktx-sl` or `ktx-daemon` release artifacts. - -**Architecture:** Keep `python/ktx-sl` and `python/ktx-daemon` as source -packages used to assemble the bundled runtime wheel. Remove direct standalone -Python wheel and source-distribution builds from the release artifact path, -manifest, readiness policy, and artifact smoke docs. The packed npm package -remains the only user-visible package; Python-backed verification continues -through the managed runtime installed from the bundled wheel. - -**Tech Stack:** Node 22 ESM scripts, `node:test`, pnpm, uv-built bundled -runtime wheel, JSON release policy, Markdown. - ---- - -## Current state - -This plan follows -`docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md`. - -The following plan files are based on that spec and are implemented in the -current tree: - -- `docs/superpowers/plans/2026-05-11-bundled-python-runtime-wheel.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-installer.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-command-integration.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-daemon-lifecycle.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-runtime.md` -- `docs/superpowers/plans/2026-05-11-public-kaelio-ktx-npm-package.md` -- `docs/superpowers/plans/2026-05-11-managed-python-runtime-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-local-embeddings-release-smoke.md` -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-local-ingest-daemon-runtime.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-docs-and-postgres-smoke-cleanup.md` -- `docs/superpowers/plans/2026-05-11-published-package-managed-runtime-smoke.md` -- `docs/superpowers/plans/2026-05-11-public-npm-release-handoff.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-prune-smoke-and-docs.md` -- `docs/superpowers/plans/2026-05-11-managed-runtime-uv-prerequisite-contract.md` - -Implementation evidence found before writing this plan includes: - -- `packages/cli/assets/python/manifest.json` and - `packages/cli/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl`. -- `packages/cli/src/managed-python-runtime.ts`, - `packages/cli/src/managed-python-command.ts`, - `packages/cli/src/managed-python-daemon.ts`, - `packages/cli/src/managed-local-embeddings.ts`, - `packages/cli/src/managed-python-http.ts`, and `packages/cli/src/runtime.ts`. -- `scripts/build-public-npm-package.mjs`, `scripts/package-artifacts.mjs`, - `scripts/published-package-smoke.mjs`, - `scripts/local-embeddings-runtime-smoke.mjs`, - `scripts/publish-public-npm-package.mjs`, and - `.github/workflows/release.yml`. -- `release-policy.json` is in `npm-public-release-ready` mode, publishes - `@kaelio/ktx`, disables Python package publishing, and encodes the hard - `uv` prerequisite. -- `README.md` and `examples/package-artifacts/README.md` document public npm - usage, managed runtime commands, `runtime prune`, and the `uv` prerequisite. - -The remaining mismatch is in the artifact release surface: - -- `scripts/package-artifacts.mjs` still runs `uv build --package ktx-sl` and - `uv build --package ktx-daemon`. -- `scripts/package-artifacts.mjs` still adds `ktx-sl` and `ktx-daemon` wheel - and source-distribution files to the artifact manifest. -- `scripts/package-artifacts.mjs` still runs a direct Python clean-install - smoke, even though the npm artifact smoke already proves Python-backed - commands through the managed runtime. -- `release-policy.json` still lists `ktx-sl` and `ktx-daemon` under - `python.packages`. -- `examples/package-artifacts/README.md` says the Python smoke installs - standalone Python artifacts directly. - -This plan removes those release artifacts. It does not delete the Python source -packages because the bundled runtime wheel builder still copies from -`python/ktx-sl/semantic_layer` and `python/ktx-daemon/src/ktx_daemon`. - -## File structure - -- Modify `scripts/package-artifacts.test.mjs`: make artifact tests expect only - `@kaelio/ktx` plus the `kaelio-ktx` bundled runtime wheel, and add a guard - that direct standalone Python artifact smoke code is gone. -- Modify `scripts/package-artifacts.mjs`: stop building standalone Python - artifacts, stop looking for their wheel and source-distribution files, remove - their release metadata, and remove the direct Python artifact verification - path. -- Modify `scripts/release-readiness.test.mjs`: update release policy fixtures - and readiness reports so the only Python release metadata is `kaelio-ktx`. -- Modify `release-policy.json`: set `python.packages` to `["kaelio-ktx"]`. -- Modify `scripts/examples-docs.test.mjs`: require docs to describe the single - npm tarball plus runtime wheel artifact shape and reject the old direct - Python-artifact smoke wording. -- Modify `README.md`: clarify that `python/ktx-sl` and `python/ktx-daemon` are - source packages, not release artifacts for the first npm release. -- Modify `examples/package-artifacts/README.md`: replace the stale standalone - Python smoke paragraph with the managed-runtime artifact contract. - -### Task 1: Make package artifact tests expect one runtime wheel - -**Files:** - -- Modify: `scripts/package-artifacts.test.mjs` -- Test: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Update package artifact imports** - -In `scripts/package-artifacts.test.mjs`, replace the import from -`./package-artifacts.mjs` with this import: - -```javascript -import { - CLI_PYTHON_ASSET_MANIFEST, - INTERNAL_NPM_WORKSPACE_PACKAGES, - RUNTIME_WHEEL_DISTRIBUTION_NAME, - RUNTIME_WHEEL_NORMALIZED_NAME, - RUNTIME_WHEEL_PACKAGE_VERSION, - artifactManifestPath, - buildArtifactCommands, - copyRuntimeWheelAssets, - findPythonArtifacts, - NPM_ARTIFACT_PACKAGES, - npmDemoSmokeSource, - npmRuntimeSmokeSource, - npmSmokePackageJson, - npmVerifySource, - packageArtifactLayout, - packageReleaseMetadata, - verifyArtifactManifest, - writeArtifactManifest, -} from './package-artifacts.mjs'; -``` - -- [ ] **Step 2: Remove standalone Python fixture setup** - -In `scripts/package-artifacts.test.mjs`, replace `writeReleaseMetadataInputs` -with this function: - -```javascript -async function writeReleaseMetadataInputs(root) { - for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { - await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); - await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { - name: packageInfo.name, - version: '0.0.0-private', - private: true, - }); - } -} -``` - -Replace `writeUploadableArtifactFixtures` with this function: - -```javascript -async function writeUploadableArtifactFixtures(layout) { - await mkdir(layout.npmDir, { recursive: true }); - await mkdir(layout.pythonDir, { recursive: true }); - - const fileContents = new Map([ - ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ - layout.npmTarballs[packageInfo.name], - `${packageInfo.name}-tarball`, - ]), - [ - join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - 'kaelio-ktx-runtime-wheel', - ], - ]); - - for (const [path, contents] of fileContents) { - await writeFile(path, contents); - } -} -``` - -- [ ] **Step 3: Change build command expectations** - -In the `buildArtifactCommands` test, replace the body with this code: - -```javascript - it('builds TypeScript packages and the runtime wheel before packing npm artifacts', () => { - const layout = packageArtifactLayout('/repo/ktx'); - const commands = buildArtifactCommands(layout); - - assert.deepEqual( - commands.slice(0, NPM_BUILD_PACKAGE_ORDER.length).map((command) => [command.command, command.args]), - NPM_BUILD_PACKAGE_ORDER.map((packageName) => ['pnpm', ['--filter', packageName, 'run', 'build']]), - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length, NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [ - command.command, - command.args, - ]), - [[process.execPath, ['scripts/build-python-runtime-wheel.mjs']]], - ); - assert.deepEqual( - commands.slice(NPM_BUILD_PACKAGE_ORDER.length + 1).map((command) => [command.command, command.args]), - [[process.execPath, ['scripts/build-public-npm-package.mjs']]], - ); - }); -``` - -- [ ] **Step 4: Change release metadata expectations** - -In the `packageReleaseMetadata` test, replace the expected array with this -array: - -```javascript - assert.deepEqual(await packageReleaseMetadata(root), [ - { - ecosystem: 'npm', - packageName: '@kaelio/ktx', - packageRoot: 'packages/cli', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, - { - ecosystem: 'python', - packageName: 'kaelio-ktx', - packageRoot: 'python/runtime-wheel', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, - ]); -``` - -- [ ] **Step 5: Change Python artifact discovery expectations** - -Replace the `findPythonArtifacts` success test with this test: - -```javascript - it('finds the bundled runtime wheel only', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); - try { - await writeFile(join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), ''); - - assert.deepEqual(await findPythonArtifacts(root), { - runtimeWheel: join(root, 'kaelio_ktx-0.1.0-py3-none-any.whl'), - }); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 6: Change artifact manifest expectations** - -Inside the artifact manifest test, replace the Python package assertion with: - -```javascript - assert.deepEqual( - manifest.packages.filter((entry) => entry.ecosystem === 'python'), - [ - { - ecosystem: 'python', - packageName: 'kaelio-ktx', - packageRoot: 'python/runtime-wheel', - packageVersion: '0.1.0', - private: false, - releaseMode: 'ci-artifact-only', - }, - ], - ); -``` - -Replace the Python file assertion with: - -```javascript - assert.deepEqual( - manifest.files - .filter((file) => file.ecosystem === 'python') - .map((file) => ({ - artifactKind: file.artifactKind, - ecosystem: file.ecosystem, - packageName: file.packageName, - packageVersion: file.packageVersion, - path: file.path, - })), - [ - { - artifactKind: 'wheel', - ecosystem: 'python', - packageName: 'kaelio-ktx', - packageVersion: '0.1.0', - path: 'python/kaelio_ktx-0.1.0-py3-none-any.whl', - }, - ], - ); -``` - -In the `verifyArtifactManifest` success test, replace the file-count assertion -with: - -```javascript - assert.equal(manifest.files.length, NPM_ARTIFACT_PACKAGES.length + 1); -``` - -- [ ] **Step 7: Replace direct Python smoke tests with a dead-code guard** - -Remove the whole `describe('pythonArtifactInstallArgs', ...)` block. - -In `describe('verification snippets', ...)`, remove the test named -`asserts the Python modules that clean installs must expose`. - -Add this test after the `verifyNpmArtifacts` test: - -```javascript -describe('standalone Python artifact cleanup', () => { - it('does not build or verify standalone Python package artifacts', async () => { - const source = await readFile(new URL('./package-artifacts.mjs', import.meta.url), 'utf8'); - - assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-sl'/); - assert.doesNotMatch(source, /uv', \['build', '--package', 'ktx-daemon'/); - assert.doesNotMatch(source, /async function verifyPythonArtifacts/); - assert.doesNotMatch(source, /pythonArtifactInstallArgs/); - assert.doesNotMatch(source, /pythonVerifySource/); - assert.doesNotMatch(source, /ktx_sl-0\.1\.0/); - assert.doesNotMatch(source, /ktx_daemon-0\.1\.0/); - }); -}); -``` - -- [ ] **Step 8: Run package artifact tests and verify failure** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: FAIL. The failures mention the extra `ktx-sl` and `ktx-daemon` -artifact commands, metadata entries, manifest files, or direct Python smoke -helpers. - -### Task 2: Remove standalone Python artifacts from package artifacts - -**Files:** - -- Modify: `scripts/package-artifacts.mjs` -- Test: `scripts/package-artifacts.test.mjs` - -- [ ] **Step 1: Remove dead constants and imports** - -In `scripts/package-artifacts.mjs`, replace the `node:path` import with this -import: - -```javascript -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; -``` - -Remove these constants: - -```javascript -const PACKAGE_VERSION = '0.0.0-private'; -const PYTHON_PACKAGE_VERSION = '0.1.0'; -``` - -Remove the whole `ordersSource` constant block. - -- [ ] **Step 2: Make npm artifact names public-package only** - -Replace `npmPackageTarballName` with this function: - -```javascript -function npmPackageTarballName(packageName) { - if (packageName !== PUBLIC_NPM_PACKAGE_NAME) { - throw new Error(`Unsupported npm artifact package: ${packageName}`); - } - return publicNpmPackageTarballName(PUBLIC_NPM_PACKAGE_VERSION); -} -``` - -- [ ] **Step 3: Remove standalone Python build commands** - -Replace `buildArtifactCommands` with this function: - -```javascript -export function buildArtifactCommands(layout) { - const packagesByName = new Map(INTERNAL_NPM_WORKSPACE_PACKAGES.map((packageInfo) => [packageInfo.name, packageInfo])); - const npmBuildCommands = NPM_ARTIFACT_BUILD_ORDER.map((packageName) => { - const packageInfo = packagesByName.get(packageName); - if (!packageInfo) { - throw new Error(`Unknown npm artifact build package: ${packageName}`); - } - return { - command: 'pnpm', - args: ['--filter', packageInfo.name, 'run', 'build'], - cwd: layout.rootDir, - }; - }); - const publicPackageCommand = { - command: process.execPath, - args: ['scripts/build-public-npm-package.mjs'], - cwd: layout.rootDir, - }; - - return [ - ...npmBuildCommands, - { - command: process.execPath, - args: ['scripts/build-python-runtime-wheel.mjs'], - cwd: layout.rootDir, - }, - publicPackageCommand, - ]; -} -``` - -- [ ] **Step 4: Discover only the bundled runtime wheel** - -Replace `findOne` and `findPythonArtifacts` with these functions: - -```javascript -function findOne(files, distributionName, suffix, label, pythonDir, version) { - const normalized = normalizePythonDistributionName(distributionName); - const found = files.find((file) => file.startsWith(`${normalized}-${version}`) && file.endsWith(suffix)); - if (!found) { - throw new Error(`Missing Python artifact: ${label}`); - } - return join(pythonDir, found); -} - -export async function findPythonArtifacts(pythonDir) { - const files = await readdir(pythonDir); - - return { - runtimeWheel: findOne( - files, - RUNTIME_WHEEL_DISTRIBUTION_NAME, - '.whl', - 'kaelio-ktx runtime wheel', - pythonDir, - RUNTIME_WHEEL_PACKAGE_VERSION, - ), - }; -} -``` - -- [ ] **Step 5: Emit release metadata only for npm and runtime wheel** - -Replace `packageReleaseMetadata` with this function: - -```javascript -export async function packageReleaseMetadata(rootDir = scriptRootDir()) { - const npmPackages = await Promise.all( - NPM_ARTIFACT_PACKAGES.map((packageInfo) => readNpmPackageMetadata(rootDir, packageInfo)), - ); - - return [ - ...npmPackages, - releaseMetadataEntry({ - ecosystem: 'python', - packageName: RUNTIME_WHEEL_DISTRIBUTION_NAME, - packageRoot: 'python/runtime-wheel', - packageVersion: RUNTIME_WHEEL_PACKAGE_VERSION, - privatePackage: false, - }), - ]; -} -``` - -- [ ] **Step 6: Remove dead TOML metadata helpers** - -Delete these helper functions from `scripts/package-artifacts.mjs` because -release metadata no longer reads standalone Python `pyproject.toml` files: - -```javascript -function readProjectBlock(toml, sourcePath) { - const lines = toml.split(/\r?\n/); - const block = []; - let inProject = false; - - for (const line of lines) { - if (/^\[project\]\s*$/.test(line)) { - inProject = true; - continue; - } - if (inProject && /^\[.*\]\s*$/.test(line)) { - break; - } - if (inProject) { - block.push(line); - } - } - - if (!inProject) { - throw new Error(`Missing [project] table in ${sourcePath}`); - } - return block.join('\n'); -} -``` - -```javascript -function readTomlStringField(projectBlock, fieldName, sourcePath) { - const match = projectBlock.match(new RegExp(`^${fieldName}\\s*=\\s*"([^"]+)"\\s*$`, 'm')); - if (!match) { - throw new Error(`Missing project.${fieldName} in ${sourcePath}`); - } - return match[1]; -} -``` - -```javascript -async function readPyprojectMetadata(path) { - const toml = await readFile(path, 'utf-8'); - const projectBlock = readProjectBlock(toml, path); - return { - name: readTomlStringField(projectBlock, 'name', path), - version: readTomlStringField(projectBlock, 'version', path), - }; -} -``` - -- [ ] **Step 7: Emit manifest records only for npm and runtime wheel** - -Replace `artifactPackageRecords` with this function: - -```javascript -function artifactPackageRecords(layout, pythonArtifacts, packages) { - const packagesByName = packageMetadataByName(packages); - const npmRecords = NPM_ARTIFACT_PACKAGES.map((packageInfo) => ({ - artifactKind: 'tarball', - artifactPath: layout.npmTarballs[packageInfo.name], - metadata: requirePackageMetadata(packagesByName, packageInfo.name), - })); - - return [ - ...npmRecords, - { - artifactKind: 'wheel', - artifactPath: pythonArtifacts.runtimeWheel, - metadata: requirePackageMetadata(packagesByName, RUNTIME_WHEEL_DISTRIBUTION_NAME), - }, - ]; -} -``` - -- [ ] **Step 8: Remove direct Python artifact verification helpers** - -Delete these exports and functions from `scripts/package-artifacts.mjs`: - -```javascript -export function pythonArtifactInstallArgs(python, pythonArtifacts) { - return ['pip', 'install', '--python', python, pythonArtifacts.runtimeWheel]; -} -``` - -```javascript -export function pythonVerifySource() { - return ` -import importlib.metadata - -import semantic_layer -import ktx_daemon - -assert importlib.metadata.version("kaelio-ktx") == "0.1.0" -assert semantic_layer is not None -assert ktx_daemon.PACKAGE_NAME == "ktx-daemon" -`; -} -``` - -```javascript -function pythonExecutable(projectDir) { - if (process.platform === 'win32') { - return join(projectDir, '.venv', 'Scripts', 'python.exe'); - } - return join(projectDir, '.venv', 'bin', 'python'); -} -``` - -```javascript -export function npmSmokePythonEnv(projectDir, baseEnv = process.env) { - const binDir = process.platform === 'win32' ? join(projectDir, '.venv', 'Scripts') : join(projectDir, '.venv', 'bin'); - const existingPath = baseEnv.PATH ?? ''; - - return { - ...baseEnv, - PATH: existingPath ? `${binDir}${delimiter}${existingPath}` : binDir, - }; -} -``` - -```javascript -async function verifyPythonArtifacts(layout, tmpRoot) { - const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); - - const projectDir = join(tmpRoot, 'python-clean-install'); - await mkdir(projectDir, { recursive: true }); - const python = pythonExecutable(projectDir); - await writeFile(join(projectDir, 'verify_python.py'), pythonVerifySource()); - - await runCommand('uv', ['venv', '.venv'], { cwd: projectDir }); - await runCommand('uv', pythonArtifactInstallArgs(python, pythonArtifacts), { - cwd: projectDir, - }); - await runCommand(python, ['verify_python.py'], { cwd: projectDir }); - await runCommand(python, ['-m', 'ktx_daemon', 'semantic-validate'], { - cwd: projectDir, - input: `${JSON.stringify({ sources: [ordersSource], dialect: 'postgres' })}\n`, - }); -} -``` - -- [ ] **Step 9: Verify artifacts through npm only** - -Replace `verifyArtifacts` with this function: - -```javascript -async function verifyArtifacts(layout) { - await verifyArtifactManifest(layout); - - const tmpRoot = await mkdtemp(join(tmpdir(), 'ktx-artifacts-')); - try { - await verifyNpmArtifacts(layout, tmpRoot); - } finally { - await rm(tmpRoot, { recursive: true, force: true }); - } -} -``` - -- [ ] **Step 10: Run package artifact tests and verify pass** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs -``` - -Expected: PASS. The output includes `# fail 0`. - -- [ ] **Step 11: Commit package artifact cleanup** - -Run: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs -git commit -m "refactor: limit release artifacts to public package runtime" -``` - -### Task 3: Align release policy and readiness reports - -**Files:** - -- Modify: `release-policy.json` -- Modify: `scripts/release-readiness.test.mjs` -- Test: `scripts/release-readiness.test.mjs` - -- [ ] **Step 1: Update release readiness fixtures** - -In `scripts/release-readiness.test.mjs`, replace -`writeReleaseMetadataInputs` with: - -```javascript -async function writeReleaseMetadataInputs(root) { - for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { - await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); - await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { - name: packageInfo.name, - version: '0.0.0-private', - private: true, - }); - } -} -``` - -Replace `writeUploadableArtifactFixtures` with: - -```javascript -async function writeUploadableArtifactFixtures(layout) { - await mkdir(layout.npmDir, { recursive: true }); - await mkdir(layout.pythonDir, { recursive: true }); - - const fileContents = new Map([ - ...NPM_ARTIFACT_PACKAGES.map((packageInfo) => [ - layout.npmTarballs[packageInfo.name], - `${packageInfo.name}-tarball`, - ]), - [join(layout.pythonDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'), 'kaelio-ktx-runtime-wheel'], - ]); - - for (const [path, contents] of fileContents) { - await writeFile(path, contents); - } -} -``` - -In `releasePolicy`, replace the `python` object with: - -```javascript - python: { - publish: false, - repository: null, - packages: ['kaelio-ktx'], - ...pythonOverrides, - }, -``` - -- [ ] **Step 2: Update readiness report expectations** - -In `scripts/release-readiness.test.mjs`, replace every expected -`packageNames` array with: - -```javascript - packageNames: ['@kaelio/ktx', 'kaelio-ktx'], -``` - -There are three report assertions to update: - -- `accepts the current ci-artifact-only policy, package metadata, and artifact manifest` -- `reports required published package smoke when release mode requires it` -- `accepts the npm public release ready policy` - -- [ ] **Step 3: Update checked release policy** - -In `release-policy.json`, replace the `python.packages` value with: - -```json - "packages": ["kaelio-ktx"] -``` - -- [ ] **Step 4: Run readiness tests and verify pass** - -Run: - -```bash -node --test scripts/release-readiness.test.mjs -``` - -Expected: PASS. The output includes `# fail 0`. - -- [ ] **Step 5: Commit release policy cleanup** - -Run: - -```bash -git add release-policy.json scripts/release-readiness.test.mjs -git commit -m "chore: align release policy with bundled runtime wheel" -``` - -### Task 4: Document the single release artifact surface - -**Files:** - -- Modify: `scripts/examples-docs.test.mjs` -- Modify: `README.md` -- Modify: `examples/package-artifacts/README.md` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Add failing docs assertions** - -In `scripts/examples-docs.test.mjs`, inside -`it('documents the public package artifact smoke shape', ...)`, add these -assertions after the existing `assert.match(readme, /managed Python runtime/);` -line: - -```javascript - assert.match(readme, /public `@kaelio\/ktx` npm tarball and the bundled `kaelio-ktx` runtime wheel/); - assert.match(readme, /does not install standalone Python packages directly/); - assert.doesNotMatch(readme, /standalone Python distributions/); - assert.doesNotMatch(readme, /installs the Python artifacts directly/); -``` - -In `it('documents public npm and managed runtime usage in the README', ...)`, -add these assertions after the existing `uv` assertions: - -```javascript - assert.match(rootReadme, /release artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` runtime wheel/); - assert.match(rootReadme, /source packages for development, not public release artifacts/); -``` - -- [ ] **Step 2: Run docs tests and verify failure** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL. The failure mentions the missing single-artifact wording in -`README.md` or `examples/package-artifacts/README.md`. - -- [ ] **Step 3: Update the package artifact example README** - -In `examples/package-artifacts/README.md`, replace: - -```markdown -The Python smoke project still installs the Python artifacts directly because -it verifies the standalone Python distributions that feed the bundled runtime -wheel. -``` - -with: - -```markdown -The artifact manifest contains the public `@kaelio/ktx` npm tarball and the -bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone -Python packages directly; Python-backed behavior is verified through the -managed runtime installed from the npm package. -``` - -- [ ] **Step 4: Update the root README release status** - -In `README.md`, in the `## Release status` section, replace this paragraph: - -```markdown -This repository builds one public npm artifact named `@kaelio/ktx`. The first -public npm handoff is policy-gated through `release-policy.json`, which keeps -Python package publishing disabled because KTX-owned Python code ships inside -the npm package as a bundled wheel. -``` - -with: - -```markdown -This repository builds one public npm artifact named `@kaelio/ktx`. The release -artifact manifest contains the public npm tarball and the bundled `kaelio-ktx` -runtime wheel. The first public npm handoff is policy-gated through -`release-policy.json`, which keeps Python package publishing disabled because -KTX-owned Python code ships inside the npm package as a bundled wheel. The -`python/ktx-sl` and `python/ktx-daemon` directories remain source packages for -development, not public release artifacts. -``` - -- [ ] **Step 5: Run docs tests and verify pass** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. The output includes `# fail 0`. - -- [ ] **Step 6: Commit docs cleanup** - -Run: - -```bash -git add README.md examples/package-artifacts/README.md scripts/examples-docs.test.mjs -git commit -m "docs: describe single public runtime artifact surface" -``` - -### Task 5: Verify the cleaned release artifact contract - -**Files:** - -- Verify: `scripts/package-artifacts.mjs` -- Verify: `scripts/package-artifacts.test.mjs` -- Verify: `scripts/release-readiness.test.mjs` -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `release-policy.json` -- Verify: `README.md` -- Verify: `examples/package-artifacts/README.md` - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -node --test scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs -``` - -Expected: PASS. The output includes `# fail 0`. - -- [ ] **Step 2: Verify stale artifact strings are gone from production/docs files** - -Run (scans only production and docs files, not test files - test files keep guard assertions that reference the removed strings): - -```bash -rg -n "uv', \\['build', '--package', 'ktx-sl'|uv', \\['build', '--package', 'ktx-daemon'|ktx_sl-0\\.1\\.0|ktx_daemon-0\\.1\\.0|pythonArtifactInstallArgs|pythonVerifySource|verifyPythonArtifacts|standalone Python distributions|installs the Python artifacts directly" scripts/package-artifacts.mjs scripts/release-readiness.mjs README.md examples/package-artifacts/README.md release-policy.json -``` - -Expected: no matches. - -- [ ] **Step 3: Verify release readiness against the current artifact manifest** - -Run: - -```bash -pnpm run release:readiness -- --json -``` - -Expected: PASS when `dist/artifacts/manifest.json` has been rebuilt after this -change. The JSON output contains: - -```json -{ - "releaseMode": "npm-public-release-ready", - "packageNames": ["@kaelio/ktx", "kaelio-ktx"], - "pythonPublishEnabled": false -} -``` - -If this command fails because the local artifact manifest was generated before -the cleanup, run: - -```bash -pnpm run artifacts:check -pnpm run release:readiness -- --json -``` - -Expected: both commands pass. The rebuilt manifest contains only -`npm/kaelio-ktx-0.1.0.tgz` and -`python/kaelio_ktx-0.1.0-py3-none-any.whl` under `files`. - -- [ ] **Step 4: Run pre-commit on changed files when configured** - -Run: - -```bash -uv run pre-commit run --files scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs release-policy.json README.md examples/package-artifacts/README.md -``` - -Expected: PASS. If pre-commit is not installed or no pre-commit config exists, -record the exact error and keep the focused Node test output from Step 1. - -- [ ] **Step 5: Commit final verification fixes if needed** - -If Step 1, Step 2, Step 3, or Step 4 required code or docs fixes, commit them: - -```bash -git add scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/release-readiness.test.mjs scripts/examples-docs.test.mjs release-policy.json README.md examples/package-artifacts/README.md -git commit -m "test: verify single public runtime artifact contract" -``` - -If no fixes were required after the previous commits, do not create an empty -commit. - -## Acceptance criteria - -- `scripts/package-artifacts.mjs` builds TypeScript packages, builds the - bundled `kaelio-ktx` runtime wheel, copies it into CLI assets, and packs the - public `@kaelio/ktx` npm tarball. -- `scripts/package-artifacts.mjs` no longer builds `ktx-sl` or `ktx-daemon` - standalone wheel or source-distribution artifacts. -- Artifact manifests contain release metadata for `@kaelio/ktx` and - `kaelio-ktx` only. -- `release-policy.json` lists only `@kaelio/ktx` under `npm.packages` and only - `kaelio-ktx` under `python.packages`. -- The artifact smoke verifies Python-backed behavior through the installed - public npm package and managed runtime, not by installing standalone Python - artifacts directly. -- Public docs state that `python/ktx-sl` and `python/ktx-daemon` remain source - packages for development, not public release artifacts. - -## Self-review - -Spec coverage: - -- The plan preserves the single public npm package requirement. -- The plan preserves the bundled KTX-owned Python wheel requirement. -- The plan keeps Python package publishing disabled. -- The plan removes the only remaining artifact path that treated KTX-owned - Python source packages as standalone release artifacts. - -Placeholder scan: - -- No steps contain placeholder implementation text. -- Every code-changing step names exact files and provides concrete replacement - snippets. - -Type and name consistency: - -- Public npm package name remains `@kaelio/ktx`. -- Bundled runtime distribution name remains `kaelio-ktx`. -- Runtime wheel filename remains `kaelio_ktx-0.1.0-py3-none-any.whl`. -- Removed standalone Python artifact names are consistently `ktx-sl` and - `ktx-daemon`. diff --git a/docs/superpowers/plans/2026-05-12-notion-warehouse-verification-gap-closure.md b/docs/superpowers/plans/2026-05-12-notion-warehouse-verification-gap-closure.md deleted file mode 100644 index 3cfdc843..00000000 --- a/docs/superpowers/plans/2026-05-12-notion-warehouse-verification-gap-closure.md +++ /dev/null @@ -1,785 +0,0 @@ -# Notion Warehouse Verification Gap Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1 gaps that prevent ingest agents, especially -Notion WorkUnits, from reliably verifying warehouse table and column -identifiers before writing wiki or semantic-layer output. - -**Architecture:** Keep the existing warehouse verification tool module and -runner wiring. Add Notion target-warehouse scoping through the local adapter -factory, make the active WorkUnit prompt name the shipped tools, enforce -`allowedConnectionNames` in `discover_data`, and teach `entity_details` to -resolve and reject column-level display targets. - -**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX local -ingest adapters, KTX file store. - ---- - -## Audit summary - -The previous implementation plan landed the main tool module and prompt -protocol, but four v1-blocking gaps remain: - -- Notion ingest sessions still allow only the Notion connection unless a - specific adapter supplies target IDs. `NotionSourceAdapter` does not supply - target warehouse IDs, so the original Notion hallucination case cannot use - `entity_details` or raw-schema `discover_data` for the warehouse connection. -- The active WorkUnit framing prompt still tells agents to call - `wiki_sl_search` and `sl_describe_table`, which are not shipped KTX tools. -- `discover_data` accepts an explicit out-of-scope `connectionName` and still - searches raw schema for that connection. -- `entity_details({ targets: [{ display: "schema.table.column" }] })` does not - resolve column display strings and does not fail explicit missing-column - targets. - -Non-blocking gaps remain out of scope for this plan: - -- Full DDL-style `entity_details` formatting with FK and profile summaries. -- AST-backed SQL read-only validation for data-modifying CTEs. -- Search over `enrichment/descriptions.json` for generated descriptions. -- Lexicographic latest-sync edge cases for non-timestamp sync IDs. -- Hard write-time validation in `wiki_write` and `emit_unmapped_fallback`. - -## File structure - -Modify these files: - -- `packages/context/src/ingest/adapters/notion/notion.adapter.ts`: add - configured target warehouse IDs and implement `listTargetConnectionIds()`. -- `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`: cover - Notion target connection ID fan-out. -- `packages/context/src/ingest/local-adapters.ts`: pass primary warehouse IDs - into `NotionSourceAdapter`. -- `packages/context/src/ingest/local-adapters.test.ts`: cover local Notion - adapter target IDs. -- `packages/context/src/ingest/adapters/notion/chunk.ts`: update Notion - WorkUnit notes to prefer the warehouse verification tools. -- `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`: update - Notion note expectations. -- `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md`: replace - stale tool names in the active WorkUnit prompt. -- `packages/context/src/ingest/ingest-prompts.test.ts`: guard the WorkUnit - prompt against stale tool names. -- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: - refuse explicit out-of-scope connection names. -- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: - cover `discover_data` scoping. -- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: - add column-aware display-target resolution. -- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: - cover column display resolution. -- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: - use column-aware resolution and report missing columns. -- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: - cover column display and missing-column behavior. - -### Task 1: Give Notion ingest access to target warehouses - -**Files:** -- Modify: `packages/context/src/ingest/adapters/notion/notion.adapter.ts` -- Modify: `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts` -- Modify: `packages/context/src/ingest/local-adapters.ts` -- Modify: `packages/context/src/ingest/local-adapters.test.ts` - -- [ ] **Step 1: Write the failing Notion adapter test** - -Add this test inside `describe('NotionSourceAdapter', ...)` in -`packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`: - -```ts -it('returns configured target warehouse connection ids', async () => { - const adapter = new NotionSourceAdapter({ - targetConnectionIds: ['warehouse', 'warehouse', 'analytics'], - }); - - await expect(adapter.listTargetConnectionIds?.(stagedDir)).resolves.toEqual([ - 'analytics', - 'warehouse', - ]); -}); -``` - -- [ ] **Step 2: Run the failing Notion adapter test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/notion/notion.adapter.test.ts -t "target warehouse connection ids" -``` - -Expected: FAIL because `NotionSourceAdapterDeps` has no -`targetConnectionIds` option and `NotionSourceAdapter` does not implement -`listTargetConnectionIds()`. - -- [ ] **Step 3: Implement Notion target connection IDs** - -Modify `packages/context/src/ingest/adapters/notion/notion.adapter.ts`: - -```ts -export interface NotionSourceAdapterDeps { - onPullSucceeded?: (ctx: NotionPullSucceededContext) => Promise; - logger?: NotionFetchLogger; - targetConnectionIds?: string[]; -} - -function uniqueSorted(values: readonly string[] | undefined): string[] { - return [...new Set(values ?? [])].sort((left, right) => - left.localeCompare(right), - ); -} -``` - -Add this method to `NotionSourceAdapter`: - -```ts - async listTargetConnectionIds(_stagedDir: string): Promise { - return uniqueSorted(this.deps.targetConnectionIds); - } -``` - -- [ ] **Step 4: Pass primary warehouses into the local Notion adapter** - -Modify the Notion adapter construction in -`packages/context/src/ingest/local-adapters.ts`: - -```ts - new NotionSourceAdapter({ - targetConnectionIds: primaryWarehouseConnectionIds(project), - ...(options.logger ? { logger: options.logger } : {}), - }), -``` - -- [ ] **Step 5: Write the local adapter fan-out test** - -Add this test to `packages/context/src/ingest/local-adapters.test.ts`: - -```ts -it('passes primary warehouse connection ids to the local Notion adapter', async () => { - const adapters = createDefaultLocalIngestAdapters( - projectWithConnections({ - notion: { - driver: 'notion', - auth_token: 'secret', - crawl_mode: 'selected_roots', - root_page_ids: ['page-1'], - }, - warehouse: { - driver: 'postgres', - url: 'postgresql://readonly@db.example.test/analytics', - }, - docs: { - driver: 'dbt', - source_dir: './dbt', - }, - } as never), - ); - - const notion = adapters.find((adapter) => adapter.source === 'notion'); - - await expect(notion?.listTargetConnectionIds?.('/tmp/staged-notion')).resolves.toEqual([ - 'warehouse', - ]); -}); -``` - -- [ ] **Step 6: Run the Notion target tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/notion/notion.adapter.test.ts -t "target warehouse connection ids" \ - src/ingest/local-adapters.test.ts -t "local Notion adapter" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/adapters/notion/notion.adapter.ts \ - packages/context/src/ingest/adapters/notion/notion.adapter.test.ts \ - packages/context/src/ingest/local-adapters.ts \ - packages/context/src/ingest/local-adapters.test.ts -git commit -m "fix(context): expose target warehouses to Notion ingest" -``` - -### Task 2: Remove stale tool names from active ingest prompts - -**Files:** -- Modify: `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md` -- Modify: `packages/context/src/ingest/ingest-prompts.test.ts` -- Modify: `packages/context/src/ingest/adapters/notion/chunk.ts` -- Modify: `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts` - -- [ ] **Step 1: Add failing prompt guards** - -Add this test to `packages/context/src/ingest/ingest-prompts.test.ts`: - -```ts -it('uses shipped warehouse verification tools in the WorkUnit prompt', async () => { - const prompt = await readFile( - new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), - 'utf-8', - ); - - expect(prompt).toContain('discover_data'); - expect(prompt).toContain('entity_details'); - expect(prompt).not.toContain('wiki_sl_search'); - expect(prompt).not.toContain('sl_describe_table'); -}); -``` - -- [ ] **Step 2: Run the failing prompt guard** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-prompts.test.ts -t "warehouse verification tools" -``` - -Expected: FAIL because the WorkUnit prompt still contains `wiki_sl_search` and -`sl_describe_table`. - -- [ ] **Step 3: Update the WorkUnit framing prompt** - -In `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md`, replace -the first `` paragraph with: - -```md -You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, Metabase card JSONs, Notion pages, or similar) and you must translate that slice into KTX semantic-layer sources and/or knowledge wiki pages, in one pass. Prior WorkUnits in this same job may have already written SL sources and wiki pages; their writes are visible on the working branch and discoverable with `discover_data`. -``` - -In workflow step 2, replace the final sentence with: - -```md -The triage skill tells you how to react when `discover_data` reveals that a prior WU already wrote something overlapping. -``` - -In workflow step 4, replace the sentence that starts -`For each raw file:` with: - -```md -4. For each raw file: call `read_raw_file` (or `read_raw_span` for slicing large files) to load content. Before writing a new SL source or wiki page, call `discover_data` for each candidate source, table, metric, or topic name to find prior-WU writes, existing wiki pages, SL sources, and raw warehouse matches; apply `ingest_triage` when you hit one, and apply any matching canonical pin before deciding whether to edit, rename, or skip. -``` - -In the `` block, replace the physical-column rule with: - -```md -- Do not invent physical column names or grain keys. For table-backed SL sources, every `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr` column must come from raw-file column declarations or warehouse-backed discovery (`discover_data`, `sl_discover`, `entity_details`). If column names are not confirmed, capture the business context in wiki instead of writing a full SL source. -``` - -- [ ] **Step 4: Update Notion WorkUnit notes** - -In `packages/context/src/ingest/adapters/notion/chunk.ts`, replace -`NOTION_SL_WRITE_GUIDANCE` with: - -```ts -const NOTION_SL_WRITE_GUIDANCE = - 'Write wiki entries with wiki_write. Wiki keys must be flat slugs like orbit-company-overview, not orbit/company-overview. Search existing wiki pages, SL sources, and raw warehouse schema for the same tables or sl_refs with discover_data before creating a new page. Only write or edit SL sources after discover_data plus sl_discover/sl_read_source or entity_details confirms a mapped non-Notion target source; if no mapped target exists, emit_unmapped_fallback and keep the fact wiki-only. Notion dataSourceCount counts Notion databases/data sources only, not warehouse/dbt mappings. If a warehouse/dbt connection exists but the named table or source is absent, use reason no_physical_table rather than no_connection_mapping. Do not create SL sources under the Notion connection just because a page mentions a warehouse table.'; -``` - -In the `reconcileNotes` array in the same file, replace: - -```ts - 'Notion dataSourceCount is Notion-only; use sl_discover for warehouse/dbt mapping decisions.', -``` - -with: - -```ts - 'Notion dataSourceCount is Notion-only; use discover_data/entity_details for warehouse/dbt mapping decisions.', -``` - -- [ ] **Step 5: Update Notion note expectations** - -In `packages/context/src/ingest/adapters/notion/notion.adapter.test.ts`, -update the note expectations in `it('chunks changed Notion pages...')`: - -```ts -expect(result.workUnits[0].notes).toContain('discover_data'); -expect(result.workUnits[0].notes).toContain('entity_details'); -``` - -Update the exact `reconcileNotes` expectation to: - -```ts -expect(result.reconcileNotes).toEqual([ - 'Notion maxKnowledgeCreatesPerRun=25', - 'Notion maxKnowledgeUpdatesPerRun=20', - 'Notion dataSourceCount is Notion-only; use discover_data/entity_details for warehouse/dbt mapping decisions.', - 'Reconcile Notion wiki pages sharing tables/sl_refs before creating distinct artifacts.', -]); -``` - -- [ ] **Step 6: Run prompt and Notion note tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-prompts.test.ts \ - src/ingest/adapters/notion/notion.adapter.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add \ - packages/context/prompts/memory_agent_bundle_ingest_work_unit.md \ - packages/context/src/ingest/ingest-prompts.test.ts \ - packages/context/src/ingest/adapters/notion/chunk.ts \ - packages/context/src/ingest/adapters/notion/notion.adapter.test.ts -git commit -m "fix(context): update ingest prompts for warehouse verification tools" -``` - -### Task 3: Enforce allowed connection scope in discover_data - -**Files:** -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` - -- [ ] **Step 1: Write the failing scoping test** - -Add this test to -`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: - -```ts -it('refuses explicit out-of-scope connection names', async () => { - const result = await tool.call({ query: 'orders', connectionName: 'billing' }, context); - - expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.'); - expect(result.structured).toEqual({ wiki: null, sl: null, raw: null }); - expect(wikiSearchTool.call).not.toHaveBeenCalled(); - expect(slDiscoverTool.call).not.toHaveBeenCalled(); - expect(catalog.searchByName).not.toHaveBeenCalled(); -}); -``` - -- [ ] **Step 2: Run the failing scoping test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -t "out-of-scope" -``` - -Expected: FAIL because `discover_data` currently searches raw schema for an -explicit `connectionName` even when it is not in `allowedConnectionNames`. - -- [ ] **Step 3: Add the scope guard** - -In -`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`, -add this helper near `totalSources()`: - -```ts -function allowedConnectionNames(context: ToolContext): ReadonlySet | null { - return context.session?.allowedConnectionNames ?? null; -} -``` - -At the top of `DiscoverDataTool.call()`, before the `sourceName` branch and -before calling any child tool, add: - -```ts - const allowed = allowedConnectionNames(context); - if (input.connectionName && allowed && !allowed.has(input.connectionName)) { - return { - markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, - structured: { wiki: null, sl: null, raw: null }, - }; - } -``` - -Then replace the raw connection-list construction with: - -```ts - const connections = input.connectionName ? [input.connectionName] : [...(allowed ?? [])].sort(); -``` - -- [ ] **Step 4: Run discover_data tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -git commit -m "fix(context): scope raw schema discovery to allowed connections" -``` - -### Task 4: Fix column-level entity_details verification - -**Files:** -- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` - -- [ ] **Step 1: Write failing catalog column-target tests** - -First update `seedLiveDatabaseScan()` in that test file so BigQuery tables have -a project/catalog. Replace the repeated inline table refs with: - -```ts -const tableRef = { - catalog: driver === 'bigquery' ? 'analytics' : null, - db: driver === 'sqlite' ? null : 'public', - name: 'orders', -}; -``` - -Use `tableRef.catalog`, `tableRef.db`, and `tableRef.name` for the seeded -table and profile table references. - -Then add these tests to -`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: - -```ts -it('resolves postgres column display strings without treating the column as a table', async () => { - await seedLiveDatabaseScan(); - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - - await expect(catalog.resolveDisplayTarget('warehouse', 'public.orders.status')).resolves.toMatchObject({ - resolved: { catalog: null, db: 'public', name: 'orders', column: 'status' }, - candidates: [], - dialect: 'postgres', - }); -}); - -it('resolves BigQuery column display strings with four parts', async () => { - await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - - await expect(catalog.resolveDisplayTarget('warehouse', 'analytics.public.orders.status')).resolves.toMatchObject({ - resolved: { catalog: 'analytics', db: 'public', name: 'orders', column: 'status' }, - candidates: [], - dialect: 'bigquery', - }); -}); -``` - -- [ ] **Step 2: Run the failing catalog tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts -t "column display" -``` - -Expected: FAIL because `resolveDisplayTarget()` does not exist. - -- [ ] **Step 3: Implement column-aware display resolution** - -In -`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`, -add this exported interface near `RawSchemaHit`: - -```ts -export interface DisplayTargetResolution { - resolved: (KtxTableRef & { column?: string }) | null; - candidates: KtxTableRef[]; - dialect: string; -} -``` - -Add these helpers near `parseDisplay()`: - -```ts -function expectedDisplayPartCount(driver: CatalogDriver): number { - if (driver === 'sqlite' || driver === 'sqlite3') { - return 1; - } - if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { - return 3; - } - return 2; -} - -function parseColumnDisplay(driver: CatalogDriver, display: string): (KtxTableRef & { column: string }) | null { - const parts = splitDisplay(display); - const tablePartCount = expectedDisplayPartCount(driver); - if (parts.length !== tablePartCount + 1) { - return null; - } - const column = parts.at(-1); - if (!column) { - return null; - } - const table = parseDisplay(driver, parts.slice(0, -1).join('.')); - return table ? { ...table, column } : null; -} -``` - -Add this method to `WarehouseCatalogService` after `resolveDisplay()`: - -```ts - async resolveDisplayTarget(connectionName: string, display: string): Promise { - const catalog = await this.loadCatalog(connectionName); - if (!catalog) { - return { resolved: null, candidates: [], dialect: 'unknown' }; - } - - const dialect = getDialectForDriver(catalog.driver).type; - const tableResolution = await this.resolveDisplay(connectionName, display); - if (tableResolution.resolved) { - return tableResolution; - } - - const parsedColumn = parseColumnDisplay(catalog.driver, display); - if (!parsedColumn) { - return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; - } - - const table = catalog.tables.find((candidate) => refsEqual(candidate, parsedColumn)); - if (!table) { - return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; - } - - return { - resolved: { - catalog: table.catalog, - db: table.db, - name: table.name, - column: parsedColumn.column, - }, - candidates: [], - dialect, - }; - } -``` - -- [ ] **Step 4: Write failing entity_details column tests** - -Add these tests to -`packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: - -```ts -it('resolves display targets that include a column name', async () => { - const result = await tool.call( - { connectionName: 'warehouse', targets: [{ display: 'public.orders.status' }] }, - context, - ); - - expect(result.markdown).toContain('### public.orders'); - expect(result.markdown).toContain('- status (text, nullable=false)'); - expect(result.markdown).not.toContain('- id (integer'); - expect(result.structured.resolved).toHaveLength(1); - expect(result.structured.resolved[0]?.columns.map((column) => column.name)).toEqual(['status']); -}); - -it('reports missing explicit columns instead of returning an empty column list', async () => { - const result = await tool.call( - { connectionName: 'warehouse', targets: [{ display: 'public.orders.plan_tier' }] }, - context, - ); - - expect(result.markdown).toContain('Column not found in scan: public.orders.plan_tier'); - expect(result.markdown).toContain('Available columns: id, status'); - expect(result.structured.resolved).toHaveLength(0); - expect(result.structured.missing).toHaveLength(1); -}); -``` - -- [ ] **Step 5: Run the failing entity_details tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -t "column" -``` - -Expected: FAIL because display column targets are treated as table names and -missing columns are not reported. - -- [ ] **Step 6: Use column-aware resolution in entity_details** - -In -`packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`, -add this helper near `appendTableMarkdown()`: - -```ts -function findColumn(detail: TableDetail, columnName: string): TableDetail['columns'][number] | null { - const normalized = columnName.toLowerCase(); - return detail.columns.find((column) => column.name.toLowerCase() === normalized) ?? null; -} -``` - -Replace the display resolution block inside the `for (const target of -input.targets)` loop with: - -```ts - const resolution = - 'display' in target - ? await catalog.resolveDisplayTarget(input.connectionName, target.display) - : { - resolved: { catalog: target.catalog, db: target.db, name: target.name, column: target.column }, - candidates: [], - dialect: '', - }; -``` - -After `const detail = await catalog.getTable(...)`, replace the existing -`resolved.push(detail); appendTableMarkdown(...)` lines with: - -```ts - const requestedColumn = resolution.resolved.column; - if (requestedColumn) { - const column = findColumn(detail, requestedColumn); - if (!column) { - missing.push({ - target, - candidates: [{ catalog: detail.catalog, db: detail.db, name: detail.name }], - }); - parts.push(`Column not found in scan: ${detail.display}.${requestedColumn}`); - parts.push(`Available columns: ${detail.columns.map((candidate) => candidate.name).join(', ')}`); - continue; - } - const scopedDetail = { ...detail, columns: [column] }; - resolved.push(scopedDetail); - appendTableMarkdown(parts, scopedDetail, column.name); - continue; - } - - resolved.push(detail); - appendTableMarkdown(parts, detail); -``` - -- [ ] **Step 7: Run warehouse verification tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -git commit -m "fix(context): verify warehouse column display targets" -``` - -### Task 5: Verify the v1 gap closure - -**Files:** -- Verify all files changed by Tasks 1-4. - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/notion/notion.adapter.test.ts \ - src/ingest/local-adapters.test.ts \ - src/ingest/ingest-prompts.test.ts \ - src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run package tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run pre-commit on changed files when configured** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/ingest/adapters/notion/notion.adapter.ts \ - packages/context/src/ingest/adapters/notion/notion.adapter.test.ts \ - packages/context/src/ingest/local-adapters.ts \ - packages/context/src/ingest/local-adapters.test.ts \ - packages/context/src/ingest/adapters/notion/chunk.ts \ - packages/context/prompts/memory_agent_bundle_ingest_work_unit.md \ - packages/context/src/ingest/ingest-prompts.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -``` - -Expected: PASS. If the repo has no pre-commit config or the local `uv` version -cannot satisfy the project pin, record the exact error and rely on focused -tests plus type-check. - -- [ ] **Step 5: Inspect final git status** - -Run: - -```bash -git status --short -``` - -Expected: only intentional files are modified. Commit any formatter-driven -changes with: - -```bash -git add packages/context -git commit -m "chore(context): verify warehouse verification v1 gaps" -``` - -## Self-review checklist - -- Spec coverage: this plan closes the remaining v1 paths for Notion warehouse - verification, active WorkUnit prompt correctness, raw discovery scoping, and - column-level identifier verification. -- Placeholder scan: no task relies on future-work markers, unnamed edge-case - handling, or cross-task shorthand. -- Type consistency: `discover_data` continues to use `connectionName`, - `sl_discover` still receives `connectionId` internally, and - `resolveDisplayTarget()` returns the same table identity plus optional - `column`. diff --git a/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md b/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md deleted file mode 100644 index d8cae96e..00000000 --- a/docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md +++ /dev/null @@ -1,957 +0,0 @@ -# Warehouse Verification Final V1 Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1 gaps that still prevent ingest agents from -reliably following warehouse verification results through to `entity_details` -and `sql_execution`. - -**Architecture:** Keep the existing warehouse verification module and runner -session scoping. Add connection names to raw discovery hits, expose primary -warehouse targets from the remaining source adapters, and make local ingest -SQL probes use the same scan connector read-only execution path as schema scan. - -**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX local -ingest runtime, KTX scan connectors. - ---- - -## Audit summary - -The first two implementation plans landed the warehouse verification tools, -prompt protocol, Notion warehouse scoping, and stale prompt-name cleanup. The -focused audit on May 12, 2026, found three remaining v1-blocking gaps: - -- `discover_data` searches multiple allowed raw warehouse scans, but raw hits do - not carry or render `connectionName`. The tool tells the agent to call - `entity_details({connectionName, targets: [...]})`, then omits the required - `connectionName` from the follow-up evidence. -- Local LookML and MetricFlow adapters do not expose primary warehouse target - IDs. The runner only adds adapter-provided targets to `allowedConnectionNames`, - so those WorkUnits cannot use raw warehouse verification unless their source - connection is itself the warehouse. -- `sql_execution` calls the local ingest connection catalog, but the catalog - either has no query executor in normal CLI ingest or calls an injected - executor without `projectDir` and connection config. The default local query - executor cannot dispatch without that config. - -Non-blocking gaps remain out of scope for this v1 plan: - -- Full DDL-style `entity_details` formatting with FK profile summaries. -- AST-backed SQL read-only validation for data-modifying CTE bodies. -- Search over generated `enrichment/descriptions.json`. -- Lexicographic latest-sync edge cases for non-timestamp sync IDs. -- Hard write-time validation in `wiki_write` and `emit_unmapped_fallback`. - -## File structure - -Modify these files: - -- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: - add `connectionName` to raw schema hit records. -- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: - render raw hit connection names and preserve them in structured output. -- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: - cover multi-connection raw discovery follow-up data. -- `packages/context/src/ingest/adapters/lookml/lookml.adapter.ts`: - accept and return configured target warehouse connection IDs. -- `packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts`: - cover LookML target warehouse IDs. -- `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts`: - accept and return configured target warehouse connection IDs. -- `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts`: - cover MetricFlow target warehouse IDs. -- `packages/context/src/ingest/local-adapters.ts`: - pass primary warehouse IDs into LookML and MetricFlow adapters. -- `packages/context/src/ingest/local-adapters.test.ts`: - cover local adapter warehouse target fan-out. -- `packages/context/src/ingest/local-bundle-runtime.ts`: - pass full project connection config to local ingest query executors. -- `packages/context/src/ingest/local-bundle-runtime.test.ts`: - cover the local ingest query executor call shape. -- `packages/context/src/ingest/local-ingest.ts`: - use the shared query executor port type. -- `packages/context/src/mcp/local-project-ports.ts`: - no behavior change expected, but type-checks against the updated local ingest - query executor type. -- `packages/cli/src/ingest.ts`: - provide a read-only scan-connector-backed query executor for normal local - ingest runs. - -Create these files: - -- `packages/cli/src/ingest-query-executor.ts`: CLI query executor that adapts - scan connectors' `executeReadOnly()` method to `KtxSqlQueryExecutorPort`. -- `packages/cli/src/ingest-query-executor.test.ts`: unit coverage for the CLI - ingest query executor. - -### Task 1: Preserve raw discovery connection names - -**Files:** -- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` - -- [ ] **Step 1: Write the failing multi-connection discovery test** - -Add this test to -`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: - -```ts - it('includes connectionName on raw schema hits so entity_details can follow up', async () => { - const multiConnectionContext: ToolContext = { - ...context, - session: { allowedConnectionNames: new Set(['warehouse', 'analytics']) } as any, - }; - catalog.searchByName.mockImplementation(async (connectionName: string, query: string) => [ - { - kind: 'table', - connectionName, - ref: { catalog: null, db: 'public', name: `${connectionName}_${query}` }, - display: `public.${connectionName}_${query}`, - matchedOn: 'name', - }, - ]); - - const result = await tool.call({ query: 'orders', limit: 10 }, multiConnectionContext); - - expect(catalog.searchByName).toHaveBeenCalledWith('analytics', 'orders', 10); - expect(catalog.searchByName).toHaveBeenCalledWith('warehouse', 'orders', 10); - expect(result.markdown).toContain('connectionName=analytics'); - expect(result.markdown).toContain('connectionName=warehouse'); - expect(result.markdown).toContain( - 'entity_details({connectionName: "analytics", targets: [{display: "public.analytics_orders"}]})', - ); - expect(result.structured.raw?.hits.map((hit) => hit.connectionName)).toEqual([ - 'analytics', - 'warehouse', - ]); - }); -``` - -- [ ] **Step 2: Run the failing discovery test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -t "connectionName on raw schema hits" -``` - -Expected: FAIL because `RawSchemaHit` has no `connectionName` property and the -markdown only renders the display string. - -- [ ] **Step 3: Add `connectionName` to raw schema hits** - -Modify the raw hit type and hit construction in -`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: - -```ts -export type RawSchemaHit = - | { - kind: 'table'; - connectionName: string; - ref: KtxTableRef; - display: string; - matchedOn: 'name' | 'db' | 'comment' | 'description'; - } - | { - kind: 'column'; - connectionName: string; - ref: KtxTableRef & { column: string }; - display: string; - matchedOn: 'name' | 'comment' | 'description'; - }; -``` - -In the table hit block, add `connectionName`: - -```ts - hits.push({ - kind: 'table', - connectionName, - ref: { catalog: table.catalog, db: table.db, name: table.name }, - display: formatDisplay(catalog.driver, table), - matchedOn: tableMatch, - }); -``` - -In the column hit block, add `connectionName`: - -```ts - hits.push({ - kind: 'column', - connectionName, - ref: { catalog: table.catalog, db: table.db, name: table.name, column: column.name }, - display: `${formatDisplay(catalog.driver, table)}.${column.name}`, - matchedOn: columnMatch, - }); -``` - -- [ ] **Step 4: Render follow-up-ready raw hits** - -Modify the raw schema markdown in -`packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: - -```ts - parts.push('## Raw Warehouse Schema', '> use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values'); - parts.push( - rawHits - .slice(0, limit) - .map( - (hit) => - `- ${hit.kind}: ${hit.display} [connectionName=${hit.connectionName}] (matched on ${hit.matchedOn}) - ` + - `follow up with \`entity_details({connectionName: "${hit.connectionName}", targets: [{display: "${hit.display}"}]})\``, - ) - .join('\n'), - ); -``` - -- [ ] **Step 5: Run the discovery test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -git commit -m "fix(context): include raw discovery connection names" -``` - -### Task 2: Expose LookML and MetricFlow warehouse targets - -**Files:** -- Modify: `packages/context/src/ingest/adapters/lookml/lookml.adapter.ts` -- Modify: `packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts` -- Modify: `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts` -- Modify: `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts` -- Modify: `packages/context/src/ingest/local-adapters.ts` -- Modify: `packages/context/src/ingest/local-adapters.test.ts` - -- [ ] **Step 1: Write failing adapter target tests** - -Add this test to -`packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts`: - -```ts - it('returns configured target warehouse connection ids', async () => { - const adapter = new LookmlSourceAdapter({ - homeDir: join(tmpRoot, 'home'), - targetConnectionIds: ['warehouse', 'analytics', 'warehouse'], - }); - - await expect(adapter.listTargetConnectionIds?.(join(tmpRoot, 'staged'))).resolves.toEqual([ - 'analytics', - 'warehouse', - ]); - }); -``` - -Add this test to -`packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts`: - -```ts - it('returns configured target warehouse connection ids', async () => { - const metricflow = new MetricflowSourceAdapter({ - homeDir: join(tmpRoot, 'cache-home'), - targetConnectionIds: ['warehouse', 'analytics', 'warehouse'], - }); - - await expect(metricflow.listTargetConnectionIds?.(stagedDir)).resolves.toEqual([ - 'analytics', - 'warehouse', - ]); - }); -``` - -- [ ] **Step 2: Run the failing adapter tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/lookml/lookml.adapter.test.ts -t "target warehouse connection ids" \ - src/ingest/adapters/metricflow/metricflow.adapter.test.ts -t "target warehouse connection ids" -``` - -Expected: FAIL because neither adapter accepts `targetConnectionIds` or -implements `listTargetConnectionIds()`. - -- [ ] **Step 3: Implement target ID support in LookML** - -Modify `packages/context/src/ingest/adapters/lookml/lookml.adapter.ts`: - -```ts -export interface LookmlSourceAdapterDeps { - homeDir: string; - targetConnectionIds?: string[]; -} - -function uniqueSorted(values: readonly string[] | undefined): string[] { - return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); -} -``` - -Add this method to `LookmlSourceAdapter`: - -```ts - async listTargetConnectionIds(_stagedDir: string): Promise { - return uniqueSorted(this.deps.targetConnectionIds); - } -``` - -- [ ] **Step 4: Implement target ID support in MetricFlow** - -Modify `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts`: - -```ts -export interface MetricflowSourceAdapterDeps { - homeDir: string; - targetConnectionIds?: string[]; -} - -function uniqueSorted(values: readonly string[] | undefined): string[] { - return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right)); -} -``` - -Add this method to `MetricflowSourceAdapter`: - -```ts - async listTargetConnectionIds(_stagedDir: string): Promise { - return uniqueSorted(this.deps.targetConnectionIds); - } -``` - -- [ ] **Step 5: Pass primary warehouses from the local adapter factory** - -Modify the LookML and MetricFlow adapter construction in -`packages/context/src/ingest/local-adapters.ts`: - -```ts - new LookmlSourceAdapter({ - homeDir: join(project.projectDir, '.ktx/cache'), - targetConnectionIds: primaryWarehouseConnectionIds(project), - }), -``` - -```ts - new MetricflowSourceAdapter({ - homeDir: join(project.projectDir, '.ktx/cache'), - targetConnectionIds: primaryWarehouseConnectionIds(project), - }), -``` - -- [ ] **Step 6: Write the local adapter fan-out test** - -Add this test to `packages/context/src/ingest/local-adapters.test.ts`: - -```ts - it('passes primary warehouse connection ids to local LookML and MetricFlow adapters', async () => { - const adapters = createDefaultLocalIngestAdapters( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'postgresql://readonly@db.example.test/analytics', - }, - lookml_docs: { - driver: 'lookml', - lookml: { - repoUrl: 'https://github.com/acme/lookml.git', - }, - }, - metrics_repo: { - driver: 'metricflow', - metricflow: { - repoUrl: 'https://github.com/acme/metrics.git', - }, - }, - } as never), - ); - - const lookml = adapters.find((adapter) => adapter.source === 'lookml'); - const metricflow = adapters.find((adapter) => adapter.source === 'metricflow'); - - await expect(lookml?.listTargetConnectionIds?.('/tmp/staged-lookml')).resolves.toEqual([ - 'warehouse', - ]); - await expect(metricflow?.listTargetConnectionIds?.('/tmp/staged-metricflow')).resolves.toEqual([ - 'warehouse', - ]); - }); -``` - -- [ ] **Step 7: Run the target fan-out tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/adapters/lookml/lookml.adapter.test.ts \ - src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ - src/ingest/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/adapters/lookml/lookml.adapter.ts \ - packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ - packages/context/src/ingest/local-adapters.ts \ - packages/context/src/ingest/local-adapters.test.ts -git commit -m "fix(context): expose warehouse targets for LookML and MetricFlow" -``` - -### Task 3: Pass full connection config to local ingest SQL execution - -**Files:** -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` -- Modify: `packages/context/src/ingest/local-ingest.ts` - -- [ ] **Step 1: Write the failing local connection catalog test** - -In `packages/context/src/ingest/local-bundle-runtime.test.ts`, change the -Vitest import to include `vi`: - -```ts -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -``` - -Extend `RuntimeWithConnectionDeps`: - -```ts -type RuntimeWithConnectionDeps = { - deps: { - connections: { - listEnabledConnections(ids: string[]): Promise>; - getConnectionById(connectionId: string): Promise<{ id: string; name: string; connectionType: string } | null>; - executeQuery(connectionId: string, sql: string): Promise; - }; - }; -}; -``` - -Add this test: - -```ts - it('passes project connection config to local ingest query executors', async () => { - const agentRunner = new AgentRunnerService({ llmProvider: { getModel: () => ({}) as never } as any }); - const queryExecutor = { - execute: vi.fn(async () => ({ - headers: ['answer'], - rows: [[1]], - totalRows: 1, - command: 'SELECT', - rowCount: 1, - })), - }; - - const runtime = createLocalBundleIngestRuntime({ - project, - adapters: [new FakeSourceAdapter()], - agentRunner, - queryExecutor, - }); - const connections = (runtime.runner as unknown as RuntimeWithConnectionDeps).deps.connections; - - await expect(connections.executeQuery('warehouse', 'select 1')).resolves.toMatchObject({ - headers: ['answer'], - }); - expect(queryExecutor.execute).toHaveBeenCalledWith({ - connectionId: 'warehouse', - projectDir: project.projectDir, - connection: project.config.connections.warehouse, - sql: 'select 1', - }); - }); -``` - -- [ ] **Step 2: Run the failing local runtime test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -t "project connection config" -``` - -Expected: FAIL because `LocalConnectionCatalog.executeQuery()` only passes -`connectionId` and `sql`. - -- [ ] **Step 3: Update local ingest query executor types** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, import the shared -query executor type: - -```ts -import { localConnectionInfoFromConfig, type KtxSqlQueryExecutorPort } from '../connections/index.js'; -``` - -Change `CreateLocalBundleIngestRuntimeOptions.queryExecutor` to: - -```ts - queryExecutor?: KtxSqlQueryExecutorPort; -``` - -Change `LocalConnectionCatalog` to store that type: - -```ts -class LocalConnectionCatalog implements SlConnectionCatalogPort { - constructor( - private readonly project: KtxLocalProject, - private readonly queryExecutor?: KtxSqlQueryExecutorPort, - ) {} -``` - -Change `executeQuery()`: - -```ts - async executeQuery(connectionId: string, sql: string): Promise { - if (!this.queryExecutor) { - throw new Error('Local ingest has no query executor configured'); - } - return this.queryExecutor.execute({ - connectionId, - projectDir: this.project.projectDir, - connection: this.project.config.connections[connectionId], - sql, - }); - } -``` - -In `packages/context/src/ingest/local-ingest.ts`, replace the local query -executor object type with the shared port: - -```ts -import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; -``` - -```ts - queryExecutor?: KtxSqlQueryExecutorPort; -``` - -- [ ] **Step 4: Run the local runtime test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -t "project connection config" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/local-bundle-runtime.test.ts \ - packages/context/src/ingest/local-ingest.ts -git commit -m "fix(context): pass connection config to ingest query executors" -``` - -### Task 4: Supply a scan-connector query executor to CLI ingest - -**Files:** -- Create: `packages/cli/src/ingest-query-executor.ts` -- Create: `packages/cli/src/ingest-query-executor.test.ts` -- Modify: `packages/cli/src/ingest.ts` - -- [ ] **Step 1: Write the CLI query executor tests** - -Create `packages/cli/src/ingest-query-executor.test.ts`: - -```ts -import type { KtxLocalProject } from '@ktx/context/project'; -import { createKtxConnectorCapabilities, type KtxScanConnector } from '@ktx/context/scan'; -import { describe, expect, it, vi } from 'vitest'; -import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; - -function project(): KtxLocalProject { - return { - projectDir: '/tmp/ktx-query-project', - config: { - project: 'warehouse', - connections: { - warehouse: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' }, - }, - }, - } as unknown as KtxLocalProject; -} - -function connector(overrides: Partial = {}): KtxScanConnector { - return { - id: 'warehouse', - driver: 'postgres', - capabilities: createKtxConnectorCapabilities({ readOnlySql: true }), - async introspect() { - throw new Error('introspect is not used by this test'); - }, - executeReadOnly: vi.fn(async () => ({ - headers: ['answer'], - rows: [[1]], - totalRows: 1, - rowCount: 1, - })), - cleanup: vi.fn(async () => {}), - ...overrides, - }; -} - -describe('createKtxCliIngestQueryExecutor', () => { - it('executes read-only SQL through the scan connector and cleans it up', async () => { - const scanConnector = connector(); - const createConnector = vi.fn(async () => scanConnector); - const executor = createKtxCliIngestQueryExecutor(project(), { createConnector }); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' }, - projectDir: '/tmp/ktx-query-project', - sql: 'select 1', - maxRows: 5, - }), - ).resolves.toMatchObject({ - headers: ['answer'], - rows: [[1]], - totalRows: 1, - command: 'SELECT', - rowCount: 1, - }); - - expect(createConnector).toHaveBeenCalledWith(project(), 'warehouse'); - expect(scanConnector.executeReadOnly).toHaveBeenCalledWith( - { connectionId: 'warehouse', sql: 'select 1', maxRows: 5 }, - { runId: 'ingest-sql-execution' }, - ); - expect(scanConnector.cleanup).toHaveBeenCalledTimes(1); - }); - - it('rejects connectors without read-only SQL support', async () => { - const scanConnector = connector({ - capabilities: createKtxConnectorCapabilities({ readOnlySql: false }), - executeReadOnly: undefined, - }); - const executor = createKtxCliIngestQueryExecutor(project(), { - createConnector: vi.fn(async () => scanConnector), - }); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres' }, - projectDir: '/tmp/ktx-query-project', - sql: 'select 1', - }), - ).rejects.toThrow('Connection "warehouse" driver "postgres" does not support read-only SQL execution.'); - expect(scanConnector.cleanup).toHaveBeenCalledTimes(1); - }); -}); -``` - -- [ ] **Step 2: Run the failing CLI query executor test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts -``` - -Expected: FAIL because `ingest-query-executor.ts` does not exist. - -- [ ] **Step 3: Add the scan-connector-backed query executor** - -Create `packages/cli/src/ingest-query-executor.ts`: - -```ts -import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import type { KtxLocalProject } from '@ktx/context/project'; -import type { KtxScanConnector, KtxScanContext } from '@ktx/context/scan'; -import { createKtxCliScanConnector } from './local-scan-connectors.js'; - -type CreateConnector = typeof createKtxCliScanConnector; - -export interface KtxCliIngestQueryExecutorDeps { - createConnector?: CreateConnector; -} - -async function cleanupConnector(connector: KtxScanConnector | null): Promise { - await connector?.cleanup?.(); -} - -export function createKtxCliIngestQueryExecutor( - project: KtxLocalProject, - deps: KtxCliIngestQueryExecutorDeps = {}, -): KtxSqlQueryExecutorPort { - const createConnector = deps.createConnector ?? createKtxCliScanConnector; - return { - async execute(input: KtxSqlQueryExecutionInput) { - let connector: KtxScanConnector | null = null; - try { - connector = await createConnector(project, input.connectionId); - if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) { - throw new Error( - `Connection "${input.connectionId}" driver "${connector.driver}" does not support read-only SQL execution.`, - ); - } - - const ctx: KtxScanContext = { runId: 'ingest-sql-execution' }; - const result = await connector.executeReadOnly( - { connectionId: input.connectionId, sql: input.sql, maxRows: input.maxRows }, - ctx, - ); - return { - headers: result.headers, - rows: result.rows, - totalRows: result.totalRows, - command: 'SELECT', - rowCount: result.rowCount, - }; - } finally { - await cleanupConnector(connector); - } - }, - }; -} -``` - -- [ ] **Step 4: Wire the CLI executor into local ingest runs** - -In `packages/cli/src/ingest.ts`, import the executor and type: - -```ts -import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import type { KtxLocalProject } from '@ktx/context/project'; -import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; -``` - -Extend `KtxIngestDeps`: - -```ts - createQueryExecutor?: (project: KtxLocalProject) => KtxSqlQueryExecutorPort; -``` - -Inside the `args.command === 'run'` branch, after `localIngestOptions` is -defined, add: - -```ts - const queryExecutor = - localIngestOptions.queryExecutor ?? - (deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(project); -``` - -Pass `queryExecutor` to both local ingest execution paths. In the Metabase -fan-out call: - -```ts - ...localIngestOptions, - queryExecutor, - trigger: 'manual_resync', -``` - -In the normal local ingest call: - -```ts - ...localIngestOptions, - queryExecutor, - pullConfigOptions: adapterOptions, -``` - -- [ ] **Step 5: Add CLI wiring coverage** - -Add this test to `packages/cli/src/ingest.test.ts`: - -```ts - it('supplies a scan-connector query executor to local ingest runs', async () => { - const io = makeIo(); - const projectDir = join(tempDir, 'query-executor-project'); - await writeWarehouseConfig(projectDir); - const queryExecutor = { - execute: vi.fn(async () => ({ - headers: [], - rows: [], - totalRows: 0, - command: 'SELECT', - rowCount: 0, - })), - }; - const runLocalIngest = vi.fn(async (input: RunLocalIngestOptions): Promise => - completedLocalBundleRun(input, 'query-executor-run'), - ); - - await expect( - runKtxIngest( - { - command: 'run', - projectDir, - connectionId: 'warehouse', - adapter: 'fake', - outputMode: 'json', - }, - io.io, - { - runLocalIngest, - createAdapters: () => [], - createQueryExecutor: () => queryExecutor, - }, - ), - ).resolves.toBe(0); - - expect(runLocalIngest).toHaveBeenCalledWith(expect.objectContaining({ queryExecutor })); - }); -``` - -- [ ] **Step 6: Run CLI query executor tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "query executor" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add \ - packages/cli/src/ingest-query-executor.ts \ - packages/cli/src/ingest-query-executor.test.ts \ - packages/cli/src/ingest.ts \ - packages/cli/src/ingest.test.ts -git commit -m "fix(cli): enable read-only SQL probes for local ingest" -``` - -### Task 5: Final verification - -**Files:** -- Verify: all files changed by Tasks 1-4. - -- [ ] **Step 1: Run focused context tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ - src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ - src/ingest/local-bundle-runtime.test.ts \ - src/ingest/local-adapters.test.ts \ - src/ingest/adapters/lookml/lookml.adapter.test.ts \ - src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ - src/ingest/ingest-bundle.runner.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: both commands pass. - -- [ ] **Step 4: Run pre-commit on changed files if configured** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - packages/context/src/ingest/adapters/lookml/lookml.adapter.ts \ - packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ - packages/context/src/ingest/local-adapters.ts \ - packages/context/src/ingest/local-adapters.test.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/local-bundle-runtime.test.ts \ - packages/context/src/ingest/local-ingest.ts \ - packages/cli/src/ingest-query-executor.ts \ - packages/cli/src/ingest-query-executor.test.ts \ - packages/cli/src/ingest.ts \ - packages/cli/src/ingest.test.ts \ - docs/superpowers/plans/2026-05-12-warehouse-verification-final-v1-closure.md -``` - -Expected: PASS. If the repository has no pre-commit config or the local `uv` -version cannot satisfy the configured toolchain, record the exact error and use -the focused test and type-check results as the closest verification. - -- [ ] **Step 5: Commit final verification fixes if any were needed** - -If verification required edits, run: - -```bash -git add -git commit -m "test: cover warehouse verification v1 closure" -``` - -If verification required no edits, do not create an empty commit. - -## Self-review - -Spec coverage: - -- Raw warehouse discovery still covers wiki, semantic-layer, and raw schema - results, and now raw hits include the connection name needed by the required - `entity_details` follow-up. -- Every local synthesis adapter with an external source connection now has a - path to target warehouse IDs: dbt and Notion already had it, Looker resolves - staged mappings, Metabase fan-out runs under target warehouse IDs, and this - plan adds LookML and MetricFlow. -- `sql_execution` remains scoped by `allowedConnectionNames`, retains the - read-only SQL wrapper, and gains a normal local ingest execution backend. - -Placeholder scan: - -- This plan contains no deferred implementation placeholders. -- Every code-changing step includes the exact test or implementation snippet to - add. - -Type consistency: - -- `connectionName` is added to `RawSchemaHit` and used by `DiscoverDataTool`. -- `targetConnectionIds` and `listTargetConnectionIds()` match the existing dbt - and Notion adapter pattern. -- Local ingest uses `KtxSqlQueryExecutorPort` consistently from CLI to context. diff --git a/docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md b/docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md deleted file mode 100644 index 42bb7f44..00000000 --- a/docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md +++ /dev/null @@ -1,1617 +0,0 @@ -# Warehouse Verification Tools Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add synthesis-time warehouse verification tools so ingest agents can verify raw warehouse tables, columns, and sample values before writing wiki pages, SL sources, `tables:` frontmatter, `sl_refs`, or unmapped fallback records. - -**Architecture:** Add a raw scan catalog service over `raw-sources//live-database//`, three BaseTool-backed ingest tools, and runner/tool-session scoping for allowed warehouse connections. Register the tools in the local ingest toolset so both WorkUnit and reconcile stages receive them through the existing `toAiSdkTools()` path. - -**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX file store, KTX semantic layer and wiki tools. - ---- - -## Audit summary - -The current repo has the original spec file only; no matching plan or implementation exists under `docs/superpowers/plans`. The following v1-blocking gaps remain: - -- `packages/context/src/connections/dialects.ts` does not exist. -- `packages/context/src/ingest/tools/warehouse-verification/` does not exist. -- `entity_details`, `sql_execution`, and `discover_data` are not available to ingest WU or reconcile toolsets. -- `ToolSession` does not carry the ingest stage's allowed warehouse connection IDs. -- Prompt updates are absent from the 11 writer skills named in the spec. -- Cleanup strings remain: `orbit_analytics.customer`, `wiki_sl_search`, and `sl_describe_table`. -- Prompt-bundling and warehouse-tool tests are absent. - -Non-blocking gaps remain out of scope for this plan: - -- Hard write-time validation in `wiki_write` and `emit_unmapped_fallback`. -- `dictionary_search`. -- `semantic_query` in synthesis toolsets. -- A raw-schema FTS index. -- A UUID identity layer for tables and columns. - -One repo-specific adjustment is required: do not import `@ktx/connector-*` -dialect classes into `@ktx/context`, because every connector package already -depends on `@ktx/context`. Add a minimal context-local dialect dispatch instead. - -## File structure - -Create these files: - -- `packages/context/src/connections/dialects.ts`: Context-local driver dispatch for identifier quoting and display formatting. -- `packages/context/src/connections/dialects.test.ts`: Driver dispatch and display-format tests. -- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts`: Reads the latest live-database scan, resolves display identifiers, and searches table and column metadata. -- `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: Fixture-backed catalog tests. -- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: `entity_details` ingest tool. -- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: Tool contract tests. -- `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`: `sql_execution` ingest tool. -- `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts`: Read-only SQL and output tests. -- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: `discover_data` ingest tool composing wiki, SL, and raw-schema search. -- `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts`: Discovery composition tests. -- `packages/context/src/ingest/tools/warehouse-verification/index.ts`: Exports tool classes and `createWarehouseVerificationTools()`. -- `packages/context/skills/_shared/identifier-verification.md`: Shared protocol text kept in the tree for review even though writer skills inline it. - -Modify these files: - -- `packages/context/src/connections/index.ts`: Export the dialect helper. -- `packages/context/src/tools/tool-session.ts`: Add `allowedConnectionNames`. -- `packages/context/src/ingest/ingest-bundle.runner.ts`: Populate `allowedConnectionNames` for WU and reconcile sessions. -- `packages/context/src/ingest/local-bundle-runtime.ts`: Register the warehouse verification tools in `LocalIngestToolsetFactory`. -- `packages/context/src/ingest/ingest-bundle.runner.test.ts`: Assert the runner scopes allowed warehouse connections. -- `packages/context/src/memory/memory-runtime-assets.test.ts`: Assert writer skills contain the protocol and banned strings are gone. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts`: Assert ingest skill packaging includes the protocol. -- `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`: Replace the fictional table example. -- `packages/context/src/sl/tools/sl-warehouse-validation.ts`: Replace the stale `sl_describe_table` hint. -- `packages/context/skills/*/SKILL.md`: Inline protocol updates for the writer skills listed in the spec. - -### Task 1: Add context-local dialect dispatch - -**Files:** -- Create: `packages/context/src/connections/dialects.ts` -- Create: `packages/context/src/connections/dialects.test.ts` -- Modify: `packages/context/src/connections/index.ts` - -- [ ] **Step 1: Write the failing dialect tests** - -Create `packages/context/src/connections/dialects.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { getDialectForDriver } from './dialects.js'; - -describe('getDialectForDriver', () => { - it.each([ - ['postgres', '"public"."orders"'], - ['postgresql', '"public"."orders"'], - ['mysql', '`public`.`orders`'], - ['clickhouse', '`public`.`orders`'], - ['sqlite', '"orders"'], - ['snowflake', '"analytics"."public"."orders"'], - ['bigquery', '`analytics`.`public`.`orders`'], - ['sqlserver', '[analytics].[public].[orders]'], - ] as const)('formats table names for %s', (driver, expected) => { - const dialect = getDialectForDriver(driver); - expect( - dialect.formatTableName({ - catalog: driver === 'snowflake' || driver === 'bigquery' || driver === 'sqlserver' ? 'analytics' : null, - db: driver === 'sqlite' ? null : 'public', - name: 'orders', - }), - ).toBe(expected); - }); - - it('throws with a supported-driver list for unknown drivers', () => { - expect(() => getDialectForDriver('oracle')).toThrow( - 'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver', - ); - }); -}); -``` - -- [ ] **Step 2: Run the failing test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts -``` - -Expected: FAIL because `./dialects.js` does not exist. - -- [ ] **Step 3: Add the minimal dialect implementation** - -Create `packages/context/src/connections/dialects.ts`: - -```ts -import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; - -export type SupportedDriver = - | 'postgres' - | 'postgresql' - | 'mysql' - | 'sqlserver' - | 'snowflake' - | 'bigquery' - | 'clickhouse' - | 'sqlite' - | 'sqlite3'; - -export interface KtxDialect { - readonly type: SupportedDriver; - quoteIdentifier(identifier: string): string; - formatTableName(table: KtxTableRef): string; - mapToDimensionType(nativeType: string): KtxSchemaDimensionType; -} - -const supportedDrivers: SupportedDriver[] = [ - 'bigquery', - 'clickhouse', - 'mysql', - 'postgres', - 'postgresql', - 'sqlite', - 'sqlite3', - 'snowflake', - 'sqlserver', -]; - -function doubleQuoted(identifier: string): string { - return `"${identifier.replace(/"/g, '""')}"`; -} - -function backtickQuoted(identifier: string): string { - return `\`${identifier.replace(/`/g, '``')}\``; -} - -function bigQueryQuoted(identifier: string): string { - return `\`${identifier.replace(/`/g, '\\`')}\``; -} - -function bracketQuoted(identifier: string): string { - return `[${identifier.replace(/\]/g, ']]')}]`; -} - -function inferDimensionType(nativeType: string): KtxSchemaDimensionType { - const normalized = nativeType.toLowerCase().trim(); - if (normalized.includes('date') || normalized.includes('time')) { - return 'time'; - } - if ( - normalized.includes('int') || - normalized.includes('num') || - normalized.includes('dec') || - normalized.includes('float') || - normalized.includes('double') || - normalized.includes('real') - ) { - return 'number'; - } - if (normalized.includes('bool') || normalized === 'bit') { - return 'boolean'; - } - return 'string'; -} - -function formatWithParts(table: KtxTableRef, quote: (identifier: string) => string, sqlite = false): string { - const parts = sqlite ? [table.name] : [table.catalog, table.db, table.name].filter((part): part is string => !!part); - return parts.map(quote).join('.'); -} - -function createDialect(type: SupportedDriver, quote: (identifier: string) => string, sqlite = false): KtxDialect { - return { - type, - quoteIdentifier: quote, - formatTableName: (table) => formatWithParts(table, quote, sqlite), - mapToDimensionType: inferDimensionType, - }; -} - -const dialects: Record = { - postgres: createDialect('postgres', doubleQuoted), - postgresql: createDialect('postgresql', doubleQuoted), - mysql: createDialect('mysql', backtickQuoted), - clickhouse: createDialect('clickhouse', backtickQuoted), - sqlite: createDialect('sqlite', doubleQuoted, true), - sqlite3: createDialect('sqlite3', doubleQuoted, true), - snowflake: createDialect('snowflake', doubleQuoted), - bigquery: createDialect('bigquery', bigQueryQuoted), - sqlserver: createDialect('sqlserver', bracketQuoted), -}; - -export function getDialectForDriver(driver: string): KtxDialect { - const normalized = driver.toLowerCase().trim(); - if (normalized in dialects) { - return dialects[normalized as SupportedDriver]; - } - throw new Error(`Unsupported warehouse driver "${driver}". Supported drivers: ${supportedDrivers.join(', ')}`); -} -``` - -Modify `packages/context/src/connections/index.ts`: - -```ts -export type { KtxDialect, SupportedDriver } from './dialects.js'; -export { getDialectForDriver } from './dialects.js'; -``` - -- [ ] **Step 4: Run the dialect tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/connections/dialects.ts packages/context/src/connections/dialects.test.ts packages/context/src/connections/index.ts -git commit -m "feat(context): add warehouse dialect dispatch" -``` - -### Task 2: Add the raw scan warehouse catalog service - -**Files:** -- Create: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` -- Create: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` - -- [ ] **Step 1: Write failing catalog tests** - -Create `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts`: - -```ts -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../../project/index.js'; -import { WarehouseCatalogService } from './warehouse-catalog.service.js'; - -describe('WarehouseCatalogService', () => { - let tempDir: string; - let project: KtxLocalProject; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-warehouse-catalog-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - async function seedLiveDatabaseScan(connectionName = 'warehouse', syncId = 'sync-2', driver = 'postgres') { - const root = `raw-sources/${connectionName}/live-database/${syncId}`; - await project.fileStore.writeFile( - `${root}/connection.json`, - JSON.stringify({ connectionId: connectionName, driver, extractedAt: '2026-05-12T00:00:00.000Z' }, null, 2), - 'ktx', - 'ktx@example.com', - 'seed connection', - ); - await project.fileStore.writeFile( - `${root}/tables/orders.json`, - JSON.stringify( - { - catalog: null, - db: driver === 'sqlite' ? null : 'public', - name: 'orders', - kind: 'table', - comment: 'Customer orders', - estimatedRows: 12, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: 'Order id', - }, - { - name: 'status', - nativeType: 'text', - normalizedType: 'text', - dimensionType: 'string', - nullable: false, - primaryKey: false, - comment: 'Order status', - }, - ], - foreignKeys: [], - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed orders', - ); - await project.fileStore.writeFile( - `${root}/enrichment/relationship-profile.json`, - JSON.stringify( - { - connectionId: connectionName, - driver, - sqlAvailable: true, - queryCount: 3, - tables: [{ table: { catalog: null, db: driver === 'sqlite' ? null : 'public', name: 'orders' }, rowCount: 12 }], - columns: { - 'orders.status': { - table: { catalog: null, db: driver === 'sqlite' ? null : 'public', name: 'orders' }, - column: 'status', - nativeType: 'text', - normalizedType: 'text', - rowCount: 12, - nullCount: 0, - distinctCount: 2, - uniquenessRatio: 0.1667, - nullRate: 0, - sampleValues: ['paid', 'refunded'], - minTextLength: 4, - maxTextLength: 8, - }, - }, - warnings: [], - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed profile', - ); - } - - it('finds the latest sync and merges table schema with relationship profile values', async () => { - await seedLiveDatabaseScan('warehouse', 'sync-1'); - await seedLiveDatabaseScan('warehouse', 'sync-2'); - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - - await expect(catalog.getLatestSyncId('warehouse')).resolves.toBe('sync-2'); - const detail = await catalog.getTable({ connectionName: 'warehouse', catalog: null, db: 'public', name: 'orders' }); - - expect(detail).toMatchObject({ - connectionName: 'warehouse', - display: 'public.orders', - rowCount: 12, - columns: [ - { name: 'id', nativeType: 'integer', primaryKey: true }, - { name: 'status', nativeType: 'text', sampleValues: ['paid', 'refunded'], distinctCount: 2 }, - ], - }); - }); - - it('returns scanAvailable=false when no live-database scan exists', async () => { - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - await expect(catalog.getTable({ connectionName: 'missing', catalog: null, db: 'public', name: 'orders' })).resolves.toBeNull(); - await expect(catalog.hasScan('missing')).resolves.toBe(false); - }); - - it('resolves postgres display strings and returns closest candidates for missing tables', async () => { - await seedLiveDatabaseScan(); - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - - await expect(catalog.resolveDisplay('warehouse', 'public.orders')).resolves.toMatchObject({ - resolved: { catalog: null, db: 'public', name: 'orders' }, - candidates: [], - dialect: 'postgres', - }); - await expect(catalog.resolveDisplay('warehouse', 'public.orderz')).resolves.toMatchObject({ - resolved: null, - candidates: [{ name: 'orders' }], - }); - }); - - it('treats two-part BigQuery identifiers as ambiguous instead of guessing', async () => { - await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - - await expect(catalog.resolveDisplay('warehouse', 'public.orders')).resolves.toMatchObject({ - resolved: null, - dialect: 'bigquery', - }); - }); - - it('searches table names, column names, comments, and descriptions', async () => { - await seedLiveDatabaseScan(); - const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); - - await expect(catalog.searchByName('warehouse', 'status', 10)).resolves.toEqual( - expect.arrayContaining([ - expect.objectContaining({ - kind: 'column', - ref: expect.objectContaining({ db: 'public', name: 'orders', column: 'status' }), - matchedOn: 'name', - }), - ]), - ); - }); -}); -``` - -- [ ] **Step 2: Run the failing catalog tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts -``` - -Expected: FAIL because the service file does not exist. - -- [ ] **Step 3: Add the catalog service** - -Create `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` with these exported shapes and behavior: - -```ts -import type { KtxFileStorePort } from '../../../core/index.js'; -import { getDialectForDriver } from '../../../connections/index.js'; -import type { KtxConnectionDriver, KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaTable, KtxTableRef } from '../../../scan/types.js'; - -export interface WarehouseCatalogServiceDeps { - fileStore: KtxFileStorePort; -} - -export interface WarehouseColumnDetail extends KtxSchemaColumn { - descriptions: Record; - rowCount: number | null; - nullCount: number | null; - distinctCount: number | null; - nullRate: number | null; - sampleValues: string[]; -} - -export interface TableDetail { - connectionName: string; - catalog: string | null; - db: string | null; - name: string; - display: string; - kind: string; - comment: string | null; - description: string | null; - rowCount: number | null; - columns: WarehouseColumnDetail[]; - foreignKeys: KtxSchemaForeignKey[]; -} - -export type RawSchemaHit = - | { kind: 'table'; ref: KtxTableRef; display: string; matchedOn: 'name' | 'db' | 'comment' | 'description' } - | { kind: 'column'; ref: KtxTableRef & { column: string }; display: string; matchedOn: 'name' | 'comment' | 'description' }; - -interface ConnectionArtifact { - driver?: KtxConnectionDriver; -} - -interface RelationshipProfileColumn { - table?: KtxTableRef; - column?: string; - rowCount?: number; - nullCount?: number; - distinctCount?: number; - nullRate?: number; - sampleValues?: unknown[]; -} - -interface RelationshipProfileArtifact { - driver?: KtxConnectionDriver; - tables?: Array<{ table?: KtxTableRef; rowCount?: number }>; - columns?: Record; -} - -interface ConnectionCatalog { - connectionName: string; - syncId: string; - driver: KtxConnectionDriver; - tables: KtxSchemaTable[]; - profile: RelationshipProfileArtifact | null; -} -``` - -The implementation must: - -- Use `fileStore.listFiles("raw-sources//live-database")` and choose the lexicographically latest path ending in `/connection.json`. -- Read every JSON file under `/tables/` rather than reconstructing a path from the table ref. This supports encoded and simple table filenames already present in tests. -- Parse display strings by driver: - - Postgres, MySQL, and ClickHouse: `schema.table`. - - SQL Server, Snowflake, and BigQuery: `catalog.schema.table`. - - SQLite: `table`. - - For BigQuery, a two-part display must return `resolved: null` and candidate matches. -- Match table refs case-insensitively, while preserving stored casing in outputs. -- Merge relationship-profile fields by `(catalog, db, name, column)`, with fallback matching on `table.name + "." + column`. -- Cache a loaded connection catalog per `connectionName` within the service instance. -- Return `null` from `getTable()` when the scan is absent or the table ref is not found. - -Use these method signatures: - -```ts -export class WarehouseCatalogService { - constructor(private readonly deps: WarehouseCatalogServiceDeps) {} - - async hasScan(connectionName: string): Promise; - async getLatestSyncId(connectionName: string): Promise; - async listTables(connectionName: string): Promise; - async getTable(ref: { connectionName: string } & KtxTableRef): Promise; - async resolveDisplay(connectionName: string, display: string): Promise<{ - resolved: KtxTableRef | null; - candidates: KtxTableRef[]; - dialect: string; - }>; - async searchByName(connectionName: string, query: string, limit: number): Promise; -} -``` - -- [ ] **Step 4: Run the catalog tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts -git commit -m "feat(context): read warehouse scan catalog" -``` - -### Task 3: Add `entity_details` - -**Files:** -- Create: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` -- Create: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` - -- [ ] **Step 1: Write failing `entity_details` tests** - -Create tests that instantiate the tool with a seeded `WarehouseCatalogService` and a `ToolContext` whose session has `allowedConnectionNames: new Set(['warehouse'])`. Test these cases: - -```ts -it('returns scoped table detail for a display target', async () => { - const result = await tool.call( - { connectionName: 'warehouse', targets: [{ display: 'public.orders' }] }, - context, - ); - expect(result.markdown).toContain('### public.orders'); - expect(result.markdown).toContain('- status (text, nullable=false)'); - expect(result.markdown).toContain('sample: ["paid","refunded"]'); - expect(result.structured.scanAvailable).toBe(true); - expect(result.structured.resolved).toHaveLength(1); -}); - -it('returns a no-scan state distinct from not found', async () => { - const result = await tool.call( - { connectionName: 'empty', targets: [{ display: 'public.orders' }] }, - { ...context, session: { ...context.session!, allowedConnectionNames: new Set(['empty']) } }, - ); - expect(result.markdown).toContain('No live-database scan available for connection "empty"; run `ktx scan` first.'); - expect(result.structured.scanAvailable).toBe(false); -}); - -it('refuses out-of-scope connections', async () => { - const result = await tool.call( - { connectionName: 'billing', targets: [{ display: 'public.orders' }] }, - context, - ); - expect(result.markdown).toContain('Connection "billing" is not available to this ingest stage.'); - expect(result.structured.scanAvailable).toBe(false); -}); -``` - -- [ ] **Step 2: Run the failing tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -``` - -Expected: FAIL because the tool file does not exist. - -- [ ] **Step 3: Implement the tool** - -Create `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: - -```ts -import { z } from 'zod'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; -import type { KtxTableRef } from '../../../scan/types.js'; -import { WarehouseCatalogService, type TableDetail } from './warehouse-catalog.service.js'; - -const targetSchema = z.union([ - z.object({ display: z.string().min(1) }), - z.object({ - catalog: z.string().nullable(), - db: z.string().nullable(), - name: z.string().min(1), - column: z.string().optional(), - }), -]); - -const entityDetailsInputSchema = z.object({ - connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), - targets: z.array(targetSchema).min(1).max(50), -}); - -type EntityDetailsInput = z.infer; - -export interface EntityDetailsStructured { - resolved: TableDetail[]; - missing: Array<{ target: unknown; candidates: KtxTableRef[] }>; - scanAvailable: boolean; -} - -function allowedConnectionNames(context: ToolContext): ReadonlySet | null { - return context.session?.allowedConnectionNames ?? null; -} - -function sampleText(values: string[]): string { - return values.length > 0 ? ` - sample: ${JSON.stringify(values.slice(0, 10))}` : ''; -} - -function appendTableMarkdown(parts: string[], detail: TableDetail, columnName?: string): void { - const columns = columnName ? detail.columns.filter((column) => column.name === columnName) : detail.columns; - parts.push(`### ${detail.display}`); - parts.push(`Type: ${detail.kind} | Native columns: ${detail.columns.length}`); - if (detail.description || detail.comment) { - parts.push(`Description: ${detail.description ?? detail.comment}`); - } - parts.push('', 'Columns:'); - for (const column of columns) { - const pk = column.primaryKey ? ', PK' : ''; - parts.push(`- ${column.name} (${column.nativeType}, nullable=${column.nullable}${pk})${sampleText(column.sampleValues)}`); - } - parts.push(''); -} - -export class EntityDetailsTool extends BaseTool { - readonly name = 'entity_details'; - - constructor(private readonly catalogFactory: (context: ToolContext) => WarehouseCatalogService) { - super(); - } - - get description(): string { - return 'Verify warehouse tables and columns from the latest live-database scan before writing them into wiki or semantic-layer output.'; - } - - get inputSchema() { - return entityDetailsInputSchema; - } - - async call(input: EntityDetailsInput, context: ToolContext): Promise> { - const allowed = allowedConnectionNames(context); - if (allowed && !allowed.has(input.connectionName)) { - return { - markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, - structured: { resolved: [], missing: [], scanAvailable: false }, - }; - } - - const catalog = this.catalogFactory(context); - const scanAvailable = await catalog.hasScan(input.connectionName); - if (!scanAvailable) { - return { - markdown: `No live-database scan available for connection "${input.connectionName}"; run \`ktx scan\` first.`, - structured: { resolved: [], missing: [], scanAvailable: false }, - }; - } - - const parts: string[] = []; - const resolved: TableDetail[] = []; - const missing: EntityDetailsStructured['missing'] = []; - - for (const target of input.targets) { - const resolution = - 'display' in target - ? await catalog.resolveDisplay(input.connectionName, target.display) - : { resolved: { catalog: target.catalog, db: target.db, name: target.name }, candidates: [], dialect: '' }; - if (!resolution.resolved) { - missing.push({ target, candidates: resolution.candidates }); - parts.push(`Not found in scan: ${'display' in target ? target.display : target.name}`); - if (resolution.candidates.length > 0) { - parts.push(`Closest matches: ${resolution.candidates.map((candidate) => candidate.name).join(', ')}`); - } - continue; - } - const detail = await catalog.getTable({ connectionName: input.connectionName, ...resolution.resolved }); - if (!detail) { - missing.push({ target, candidates: resolution.candidates }); - continue; - } - resolved.push(detail); - appendTableMarkdown(parts, detail, 'column' in target ? target.column : undefined); - } - - return { - markdown: parts.join('\n').trim(), - structured: { resolved, missing, scanAvailable: true }, - }; - } -} -``` - -- [ ] **Step 4: Run the `entity_details` tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -git commit -m "feat(context): add entity details verification tool" -``` - -### Task 4: Add `sql_execution` - -**Files:** -- Create: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts` -- Create: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts` - -- [ ] **Step 1: Write failing `sql_execution` tests** - -Create tests for: - -```ts -it('wraps read-only SQL with a capped row limit', async () => { - connections.executeQuery.mockResolvedValue({ headers: ['status'], rows: [['paid']], totalRows: 1 }); - const result = await tool.call( - { connectionName: 'warehouse', sql: 'select status from public.orders', rowLimit: 5 }, - context, - ); - expect(connections.executeQuery).toHaveBeenCalledWith( - 'warehouse', - 'select * from (select status from public.orders) as ktx_query_result limit 5', - ); - expect(result.markdown).toContain('| status |'); - expect(result.structured.wrappedSql).toContain('limit 5'); -}); - -it.each(['insert into x values (1)', 'drop table x', 'vacuum'])('rejects mutating SQL: %s', async (sql) => { - const result = await tool.call({ connectionName: 'warehouse', sql }, context); - expect(result.markdown).toContain('Only read-only SELECT/WITH queries can be executed locally.'); - expect(connections.executeQuery).not.toHaveBeenCalled(); -}); - -it('surfaces connector errors verbatim', async () => { - connections.executeQuery.mockRejectedValue(new Error('relation "orbit_analytics.customer" does not exist')); - const result = await tool.call( - { connectionName: 'warehouse', sql: 'select 1 from orbit_analytics.customer', rowLimit: 1 }, - context, - ); - expect(result.markdown).toContain('relation "orbit_analytics.customer" does not exist'); - expect(result.structured.error).toContain('relation "orbit_analytics.customer" does not exist'); -}); -``` - -- [ ] **Step 2: Run the failing tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts -``` - -Expected: FAIL because the tool file does not exist. - -- [ ] **Step 3: Implement the tool** - -Create `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`: - -```ts -import { z } from 'zod'; -import { assertReadOnlySql, limitSqlForExecution } from '../../../connections/index.js'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; - -const sqlExecutionInputSchema = z.object({ - connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), - sql: z.string().min(1), - rowLimit: z.number().int().positive().max(1000).optional().default(100), -}); - -type SqlExecutionInput = z.infer; - -export interface SqlExecutionStructured { - headers: string[]; - rows: unknown[][]; - rowCount: number; - truncated: boolean; - sql: string; - wrappedSql: string; - error?: string; -} - -function markdownTable(headers: string[], rows: unknown[][], totalRows: number): string { - if (headers.length === 0) { - return rows.length === 0 ? 'Query returned no rows.' : JSON.stringify(rows.slice(0, 20)); - } - const visible = rows.slice(0, 20); - const lines = [ - `| ${headers.join(' | ')} |`, - `| ${headers.map(() => '---').join(' | ')} |`, - ...visible.map((row) => `| ${row.map((value) => String(value ?? '')).join(' | ')} |`), - ]; - if (totalRows > visible.length) { - lines.push(`... +${totalRows - visible.length} more rows`); - } - return lines.join('\n'); -} - -export class SqlExecutionTool extends BaseTool { - readonly name = 'sql_execution'; - - constructor(private readonly connections: SlConnectionCatalogPort) { - super(); - } - - get description(): string { - return 'Run a single read-only SELECT or WITH probe against an allowed warehouse connection and return a capped markdown table or the warehouse error.'; - } - - get inputSchema() { - return sqlExecutionInputSchema; - } - - async call(input: SqlExecutionInput, context: ToolContext): Promise> { - const allowed = context.session?.allowedConnectionNames; - if (allowed && !allowed.has(input.connectionName)) { - return { - markdown: `Connection "${input.connectionName}" is not available to this ingest stage.`, - structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql: input.sql, wrappedSql: '', error: 'connection_not_allowed' }, - }; - } - - let sql: string; - let wrappedSql: string; - try { - sql = assertReadOnlySql(input.sql); - wrappedSql = limitSqlForExecution(sql, input.rowLimit); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - markdown: message, - structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql: input.sql, wrappedSql: '', error: message }, - }; - } - - try { - const result = await this.connections.executeQuery(input.connectionName, wrappedSql); - const headers = result.headers ?? []; - const rows = result.rows ?? []; - const rowCount = result.totalRows ?? rows.length; - return { - markdown: markdownTable(headers, rows, rowCount), - structured: { headers, rows, rowCount, truncated: rowCount > rows.length, sql, wrappedSql }, - }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - markdown: `SQL execution failed: ${message}`, - structured: { headers: [], rows: [], rowCount: 0, truncated: false, sql, wrappedSql, error: message }, - }; - } - } -} -``` - -- [ ] **Step 4: Run the `sql_execution` tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts -git commit -m "feat(context): add ingest SQL verification tool" -``` - -### Task 5: Add `discover_data` - -**Files:** -- Create: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` -- Create: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` -- Create: `packages/context/src/ingest/tools/warehouse-verification/index.ts` - -- [ ] **Step 1: Write failing `discover_data` tests** - -Create tests with fake `wikiSearchTool.call`, `slDiscoverTool.call`, and `WarehouseCatalogService.searchByName`. Cover: - -```ts -it('groups wiki, semantic layer, and raw schema hits with routing hints', async () => { - const result = await tool.call({ query: 'orders', connectionName: 'warehouse', limit: 5 }, context); - expect(result.markdown).toContain('## Wiki Pages'); - expect(result.markdown).toContain('use `wiki_read(blockKey)` for full content'); - expect(result.markdown).toContain('## Semantic Layer Sources'); - expect(result.markdown).toContain('use `sl_read_source(sourceName)` for the YAML'); - expect(result.markdown).toContain('## Raw Warehouse Schema'); - expect(result.markdown).toContain('use `entity_details({connectionName, targets: [{display}]})`'); - expect(result.structured.raw?.hits).toHaveLength(1); -}); - -it('delegates sourceName inspect mode to sl_discover only', async () => { - const result = await tool.call({ sourceName: 'orders', connectionName: 'warehouse' }, context); - expect(slDiscoverTool.call).toHaveBeenCalledWith({ sourceName: 'orders', connectionId: 'warehouse' }, context); - expect(wikiSearchTool.call).not.toHaveBeenCalled(); - expect(catalog.searchByName).not.toHaveBeenCalled(); - expect(result.markdown).toContain('source detail'); -}); - -it('returns the empty-state message when all sections are empty', async () => { - const result = await tool.call({ query: 'customer source', connectionName: 'warehouse' }, emptyContext); - expect(result.markdown).toContain('No matches for "customer source" across wiki, semantic layer, or raw warehouse schema.'); -}); -``` - -- [ ] **Step 2: Run the failing tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -``` - -Expected: FAIL because the tool file does not exist. - -- [ ] **Step 3: Implement the tool and index export** - -Create `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts`: - -```ts -import { z } from 'zod'; -import type { BaseTool, ToolContext, ToolOutput } from '../../../tools/index.js'; -import { BaseTool as ToolBase } from '../../../tools/index.js'; -import { WarehouseCatalogService, type RawSchemaHit } from './warehouse-catalog.service.js'; - -const discoverDataInputSchema = z.object({ - query: z.string().optional(), - connectionName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(), - limit: z.number().int().positive().max(50).optional().default(10), - sourceName: z.string().optional(), -}); - -type DiscoverDataInput = z.infer; - -export interface DiscoverDataStructured { - wiki: unknown | null; - sl: unknown | null; - raw: { hits: RawSchemaHit[] } | null; -} - -interface DiscoverDataDeps { - wikiSearchTool: BaseTool; - slDiscoverTool: BaseTool; - catalogFactory: (context: ToolContext) => WarehouseCatalogService; -} - -export class DiscoverDataTool extends ToolBase { - readonly name = 'discover_data'; - - constructor(private readonly deps: DiscoverDataDeps) { - super(); - } - - get description(): string { - return 'Discover existing wiki pages, semantic layer sources, and raw warehouse schema hits before writing ingest output.'; - } - - get inputSchema() { - return discoverDataInputSchema; - } - - async call(input: DiscoverDataInput, context: ToolContext): Promise> { - if (input.sourceName) { - const sl = await this.deps.slDiscoverTool.call( - { sourceName: input.sourceName, connectionId: input.connectionName }, - context, - ); - return { markdown: sl.markdown, structured: { wiki: null, sl: sl.structured, raw: null } }; - } - - const query = input.query?.trim() || ''; - const limit = input.limit ?? 10; - const parts: string[] = []; - let wiki: unknown | null = null; - let sl: unknown | null = null; - let raw: DiscoverDataStructured['raw'] = null; - - if (query) { - const wikiResult = await this.deps.wikiSearchTool.call({ query, limit }, context); - if (wikiResult.structured?.totalFound > 0) { - parts.push('## Wiki Pages', '> use `wiki_read(blockKey)` for full content', wikiResult.markdown, ''); - wiki = wikiResult.structured; - } - } - - const slResult = await this.deps.slDiscoverTool.call( - { query: query || undefined, connectionId: input.connectionName }, - context, - ); - if (slResult.structured?.totalSources > 0) { - parts.push('## Semantic Layer Sources', '> use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details', slResult.markdown, ''); - sl = slResult.structured; - } - - const catalog = this.deps.catalogFactory(context); - const connections = input.connectionName - ? [input.connectionName] - : [...(context.session?.allowedConnectionNames ?? [])].sort(); - const rawHits: RawSchemaHit[] = []; - for (const connectionName of connections) { - rawHits.push(...(await catalog.searchByName(connectionName, query, limit))); - } - if (rawHits.length > 0) { - parts.push('## Raw Warehouse Schema', '> use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values'); - parts.push( - rawHits - .slice(0, limit) - .map((hit) => `- ${hit.kind}: ${hit.display} (matched on ${hit.matchedOn})`) - .join('\n'), - ); - raw = { hits: rawHits.slice(0, limit) }; - } - - if (parts.length === 0) { - return { - markdown: `No matches for "${query}" across wiki, semantic layer, or raw warehouse schema. Try broader terms; this concept may not exist yet.`, - structured: { wiki, sl, raw }, - }; - } - - return { markdown: parts.join('\n'), structured: { wiki, sl, raw } }; - } -} -``` - -Create `packages/context/src/ingest/tools/warehouse-verification/index.ts`: - -```ts -import type { BaseTool, ToolContext } from '../../../tools/index.js'; -import type { KtxFileStorePort } from '../../../core/index.js'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; -import { DiscoverDataTool } from './discover-data.tool.js'; -import { EntityDetailsTool } from './entity-details.tool.js'; -import { SqlExecutionTool } from './sql-execution.tool.js'; -import { WarehouseCatalogService } from './warehouse-catalog.service.js'; - -export { DiscoverDataTool } from './discover-data.tool.js'; -export { EntityDetailsTool } from './entity-details.tool.js'; -export { SqlExecutionTool } from './sql-execution.tool.js'; -export { WarehouseCatalogService } from './warehouse-catalog.service.js'; -export type { TableDetail, WarehouseColumnDetail, RawSchemaHit } from './warehouse-catalog.service.js'; - -export function createWarehouseVerificationTools(deps: { - connections: SlConnectionCatalogPort; - fallbackFileStore: KtxFileStorePort; - wikiSearchTool: BaseTool; - slDiscoverTool: BaseTool; -}): BaseTool[] { - const catalogFactory = (context: ToolContext) => - new WarehouseCatalogService({ - fileStore: context.session?.configService ?? deps.fallbackFileStore, - }); - return [ - new EntityDetailsTool(catalogFactory), - new SqlExecutionTool(deps.connections), - new DiscoverDataTool({ - wikiSearchTool: deps.wikiSearchTool, - slDiscoverTool: deps.slDiscoverTool, - catalogFactory, - }), - ]; -} -``` - -- [ ] **Step 4: Run the `discover_data` tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts packages/context/src/ingest/tools/warehouse-verification/index.ts -git commit -m "feat(context): add raw warehouse discovery tool" -``` - -### Task 6: Wire tools into ingest sessions - -**Files:** -- Modify: `packages/context/src/tools/tool-session.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.test.ts` - -- [ ] **Step 1: Write failing scoping test** - -Add to `packages/context/src/ingest/ingest-bundle.runner.test.ts`: - -```ts -it('threads target warehouse connection names into WorkUnit and reconcile tool sessions', async () => { - const deps = makeDeps(); - const sessions: any[] = []; - deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['warehouse']); - deps.toolsetFactory.createIngestWuToolset.mockImplementation((toolSession: any) => { - sessions.push(toolSession); - return { - toAiSdkTools: vi.fn().mockReturnValue({}), - getAllTools: vi.fn().mockReturnValue([]), - getToolNames: vi.fn().mockReturnValue([]), - }; - }); - deps.agentRunner.runLoop.mockResolvedValue({ stopReason: 'natural' }); - - const runner = buildRunner(deps); - (runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({ - currentHashes: new Map([['a.yml', 'h1']]), - rawDirInWorktree: 'raw-sources/notion/fake/s', - }); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x'); - - await runner.run({ - jobId: 'j1', - connectionId: 'notion', - sourceKey: 'fake', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload-x' }, - }); - - expect([...sessions[0].allowedConnectionNames].sort()).toEqual(['notion', 'warehouse']); -}); -``` - -- [ ] **Step 2: Run the failing runner test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.test.ts -t "threads target warehouse connection names" -``` - -Expected: FAIL because `allowedConnectionNames` is absent. - -- [ ] **Step 3: Thread allowed connection names** - -Modify `packages/context/src/tools/tool-session.ts`: - -```ts - allowedRawPaths?: ReadonlySet; - allowedConnectionNames?: ReadonlySet; - semanticLayerService: SemanticLayerService; -``` - -Modify WU session creation in `packages/context/src/ingest/ingest-bundle.runner.ts`: - -```ts - allowedRawPaths: new Set(wu.rawFiles), - allowedConnectionNames: new Set(slConnectionIds), - semanticLayerService: scopedSemanticLayerService, -``` - -Modify reconcile session creation in the same file: - -```ts - allowedRawPaths: reconciliationAllowedRawPaths, - allowedConnectionNames: new Set(slConnectionIds), - semanticLayerService: rcScopedSl, -``` - -- [ ] **Step 4: Register the tools in the local ingest toolset** - -Modify `packages/context/src/ingest/local-bundle-runtime.ts`: - -```ts -import { - createWarehouseVerificationTools, -} from './tools/warehouse-verification/index.js'; -``` - -Refactor the existing inline wiki and SL tool instances in `LocalIngestToolsetFactory` so `wikiSearchTool` and `slDiscoverTool` are named constants, then add the warehouse tools: - -```ts - const wikiSearchTool = new WikiSearchTool({ - search: async (input) => { - const results = await searchLocalKnowledgePages(deps.project, { - userId: input.userId, - query: input.query, - limit: input.limit, - embeddingService: deps.embedding, - }); - return { - results: results.slice(0, input.limit).map((result) => ({ - key: result.key, - path: result.path, - summary: result.summary, - score: result.score, - matchReasons: result.matchReasons, - lanes: result.lanes, - })), - totalFound: results.length, - }; - }, - }); - const slDiscoverTool = new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 }); - const warehouseVerificationTools = createWarehouseVerificationTools({ - connections: deps.connections, - fallbackFileStore: deps.project.fileStore, - wikiSearchTool, - slDiscoverTool, - }); - - this.baseTools = [ - new WikiReadTool(deps.wikiService, deps.knowledgeIndex), - wikiSearchTool, - new WikiListTagsTool(deps.wikiService, deps.knowledgeIndex), - new WikiWriteTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), - new WikiRemoveTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), - slDiscoverTool, - new SlEditSourceTool(slDeps), - new SlReadSourceTool(slDeps), - new SlWriteSourceTool(slDeps), - new SlValidateTool(slDeps), - new SlRollbackTool(deps.slSourcesRepository, deps.connections, 0), - ...warehouseVerificationTools, - ]; -``` - -- [ ] **Step 5: Run integration and toolset tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.test.ts -t "threads target warehouse connection names" -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -Run: - -```bash -git add packages/context/src/tools/tool-session.ts packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/local-bundle-runtime.ts packages/context/src/ingest/ingest-bundle.runner.test.ts -git commit -m "feat(context): expose warehouse verification tools to ingest" -``` - -### Task 7: Update writer prompts and cleanup stale references - -**Files:** -- Create: `packages/context/skills/_shared/identifier-verification.md` -- Modify: `packages/context/skills/notion_synthesize/SKILL.md` -- Modify: `packages/context/skills/dbt_ingest/SKILL.md` -- Modify: `packages/context/skills/lookml_ingest/SKILL.md` -- Modify: `packages/context/skills/looker_ingest/SKILL.md` -- Modify: `packages/context/skills/metabase_ingest/SKILL.md` -- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` -- Modify: `packages/context/skills/live_database_ingest/SKILL.md` -- Modify: `packages/context/skills/historic_sql_table_digest/SKILL.md` -- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md` -- Modify: `packages/context/skills/knowledge_capture/SKILL.md` -- Modify: `packages/context/skills/sl_capture/SKILL.md` -- Modify: `packages/context/skills/sl/SKILL.md` -- Modify: `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts` -- Modify: `packages/context/src/sl/tools/sl-warehouse-validation.ts` - -- [ ] **Step 1: Add the shared protocol file** - -Create `packages/context/skills/_shared/identifier-verification.md`: - -```md -## Identifier Verification Protocol - -Before writing a wiki page or SL source on any topic: - -1. `discover_data({query: ""})` - see what wikis, SL sources, and raw - tables already exist. Prefer updating existing pages over creating new ones. - -Before emitting any `schema.table` or `schema.table.column` into a wiki body, -SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: - -2. `entity_details({connectionName, targets: [{display: ""}]})` - - confirm the identifier resolves; inspect native types, FK/PK, and - sampleValues. -3. For literal values from the source, such as status codes or plan tiers, - check whether they appear in `entity_details` sampleValues for the relevant - column. If sampleValues is short or the sample may have missed real values, - run a `sql_execution` probe: - `SELECT DISTINCT FROM LIMIT 50`. -4. If the candidate identifier still does not resolve, do one of: - - Use `sql_execution` with `SELECT 1 FROM LIMIT 0`. If it errors, the - identifier is fictional. - - Wrap the identifier in `[unverified - from ]` in the wiki body, - citing the exact raw path that mentioned it. - - When recording `emit_unmapped_fallback` with `no_physical_table`, include - the failing probe error in `clarification`. -5. Never copy `.` placeholder strings from these instructions - into output. -``` - -- [ ] **Step 2: Inline the protocol into writer skills** - -Add the same protocol block to these skills: - -```text -packages/context/skills/notion_synthesize/SKILL.md -packages/context/skills/dbt_ingest/SKILL.md -packages/context/skills/lookml_ingest/SKILL.md -packages/context/skills/looker_ingest/SKILL.md -packages/context/skills/metabase_ingest/SKILL.md -packages/context/skills/metricflow_ingest/SKILL.md -packages/context/skills/live_database_ingest/SKILL.md -packages/context/skills/historic_sql_patterns/SKILL.md -packages/context/skills/knowledge_capture/SKILL.md -packages/context/skills/sl_capture/SKILL.md -``` - -For `packages/context/skills/historic_sql_table_digest/SKILL.md`, add this shorter block: - -```md -## Identifier Verification Protocol - -Only mention columns visible in the table's scan record. Use -`entity_details({connectionName, targets: [{display: ""}]})` if -the table or column attribution is uncertain. Do not infer join columns or -filters from neighboring SQL unless the scan record confirms the column exists -on the named table. -``` - -For `packages/context/skills/sl/SKILL.md`, add this cross-reference: - -```md -For capture-time identifier verification, load `sl_capture`. Synthesis writer -skills must verify warehouse identifiers with `discover_data`, -`entity_details`, and `sql_execution` before emitting table or column names. -``` - -- [ ] **Step 3: Apply per-skill edits** - -Make these exact content changes: - -- In `notion_synthesize`, add `discover_data`, `entity_details`, and `sql_execution` to the `Allowed:` line. Replace `tableRef: "orbit_analytics.customer"` with `tableRef: ".
    "`. -- In `dbt_ingest`, replace `wiki_sl_search` with `discover_data` and `sl_describe_table` with `entity_details`. -- In `lookml_ingest`, add: `Verify each sql_table_name from the LookML view with entity_details before mapping to an SL source.` -- In `looker_ingest`, add: `For every Looker field reference, call entity_details on the underlying schema.table.column before promoting it to sl_refs or quoting it in wiki body.` -- In `metabase_ingest`, add: `Before writing a wiki page derived from a Metabase question SQL, verify each schema.table.column mentioned with entity_details.` -- In `metricflow_ingest`, add: `Verify each MetricFlow model source table with entity_details before producing the corresponding sl_write_source.` -- In `live_database_ingest`, add: `Sample values come from the scan record; do not invent values not present in relationship-profile.json.` -- In `historic_sql_patterns`, add: `Every join column mentioned in pattern descriptions must be verified via entity_details for both sides of the join.` -- In `knowledge_capture`, update the workflow to call `discover_data` first when a page relates to data or SL concepts. -- In `sl_capture`, add: `Before sl_write_source, call entity_details on the target table to confirm column names and types match the YAML being written.` - -- [ ] **Step 4: Remove stale code and prompt strings** - -Modify `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`: - -```ts -.describe('The fully-qualified table or source reference that triggered the fallback (e.g. ".
    "). Used to generate canonical detail text.'), -``` - -Modify `packages/context/src/sl/tools/sl-warehouse-validation.ts`: - -```ts - `that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`, -``` - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/skills packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts packages/context/src/sl/tools/sl-warehouse-validation.ts -git commit -m "docs(context): add ingest identifier verification protocol" -``` - -### Task 8: Add prompt-bundling and banned-string tests - -**Files:** -- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` -- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Add failing asset tests** - -Add to `packages/context/src/memory/memory-runtime-assets.test.ts`: - -```ts -const verificationWriterSkills = [ - 'notion_synthesize', - 'dbt_ingest', - 'lookml_ingest', - 'looker_ingest', - 'metabase_ingest', - 'metricflow_ingest', - 'live_database_ingest', - 'historic_sql_table_digest', - 'historic_sql_patterns', - 'knowledge_capture', - 'sl_capture', -] as const; - -it('ships identifier verification protocol in every synthesis writer skill', async () => { - for (const skillName of verificationWriterSkills) { - const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); - expect(body).toContain('## Identifier Verification Protocol'); - expect(body).toMatch(/discover_data|entity_details/); - } -}); - -it('does not ship stale warehouse verification tool names or fictional identifiers', async () => { - for (const skillName of verificationWriterSkills) { - const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); - expect(body).not.toContain('orbit_analytics.customer'); - expect(body).not.toContain('wiki_sl_search'); - expect(body).not.toContain('sl_describe_table'); - } -}); -``` - -Add to `packages/context/src/ingest/ingest-runtime-assets.test.ts`: - -```ts -it('packages identifier verification prompt assets', async () => { - const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); - expect(shared).toContain('## Identifier Verification Protocol'); - expect(shared).toContain('discover_data'); - expect(shared).toContain('entity_details'); - expect(shared).toContain('sql_execution'); -}); -``` - -- [ ] **Step 2: Run the asset tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS after Task 7. - -- [ ] **Step 3: Commit** - -Run: - -```bash -git add packages/context/src/memory/memory-runtime-assets.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts -git commit -m "test(context): guard ingest identifier verification prompts" -``` - -### Task 9: Run the full v1 verification set - -**Files:** -- Verify all files changed by Tasks 1-8. - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/connections/dialects.test.ts \ - src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ - src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ - src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - src/ingest/ingest-bundle.runner.test.ts \ - src/memory/memory-runtime-assets.test.ts \ - src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run package tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run pre-commit on changed files when configured** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/connections/dialects.ts \ - packages/context/src/connections/dialects.test.ts \ - packages/context/src/connections/index.ts \ - packages/context/src/tools/tool-session.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/ingest-bundle.runner.test.ts \ - packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts \ - packages/context/src/sl/tools/sl-warehouse-validation.ts \ - packages/context/src/memory/memory-runtime-assets.test.ts \ - packages/context/src/ingest/ingest-runtime-assets.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts \ - packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - packages/context/src/ingest/tools/warehouse-verification/index.ts \ - packages/context/skills/_shared/identifier-verification.md \ - packages/context/skills/notion_synthesize/SKILL.md \ - packages/context/skills/dbt_ingest/SKILL.md \ - packages/context/skills/lookml_ingest/SKILL.md \ - packages/context/skills/looker_ingest/SKILL.md \ - packages/context/skills/metabase_ingest/SKILL.md \ - packages/context/skills/metricflow_ingest/SKILL.md \ - packages/context/skills/live_database_ingest/SKILL.md \ - packages/context/skills/historic_sql_table_digest/SKILL.md \ - packages/context/skills/historic_sql_patterns/SKILL.md \ - packages/context/skills/knowledge_capture/SKILL.md \ - packages/context/skills/sl_capture/SKILL.md \ - packages/context/skills/sl/SKILL.md -``` - -Expected: PASS. If the repo has no pre-commit config or the local `uv` version cannot satisfy the project pin, record the exact error and rely on the focused tests plus type-check. - -- [ ] **Step 5: Commit final verification notes if any files changed during checks** - -Run: - -```bash -git status --short -``` - -Expected: only intentional files are modified. Commit any formatter-driven edits with: - -```bash -git add packages/context -git commit -m "chore(context): verify warehouse verification tools" -``` - -## Self-review checklist - -- Spec coverage: the plan covers dialect dispatch, raw scan catalog reads, `entity_details`, `sql_execution`, `discover_data`, WU and reconcile availability, prompt updates, cleanups, and tests. -- Placeholder scan: no task relies on unnamed future work. -- Type consistency: tool inputs use `connectionName`; existing `sl_discover` calls receive `connectionId` internally; raw SQL execution uses `SlConnectionCatalogPort.executeQuery()` because `SemanticLayerService.executeQuery()` currently accepts semantic-layer query input, not raw SQL. diff --git a/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md b/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md deleted file mode 100644 index 107379f8..00000000 --- a/docs/superpowers/plans/2026-05-13-cli-command-tree-script.md +++ /dev/null @@ -1,580 +0,0 @@ -# CLI Command-Tree Script Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a build-time script that prints the full `ktx` CLI command tree (name, aliases, description per node) as an indented text tree, for docs and discovery - without adding a runtime `ktx` subcommand. - -**Architecture:** Commander.js exposes every registered command as a `Command` instance with `.commands`, `.name()`, `.aliases()`, `.description()` - we walk that tree. The current `runCommanderKtxCli` in `packages/cli/src/cli-program.ts` builds the program inline; we extract that assembly into a pure `buildKtxProgram(...)` helper that any caller can use to materialize the configured root `Command` without parsing argv. A new pure module `command-tree.ts` walks the `Command` into plain data and renders it as indented text. A new TypeScript entrypoint `print-command-tree.ts` compiles alongside `bin.ts` into `dist/print-command-tree.js`, instantiates the program with stub IO/deps, and writes the rendered tree to stdout. A pnpm script under `@ktx/cli` exposes it as `pnpm --filter @ktx/cli run docs:commands`. - -**Tech Stack:** TypeScript (NodeNext ESM), Node 22, Commander 14 via `@commander-js/extra-typings`, vitest 4. - ---- - -## File Map - -- **Modify:** `packages/cli/src/cli-program.ts` - extract `buildKtxProgram` from `runCommanderKtxCli`. -- **Create:** `packages/cli/src/cli-program.test.ts` - vitest tests for the new helper. -- **Create:** `packages/cli/src/command-tree.ts` - pure `walkCommandTree` + `formatCommandTree`. -- **Create:** `packages/cli/src/command-tree.test.ts` - vitest tests against ad-hoc Command trees. -- **Create:** `packages/cli/src/print-command-tree.ts` - script entrypoint; thin glue. -- **Create:** `packages/cli/src/print-command-tree.test.ts` - vitest test that calls the script's exported `main()` with a fake stdout and asserts the rendered tree includes known top-level commands. -- **Modify:** `packages/cli/package.json` - add `docs:commands` script and include the new entry in tsc build output (no change needed if `tsconfig` already globs `src/**/*.ts`, but verify). -- **Modify:** `packages/cli/README.md` (if it exists; otherwise skip) - document `pnpm run docs:commands`. - -Files that change together (cli-program + its test, command-tree + its test, print-command-tree + its test) live next to each other under `packages/cli/src/`, matching the existing convention (e.g. `bin.ts`, `cli-runtime.ts`, `runtime.ts` + `runtime.test.ts`). - ---- - -## Task 1: Extract `buildKtxProgram` from `runCommanderKtxCli` - -Refactor only - no behavior change. The current code in `cli-program.ts` interleaves program construction with `parseAsync` dispatch. Splitting them lets the new script reuse construction without invoking the CLI. - -**Files:** -- Modify: `packages/cli/src/cli-program.ts:197-275` (function `runCommanderKtxCli`) -- Create: `packages/cli/src/cli-program.test.ts` - -- [ ] **Step 1: Write the failing test** - -Create `packages/cli/src/cli-program.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import type { Command } from '@commander-js/extra-typings'; -import { buildKtxProgram } from './cli-program.js'; -import type { KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; - -function stubIo(): KtxCliIo { - return { - stdout: { isTTY: false, columns: 80, write: () => {} }, - stderr: { write: () => {} }, - }; -} - -function stubPackageInfo(): KtxCliPackageInfo { - return { name: '@ktx/cli', version: '0.0.0-test', contextPackageName: '@ktx/context' }; -} - -describe('buildKtxProgram', () => { - it('returns a Command named "ktx" with all registered top-level subcommands', () => { - const program: Command = buildKtxProgram({ - io: stubIo(), - deps: {}, - packageInfo: stubPackageInfo(), - runInit: async () => 0, - }); - - expect(program.name()).toBe('ktx'); - const topLevel = program.commands.map((c) => c.name()).sort(); - // Sanity check: at least these registrar surfaces must be present. - for (const expected of ['setup', 'serve', 'sl', 'dev']) { - expect(topLevel).toContain(expected); - } - }); - - it('does not parse argv or invoke action handlers', async () => { - // Build should be a pure call; no rejections, no side-effects to stdout. - let wrote = ''; - const io: KtxCliIo = { - stdout: { isTTY: false, columns: 80, write: (chunk) => { wrote += chunk; } }, - stderr: { write: (chunk) => { wrote += chunk; } }, - }; - buildKtxProgram({ io, deps: {}, packageInfo: stubPackageInfo(), runInit: async () => 0 }); - expect(wrote).toBe(''); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pnpm --filter @ktx/cli exec vitest run src/cli-program.test.ts` - -Expected: FAIL - `buildKtxProgram is not exported from './cli-program.js'` (or similar TS/ESM error). - -- [ ] **Step 3: Extract `buildKtxProgram` from `runCommanderKtxCli`** - -Edit `packages/cli/src/cli-program.ts`. Add a new exported function above `runCommanderKtxCli`: - -```typescript -export interface BuildKtxProgramOptions { - io: KtxCliIo; - deps: KtxCliDeps; - packageInfo: KtxCliPackageInfo; - runInit: (args: { projectDir: string; projectName?: string; force: boolean }, io: KtxCliIo) => Promise; - setExitCode?: (code: number) => void; -} - -export function buildKtxProgram(options: BuildKtxProgramOptions): Command { - const program = createBaseProgram(options.packageInfo, options.io); - const context: KtxCliCommandContext = { - io: options.io, - deps: options.deps, - packageInfo: options.packageInfo, - setExitCode: options.setExitCode ?? (() => {}), - runInit: options.runInit, - writeDebug: (command, commandContext) => { - writeDebug(options.io, commandContext, command); - }, - }; - - registerSetupCommands(program, context); - registerConnectionCommands(program, context); - registerPublicIngestCommands(program, context); - registerWikiCommands(program, context); - registerSlCommands(program, context); - registerRuntimeCommands(program, context); - registerServeCommands(program, context); - registerStatusCommands(program, context); - registerAgentCommands(program, context); - registerDevCommands(program, context); - - return program; -} -``` - -Then rewrite the body of `runCommanderKtxCli` (lines 197-275) to delegate program assembly. Replace the block from `const program = createBaseProgram(info, io);` (line 206) through `registerDevCommands(program, context);` (line 248) with: - -```typescript - profileMark('commander:entry'); - let exitCode = 0; - const program = buildKtxProgram({ - io, - deps, - packageInfo: info, - runInit: options.runInit, - setExitCode: (code: number) => { - exitCode = code; - }, - }); - profileMark('commander:program-built'); - const context: KtxCliCommandContext = { - io, - deps, - packageInfo: info, - setExitCode: (code: number) => { - exitCode = code; - }, - runInit: options.runInit, - writeDebug: (command: string, commandContext: CommandWithGlobalOptions) => { - writeDebug(io, commandContext, command); - }, - }; -``` - -Keep the `context` re-declaration only if subsequent code (the `if (argv.length === 0)` branch that calls `runBareInteractiveCommand(program, io, context)`) still needs it. It does - `runBareInteractiveCommand` consumes `context`. Keep `context` exactly as it was after the deletion; do not change `runBareInteractiveCommand`'s signature or behavior. Drop the now-removed individual `register*` calls and their `profileMark` lines from `runCommanderKtxCli`. - -- [ ] **Step 4: Run the new test to verify it passes** - -Run: `pnpm --filter @ktx/cli exec vitest run src/cli-program.test.ts` - -Expected: PASS - both `it` blocks green. - -- [ ] **Step 5: Run the full CLI test suite to confirm no regression** - -Run: `pnpm --filter @ktx/cli run test 2>&1 | tee /tmp/ktx-cli-test-output.log` - -Expected: PASS overall. Inspect the log if any previously-passing test now fails - most likely a missing register call (compare to lines 221-249 of the pre-change file). - -- [ ] **Step 6: Type-check** - -Run: `pnpm --filter @ktx/cli run type-check` - -Expected: no errors. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/cli-program.ts packages/cli/src/cli-program.test.ts -git commit -m "refactor(cli): extract buildKtxProgram for reuse outside runCommanderKtxCli" -``` - ---- - -## Task 2: Pure tree walker `walkCommandTree` - -Take a Commander `Command` and produce plain data: `{ name, description, aliases, children }`. No formatting yet. Pure function - depends only on the public `Command` API. - -**Files:** -- Create: `packages/cli/src/command-tree.ts` -- Create: `packages/cli/src/command-tree.test.ts` - -- [ ] **Step 1: Write the failing test** - -Create `packages/cli/src/command-tree.test.ts`: - -```typescript -import { Command } from '@commander-js/extra-typings'; -import { describe, expect, it } from 'vitest'; -import { walkCommandTree } from './command-tree.js'; - -describe('walkCommandTree', () => { - it('captures name, description, aliases, and nested children', () => { - const root = new Command('root').description('the root'); - const child = new Command('child').description('a child').alias('c').alias('ch'); - const grandchild = new Command('grand').description('a grandchild'); - child.addCommand(grandchild); - root.addCommand(child); - - const tree = walkCommandTree(root); - - expect(tree).toEqual({ - name: 'root', - description: 'the root', - aliases: [], - children: [ - { - name: 'child', - description: 'a child', - aliases: ['c', 'ch'], - children: [ - { name: 'grand', description: 'a grandchild', aliases: [], children: [] }, - ], - }, - ], - }); - }); - - it('returns an empty children array when there are no subcommands', () => { - const leaf = new Command('leaf').description('alone'); - expect(walkCommandTree(leaf)).toEqual({ - name: 'leaf', - description: 'alone', - aliases: [], - children: [], - }); - }); - - it('uses an empty string when description is unset', () => { - const cmd = new Command('bare'); - expect(walkCommandTree(cmd).description).toBe(''); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts` - -Expected: FAIL - `walkCommandTree` cannot be resolved. - -- [ ] **Step 3: Implement `walkCommandTree`** - -Create `packages/cli/src/command-tree.ts`: - -```typescript -import type { Command } from '@commander-js/extra-typings'; - -export interface CommandTreeNode { - name: string; - description: string; - aliases: string[]; - children: CommandTreeNode[]; -} - -export function walkCommandTree(command: Command): CommandTreeNode { - return { - name: command.name(), - description: command.description(), - aliases: command.aliases(), - children: command.commands.map((child) => walkCommandTree(child as Command)), - }; -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts` - -Expected: PASS (3 of 3). - -- [ ] **Step 5: Type-check** - -Run: `pnpm --filter @ktx/cli run type-check` - -Expected: no errors. - ---- - -## Task 3: Indented-text renderer `formatCommandTree` - -Render a `CommandTreeNode` as plain text. Each node on its own line: `[ (alias1, alias2)][ - description]`. Indent is two spaces per depth level. Children sorted alphabetically by name to keep output stable across changes that reorder registrar calls. - -**Files:** -- Modify: `packages/cli/src/command-tree.ts` -- Modify: `packages/cli/src/command-tree.test.ts` - -- [ ] **Step 1: Write the failing test** - -Append to `packages/cli/src/command-tree.test.ts`: - -```typescript -import { formatCommandTree } from './command-tree.js'; - -describe('formatCommandTree', () => { - it('renders a single node with no children', () => { - const node = { name: 'solo', description: 'just me', aliases: [], children: [] }; - expect(formatCommandTree(node)).toBe('solo - just me\n'); - }); - - it('renders aliases in parentheses before the description', () => { - const node = { name: 'cmd', description: 'does things', aliases: ['c', 'co'], children: [] }; - expect(formatCommandTree(node)).toBe('cmd (c, co) - does things\n'); - }); - - it('omits the dash when description is empty', () => { - const node = { name: 'bare', description: '', aliases: [], children: [] }; - expect(formatCommandTree(node)).toBe('bare\n'); - }); - - it('indents children by two spaces per depth level and sorts siblings alphabetically', () => { - const tree = { - name: 'root', - description: 'top', - aliases: [], - children: [ - { name: 'beta', description: 'b', aliases: [], children: [] }, - { name: 'alpha', description: 'a', aliases: ['al'], children: [ - { name: 'inner', description: 'i', aliases: [], children: [] }, - ] }, - ], - }; - expect(formatCommandTree(tree)).toBe( - 'root - top\n' + - ' alpha (al) - a\n' + - ' inner - i\n' + - ' beta - b\n', - ); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts` - -Expected: FAIL - `formatCommandTree` is not exported. - -- [ ] **Step 3: Implement `formatCommandTree`** - -Append to `packages/cli/src/command-tree.ts`: - -```typescript -export function formatCommandTree(node: CommandTreeNode): string { - const lines: string[] = []; - appendNode(node, 0, lines); - return `${lines.join('\n')}\n`; -} - -function appendNode(node: CommandTreeNode, depth: number, lines: string[]): void { - const indent = ' '.repeat(depth); - const aliasPart = node.aliases.length > 0 ? ` (${node.aliases.join(', ')})` : ''; - const descPart = node.description.length > 0 ? ` - ${node.description}` : ''; - lines.push(`${indent}${node.name}${aliasPart}${descPart}`); - - const sortedChildren = [...node.children].sort((a, b) => a.name.localeCompare(b.name)); - for (const child of sortedChildren) { - appendNode(child, depth + 1, lines); - } -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts` - -Expected: PASS (7 of 7 across walkCommandTree + formatCommandTree). - -- [ ] **Step 5: Type-check** - -Run: `pnpm --filter @ktx/cli run type-check` - -Expected: no errors. - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/src/command-tree.ts packages/cli/src/command-tree.test.ts -git commit -m "feat(cli): add walkCommandTree and formatCommandTree helpers" -``` - ---- - -## Task 4: Script entrypoint `print-command-tree.ts` - -Thin glue: build the program with stub IO/deps, walk, format, write to a provided stdout. Export a `main(stdout)` function for unit testing; only auto-run when invoked as a script. - -**Files:** -- Create: `packages/cli/src/print-command-tree.ts` -- Create: `packages/cli/src/print-command-tree.test.ts` - -- [ ] **Step 1: Write the failing test** - -Create `packages/cli/src/print-command-tree.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { renderKtxCommandTree } from './print-command-tree.js'; - -describe('renderKtxCommandTree', () => { - it('renders an indented tree rooted at "ktx" with known top-level commands', () => { - const output = renderKtxCommandTree(); - - const lines = output.split('\n'); - expect(lines[0]).toMatch(/^ktx( |$|\s-)/); - - // Top-level commands are indented exactly two spaces. - const topLevel = lines - .filter((line) => /^ {2}\S/.test(line)) - .map((line) => line.trim().split(' ')[0]); - - for (const expected of ['setup', 'serve', 'sl', 'dev']) { - expect(topLevel).toContain(expected); - } - }); - - it('ends with a single trailing newline', () => { - const output = renderKtxCommandTree(); - expect(output.endsWith('\n')).toBe(true); - expect(output.endsWith('\n\n')).toBe(false); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pnpm --filter @ktx/cli exec vitest run src/print-command-tree.test.ts` - -Expected: FAIL - module not found. - -- [ ] **Step 3: Implement the script** - -Create `packages/cli/src/print-command-tree.ts`: - -```typescript -import { fileURLToPath } from 'node:url'; -import { buildKtxProgram } from './cli-program.js'; -import type { KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; -import { formatCommandTree, walkCommandTree } from './command-tree.js'; - -function silentIo(): KtxCliIo { - return { - stdout: { isTTY: false, columns: 80, write: () => {} }, - stderr: { write: () => {} }, - }; -} - -function stubPackageInfo(): KtxCliPackageInfo { - return { name: '@ktx/cli', version: '0.0.0-docs', contextPackageName: '@ktx/context' }; -} - -export function renderKtxCommandTree(): string { - const program = buildKtxProgram({ - io: silentIo(), - deps: {}, - packageInfo: stubPackageInfo(), - runInit: async () => 0, - }); - return formatCommandTree(walkCommandTree(program)); -} - -export function main(stdout: { write(chunk: string): void }): void { - stdout.write(renderKtxCommandTree()); -} - -const invokedAsScript = - typeof process !== 'undefined' && - Array.isArray(process.argv) && - process.argv[1] !== undefined && - fileURLToPath(import.meta.url) === process.argv[1]; - -if (invokedAsScript) { - main(process.stdout); -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pnpm --filter @ktx/cli exec vitest run src/print-command-tree.test.ts` - -Expected: PASS - both assertions green. - -- [ ] **Step 5: Type-check** - -Run: `pnpm --filter @ktx/cli run type-check` - -Expected: no errors. - -- [ ] **Step 6: Build and run the script end-to-end** - -Run: -```bash -pnpm --filter @ktx/cli run build -node packages/cli/dist/print-command-tree.js | head -20 -``` - -Expected: first line begins with `ktx`, followed by indented top-level commands (`setup`, `serve`, `sl`, `dev`, etc.). No errors on stderr. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/print-command-tree.ts packages/cli/src/print-command-tree.test.ts -git commit -m "feat(cli): add print-command-tree build-time script" -``` - ---- - -## Task 5: Wire pnpm script and document - -Expose the script through pnpm so contributors and CI don't need to remember the `node dist/…` path. - -**Files:** -- Modify: `packages/cli/package.json` (add `docs:commands` to `scripts`) - -- [ ] **Step 1: Inspect existing scripts block** - -Run: `node -e "const p=require('./packages/cli/package.json'); console.log(JSON.stringify(p.scripts, null, 2))"` - -Note the current keys (`build`, `smoke`, `test`, `test:slow`, `type-check`, `assets:demo`). Add a new entry that depends on `build`. - -- [ ] **Step 2: Add the `docs:commands` script** - -Edit `packages/cli/package.json`. In the `"scripts"` object, add (after `"build"`): - -```json -"docs:commands": "pnpm run build && node dist/print-command-tree.js", -``` - -Keep alphabetical-ish ordering consistent with the existing block; if other scripts use `&&` chains for build prerequisites, match the style. - -- [ ] **Step 3: Verify the script runs** - -Run: `pnpm --filter @ktx/cli run docs:commands | head -30` - -Expected: builds the CLI, then prints the tree (first line `ktx ...`, two-space-indented children below). - -- [ ] **Step 4: Verify nothing else broke** - -Run in parallel: -- `pnpm --filter @ktx/cli run type-check` -- `pnpm --filter @ktx/cli run test` - -Expected: both PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/package.json -git commit -m "chore(cli): add docs:commands pnpm script" -``` - ---- - -## Verification Summary - -After all tasks, confirm: - -- [ ] `pnpm --filter @ktx/cli run type-check` - clean -- [ ] `pnpm --filter @ktx/cli run test` - green, including new tests in `cli-program.test.ts`, `command-tree.test.ts`, `print-command-tree.test.ts` -- [ ] `pnpm --filter @ktx/cli run docs:commands` - prints `ktx` followed by indented subcommand tree -- [ ] `git status --short` - only the files listed in the File Map are modified or created; no incidental edits - -If any check fails, fix in place and re-run before declaring done. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-public-cli-surface.md b/docs/superpowers/plans/2026-05-13-unified-ingest-public-cli-surface.md deleted file mode 100644 index 279d02a8..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-public-cli-surface.md +++ /dev/null @@ -1,1584 +0,0 @@ -# Unified Ingest Public CLI Surface Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `ktx ingest` the public foreground context-build command for one connection or all configured connections. - -**Architecture:** Reuse the existing `public-ingest.ts` orchestration as the public command engine, then extend it to resolve database depth, query-history intent, warnings, and adapter bypasses for connection-centric ingest. Keep low-level `scan` and adapter-backed `ingest run` available as hidden debug surfaces while normal help, output, generated config, and setup recovery text point to `ktx ingest `. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages, existing scan and local ingest adapters. - ---- - -## Current audit - -The unified ingest spec is not v1-complete. Relevant implemented pieces exist, -but they are not wired as the public product surface: - -- `packages/cli/src/public-ingest.ts` can plan database connections before - source connections and can call scan or source ingest internals. -- `packages/cli/src/context-build-view.ts` renders a foreground progress view - and captures target progress. -- Historic SQL internals exist in `packages/context/src/ingest/adapters/historic-sql` - and CLI adapter wiring exists in `packages/cli/src/local-adapters.ts`. -- The public CLI still rejects `ktx ingest warehouse`; see - `packages/cli/src/index.test.ts`, test name - `rejects removed public ingest shorthand`. -- Root help still exposes `scan`, `ktx ingest --help` still exposes `run` and - `watch`, and generated default config still includes `live-database`. - -This plan addresses the first v1-blocking slice: the public command surface, -connection-centric execution, public depth flags, query-history run overrides, -hidden legacy debug commands, and stale public wording. Setup depth prompting -and foreground-only state cleanup remain separate v1-blocking work after this -slice. - -## File structure - -- Modify `packages/cli/src/cli-runtime.ts`: add an injectable - `publicIngest` dependency for Commander tests and for command routing. -- Modify `packages/cli/src/commands/ingest-commands.ts`: make the parent - `ktx ingest` command accept `[connectionId]`, `--all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days`; hide legacy `run` and `watch`. -- Modify `packages/cli/src/commands/scan-commands.ts`: hide `ktx scan` from - root help while keeping direct debug invocation. -- Modify `packages/cli/src/cli-program.ts`: remove `scan` from normal - project-aware root command help expectations only where user-facing. -- Modify `packages/cli/src/public-ingest.ts`: resolve target type, depth, - query-history settings, warnings, readiness failures, and adapter bypasses. -- Modify `packages/cli/src/context-build-view.ts`: rename public labels and - public operation text away from scan terminology. -- Modify `packages/cli/src/ingest.ts`: allow connection-centric ingest to run - an inferred adapter without requiring `ingest.adapters`. -- Modify `packages/cli/src/local-adapters.ts`: accept current-run - query-history overrides for `context.queryHistory` without rewriting config. -- Modify `packages/context/src/project/config.ts`: stop generating - `live-database` and source adapters in default `ktx.yaml`. -- Modify `packages/cli/src/setup-sources.ts`: replace stale recovery command - suggestions with `ktx ingest `. -- Modify `README.md` and script assertions that document normal public command - output. - -## Tasks - -### Task 1: Route the public `ktx ingest` command - -**Files:** -- Modify: `packages/cli/src/cli-runtime.ts` -- Modify: `packages/cli/src/commands/ingest-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Modify: `packages/cli/src/dev.test.ts` - -- [ ] **Step 1: Write failing Commander routing tests** - -In `packages/cli/src/index.test.ts`, replace the test named -`rejects removed public ingest shorthand` with: - -```ts - it('routes public connection-centric ingest shorthand', async () => { - const testIo = makeIo(); - const publicIngest = vi.fn().mockResolvedValue(0); - - await expect( - runKtxCli(['--project-dir', '/tmp/project', 'ingest', 'warehouse', '--fast', '--no-input'], testIo.io, { - publicIngest, - }), - ).resolves.toBe(0); - - expect(publicIngest).toHaveBeenCalledWith( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - depth: 'fast', - queryHistory: 'default', - }, - testIo.io, - ); - expect(testIo.stderr()).toBe('Project: /tmp/project\n'); - }); - - it('routes public ingest --all --deep with JSON output', async () => { - const testIo = makeIo(); - const publicIngest = vi.fn().mockResolvedValue(0); - - await expect( - runKtxCli(['--project-dir', '/tmp/project', 'ingest', '--all', '--deep', '--json'], testIo.io, { - publicIngest, - }), - ).resolves.toBe(0); - - expect(publicIngest).toHaveBeenCalledWith( - { - command: 'run', - projectDir: '/tmp/project', - all: true, - json: true, - inputMode: 'auto', - depth: 'deep', - queryHistory: 'default', - }, - testIo.io, - ); - expect(testIo.stderr()).toBe(''); - }); - - it('rejects mutually exclusive public ingest depth flags before dispatch', async () => { - const testIo = makeIo(); - const publicIngest = vi.fn().mockResolvedValue(0); - - await expect( - runKtxCli(['--project-dir', '/tmp/project', 'ingest', 'warehouse', '--fast', '--deep'], testIo.io, { - publicIngest, - }), - ).resolves.toBe(1); - - expect(publicIngest).not.toHaveBeenCalled(); - expect(testIo.stderr()).toContain("option '--deep' cannot be used with option '--fast'"); - }); -``` - -In the existing ingest help test, change the expected help assertions to: - -```ts - expect(testIo.stdout()).toContain('Usage: ktx ingest [options] [connectionId]'); - expect(testIo.stdout()).toContain('Build or inspect KTX context'); - expect(testIo.stdout()).toContain('--all'); - expect(testIo.stdout()).toContain('--fast'); - expect(testIo.stdout()).toContain('--deep'); - expect(testIo.stdout()).toContain('--query-history'); - expect(testIo.stdout()).toContain('--no-query-history'); - expect(testIo.stdout()).toContain('--query-history-window-days '); - expect(testIo.stdout()).toContain('status'); - expect(testIo.stdout()).toContain('replay'); - expect(testIo.stdout()).not.toContain('run'); - expect(testIo.stdout()).not.toContain('watch'); -``` - -In `packages/cli/src/dev.test.ts`, update the generated nested help case for -`['ingest', 'run', '--help']` so it no longer treats legacy run help as a -normal generated public help case. Add this direct hidden-command regression -test instead: - -```ts - it('keeps legacy adapter-backed ingest run callable but hidden from ingest help', async () => { - const helpIo = makeIo(); - const runIo = makeIo(); - const ingest = vi.fn(async () => 0); - - await expect(runKtxCli(['ingest', '--help'], helpIo.io, { ingest })).resolves.toBe(0); - await expect( - runKtxCli( - ['ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase', '--project-dir', '/tmp/project'], - runIo.io, - { ingest }, - ), - ).resolves.toBe(0); - - expect(helpIo.stdout()).not.toContain('run'); - expect(ingest).toHaveBeenCalledWith( - expect.objectContaining({ command: 'run', connectionId: 'warehouse', adapter: 'metabase' }), - runIo.io, - ); - }); -``` - -- [ ] **Step 2: Run the failing Commander tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/dev.test.ts -t "public connection-centric ingest|public ingest --all|mutually exclusive public ingest|legacy adapter-backed ingest run|prints ingest help" -``` - -Expected: FAIL because `KtxCliDeps` has no `publicIngest`, `ktx ingest -warehouse` is unknown, and `run`/`watch` are still visible in help. - -- [ ] **Step 3: Add the injectable public ingest dependency** - -In `packages/cli/src/cli-runtime.ts`, add this import near the existing CLI -argument type imports: - -```ts -import type { KtxPublicIngestArgs } from './public-ingest.js'; -``` - -In `KtxCliDeps`, add: - -```ts - publicIngest?: (args: KtxPublicIngestArgs, io: KtxCliIo) => Promise; -``` - -- [ ] **Step 4: Register parent `ktx ingest` options and hidden legacy commands** - -In `packages/cli/src/commands/ingest-commands.ts`, add: - -```ts -import type { KtxPublicIngestArgs } from '../public-ingest.js'; -import { parsePositiveIntegerOption } from '../cli-program.js'; -``` - -Replace the current `const ingest = program.command('ingest')...` block with: - -```ts - const ingest = program - .command('ingest') - .description('Build or inspect KTX context') - .argument('[connectionId]', 'Configured connection id to ingest') - .option('--all', 'Ingest all configured connections', false) - .addOption(new Option('--fast', 'Use deterministic database schema ingest').conflicts('deep')) - .addOption(new Option('--deep', 'Use AI-enriched database ingest').conflicts('fast')) - .addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory')) - .addOption(new Option('--no-query-history', 'Skip database query-history usage patterns')) - .option('--query-history-window-days ', 'Query-history lookback window for this run', parsePositiveIntegerOption) - .addOption(new Option('--plain', 'Print plain text output').conflicts(['json'])) - .addOption(new Option('--json', 'Print JSON output').conflicts(['plain'])) - .option('--no-input', 'Disable interactive terminal input') - .showHelpAfterError(); - - ingest.action(async (connectionId: string | undefined, options, command) => { - const { runKtxPublicIngest } = await import('../public-ingest.js'); - const queryHistory = - options.queryHistory === true ? 'enabled' : options.queryHistory === false ? 'disabled' : 'default'; - const args: KtxPublicIngestArgs = { - command: 'run', - projectDir: resolveCommandProjectDir(command), - ...(connectionId ? { targetConnectionId: connectionId } : {}), - all: options.all === true, - json: options.json === true, - inputMode: options.input === false ? 'disabled' : 'auto', - ...(options.fast === true ? { depth: 'fast' as const } : {}), - ...(options.deep === true ? { depth: 'deep' as const } : {}), - queryHistory, - ...(options.queryHistoryWindowDays !== undefined - ? { queryHistoryWindowDays: options.queryHistoryWindowDays } - : {}), - }; - context.setExitCode(await (context.deps.publicIngest ?? runKtxPublicIngest)(args, context.io)); - }); -``` - -Then hide the legacy `run` and `watch` subcommands by changing: - -```ts - .command('run') -``` - -to: - -```ts - .command('run', { hidden: true }) -``` - -and changing: - -```ts - .command('watch') -``` - -to: - -```ts - .command('watch', { hidden: true }) -``` - -- [ ] **Step 5: Run Commander tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/dev.test.ts -``` - -Expected: PASS after updating any remaining help text expectations that still -assume public `run` or `watch`. - -- [ ] **Step 6: Commit public route wiring** - -Run: - -```bash -git add packages/cli/src/cli-runtime.ts packages/cli/src/commands/ingest-commands.ts packages/cli/src/index.test.ts packages/cli/src/dev.test.ts -git commit -m "feat(cli): route public connection ingest command" -``` - -### Task 2: Hide top-level `scan` from normal help - -**Files:** -- Modify: `packages/cli/src/commands/scan-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Modify: `packages/cli/src/dev.test.ts` -- Modify: `packages/cli/src/cli-program.ts` - -- [ ] **Step 1: Update public help tests** - -In `packages/cli/src/index.test.ts`, in the test `prints the public command -surface in root help`, change the visible command list: - -```ts - for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status']) { - expect(testIo.stdout()).toContain(`${command}`); - } - expect(testIo.stdout()).not.toMatch(/^ scan\s/m); -``` - -In `packages/cli/src/dev.test.ts`, keep the direct `['scan', '--help']` case -so the hidden debug command is still callable. - -- [ ] **Step 2: Run the failing scan help tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/dev.test.ts -t "public command surface|generated nested help" -``` - -Expected: FAIL because root help still prints `scan`. - -- [ ] **Step 3: Hide the scan command** - -In `packages/cli/src/commands/scan-commands.ts`, change: - -```ts - program - .command('scan') -``` - -to: - -```ts - program - .command('scan', { hidden: true }) -``` - -In `packages/cli/src/cli-program.ts`, leave `scan` in -`PROJECT_AWARE_ROOT_COMMANDS` so hidden direct invocations still receive -project-dir behavior: - -```ts -const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'scan']); -``` - -- [ ] **Step 4: Run scan help tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/dev.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit scan help hiding** - -Run: - -```bash -git add packages/cli/src/commands/scan-commands.ts packages/cli/src/index.test.ts packages/cli/src/dev.test.ts packages/cli/src/cli-program.ts -git commit -m "feat(cli): hide standalone scan from public help" -``` - -### Task 3: Resolve public ingest depth, warnings, and query-history intent - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write failing public ingest planner tests** - -In `packages/cli/src/public-ingest.test.ts`, add these tests inside -`describe('buildPublicIngestPlan', ...)`: - -```ts - it('resolves database depth from flags, stored context, and defaults', () => { - const project = projectWithConnections({ - fast_default: { driver: 'postgres' }, - deep_default: { driver: 'postgres', context: { depth: 'deep' } }, - docs: { driver: 'notion' }, - }); - - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'fast_default', - all: false, - queryHistory: 'default', - }).targets[0], - ).toMatchObject({ connectionId: 'fast_default', databaseDepth: 'fast', queryHistory: { enabled: false } }); - - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'deep_default', - all: false, - queryHistory: 'default', - }).targets[0], - ).toMatchObject({ connectionId: 'deep_default', databaseDepth: 'deep' }); - - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'docs', - all: false, - depth: 'deep', - queryHistory: 'default', - }).warnings, - ).toEqual(['--deep affects database ingest only; ignoring it for docs.']); - }); - - it('upgrades effective depth when query history is explicitly enabled', () => { - const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { queryHistory: { enabled: false } } }, - }); - - const plan = buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - depth: 'fast', - queryHistory: 'enabled', - queryHistoryWindowDays: 30, - }); - - expect(plan.targets[0]).toMatchObject({ - connectionId: 'warehouse', - databaseDepth: 'deep', - queryHistory: { enabled: true, windowDays: 30, dialect: 'postgres' }, - }); - expect(plan.warnings).toEqual(['--query-history requires deep ingest; running warehouse with --deep.']); - }); - - it('warns and skips query history for unsupported database drivers', () => { - const project = projectWithConnections({ local: { driver: 'sqlite' } }); - - const plan = buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'local', - all: false, - queryHistory: 'enabled', - }); - - expect(plan.targets[0]).toMatchObject({ - connectionId: 'local', - databaseDepth: 'fast', - queryHistory: { enabled: false, unsupported: true }, - }); - expect(plan.warnings).toEqual(['--query-history is not supported for sqlite; running schema ingest for local.']); - }); -``` - -- [ ] **Step 2: Run the failing public ingest planner tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "resolves database depth|upgrades effective depth|unsupported database drivers" -``` - -Expected: FAIL because `depth`, `queryHistory`, `databaseDepth`, and plan -warnings do not exist. - -- [ ] **Step 3: Extend public ingest types** - -In `packages/cli/src/public-ingest.ts`, replace the public step and args types -near the top with: - -```ts -type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update'; -type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run'; -type KtxPublicIngestInputMode = 'auto' | 'disabled'; -type KtxPublicIngestDepth = 'fast' | 'deep'; -type KtxPublicIngestQueryHistoryFlag = 'default' | 'enabled' | 'disabled'; -type HistoricSqlDialect = 'postgres' | 'bigquery' | 'snowflake'; -``` - -In the `command: 'run'` variant of `KtxPublicIngestArgs`, add: - -```ts - depth?: KtxPublicIngestDepth; - queryHistory?: KtxPublicIngestQueryHistoryFlag; - queryHistoryWindowDays?: number; -``` - -Replace `KtxPublicIngestPlanTarget` with: - -```ts -export interface KtxPublicIngestPlanTarget { - connectionId: string; - driver: string; - operation: 'database-ingest' | 'source-ingest'; - adapter?: string; - sourceDir?: string; - debugCommand: string; - steps: KtxPublicIngestStepName[]; - databaseDepth?: KtxPublicIngestDepth; - queryHistory?: { - enabled: boolean; - dialect?: HistoricSqlDialect; - windowDays?: number; - unsupported?: boolean; - skippedStoredByFast?: boolean; - }; -} -``` - -Add warnings to `KtxPublicIngestPlan`: - -```ts -export interface KtxPublicIngestPlan { - projectDir: string; - targets: KtxPublicIngestPlanTarget[]; - warnings: string[]; -} -``` - -- [ ] **Step 4: Add depth and query-history resolver helpers** - -Add these helpers after `warehouseDrivers`: - -```ts -const queryHistoryDialectByDriver = new Map([ - ['postgres', 'postgres'], - ['postgresql', 'postgres'], - ['bigquery', 'bigquery'], - ['snowflake', 'snowflake'], -]); - -function connectionContext(connection: KtxProjectConnectionConfig): Record { - const value = connection.context; - return typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record) : {}; -} - -function storedDepth(connection: KtxProjectConnectionConfig): KtxPublicIngestDepth | undefined { - const value = connectionContext(connection).depth; - return value === 'fast' || value === 'deep' ? value : undefined; -} - -function storedQueryHistory(connection: KtxProjectConnectionConfig): Record { - const value = connectionContext(connection).queryHistory; - return typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record) : {}; -} - -function positiveInteger(value: unknown): number | undefined { - return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; -} -``` - -Add: - -```ts -function resolveDatabaseTargetOptions(input: { - connectionId: string; - driver: string; - connection: KtxProjectConnectionConfig; - args: { - depth?: KtxPublicIngestDepth; - queryHistory?: KtxPublicIngestQueryHistoryFlag; - queryHistoryWindowDays?: number; - }; - warnings: string[]; -}): Pick { - const storedQh = storedQueryHistory(input.connection); - const dialect = queryHistoryDialectByDriver.get(input.driver); - const explicitQueryHistory = input.args.queryHistory ?? 'default'; - const storedEnabled = storedQh.enabled === true; - const requestedQh = explicitQueryHistory === 'enabled' || (explicitQueryHistory === 'default' && storedEnabled); - let depth = input.args.depth ?? storedDepth(input.connection) ?? 'fast'; - const queryHistory = { - enabled: false, - ...(input.args.queryHistoryWindowDays !== undefined - ? { windowDays: input.args.queryHistoryWindowDays } - : positiveInteger(storedQh.windowDays) !== undefined - ? { windowDays: positiveInteger(storedQh.windowDays) } - : {}), - }; - - if (requestedQh && !dialect) { - input.warnings.push( - explicitQueryHistory === 'enabled' || input.args.queryHistoryWindowDays !== undefined - ? `--query-history is not supported for ${input.driver}; running schema ingest for ${input.connectionId}.` - : `${input.connectionId} has query history enabled in ktx.yaml, but ${input.driver} does not support it; running schema ingest.`, - ); - return { - databaseDepth: depth, - queryHistory: { ...queryHistory, unsupported: true }, - steps: ['database-schema'], - }; - } - - if (requestedQh && dialect) { - if (depth === 'fast') { - input.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`); - } - depth = 'deep'; - return { - databaseDepth: depth, - queryHistory: { ...queryHistory, enabled: true, dialect }, - steps: ['database-schema', 'query-history'], - }; - } - - if (input.args.depth === 'fast' && explicitQueryHistory !== 'enabled' && storedEnabled) { - input.warnings.push( - `${input.connectionId} has query history enabled in ktx.yaml, but --fast skips query-history processing.`, - ); - return { - databaseDepth: 'fast', - queryHistory: { ...queryHistory, skippedStoredByFast: true }, - steps: ['database-schema'], - }; - } - - return { - databaseDepth: depth, - queryHistory, - steps: ['database-schema'], - }; -} -``` - -- [ ] **Step 5: Use the resolver in plan construction** - -Change `targetForConnection` to accept args and warnings: - -```ts -function targetForConnection( - connectionId: string, - connection: KtxProjectConnectionConfig, - args: { - depth?: KtxPublicIngestDepth; - queryHistory?: KtxPublicIngestQueryHistoryFlag; - queryHistoryWindowDays?: number; - }, - warnings: string[], -): KtxPublicIngestPlanTarget { -``` - -In the source-adapter branch, before returning, add: - -```ts - if (args.depth) { - warnings.push(`--${args.depth} affects database ingest only; ignoring it for ${connectionId}.`); - } - if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) { - warnings.push(`--query-history affects database ingest only; ignoring it for ${connectionId}.`); - } -``` - -Change the source debug command to: - -```ts - debugCommand: `ktx ingest ${connectionId} --debug`, -``` - -In the warehouse branch, replace the return object with: - -```ts - const options = resolveDatabaseTargetOptions({ connectionId, driver, connection, args, warnings }); - return { - connectionId, - driver, - operation: 'database-ingest', - debugCommand: `ktx ingest ${connectionId} --debug`, - ...options, - }; -``` - -In `buildPublicIngestPlan`, add warnings and return them: - -```ts - const warnings: string[] = []; - const targets = selected.map(([connectionId, connection]) => targetForConnection(connectionId, connection, args, warnings)); - return { - projectDir: args.projectDir, - targets: [ - ...targets.filter((t) => t.operation === 'database-ingest'), - ...targets.filter((t) => t.operation === 'source-ingest'), - ], - warnings, - }; -``` - -- [ ] **Step 6: Run planner tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "buildPublicIngestPlan" -``` - -Expected: PASS after updating older expected target snapshots from -`operation: 'scan'` to `operation: 'database-ingest'`, from `steps: ['scan']` -to `steps: ['database-schema']`, and adding `warnings: []`. - -- [ ] **Step 7: Commit public ingest planning** - -Run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "feat(cli): plan public ingest depth and query history" -``` - -### Task 4: Execute database depth and query-history facets - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/public-ingest.test.ts` -- Modify: `packages/cli/src/ingest.ts` - -- [ ] **Step 1: Write failing execution tests** - -In `packages/cli/src/public-ingest.test.ts`, add: - -```ts - it('maps fast and deep database targets to scan internals', async () => { - const io = makeIo(); - const project = projectWithConnections({ - fast: { driver: 'postgres' }, - deep: { driver: 'postgres', context: { depth: 'deep' } }, - }); - const runScan = vi.fn(async () => 0); - - await expect( - runKtxPublicIngest( - { command: 'run', projectDir: '/tmp/project', all: true, json: false, inputMode: 'disabled', queryHistory: 'default' }, - io.io, - { loadProject: vi.fn(async () => project), runScan }, - ), - ).resolves.toBe(0); - - expect(runScan).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ connectionId: 'deep', mode: 'enriched', detectRelationships: true }), - expect.anything(), - ); - expect(runScan).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ connectionId: 'fast', mode: 'structural', detectRelationships: false }), - expect.anything(), - ); - }); - - it('runs query history after schema ingest with current-run window override', async () => { - const io = makeIo(); - const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true, windowDays: 90 } } }, - }); - const runScan = vi.fn(async () => 0); - const runIngest = vi.fn(async () => 0); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - queryHistory: 'enabled', - queryHistoryWindowDays: 30, - }, - io.io, - { loadProject: vi.fn(async () => project), runScan, runIngest }, - ), - ).resolves.toBe(0); - - expect(runScan).toHaveBeenCalledWith( - expect.objectContaining({ connectionId: 'warehouse', mode: 'enriched' }), - expect.anything(), - ); - expect(runIngest).toHaveBeenCalledWith( - expect.objectContaining({ - command: 'run', - connectionId: 'warehouse', - adapter: 'historic-sql', - allowImplicitAdapter: true, - historicSqlPullConfigOverride: expect.objectContaining({ dialect: 'postgres', windowDays: 30 }), - }), - expect.anything(), - ); - }); -``` - -- [ ] **Step 2: Run the failing execution tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "maps fast and deep|runs query history" -``` - -Expected: FAIL because execution still uses `scanMode`, no query-history step -exists, and `KtxIngestArgs` has no implicit-adapter fields. - -- [ ] **Step 3: Add implicit adapter and query-history override fields** - -In `packages/cli/src/ingest.ts`, extend the `command: 'run'` args type: - -```ts - allowImplicitAdapter?: boolean; - historicSqlPullConfigOverride?: Record; -``` - -In the `adapterOptions` object inside `runKtxIngest`, add: - -```ts - ...(args.historicSqlPullConfigOverride - ? { historicSqlPullConfigOverride: args.historicSqlPullConfigOverride } - : {}), -``` - -Before calling `executeLocalIngest`, create the project used for local ingest: - -```ts - const ingestProject = - args.allowImplicitAdapter && !project.config.ingest.adapters.includes(args.adapter) - ? { - ...project, - config: { - ...project.config, - ingest: { - ...project.config.ingest, - adapters: [...project.config.ingest.adapters, args.adapter], - }, - }, - } - : project; -``` - -Then pass `ingestProject` instead of `project` to `runLocalMetabaseIngest`, -`createAdapters`, `createQueryExecutor`, and `executeLocalIngest` in the -`command: 'run'` branch. - -Keep `packages/context/src/ingest/local-ingest.ts` unchanged. The public path -satisfies its strict `assertConfigured()` contract by passing an in-memory -project config whose adapter list includes the inferred adapter for this run. - -- [ ] **Step 4: Execute database targets from effective depth** - -In `packages/cli/src/public-ingest.ts`, update the database branch of -`executePublicIngestTarget`: - -```ts - if (target.operation === 'database-ingest') { - const { runKtxScan } = await import('./scan.js'); - const scanArgs: KtxScanArgs = { - command: 'run', - projectDir: args.projectDir, - connectionId: target.connectionId, - mode: target.databaseDepth === 'deep' ? 'enriched' : 'structural', - detectRelationships: target.databaseDepth === 'deep' ? true : false, - dryRun: false, - }; - const runScan = deps.runScan ?? runKtxScan; - const scanExitCode = deps.scanProgress - ? await runScan(scanArgs, io, { progress: deps.scanProgress }) - : await runScan(scanArgs, io); - if (scanExitCode !== 0) { - return markTargetResult(target, 'failed', 'database-schema'); - } - - if (target.queryHistory?.enabled === true) { - const { runKtxIngest } = await import('./ingest.js'); - const runIngest = deps.runIngest ?? runKtxIngest; - const ingestArgs: KtxIngestArgs = { - command: 'run', - projectDir: args.projectDir, - connectionId: target.connectionId, - adapter: 'historic-sql', - outputMode: sourceIngestOutputMode(args, io), - inputMode: args.inputMode, - allowImplicitAdapter: true, - historicSqlPullConfigOverride: { - dialect: target.queryHistory.dialect, - ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}), - }, - }; - const qhExitCode = await runIngest(ingestArgs, io); - if (qhExitCode !== 0) { - return markTargetResult(target, 'failed', 'query-history'); - } - } - - return markTargetResult(target, 'done'); - } -``` - -Update `markTargetResult` to accept the failed operation: - -```ts -function markTargetResult( - target: KtxPublicIngestPlanTarget, - status: 'done' | 'failed', - failedOperation?: KtxPublicIngestStepName, -): KtxPublicIngestTargetResult { -``` - -Inside the function, replace the failed-operation selection with: - -```ts - const selectedFailedOperation = - failedOperation ?? (target.operation === 'database-ingest' ? 'database-schema' : 'source-ingest'); -``` - -Then use `selectedFailedOperation` in the failed-step comparison and detail. - -- [ ] **Step 5: Print plan warnings before results** - -In `runKtxPublicIngest`, after building `plan` and before executing targets, -add: - -```ts - if (!args.json && plan.warnings.length > 0) { - for (const warning of plan.warnings) { - io.stderr.write(`Warning: ${warning}\n`); - } - } -``` - -For JSON output, the existing `{ plan, results }` payload now includes -`plan.warnings`. - -- [ ] **Step 6: Run execution tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/ingest.test.ts -``` - -Expected: PASS after updating public result table labels from `Scan` to -`Database` or `Schema` in existing assertions. - -- [ ] **Step 7: Commit public execution behavior** - -Run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/ingest.ts -git commit -m "feat(cli): execute public database ingest facets" -``` - -### Task 5: Accept `context.queryHistory` in historic-SQL adapter plumbing - -**Files:** -- Modify: `packages/cli/src/local-adapters.ts` -- Modify: `packages/cli/src/local-adapters.test.ts` -- Modify: `packages/context/src/ingest/local-adapters.ts` -- Modify: `packages/context/src/ingest/local-adapters.test.ts` - -- [ ] **Step 1: Write failing query-history config tests** - -In `packages/context/src/ingest/local-adapters.test.ts`, add: - -```ts - it('maps connection context.queryHistory to historic-sql pull config', async () => { - const project = projectWithConnections({ - warehouse: { - driver: 'postgres', - context: { - queryHistory: { - enabled: true, - windowDays: 45, - minExecutions: 7, - filters: { dropTrivialProbes: true }, - }, - }, - }, - }); - const adapter = { source: 'historic-sql' } as never; - - await expect(localPullConfigForAdapter(project, adapter, 'warehouse')).resolves.toMatchObject({ - dialect: 'postgres', - windowDays: 45, - minExecutions: 7, - filters: { dropTrivialProbes: true }, - }); - }); - - it('prefers context.queryHistory over legacy historicSql', async () => { - const project = projectWithConnections({ - warehouse: { - driver: 'postgres', - historicSql: { enabled: true, dialect: 'postgres', windowDays: 90 }, - context: { queryHistory: { enabled: true, windowDays: 30 } }, - }, - }); - const adapter = { source: 'historic-sql' } as never; - - await expect(localPullConfigForAdapter(project, adapter, 'warehouse')).resolves.toMatchObject({ - dialect: 'postgres', - windowDays: 30, - }); - }); -``` - -In `packages/cli/src/local-adapters.test.ts`, add a test that creates a -Postgres connection with `context.queryHistory.enabled: true`, calls -`createKtxCliLocalIngestAdapters(project, { historicSqlConnectionId: -'warehouse' })`, and expects one adapter with `source === 'historic-sql'`. - -- [ ] **Step 2: Run the failing adapter tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-adapters.test.ts -pnpm --filter @ktx/cli exec vitest run src/local-adapters.test.ts -``` - -Expected: FAIL because both layers only look at `connection.historicSql`. - -- [ ] **Step 3: Add context-query-history mapping in context local adapters** - -In `packages/context/src/ingest/local-adapters.ts`, add: - -```ts -const historicSqlDialectByDriver = new Map([ - ['postgres', 'postgres'], - ['postgresql', 'postgres'], - ['bigquery', 'bigquery'], - ['snowflake', 'snowflake'], -]); - -function queryHistoryRecord(connection: unknown): Record | null { - if (!isRecord(connection)) return null; - const context = isRecord(connection.context) ? connection.context : null; - const queryHistory = isRecord(context?.queryHistory) ? context.queryHistory : null; - return queryHistory; -} - -function queryHistoryPullConfig(connection: unknown): Record | null { - const queryHistory = queryHistoryRecord(connection); - if (queryHistory?.enabled !== true || !isRecord(connection)) return null; - const dialect = historicSqlDialectByDriver.get(String(connection.driver ?? '').toLowerCase()); - if (!dialect) return null; - return { ...queryHistory, dialect }; -} -``` - -In `localPullConfigForAdapter`, replace the historic-SQL block with: - -```ts - if (adapter.source === HISTORIC_SQL_SOURCE_KEY) { - const queryHistory = queryHistoryPullConfig(connection); - if (queryHistory) { - return historicSqlUnifiedPullConfigSchema.parse(queryHistory); - } - const historicSql = isRecord(connection?.historicSql) ? connection.historicSql : null; - if (historicSql?.enabled !== true) { - throw new Error(`Connection "${connectionId}" does not have context.queryHistory.enabled: true`); - } - return historicSqlUnifiedPullConfigSchema.parse({ - ...historicSql, - }); - } -``` - -- [ ] **Step 4: Add context-query-history detection in CLI local adapters** - -In `packages/cli/src/local-adapters.ts`, replace `enabledHistoricSqlDialect` -with: - -```ts -function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery' | 'snowflake' | null { - const direct = historicSqlRecord(connection); - const context = - connection && typeof connection === 'object' && !Array.isArray(connection) - ? (connection as { context?: unknown }).context - : null; - const queryHistory = - context && typeof context === 'object' && !Array.isArray(context) - ? (context as { queryHistory?: unknown }).queryHistory - : null; - const enabled = - queryHistory && typeof queryHistory === 'object' && !Array.isArray(queryHistory) - ? (queryHistory as { enabled?: unknown }).enabled === true - : direct?.enabled === true; - if (!enabled) { - return null; - } - const driver = String((connection as { driver?: unknown })?.driver ?? '').toLowerCase(); - if (driver === 'postgres' || driver === 'postgresql') return 'postgres'; - if (driver === 'bigquery') return 'bigquery'; - if (driver === 'snowflake') return 'snowflake'; - const legacyDialect = String(direct?.dialect ?? '').toLowerCase(); - return legacyDialect === 'postgres' || legacyDialect === 'bigquery' || legacyDialect === 'snowflake' - ? legacyDialect - : null; -} -``` - -- [ ] **Step 5: Run adapter tests again** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-adapters.test.ts -pnpm --filter @ktx/cli exec vitest run src/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit query-history adapter config** - -Run: - -```bash -git add packages/context/src/ingest/local-adapters.ts packages/context/src/ingest/local-adapters.test.ts packages/cli/src/local-adapters.ts packages/cli/src/local-adapters.test.ts -git commit -m "feat(ingest): read connection query history config" -``` - -### Task 6: Remove normal `live-database`, adapter, and scan wording from public output - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/context-build-view.ts` -- Modify: `packages/cli/src/context-build-view.test.ts` -- Modify: `packages/cli/src/setup-sources.ts` -- Modify: `packages/cli/src/setup-sources.test.ts` - -- [ ] **Step 1: Write failing wording tests** - -In `packages/cli/src/context-build-view.test.ts`, change the group label -assertions from `Primary sources:` to `Databases:` and update the running -database detail test to expect `reading schema` instead of `scanning...`. - -Add this setup recovery assertion in the test covering failed initial source -ingest in `packages/cli/src/setup-sources.test.ts`: - -```ts - expect(io.stdout()).toContain(`Run later: ktx ingest ${connectionId}`); - expect(io.stdout()).not.toContain('ktx ingest run --connection-id'); - expect(io.stdout()).not.toContain('--adapter'); -``` - -- [ ] **Step 2: Run failing wording tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/setup-sources.test.ts -t "Databases|reading schema|Run later" -``` - -Expected: FAIL because labels still say `Primary sources`, running database -detail says `scanning...`, and setup recovery still suggests adapter-backed -ingest. - -- [ ] **Step 3: Update public render labels** - -In `packages/cli/src/context-build-view.ts`, change: - -```ts - ...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width), -``` - -to: - -```ts - ...renderTargetGroup('Databases', state.primarySources, state.frame, styled, width), -``` - -In `targetDetail`, change: - -```ts - ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...'); -``` - -to: - -```ts - ?? (target.target.operation === 'database-ingest' ? 'reading schema' : 'ingesting...'); -``` - -Update type comparisons in this file from `'scan'` to `'database-ingest'` for -public target operation checks. - -- [ ] **Step 4: Update setup source recovery text** - -In `packages/cli/src/setup-sources.ts`, replace: - -```ts - input.io.stdout.write(`│ Run later: ktx ingest run --connection-id ${input.connectionId} --adapter \n`); -``` - -with: - -```ts - input.io.stdout.write(`│ Run later: ktx ingest ${input.connectionId}\n`); -``` - -- [ ] **Step 5: Run wording tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/setup-sources.test.ts -``` - -Expected: PASS after updating existing snapshots for the new public operation -name. - -- [ ] **Step 6: Commit public wording cleanup** - -Run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/setup-sources.ts packages/cli/src/setup-sources.test.ts -git commit -m "fix(cli): use public ingest wording" -``` - -### Task 7: Stop generating adapter allow-list entries in normal config - -**Files:** -- Modify: `packages/context/src/project/config.ts` -- Modify: `packages/context/src/project/config.test.ts` -- Modify: `packages/cli/src/setup-sources.ts` -- Modify: `packages/cli/src/setup-sources.test.ts` -- Modify: `packages/cli/src/setup-databases.ts` -- Modify: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Write failing config tests** - -In `packages/context/src/project/config.test.ts`, update default assertions: - -```ts - ingest: { - adapters: [], -``` - -and: - -```ts - expect(serialized).not.toContain('live-database'); - expect(parsed.ingest.adapters).toEqual([]); -``` - -In setup database and source tests, add assertions after generated config is -read: - -```ts - expect(configText).not.toContain('live-database'); - expect(configText).not.toContain('historic-sql'); - expect(configText).not.toMatch(/^\s+adapters:/m); -``` - -- [ ] **Step 2: Run failing config tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts src/setup-sources.test.ts -``` - -Expected: FAIL because defaults and setup still write adapter entries. - -- [ ] **Step 3: Change default config** - -In `packages/context/src/project/config.ts`, change: - -```ts - adapters: ['live-database', 'lookml', 'metabase', 'metricflow', 'notion'], -``` - -to: - -```ts - adapters: [], -``` - -- [ ] **Step 4: Stop setup from appending normal source adapters** - -In `packages/cli/src/setup-sources.ts`, change `writeSourceConnection` so the -new config only writes `connections`: - -```ts - const nextConfig = { - ...project.config, - connections: { - ...project.config.connections, - [connectionId]: connection, - }, - }; -``` - -Remove the `adapters` mutation in that helper and remove adapter rollback code -that only exists to undo automatic adapter appends. - -- [ ] **Step 5: Stop Historic SQL setup from appending adapters** - -In `packages/cli/src/setup-databases.ts`, change `ensureHistoricSqlIngestDefaults` -so it only raises `ingest.workUnits.maxConcurrency`: - -```ts -async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { - const project = await loadKtxProject({ projectDir }); - const maxConcurrency = Math.max( - project.config.ingest.workUnits.maxConcurrency, - HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY, - ); - if (maxConcurrency === project.config.ingest.workUnits.maxConcurrency) { - return; - } - await writeFile( - project.configPath, - serializeKtxProjectConfig({ - ...project.config, - ingest: { - ...project.config.ingest, - workUnits: { - ...project.config.ingest.workUnits, - maxConcurrency, - }, - }, - }), - 'utf-8', - ); -} -``` - -- [ ] **Step 6: Run config tests again** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts src/setup-sources.test.ts src/public-ingest.test.ts -``` - -Expected: PASS. Public source ingest still works because Task 4 synthesizes the -inferred adapter for public connection-centric runs. - -- [ ] **Step 7: Commit config cleanup** - -Run: - -```bash -git add packages/context/src/project/config.ts packages/context/src/project/config.test.ts packages/cli/src/setup-sources.ts packages/cli/src/setup-sources.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "fix(config): stop generating ingest adapter allow lists" -``` - -### Task 8: Update public docs and script assertions - -**Files:** -- Modify: `README.md` -- Modify: `scripts/examples-docs.test.mjs` -- Modify: `scripts/package-artifacts.mjs` -- Modify: `scripts/package-artifacts.test.mjs` -- Modify: `scripts/installed-live-database-smoke.mjs` -- Modify: `scripts/installed-live-database-smoke.test.mjs` - -- [ ] **Step 1: Write failing docs assertion changes** - -In `scripts/examples-docs.test.mjs`, replace assertions that require -`ktx scan `, `ktx scan [options]`, and -`live-database/` in normal README output with assertions for: - -```js -assert.match(buildingContext, /ktx ingest /); -assert.match(buildingContext, /ktx ingest --all/); -assert.doesNotMatch(rootReadme, /live-database\//); -assert.doesNotMatch(rootReadme, /ktx scan/); -``` - -In package artifact smoke tests, change normal public smoke labels from -`ktx scan structural` and `ktx scan enriched` to `ktx ingest fast` and -`ktx ingest deep`. - -- [ ] **Step 2: Run failing docs/script tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs scripts/package-artifacts.test.mjs scripts/installed-live-database-smoke.test.mjs -``` - -Expected: FAIL because docs and smoke scripts still mention `scan` and -`live-database`. - -- [ ] **Step 3: Update README public examples** - -In `README.md`, replace normal context-build examples: - -```md -ktx scan warehouse --project-dir "$PROJECT_DIR" -``` - -with: - -```md -ktx ingest warehouse --project-dir "$PROJECT_DIR" --fast -``` - -Replace enriched examples with: - -```md -ktx ingest warehouse --project-dir "$PROJECT_DIR" --deep -``` - -Replace adapter-backed ingest examples for normal users with: - -```md -ktx ingest notion --project-dir "$PROJECT_DIR" -``` - -Keep internal artifact paths only in sections explicitly labeled as debug or -implementation details. - -- [ ] **Step 4: Update smoke scripts to use public ingest** - -In `scripts/package-artifacts.mjs`, replace public scan smoke invocations with: - -```js -const structuralScan = await run('pnpm', [ - 'exec', - 'ktx', - 'ingest', - 'warehouse', - '--project-dir', - projectDir, - '--fast', - '--no-input', -]); -``` - -and: - -```js -const enrichedScan = await run('pnpm', [ - 'exec', - 'ktx', - 'ingest', - 'warehouse', - '--project-dir', - projectDir, - '--deep', - '--no-input', -]); -``` - -Update expected output matches from `Mode: structural` and `Mode: enriched` to -the public result summary that `runKtxPublicIngest` prints, for example -`Database schema` or `database-schema done` depending on the final Task 4 -rendering. - -In `scripts/installed-live-database-smoke.mjs`, keep the file name if renaming -would churn scripts, but change the public CLI invocation from adapter-backed -`ktx ingest run --adapter live-database` to: - -```js -return ['exec', 'ktx', 'ingest', connectionId, '--project-dir', projectDir, '--fast', '--no-input']; -``` - -- [ ] **Step 5: Run docs/script tests again** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs scripts/package-artifacts.test.mjs scripts/installed-live-database-smoke.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 6: Commit docs and smoke cleanup** - -Run: - -```bash -git add README.md scripts/examples-docs.test.mjs scripts/package-artifacts.mjs scripts/package-artifacts.test.mjs scripts/installed-live-database-smoke.mjs scripts/installed-live-database-smoke.test.mjs -git commit -m "docs: document public ingest command" -``` - -### Task 9: Run final verification - -**Files:** -- Verify only. - -- [ ] **Step 1: Run focused CLI and context tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/dev.test.ts src/public-ingest.test.ts src/context-build-view.test.ts src/ingest.test.ts src/local-adapters.test.ts src/setup-sources.test.ts src/setup-databases.test.ts -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts src/ingest/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run workspace type checks for touched packages** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run docs and script tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs scripts/package-artifacts.test.mjs scripts/installed-live-database-smoke.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only pre-existing findings unrelated to the files changed -in this plan. - -- [ ] **Step 5: Commit any verification-only expectation fixes** - -If verification required expectation-only changes, run: - -```bash -git add packages/cli/src packages/context/src scripts README.md -git commit -m "test: align ingest surface expectations" -``` - -If there were no changes, do not create an empty commit. - -## Self-review notes - -Spec coverage in this plan: - -- Covers `ktx ingest ` and `ktx ingest --all`. -- Covers public `--fast` and `--deep` mapping to structural and enriched scan - internals. -- Covers hidden legacy `scan`, `ingest run`, and `ingest watch` help behavior. -- Covers adapter allow-list bypass for public connection-centric ingest. -- Covers current-run query-history enablement and window override. -- Covers normal generated config removing adapter allow lists. -- Covers normal help, docs, setup recovery text, and progress wording. - -Known v1-blocking work not included in this plan: - -- Setup must ask for and store `connections..context.depth`. -- Setup readiness must treat fast and deep contexts differently. -- Setup context state must remove detach, watch, resume, stop, paused, and - background subprocess behavior. -- Config rewrite must migrate legacy `connection.historicSql` into - `connection.context.queryHistory`. -- Config/setup validation must reject connection ids that collide with - surviving ingest subcommands. - -Placeholder scan: no task uses deferred code markers or unnamed edge handling. -Each implementation task names exact files, tests, commands, and the concrete -code shape to add. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-closure.md deleted file mode 100644 index f7e293b4..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-closure.md +++ /dev/null @@ -1,1865 +0,0 @@ -# Unified Ingest V1 Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking gaps in the unified `ktx ingest` -redesign after the public CLI surface slice. - -**Architecture:** Keep the implemented connection-centric `ktx ingest` command -as the public entry point, and add the missing policy layer around it: depth -readiness, setup depth defaults, foreground-only context builds, legacy -query-history config migration, and reserved connection-id validation. Put -shared depth policy in a small CLI helper so public ingest and setup use the -same rules. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages, -YAML config serialization. - ---- - -## Current audit - -The implemented `2026-05-13-unified-ingest-public-cli-surface.md` slice covers -the first public-surface layer: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` are routed in - `packages/cli/src/commands/ingest-commands.ts`. -- `ktx scan`, `ktx ingest run`, and `ktx ingest watch` are hidden from normal - help while still callable as debug or stored-report surfaces. -- `packages/cli/src/public-ingest.ts` plans database targets before source - targets, maps fast/deep to scan internals, runs query history after schema - ingest, and bypasses adapter allow lists for inferred public adapters. -- `packages/context/src/project/config.ts` no longer generates normal - `ingest.adapters` entries. -- README and smoke scripts now document public `ktx ingest` examples. - -Remaining v1-blocking gaps against the original spec: - -- Explicit or stored `deep` currently does not fail before work starts when - model, scan-enrichment, or scan-embedding config is missing. -- Deep database ingest always passes `detectRelationships: true` instead of - honoring `scan.relationships.enabled`. -- `ktx setup` does not ask for or store - `connections..context.depth`, and still forces enriched context builds. -- Setup readiness still requires enriched AI artifacts for every database - context and blocks all context builds when AI inputs are missing, even when - the effective depth is `fast`. -- Setup still writes legacy `connections..historicSql` instead of - canonical `connections..context.queryHistory`. -- Legacy `historicSql` migration is not performed by setup. -- Context build still supports detach, watch, resume, stop, paused/detached - state, and background subprocesses. -- Setup/config validation does not reject connection ids that collide with - surviving ingest subcommands: `status`, `replay`, `run`, and `watch`. - -Non-blocking gaps after this plan: - -- Internal package names, adapter names, raw-source artifact paths, and debug - output can still use `scan`, `live-database`, and `historic-sql`. -- The hidden debug `ktx scan` and hidden adapter-backed `ktx ingest run` - commands can remain until an explicit internal cleanup plan removes them. -- MCP scan tool names and low-level scan tests can continue to use scan - terminology because the original spec only requires normal CLI/help/output - cleanup for v1. - -## File structure - -- Create `packages/cli/src/ingest-depth.ts`: shared database driver detection, - depth defaults, deep-readiness checks, and context-depth config helpers. -- Modify `packages/cli/src/public-ingest.ts`: use shared depth policy, add - preflight failures, and pass relationship detection only when enabled. -- Modify `packages/cli/src/public-ingest.test.ts`: cover deep preflight, - per-target `--all` isolation, and relationship flag mapping. -- Modify `packages/cli/src/setup-databases.ts`: write - `context.queryHistory`, migrate legacy `historicSql`, and read the canonical - shape for query-history probe behavior. -- Modify `packages/cli/src/setup-databases.test.ts`: replace legacy - `historicSql` expectations with canonical `context.queryHistory` - expectations and migration coverage. -- Modify `packages/cli/src/setup-context.ts`: prompt/store context depth, - remove foreground detach/background logic, normalize legacy state, and make - readiness depth-aware. -- Modify `packages/cli/src/setup-context.test.ts`: cover fast readiness, deep - readiness, stored depth, foreground-only state, and removed watch/detach - affordances. -- Modify `packages/cli/src/context-build-view.ts`: remove detach hint and - background subprocess support. -- Modify `packages/cli/src/context-build-view.test.ts`: assert foreground-only - progress copy. -- Modify `packages/context/src/project/config.ts`: reject reserved connection - ids during config parse. -- Modify `packages/context/src/project/index.ts`: export reserved-id helpers - for setup flows. -- Modify `packages/context/src/project/config.test.ts`: cover reserved - connection ids. -- Modify `packages/cli/src/setup-sources.ts`, - `packages/cli/src/setup-sources.test.ts`, - `packages/cli/src/commands/setup-commands.ts`, and - `packages/cli/src/index.test.ts`: reject reserved ids during setup prompts - and setup flags. - -## Tasks - -### Task 1: Add depth policy and public deep preflight - -**Files:** -- Create: `packages/cli/src/ingest-depth.ts` -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write failing public ingest preflight tests** - -In `packages/cli/src/public-ingest.test.ts`, add this helper after -`projectWithConnections`: - -```ts -function deepReadyProject(connections: KtxProjectConfig['connections'], relationshipsEnabled = true): KtxPublicIngestProject { - const config = buildDefaultKtxProjectConfig('warehouse'); - return { - projectDir: '/tmp/project', - config: { - ...config, - connections, - llm: { - ...config.llm, - provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret - models: { default: 'gpt-test' }, - }, - scan: { - ...config.scan, - enrichment: { - mode: 'llm', - embeddings: { - backend: 'openai', - model: 'text-embedding-3-small', - dimensions: 1536, - }, - }, - relationships: { - ...config.scan.relationships, - enabled: relationshipsEnabled, - }, - }, - }, - }; -} -``` - -Add these tests inside the `buildPublicIngestPlan` describe block: - -```ts - it('records a preflight failure for deep database ingest when readiness config is missing', () => { - const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }); - - const plan = buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - queryHistory: 'default', - }); - - expect(plan.targets[0]).toMatchObject({ - connectionId: 'warehouse', - databaseDepth: 'deep', - preflightFailure: - 'warehouse requires deep ingest readiness: model configuration, scan enrichment mode, scan embeddings. Run ktx setup or rerun with --fast.', - }); - }); - - it('honors scan.relationships.enabled when planning deep database ingest', () => { - const plan = buildPublicIngestPlan( - deepReadyProject({ warehouse: { driver: 'postgres', context: { depth: 'deep' } } }, false), - { - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - queryHistory: 'default', - }, - ); - - expect(plan.targets[0]).toMatchObject({ - connectionId: 'warehouse', - databaseDepth: 'deep', - detectRelationships: false, - }); - }); -``` - -Add this test inside the `runKtxPublicIngest` describe block: - -```ts - it('fails deep-readiness targets before work starts while continuing independent --all targets', async () => { - const io = makeIo(); - const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - docs: { driver: 'notion' }, - }); - const runScan = vi.fn(async () => 0); - const runIngest = vi.fn(async () => 0); - - await expect( - runKtxPublicIngest( - { command: 'run', projectDir: '/tmp/project', all: true, json: false, inputMode: 'disabled' }, - io.io, - { loadProject: vi.fn(async () => project), runScan, runIngest }, - ), - ).resolves.toBe(1); - - expect(runScan).not.toHaveBeenCalled(); - expect(runIngest).toHaveBeenCalledWith( - expect.objectContaining({ command: 'run', connectionId: 'docs', adapter: 'notion' }), - expect.anything(), - ); - expect(io.stdout()).toContain('warehouse requires deep ingest readiness'); - }); -``` - -- [ ] **Step 2: Run the failing public ingest tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "preflight failure|relationships.enabled|deep-readiness" -``` - -Expected: FAIL because `preflightFailure`, shared depth policy, and -relationship-aware deep planning do not exist. - -- [ ] **Step 3: Create shared depth policy** - -Create `packages/cli/src/ingest-depth.ts`: - -```ts -import type { KtxProjectConfig, KtxProjectConnectionConfig } from '@ktx/context/project'; - -export type KtxDatabaseContextDepth = 'fast' | 'deep'; - -export const KTX_DATABASE_DRIVER_IDS = new Set([ - 'sqlite', - 'postgres', - 'postgresql', - 'mysql', - 'clickhouse', - 'sqlserver', - 'bigquery', - 'snowflake', -]); - -export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string { - return String(connection.driver ?? '').trim().toLowerCase(); -} - -export function isDatabaseDriver(driver: string): boolean { - return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase()); -} - -export function connectionContextRecord(connection: KtxProjectConnectionConfig): Record { - const context = connection.context; - return typeof context === 'object' && context !== null && !Array.isArray(context) - ? (context as Record) - : {}; -} - -export function databaseContextDepth(connection: KtxProjectConnectionConfig): KtxDatabaseContextDepth | undefined { - const depth = connectionContextRecord(connection).depth; - return depth === 'fast' || depth === 'deep' ? depth : undefined; -} - -export function withDatabaseContextDepth( - connection: KtxProjectConnectionConfig, - depth: KtxDatabaseContextDepth, -): KtxProjectConnectionConfig { - return { - ...connection, - context: { - ...connectionContextRecord(connection), - depth, - }, - }; -} - -export function deepReadinessGaps(config: KtxProjectConfig): string[] { - const gaps: string[] = []; - if (config.llm.provider.backend === 'none' || !config.llm.models.default) { - gaps.push('model configuration'); - } - - if (config.scan.enrichment.mode !== 'llm') { - gaps.push('scan enrichment mode'); - } - - const embeddings = config.scan.enrichment.embeddings; - if ( - !embeddings || - embeddings.backend === 'none' || - embeddings.backend === 'deterministic' || - !embeddings.model || - embeddings.dimensions <= 0 - ) { - gaps.push('scan embeddings'); - } - - return gaps; -} - -export function recommendedDatabaseContextDepth(config: KtxProjectConfig): KtxDatabaseContextDepth { - return deepReadinessGaps(config).length === 0 ? 'deep' : 'fast'; -} -``` - -- [ ] **Step 4: Apply preflight and relationship policy in public ingest** - -In `packages/cli/src/public-ingest.ts`, replace the local depth and warehouse -driver definitions with imports: - -```ts -import { - type KtxDatabaseContextDepth, - databaseContextDepth, - deepReadinessGaps, - isDatabaseDriver, - normalizeConnectionDriver, -} from './ingest-depth.js'; -``` - -Change `type KtxPublicIngestDepth = 'fast' | 'deep';` to: - -```ts -type KtxPublicIngestDepth = KtxDatabaseContextDepth; -``` - -Remove the local `warehouseDrivers`, `normalizedDriver`, -`connectionContext`, and `storedDepth` helpers. - -Add these fields to `KtxPublicIngestPlanTarget`: - -```ts - detectRelationships?: boolean; - preflightFailure?: string; -``` - -In `resolveDatabaseTargetOptions`, replace: - -```ts - let depth = input.args.depth ?? depthFromLegacyScanMode(input.args.scanMode) ?? storedDepth(input.connection) ?? 'fast'; -``` - -with: - -```ts - let depth = - input.args.depth ?? depthFromLegacyScanMode(input.args.scanMode) ?? databaseContextDepth(input.connection) ?? 'fast'; -``` - -Change `targetForConnection` to accept the project config: - -```ts -function targetForConnection( - connectionId: string, - connection: KtxProjectConnectionConfig, - projectConfig: KtxPublicIngestProject['config'], - args: { - depth?: KtxPublicIngestDepth; - queryHistory?: KtxPublicIngestQueryHistoryFlag; - queryHistoryWindowDays?: number; - scanMode?: Extract['mode']; - }, - warnings: string[], -): KtxPublicIngestPlanTarget { -``` - -Use shared driver detection: - -```ts - const driver = normalizeConnectionDriver(connection); -``` - -Replace the warehouse branch with: - -```ts - if (isDatabaseDriver(driver)) { - const options = resolveDatabaseTargetOptions({ connectionId, driver, connection, args, warnings }); - const gaps = options.databaseDepth === 'deep' ? deepReadinessGaps(projectConfig) : []; - return { - connectionId, - driver, - operation: 'database-ingest', - debugCommand: `ktx ingest ${connectionId} --debug`, - detectRelationships: options.databaseDepth === 'deep' && projectConfig.scan.relationships.enabled, - ...(gaps.length > 0 - ? { - preflightFailure: `${connectionId} requires deep ingest readiness: ${gaps.join( - ', ', - )}. Run ktx setup or rerun with --fast.`, - } - : {}), - ...options, - }; - } -``` - -In `buildPublicIngestPlan`, pass `project.config`: - -```ts - const targets = selected.map(([connectionId, connection]) => - targetForConnection(connectionId, connection, project.config, args, warnings), - ); -``` - -At the start of `executePublicIngestTarget`, add: - -```ts - if (target.preflightFailure) { - return { - connectionId: target.connectionId, - driver: target.driver, - steps: defaultSteps(target).map((step) => - step.operation === 'database-schema' - ? { - ...step, - status: 'failed', - detail: target.preflightFailure, - } - : step, - ), - }; - } -``` - -Change database scan args from: - -```ts - detectRelationships: target.databaseDepth === 'deep' ? true : false, -``` - -to: - -```ts - detectRelationships: target.detectRelationships === true, -``` - -- [ ] **Step 5: Run public ingest tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit depth preflight** - -Run: - -```bash -git add packages/cli/src/ingest-depth.ts packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "feat(cli): preflight deep public ingest readiness" -``` - -### Task 2: Store query history under `context.queryHistory` - -**Files:** -- Modify: `packages/cli/src/setup-databases.ts` -- Test: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Write failing setup query-history shape tests** - -In `packages/cli/src/setup-databases.test.ts`, update the existing tests that -expect `config.connections..historicSql` so they expect -`config.connections..context.queryHistory` instead. - -Add this test near the existing Historic SQL setup tests: - -```ts - it('migrates legacy historicSql to context.queryHistory during database setup', async () => { - await writeProjectConfig(tempDir, { - connections: { - warehouse: { - driver: 'postgres', - readonly: true, - historicSql: { - enabled: true, - dialect: 'postgres', - windowDays: 45, - minExecutions: 9, - concurrency: 3, - staleArchiveAfterDays: 120, - filters: { - dropTrivialProbes: true, - serviceAccounts: { mode: 'exclude', patterns: ['^svc_'] }, - orchestrators: { mode: 'exclude', patterns: ['airflow'] }, - dropFailedBelow: 2, - }, - redactionPatterns: ['(?i)secret'], - }, - }, - }, - }); - - const io = makeIo(); - - await expect( - runKtxSetupDatabasesStep( - { - projectDir: tempDir, - inputMode: 'disabled', - databaseConnectionIds: ['warehouse'], - skipConnectionTest: true, - skipInitialScan: true, - }, - io.io, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse.historicSql).toBeUndefined(); - expect(config.connections.warehouse.context).toMatchObject({ - queryHistory: { - enabled: true, - windowDays: 45, - minExecutions: 9, - concurrency: 3, - staleArchiveAfterDays: 120, - filters: { - dropTrivialProbes: true, - serviceAccounts: { mode: 'exclude', patterns: ['^svc_'] }, - orchestrators: { mode: 'exclude', patterns: ['airflow'] }, - dropFailedBelow: 2, - }, - redactionPatterns: ['(?i)secret'], - }, - }); - }); -``` - -- [ ] **Step 2: Run failing setup database tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts -t "queryHistory|historicSql|migrates legacy" -``` - -Expected: FAIL because setup still writes and reads `historicSql`. - -- [ ] **Step 3: Add query-history config helpers** - -In `packages/cli/src/setup-databases.ts`, add these helpers after -`historicSqlConfigRecord`: - -```ts -function contextRecord(connection: KtxProjectConnectionConfig | undefined): Record { - const context = connection?.context; - return context && typeof context === 'object' && !Array.isArray(context) ? (context as Record) : {}; -} - -function queryHistoryConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record | null { - const queryHistory = contextRecord(connection).queryHistory; - return queryHistory && typeof queryHistory === 'object' && !Array.isArray(queryHistory) - ? (queryHistory as Record) - : null; -} - -function stripLegacyHistoricSql(connection: KtxProjectConnectionConfig): KtxProjectConnectionConfig { - const { historicSql: _historicSql, ...rest } = connection as KtxProjectConnectionConfig & { - historicSql?: unknown; - }; - return rest; -} - -function withQueryHistoryConfig( - connection: KtxProjectConnectionConfig, - queryHistory: Record, -): KtxProjectConnectionConfig { - return { - ...stripLegacyHistoricSql(connection), - context: { - ...contextRecord(connection), - queryHistory, - }, - }; -} - -function migrateLegacyHistoricSqlConnection(connection: KtxProjectConnectionConfig): KtxProjectConnectionConfig { - const existingQueryHistory = queryHistoryConfigRecord(connection); - const legacy = historicSqlConfigRecord(connection); - if (existingQueryHistory || !legacy) { - return existingQueryHistory ? stripLegacyHistoricSql(connection) : connection; - } - const { dialect: _dialect, ...queryHistory } = legacy; - return withQueryHistoryConfig(connection, queryHistory); -} -``` - -- [ ] **Step 4: Write canonical query-history config from setup** - -In `applyHistoricSqlConfigToConnection`, replace each returned `historicSql` -object with a call to `withQueryHistoryConfig(input.connection, queryHistory)`. - -For disabled query history, return: - -```ts - return withQueryHistoryConfig(input.connection, { ...existing, enabled: false }); -``` - -For Postgres enabled query history, return: - -```ts - return withQueryHistoryConfig(input.connection, { - ...common, - minExecutions: input.args.historicSqlMinExecutions ?? 5, - }); -``` - -For BigQuery and Snowflake enabled query history, return: - -```ts - return withQueryHistoryConfig(input.connection, { - ...common, - windowDays: input.args.historicSqlWindowDays ?? 90, - redactionPatterns: input.args.historicSqlRedactionPatterns ?? [], - }); -``` - -Change `common` so it does not include `dialect`: - -```ts - const common: Record = { - ...existing, - enabled: true, - filters: historicSqlFiltersForSetup(input.args.historicSqlServiceAccountPatterns), - }; -``` - -Where `existing` is built, prefer canonical config: - -```ts - const existing = queryHistoryConfigRecord(input.connection) ?? historicSqlConfigRecord(input.connection) ?? {}; -``` - -- [ ] **Step 5: Migrate legacy blocks during setup writes** - -In `writeConnectionConfig`, normalize all project connections before writing: - -```ts - const migratedConnections = Object.fromEntries( - Object.entries(project.config.connections).map(([connectionId, connection]) => [ - connectionId, - migrateLegacyHistoricSqlConnection(connection), - ]), - ); - const nextConnection = migrateLegacyHistoricSqlConnection(input.connection); - const config = { - ...project.config, - connections: { - ...migratedConnections, - [input.connectionId]: nextConnection, - }, - }; -``` - -Change the post-write Historic SQL defaults check to read canonical config: - -```ts - const queryHistory = queryHistoryConfigRecord(nextConnection); - if (queryHistory?.enabled === true) { - await ensureHistoricSqlIngestDefaults(input.projectDir); - } -``` - -Update `historicSqlConfigRecord` callers used for probe decisions to prefer -`queryHistoryConfigRecord(connection)` and only fall back to legacy -`historicSqlConfigRecord(connection)`. - -- [ ] **Step 6: Run setup database tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts -``` - -Expected: PASS after updating assertions from `historicSql` to -`context.queryHistory`. - -- [ ] **Step 7: Commit setup query-history config** - -Run: - -```bash -git add packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "feat(setup): store query history in connection context" -``` - -### Task 3: Store setup database context depth - -**Files:** -- Modify: `packages/cli/src/setup-context.ts` -- Test: `packages/cli/src/setup-context.test.ts` - -- [ ] **Step 1: Write failing setup depth tests** - -In `packages/cli/src/setup-context.test.ts`, replace the test named -`does not treat schema-only scan shards as completed setup context` with: - -```ts - it('treats fast database context as ready from schema manifest shards without AI artifacts', async () => { - await writeReadyProject(tempDir, { - connections: { - warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } }, - }, - llm: { provider: { backend: 'none' }, models: {} }, - scan: { enrichment: { mode: 'none' } }, - }); - await mkdir(join(tempDir, 'semantic-layer', 'warehouse', '_schema'), { recursive: true }); - await writeFile(join(tempDir, 'semantic-layer', 'warehouse', '_schema', 'public.yaml'), 'tables: {}\n'); - await writeScanReport(tempDir, '2026-05-09T10:00:00.000Z', { - mode: 'structural', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'], - }); - const io = makeIo(); - const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { runContextBuild: runContextBuildMock }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(runContextBuildMock).not.toHaveBeenCalled(); - expect(io.stdout()).toContain('Existing context artifacts were found from setup ingest.'); - }); -``` - -Add these tests near the existing setup context build tests: - -```ts - it('stores fast context depth non-interactively when deep readiness is missing', async () => { - await writeReadyProject(tempDir, { - connections: { warehouse: { driver: 'postgres', readonly: true } }, - llm: { provider: { backend: 'none' }, models: {} }, - scan: { enrichment: { mode: 'none' } }, - }); - const io = makeIo(); - const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); - const verifyContextReady = vi.fn(async () => ({ - ready: true, - agentContextReady: true, - semanticSearchReady: true, - details: ['ready'], - })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { runContextBuild: runContextBuildMock, verifyContextReady }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse.context).toMatchObject({ depth: 'fast' }); - expect(runContextBuildMock).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ projectDir: tempDir, inputMode: 'disabled' }), - expect.anything(), - expect.anything(), - ); - expect(runContextBuildMock.mock.calls[0]?.[1]).not.toMatchObject({ - scanMode: 'enriched', - detectRelationships: true, - }); - }); - - it('prompts for database context depth after final readiness is known', async () => { - await writeReadyProject(tempDir, { - connections: { warehouse: { driver: 'postgres', readonly: true } }, - llm: { - provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret - models: { default: 'gpt-test' }, - }, - scan: { - enrichment: { - mode: 'llm', - embeddings: { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536 }, - }, - }, - }); - const io = makeIo(); - const select = vi.fn(async () => 'deep'); - const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); - const verifyContextReady = vi.fn(async () => ({ - ready: true, - agentContextReady: true, - semanticSearchReady: true, - details: ['ready'], - })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'auto' }, - io.io, - { - prompts: { select, cancel: vi.fn() }, - runContextBuild: runContextBuildMock, - verifyContextReady, - }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(select).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('How much database context should KTX build?'), - }), - ); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse.context).toMatchObject({ depth: 'deep' }); - }); -``` - -- [ ] **Step 2: Run failing setup depth tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-context.test.ts -t "fast database context|stores fast context depth|prompts for database context depth" -``` - -Expected: FAIL because setup has no depth prompt/storage and still gates all -context builds on AI readiness. - -- [ ] **Step 3: Add setup depth helpers** - -In `packages/cli/src/setup-context.ts`, add imports: - -```ts -import { - type KtxDatabaseContextDepth, - databaseContextDepth, - deepReadinessGaps, - isDatabaseDriver, - normalizeConnectionDriver, - recommendedDatabaseContextDepth, - withDatabaseContextDepth, -} from './ingest-depth.js'; -``` - -Add these helpers after `listContextTargets`: - -```ts -function databaseConnectionsNeedingDepth(project: KtxLocalProject): string[] { - return Object.entries(project.config.connections) - .filter(([, connection]) => isDatabaseDriver(normalizeConnectionDriver(connection))) - .filter(([, connection]) => databaseContextDepth(connection) === undefined) - .map(([connectionId]) => connectionId) - .sort((left, right) => left.localeCompare(right)); -} - -async function writeDatabaseContextDepths( - project: KtxLocalProject, - connectionIds: string[], - depth: KtxDatabaseContextDepth, -): Promise { - if (connectionIds.length === 0) { - return project; - } - const nextConnections = { ...project.config.connections }; - for (const connectionId of connectionIds) { - const connection = nextConnections[connectionId]; - if (connection) { - nextConnections[connectionId] = withDatabaseContextDepth(connection, depth); - } - } - const nextConfig = { ...project.config, connections: nextConnections }; - await writeFile(project.configPath, serializeKtxProjectConfig(nextConfig), 'utf-8'); - return await loadKtxProject({ projectDir: project.projectDir }); -} - -async function ensureSetupDatabaseContextDepths(input: { - project: KtxLocalProject; - args: KtxSetupContextStepArgs; - prompts: KtxSetupContextPromptAdapter; -}): Promise { - const missingDepthConnectionIds = databaseConnectionsNeedingDepth(input.project); - if (missingDepthConnectionIds.length === 0) { - return input.project; - } - - const recommended = recommendedDatabaseContextDepth(input.project.config); - if (input.args.inputMode === 'disabled') { - return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, recommended); - } - - const deepReady = deepReadinessGaps(input.project.config).length === 0; - const options = - recommended === 'deep' - ? [ - { value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' }, - { value: 'fast', label: 'Fast: schema only, no AI, quickest' }, - { value: 'back', label: 'Back' }, - ] - : [ - { value: 'fast', label: 'Fast: schema only, no AI, quickest' }, - { value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' }, - { value: 'back', label: 'Back' }, - ]; - - const choice = await input.prompts.select({ - message: - 'How much database context should KTX build?\n\n' + - (deepReady - ? 'Deep is available because model, embedding, and scan enrichment are configured.' - : 'Fast is recommended because model, embedding, or scan enrichment is not configured.'), - options, - }); - if (choice === 'back') { - return 'back'; - } - return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, choice as KtxDatabaseContextDepth); -} -``` - -- [ ] **Step 4: Use stored depth in setup context builds** - -In `runKtxSetupContextStep`, after loading `project` and before reading the -existing setup context state, change `const project` to `let project`, then -add: - -```ts - const depthProject = await ensureSetupDatabaseContextDepths({ - project, - args, - prompts: deps.prompts ?? createPromptAdapter(), - }); - if (depthProject === 'back') { - return { status: 'back', projectDir: args.projectDir }; - } - project = depthProject; -``` - -Remove the unconditional missing-capability gate: - -```ts - const missing = missingCapabilities(project); - if (missing.length > 0) { - if (args.allowEmpty === true) { - return { status: 'skipped', projectDir: args.projectDir }; - } - writeMissingCapabilities(missing, io); - return { status: 'missing-input', projectDir: args.projectDir }; - } -``` - -Replace it with a deep-only target preflight gate: - -```ts - const preflightPlan = buildPublicIngestPlan(project, { projectDir: project.projectDir, all: true }); - const preflightFailures = preflightPlan.targets.flatMap((target) => - target.preflightFailure ? [`${target.connectionId}: ${target.preflightFailure}`] : [], - ); - if (preflightFailures.length > 0) { - if (args.allowEmpty === true) { - return { status: 'skipped', projectDir: args.projectDir }; - } - writeMissingCapabilities(preflightFailures, io); - return { status: 'missing-input', projectDir: args.projectDir }; - } -``` - -In `runBuild`, change the `runContextBuild` call from: - -```ts - { - projectDir: args.projectDir, - inputMode: args.inputMode, - scanMode: 'enriched', - detectRelationships: true, - }, -``` - -to: - -```ts - { - projectDir: args.projectDir, - inputMode: args.inputMode, - }, -``` - -- [ ] **Step 5: Run setup context depth tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-context.test.ts -t "fast database context|stores fast context depth|prompts for database context depth" -``` - -Expected: PASS after updating helper fixtures to accept the override shape in -the new tests. - -- [ ] **Step 6: Commit setup context depth** - -Run: - -```bash -git add packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts -git commit -m "feat(setup): store database context depth" -``` - -### Task 4: Make setup readiness depth-aware - -**Files:** -- Modify: `packages/cli/src/setup-context.ts` -- Test: `packages/cli/src/setup-context.test.ts` - -- [ ] **Step 1: Write failing depth-aware readiness tests** - -In `packages/cli/src/setup-context.test.ts`, add: - -```ts - it('requires completed relationships for deep context when relationship discovery is enabled', async () => { - await writeReadyProject(tempDir, { - connections: { - warehouse: { driver: 'postgres', readonly: true, context: { depth: 'deep' } }, - }, - scan: { relationships: { enabled: true } }, - }); - await mkdir(join(tempDir, 'semantic-layer', 'dbt-main'), { recursive: true }); - await writeFile(join(tempDir, 'semantic-layer', 'dbt-main', 'mart_revenue_daily.yaml'), 'name: mart_revenue_daily\n'); - await writeReadyEnrichedScanReport(tempDir, '2026-05-09T10:00:00.000Z', { - completedStages: ['descriptions', 'embeddings'], - relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 }, - }); - const io = makeIo(); - const runContextBuildMock = vi.fn(async () => { - await writeReadyEnrichedScanReport(tempDir, '2026-05-09T10:01:00.000Z', { - completedStages: ['descriptions', 'embeddings', 'relationships'], - relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 }, - }); - return { exitCode: 0 }; - }); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { runContextBuild: runContextBuildMock }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(runContextBuildMock).toHaveBeenCalledOnce(); - }); - - it('does not require relationships for deep context when relationship discovery is disabled', async () => { - await writeReadyProject(tempDir, { - connections: { - warehouse: { driver: 'postgres', readonly: true, context: { depth: 'deep' } }, - }, - scan: { relationships: { enabled: false } }, - }); - await mkdir(join(tempDir, 'semantic-layer', 'dbt-main'), { recursive: true }); - await writeFile(join(tempDir, 'semantic-layer', 'dbt-main', 'mart_revenue_daily.yaml'), 'name: mart_revenue_daily\n'); - await writeReadyEnrichedScanReport(tempDir, '2026-05-09T10:00:00.000Z', { - completedStages: ['descriptions', 'embeddings'], - }); - const io = makeIo(); - const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { runContextBuild: runContextBuildMock }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(runContextBuildMock).not.toHaveBeenCalled(); - }); -``` - -- [ ] **Step 2: Run failing depth-aware readiness tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-context.test.ts -t "requires completed relationships|does not require relationships" -``` - -Expected: FAIL because readiness only checks enriched descriptions and -embeddings. - -- [ ] **Step 3: Replace scan readiness helpers** - -In `packages/cli/src/setup-context.ts`, replace -`scanReportHasCompletedDescriptionEnrichment` with: - -```ts -function scanReportHasSchemaManifest(report: unknown, connectionId: string): boolean { - if (!isRecord(report)) { - return false; - } - if (report.connectionId !== connectionId || report.dryRun === true) { - return false; - } - return stringArrayValue(isRecord(report.artifactPaths) ? report.artifactPaths.manifestShards : undefined).length > 0; -} - -function scanReportHasCompletedDeepEnrichment( - report: unknown, - connectionId: string, - relationshipsRequired: boolean, -): boolean { - if (!isRecord(report)) { - return false; - } - if (report.connectionId !== connectionId || report.mode !== 'enriched' || report.dryRun === true) { - return false; - } - if (!isRecord(report.enrichment) || !isRecord(report.enrichmentState) || !isRecord(report.artifactPaths)) { - return false; - } - const completedStages = stringArrayValue(report.enrichmentState.completedStages); - return ( - report.enrichment.tableDescriptions === 'completed' && - report.enrichment.columnDescriptions === 'completed' && - report.enrichment.embeddings === 'completed' && - completedStages.includes('descriptions') && - completedStages.includes('embeddings') && - (!relationshipsRequired || completedStages.includes('relationships')) && - stringArrayValue(report.artifactPaths.manifestShards).length > 0 - ); -} - -function scanReportSatisfiesDepth(input: { - report: unknown; - connectionId: string; - depth: KtxDatabaseContextDepth; - relationshipsRequired: boolean; -}): boolean { - if (input.depth === 'fast') { - return scanReportHasSchemaManifest(input.report, input.connectionId); - } - return scanReportHasCompletedDeepEnrichment(input.report, input.connectionId, input.relationshipsRequired); -} -``` - -Replace `verifyPrimarySourceScans` with: - -```ts -async function verifyPrimarySourceScans( - project: KtxLocalProject, - connectionIds: string[], -): Promise<{ ready: boolean; details: string[] }> { - const details: string[] = []; - const relationshipsRequired = project.config.scan.relationships.enabled; - for (const connectionId of connectionIds) { - const connection = project.config.connections[connectionId]; - const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast'; - const report = await readLatestScanReport(project.projectDir, connectionId); - if (!scanReportSatisfiesDepth({ report, connectionId, depth, relationshipsRequired })) { - details.push( - depth === 'fast' - ? `${connectionId}: schema context has not completed.` - : `${connectionId}: deep database context has not completed.`, - ); - } - } - return { ready: details.length === 0, details }; -} -``` - -In `defaultVerifyContextReady`, change: - -```ts - const primarySourceScans = await verifyPrimarySourceScans(projectDir, targets.primarySourceConnectionIds); -``` - -to: - -```ts - const primarySourceScans = await verifyPrimarySourceScans(project, targets.primarySourceConnectionIds); -``` - -- [ ] **Step 4: Update success wording away from scan** - -In `writeSuccess`, replace: - -```ts - io.stdout.write(` ${connectionId}: enriched scan complete\n`); -``` - -with: - -```ts - const connection = project.config.connections[connectionId]; - const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast'; - io.stdout.write(` ${connectionId}: ${depth === 'deep' ? 'deep context complete' : 'schema context complete'}\n`); -``` - -Change the function signature to accept `project`: - -```ts -function writeSuccess( - project: KtxLocalProject, - readiness: KtxSetupContextReadiness, - targets: KtxSetupContextTargets, - io: KtxCliIo, -): void { -``` - -Change the caller from: - -```ts - writeSuccess(readiness, targets, io); -``` - -to: - -```ts - writeSuccess(project, readiness, targets, io); -``` - -- [ ] **Step 5: Run setup context readiness tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-context.test.ts -``` - -Expected: PASS after updating old test names and assertions that referred to -`enriched scan complete`. - -- [ ] **Step 6: Commit depth-aware readiness** - -Run: - -```bash -git add packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts -git commit -m "feat(setup): verify context readiness by database depth" -``` - -### Task 5: Remove background context-build control - -**Files:** -- Modify: `packages/cli/src/context-build-view.ts` -- Modify: `packages/cli/src/setup-context.ts` -- Test: `packages/cli/src/context-build-view.test.ts` -- Test: `packages/cli/src/setup-context.test.ts` -- Test: `packages/cli/src/setup.test.ts` - -- [ ] **Step 1: Write failing foreground-only tests** - -In `packages/cli/src/context-build-view.test.ts`, add: - -```ts - it('renders foreground-only progress hints without detach or resume commands', () => { - const state = initViewState([ - { - connectionId: 'warehouse', - driver: 'postgres', - operation: 'database-ingest', - debugCommand: 'ktx ingest warehouse --debug', - steps: ['database-schema'], - }, - ]); - state.primarySources[0]!.status = 'running'; - - const rendered = renderContextBuildView(state, { styled: false, showHint: true, projectDir: '/tmp/project' }); - - expect(rendered).toContain('Ctrl+C to stop'); - expect(rendered).not.toContain('d to detach'); - expect(rendered).not.toContain('resume'); - }); -``` - -In `packages/cli/src/setup-context.test.ts`, replace tests that expect -detached/watch behavior with: - -```ts - it('normalizes legacy detached and paused setup context states to stale', async () => { - await writeKtxSetupContextState(tempDir, { - runId: 'setup-context-local-old', - status: 'detached' as never, - startedAt: '2026-05-09T09:00:00.000Z', - updatedAt: '2026-05-09T09:00:00.000Z', - primarySourceConnectionIds: ['warehouse'], - contextSourceConnectionIds: [], - reportIds: [], - artifactPaths: [], - retryableFailedTargets: [], - commands: contextBuildCommands(tempDir, 'setup-context-local-old'), - }); - - await expect(readKtxSetupContextState(tempDir)).resolves.toMatchObject({ - status: 'stale', - failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.', - }); - }); - - it('starts a fresh foreground build when a stale running state is found', async () => { - await writeReadyProject(tempDir, { - connections: { warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } } }, - }); - await writeKtxSetupContextState(tempDir, { - runId: 'setup-context-local-running', - status: 'running', - startedAt: '2026-05-09T09:00:00.000Z', - updatedAt: '2026-05-09T09:00:00.000Z', - primarySourceConnectionIds: ['warehouse'], - contextSourceConnectionIds: [], - reportIds: [], - artifactPaths: [], - retryableFailedTargets: [], - commands: contextBuildCommands(tempDir, 'setup-context-local-running'), - }); - const io = makeIo(); - const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); - const verifyContextReady = vi.fn(async () => ({ - ready: true, - agentContextReady: true, - semanticSearchReady: true, - details: ['ready'], - })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { runContextBuild: runContextBuildMock, verifyContextReady }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(runContextBuildMock).toHaveBeenCalledOnce(); - }); -``` - -- [ ] **Step 2: Run failing foreground-only tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/setup-context.test.ts src/setup.test.ts -t "foreground-only|legacy detached|stale running|detached|watch" -``` - -Expected: FAIL because detach, watch, paused, and background logic still -exist. - -- [ ] **Step 3: Remove detach and background spawning from the progress view** - -In `packages/cli/src/context-build-view.ts`, remove these imports: - -```ts -import { spawn } from 'node:child_process'; -import { mkdirSync, openSync } from 'node:fs'; -``` - -Delete these functions: - -```ts -function resolveKtxEntryScript(): string | null -function spawnBackgroundBuild(projectDir: string): { logPath: string } | null -export function defaultSetupKeystroke( - onDetach: () => void, - onCtrlC: () => void, -): (() => void) | null -``` - -Change the default hint in `renderContextBuildView`: - -```ts - const hintContent = options.hintText ?? 'Ctrl+C to stop'; -``` - -Remove these fields from `ContextBuildDeps`: - -```ts - setupKeystroke?: (onDetach: () => void, onCtrlC: () => void) => (() => void) | null; - onDetach?: () => void; -``` - -Change `ContextBuildResult` to: - -```ts -export interface ContextBuildResult { - exitCode: number; - reportIds?: string[]; - artifactPaths?: string[]; -} -``` - -In `runContextBuild`, delete the `detached`, `exiting`, `cleanupKeystroke`, -and `setupKeystroke` block. Keep the `try/finally` cleanup for -`spinnerInterval`. - -Delete this branch: - -```ts - if (detached) { - return { exitCode: 0, detached: true }; - } -``` - -Return: - -```ts - return { - exitCode: hasFailure ? 1 : 0, - ...(reportIds.size > 0 ? { reportIds: [...reportIds] } : {}), - ...(artifactPaths.size > 0 ? { artifactPaths: [...artifactPaths] } : {}), - }; -``` - -- [ ] **Step 4: Normalize setup context state to foreground-only statuses** - -In `packages/cli/src/setup-context.ts`, remove `detached` and `paused` from -`KtxSetupContextBuildStatus` and `KtxSetupContextResult`. - -Change `KtxSetupContextCommands` to: - -```ts -export interface KtxSetupContextCommands { - build: string; - status: string; -} -``` - -Change `contextBuildCommands` to return: - -```ts - return { - build: `ktx setup --project-dir ${resolvedProjectDir}`, - status: `ktx status --project-dir ${resolvedProjectDir}`, - }; -``` - -In `normalizeState`, normalize legacy states: - -```ts - const rawStatus = record.status ?? 'not_started'; - const legacyActive = rawStatus === 'detached' || rawStatus === 'paused' || rawStatus === 'running'; - const status: KtxSetupContextBuildStatus = legacyActive ? 'stale' : rawStatus; -``` - -Add a default failure reason for legacy active states: - -```ts - ...(typeof record.failureReason === 'string' - ? { failureReason: record.failureReason } - : legacyActive - ? { failureReason: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.' } - : {}), -``` - -In `setupContextStatusFromState`, remove `watchCommand`: - -```ts - ...(state.runId ? { statusCommand: state.commands.status } : {}), -``` - -In `runBuild`, remove `onDetach` handling and remove the -`buildResult.detached` branch. - -Delete `isActiveStatus`, `watchExitCode`, `defaultSleep`, `writeContextStatus`, -`watchContextStatus`, `watchContextStatusText`, -`watchContextStatusWithProgressView`, and `setupResultFromWatchedState`. - -In `runKtxSetupContextStep`, remove the branch that prompts: - -```ts - 'A context build is running in the background.\n\n' + - 'You can watch it until it finishes, check its status once, or start a fresh build.' -``` - -Replace it with: - -```ts - if (existingState.status === 'stale') { - io.stdout.write('Previous context build state is stale; starting a fresh foreground build.\n'); - } -``` - -- [ ] **Step 5: Update setup tests that referenced detached/watch** - -In `packages/cli/src/setup.test.ts`, replace expectations for returned -`status: 'detached'` from the context step with `status: 'failed'` only when -the mocked context step returns failed. Remove tests named: - -- `does not install agents when full setup context build is detached` -- `skips entry menu and auto-watches when context build is active and showEntryMenu is true` - -Replace them with one test: - -```ts - it('does not offer background watch choices from setup status', async () => { - const tempDir = await makeTempProject(); - await writeKtxSetupContextState(tempDir, { - runId: 'setup-context-local-stale', - status: 'running', - startedAt: '2026-05-09T09:00:00.000Z', - updatedAt: '2026-05-09T09:00:00.000Z', - primarySourceConnectionIds: ['warehouse'], - contextSourceConnectionIds: [], - reportIds: [], - artifactPaths: [], - retryableFailedTargets: [], - commands: contextBuildCommands(tempDir, 'setup-context-local-stale'), - }); - - const result = await runKtxSetupStatus({ projectDir: tempDir }, makeIo().io); - - expect(result).toBe(0); - const state = await readKtxSetupContextState(tempDir); - expect(state.status).toBe('stale'); - }); -``` - -- [ ] **Step 6: Run foreground-only tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/setup-context.test.ts src/setup.test.ts -``` - -Expected: PASS after removing stale detached/watch assertions. - -- [ ] **Step 7: Commit foreground-only cleanup** - -Run: - -```bash -git add packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts packages/cli/src/setup.test.ts -git commit -m "fix(setup): keep context build foreground only" -``` - -### Task 6: Reject ingest subcommand connection ids - -**Files:** -- Modify: `packages/context/src/project/config.ts` -- Modify: `packages/context/src/project/index.ts` -- Modify: `packages/context/src/project/config.test.ts` -- Modify: `packages/cli/src/setup-sources.ts` -- Modify: `packages/cli/src/setup-sources.test.ts` -- Modify: `packages/cli/src/commands/setup-commands.ts` -- Modify: `packages/cli/src/index.test.ts` -- Modify: `packages/cli/src/setup-databases.ts` -- Modify: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Write failing reserved-id tests** - -In `packages/context/src/project/config.test.ts`, add: - -```ts - it.each(['status', 'replay', 'run', 'watch'])( - 'rejects reserved ingest connection id "%s"', - (connectionId) => { - expect(() => - parseKtxProjectConfig(` -project: reserved-test -connections: - ${connectionId}: - driver: postgres -`), - ).toThrow(`"${connectionId}" is reserved for ktx ingest ${connectionId}`); - }, - ); -``` - -In `packages/cli/src/index.test.ts`, add a Commander setup flag test: - -```ts - it('rejects reserved setup database connection ids before dispatch', async () => { - const testIo = makeIo(); - const setup = vi.fn(async () => 0); - - await expect( - runKtxCli(['setup', '--new-database-connection-id', 'status', '--no-input'], testIo.io, { setup }), - ).resolves.toBe(1); - - expect(setup).not.toHaveBeenCalled(); - expect(testIo.stderr()).toContain('"status" is reserved for ktx ingest status; choose a different connection id.'); - }); -``` - -In `packages/cli/src/setup-sources.test.ts`, add a prompt test that enters -`status` for a Notion connection id and expects the step to fail with the same -message. - -In `packages/cli/src/setup-databases.test.ts`, add a non-interactive test that -passes `databaseConnectionId: 'replay'` and expects `status: 'failed'` with the -same reserved-id message. - -- [ ] **Step 2: Run failing reserved-id tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts -t "reserved ingest connection" -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/setup-sources.test.ts src/setup-databases.test.ts -t "reserved" -``` - -Expected: FAIL because only the unsafe-character regex exists. - -- [ ] **Step 3: Add reserved-id validation to project config** - -In `packages/context/src/project/config.ts`, add after `isRecord`: - -```ts -const RESERVED_INGEST_CONNECTION_IDS = new Map([ - ['status', 'ktx ingest status'], - ['replay', 'ktx ingest replay'], - ['run', 'ktx ingest run'], - ['watch', 'ktx ingest watch'], -]); - -export function reservedKtxIngestConnectionIdMessage(connectionId: string): string | null { - const command = RESERVED_INGEST_CONNECTION_IDS.get(connectionId); - return command ? `"${connectionId}" is reserved for ${command}; choose a different connection id.` : null; -} - -export function assertKtxConnectionIdIsNotReserved(connectionId: string): void { - const message = reservedKtxIngestConnectionIdMessage(connectionId); - if (message) { - throw new Error(message); - } -} -``` - -In `parseKtxProjectConfig`, before returning the parsed object, validate -connection ids: - -```ts - const parsedConnections = isRecord(parsed.connections) - ? (parsed.connections as Record) - : defaults.connections; - for (const connectionId of Object.keys(parsedConnections)) { - assertKtxConnectionIdIsNotReserved(connectionId); - } -``` - -Then change the returned `connections` field to: - -```ts - connections: parsedConnections, -``` - -In `packages/context/src/project/index.ts`, export the helpers: - -```ts -export { - assertKtxConnectionIdIsNotReserved, - buildDefaultKtxProjectConfig, - parseKtxProjectConfig, - reservedKtxIngestConnectionIdMessage, - serializeKtxProjectConfig, -} from './config.js'; -``` - -- [ ] **Step 4: Use reserved-id validation in setup** - -In `packages/cli/src/setup-sources.ts`, import: - -```ts -import { assertKtxConnectionIdIsNotReserved } from '@ktx/context/project'; -``` - -Change `assertSafeConnectionId`: - -```ts -function assertSafeConnectionId(connectionId: string): void { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) { - throw new Error(`Unsafe connection id: ${connectionId}`); - } - assertKtxConnectionIdIsNotReserved(connectionId); -} -``` - -In `packages/cli/src/setup-databases.ts`, import -`assertKtxConnectionIdIsNotReserved` and add: - -```ts -function assertSafeDatabaseConnectionId(connectionId: string): void { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) { - throw new Error(`Unsafe connection id: ${connectionId}`); - } - assertKtxConnectionIdIsNotReserved(connectionId); -} -``` - -In `chooseConnectionIdForDriver`, validate every new id before returning: - -```ts - assertSafeDatabaseConnectionId(input.args.databaseConnectionId); - return { kind: 'new', connectionId: input.args.databaseConnectionId }; -``` - -and: - -```ts - assertSafeDatabaseConnectionId(connectionId); - return connectionId ? { kind: 'new', connectionId } : 'missing-input'; -``` - -In `packages/cli/src/commands/setup-commands.ts`, update -`--new-database-connection-id` parsing: - -```ts - .option('--new-database-connection-id ', 'Connection id for one new database connection', (value) => { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - const reservedMessage = reservedKtxIngestConnectionIdMessage(value); - if (reservedMessage) { - throw new InvalidArgumentError(reservedMessage); - } - return value; - }) -``` - -Add the import: - -```ts -import { reservedKtxIngestConnectionIdMessage } from '@ktx/context/project'; -``` - -- [ ] **Step 5: Run reserved-id tests again** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts -t "reserved ingest connection" -pnpm --filter @ktx/cli exec vitest run src/index.test.ts src/setup-sources.test.ts src/setup-databases.test.ts -t "reserved" -``` - -Expected: PASS. - -- [ ] **Step 6: Commit reserved-id validation** - -Run: - -```bash -git add packages/context/src/project/config.ts packages/context/src/project/index.ts packages/context/src/project/config.test.ts packages/cli/src/setup-sources.ts packages/cli/src/setup-sources.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/index.test.ts -git commit -m "fix(config): reject reserved ingest connection ids" -``` - -### Task 7: Final verification - -**Files:** -- Verify only. - -- [ ] **Step 1: Run focused TypeScript tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/setup-context.test.ts src/context-build-view.test.ts src/setup-databases.test.ts src/setup-sources.test.ts src/setup.test.ts src/index.test.ts -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts src/ingest/local-adapters.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run docs and script tests touched by unified ingest** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs scripts/package-artifacts.test.mjs scripts/installed-live-database-smoke.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If the check reports unrelated pre-existing findings, record -the exact findings in the implementation notes and do not silence them with a -broad ignore. - -- [ ] **Step 5: Run pre-commit for changed files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/cli/src/ingest-depth.ts \ - packages/cli/src/public-ingest.ts \ - packages/cli/src/public-ingest.test.ts \ - packages/cli/src/setup-context.ts \ - packages/cli/src/setup-context.test.ts \ - packages/cli/src/context-build-view.ts \ - packages/cli/src/context-build-view.test.ts \ - packages/cli/src/setup-databases.ts \ - packages/cli/src/setup-databases.test.ts \ - packages/cli/src/setup-sources.ts \ - packages/cli/src/setup-sources.test.ts \ - packages/cli/src/commands/setup-commands.ts \ - packages/cli/src/index.test.ts \ - packages/context/src/project/config.ts \ - packages/context/src/project/config.test.ts \ - packages/context/src/project/index.ts -``` - -Expected: PASS. If local `uv` cannot satisfy the pinned project version, state -the version mismatch and run the TypeScript checks above as the closest -available verification. - -- [ ] **Step 6: Commit verification-only fixes** - -If verification required expectation or formatting changes, run: - -```bash -git add packages/cli/src packages/context/src scripts README.md -git commit -m "test: close unified ingest v1 expectations" -``` - -If no files changed during verification, do not create an empty commit. - -## Self-review notes - -Spec coverage in this plan: - -- Covers deep readiness failures before work starts for explicit or stored - `deep` and for query-history depth upgrades. -- Covers `scan.relationships.enabled` in deep database ingest. -- Covers setup depth prompting and storage under - `connections..context.depth`. -- Covers fast readiness without AI descriptions or embeddings. -- Covers deep readiness with relationship-stage gating only when relationship - discovery is enabled. -- Covers generated setup query-history config under - `connections..context.queryHistory`. -- Covers setup migration from legacy `connection.historicSql`. -- Covers foreground-only context build by removing detach, watch, resume, stop, - paused/detached state, and background subprocess behavior. -- Covers reserved ingest subcommand ids in setup and config validation. - -Placeholder scan: no deferred markers, unnamed edge handling, or undefined -types remain in the plan. The plan uses concrete file paths, commands, and -code shapes for each implementation task. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-docs-site-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-docs-site-closure.md deleted file mode 100644 index c1edc24e..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-docs-site-closure.md +++ /dev/null @@ -1,829 +0,0 @@ -# Unified Ingest V1 Docs Site Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the remaining public documentation surfaces that still present -`ktx scan`, adapter-backed `ktx ingest run`, `ktx ingest watch`, -`live-database`, or `Historic SQL` as normal v1 user workflows. - -**Architecture:** Keep the implemented CLI behavior unchanged. Update the -Fumadocs content, example READMEs, and documentation regression tests so public -guidance uses connection-centric `ktx ingest `, `ktx ingest ---all`, `--fast`, `--deep`, `--query-history`, `ktx ingest status`, and -`ktx ingest replay`. - -**Tech Stack:** Markdown, MDX frontmatter, Fumadocs page metadata, Node test -runner, pnpm workspace scripts. - ---- - -## Current audit - -The four implemented unified-ingest plans cover the CLI and setup v1 surface: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` route through `public-ingest.ts`. -- Database targets run before source targets, public source ingest bypasses - adapter allow-lists, and public database ingest captures internal scan output. -- `ktx scan`, `ktx ingest run`, and `ktx ingest watch` are hidden from normal - help. -- Setup stores `connections..context.depth`, writes - `connections..context.queryHistory`, rejects reserved ingest ids, and - uses foreground-only context-build state. - -### V1-blocking gaps - -- `docs-site/content/docs/cli-reference/ktx-ingest.mdx` still documents - adapter-level `ktx ingest run`, `--adapter`, `ktx ingest watch`, and - `live-database`. -- `docs-site/content/docs/cli-reference/ktx-scan.mdx` still presents - `ktx scan` as a public command, and - `docs-site/content/docs/cli-reference/meta.json` still publishes it in the - CLI reference. -- `docs-site/content/docs/cli-reference/ktx-dev.mdx` still links to root - `ktx scan` as a normal command. -- `docs-site/content/docs/guides/building-context.mdx` still has an adapter - table that lists `historic-sql` and `live-database`, and it still documents - `ktx ingest watch` as the visual progress path. -- `docs-site/content/docs/integrations/context-sources.mdx` still instructs - users to run - `ktx ingest run --connection-id --adapter `. -- `docs-site/content/docs/concepts/context-as-code.mdx` still recommends - scheduled - `ktx ingest run --connection-id --adapter --no-input`. -- `docs-site/content/docs/getting-started/quickstart.mdx` still says setup - runs structural/enriched scans, exposes Historic SQL flags, and describes - detach/background context-build behavior. -- `docs-site/content/docs/integrations/primary-sources.mdx` still uses the - legacy `historicSql` config shape and Historic SQL wording for supported - query-history drivers. -- `examples/README.md` and `examples/local-warehouse/README.md` still present - `ktx ingest run --adapter fake` as the example command. - -### Non-blocking gaps - -- Hidden debug commands can continue to call `ktx scan`, - `ktx ingest run`, and `ktx ingest watch`. -- Internal source keys, raw artifact paths, tests, scripts, and developer-only - package taxonomy can continue to use `scan`, `live-database`, and - `historic-sql`. -- Contributor docs can continue to mention scan internals when describing - package ownership or connector implementation details. -- The `examples/local-warehouse/ktx.yaml` fake adapter fixture can remain for - CLI smoke tests if the public example docs stop recommending it as a normal - user workflow. - -## File structure - -- Modify `scripts/examples-docs.test.mjs`: add regression assertions for - docs-site and example README unified-ingest wording. -- Modify `docs-site/content/docs/cli-reference/ktx-ingest.mdx`: rewrite the - page around the connection-centric public command. -- Delete `docs-site/content/docs/cli-reference/ktx-scan.mdx`: remove the - public scan reference page. -- Modify `docs-site/content/docs/cli-reference/meta.json`: remove - `ktx-scan` from published CLI reference pages. -- Modify `docs-site/content/docs/cli-reference/ktx-dev.mdx`: remove the - root-scan link and clarify that database context is built by `ktx ingest`. -- Modify `docs-site/content/docs/guides/building-context.mdx`: remove - adapter tables and live watch guidance; describe status/replay only. -- Modify `docs-site/content/docs/integrations/context-sources.mdx`: replace - adapter-backed ingest commands with `ktx ingest `. -- Modify `docs-site/content/docs/concepts/context-as-code.mdx`: replace - scheduled adapter-backed ingest guidance with `ktx ingest --all`. -- Modify `docs-site/content/docs/getting-started/quickstart.mdx`: update setup - language for schema context, depth, query history, and foreground-only - progress. -- Modify `docs-site/content/docs/integrations/primary-sources.mdx`: replace - `historicSql` with `context.queryHistory` and query-history wording. -- Modify `examples/README.md`: stop advertising the fake adapter command as a - public example workflow. -- Modify `examples/local-warehouse/README.md`: mark the fake adapter fixture as - contributor-only and point users to public ingest docs. - -## Tasks - -### Task 1: Add stale public-doc regression tests - -**Files:** -- Modify: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Add failing docs-site unified-ingest assertions** - -In `scripts/examples-docs.test.mjs`, replace the existing test named -`documents public context build workflows in the docs site` with: - -```js - it('documents unified public ingest workflows in the docs site', async () => { - const rootReadme = await readText('README.md'); - const cliMeta = await readText('docs-site/content/docs/cli-reference/meta.json'); - const ingestReference = await readText('docs-site/content/docs/cli-reference/ktx-ingest.mdx'); - const devReference = await readText('docs-site/content/docs/cli-reference/ktx-dev.mdx'); - const buildingContext = await readText('docs-site/content/docs/guides/building-context.mdx'); - const contextSources = await readText('docs-site/content/docs/integrations/context-sources.mdx'); - const contextAsCode = await readText('docs-site/content/docs/concepts/context-as-code.mdx'); - const quickstart = await readText('docs-site/content/docs/getting-started/quickstart.mdx'); - const primarySources = await readText('docs-site/content/docs/integrations/primary-sources.mdx'); - const examplesIndex = await readText('examples/README.md'); - const localWarehouseReadme = await readText('examples/local-warehouse/README.md'); - - assert.match(ingestReference, /ktx ingest /); - assert.match(ingestReference, /ktx ingest --all --deep/); - assert.match(ingestReference, /--query-history-window-days /); - assert.match(buildingContext, /ktx ingest /); - assert.match(buildingContext, /ktx ingest --all/); - assert.match(buildingContext, /ktx ingest replay /); - assert.match(contextSources, /ktx ingest /); - assert.match(contextAsCode, /ktx ingest --all --no-input/); - assert.match(quickstart, /schema context/); - assert.match(primarySources, /context:\\n queryHistory:/); - - assert.doesNotMatch(cliMeta, /ktx-scan/); - assert.doesNotMatch(ingestReference, /ktx ingest run/); - assert.doesNotMatch(ingestReference, /--adapter/); - assert.doesNotMatch(ingestReference, /ktx ingest watch/); - assert.doesNotMatch(ingestReference, /live-database/); - assert.doesNotMatch(devReference, /ktx scan/); - assert.doesNotMatch(buildingContext, /ktx ingest watch/); - assert.doesNotMatch(buildingContext, /historic-sql/); - assert.doesNotMatch(buildingContext, /live-database/); - assert.doesNotMatch(contextSources, /ktx ingest run --connection-id/); - assert.doesNotMatch(contextSources, /--adapter /); - assert.doesNotMatch(contextAsCode, /ktx ingest run --connection-id/); - assert.doesNotMatch(quickstart, /Historic SQL/); - assert.doesNotMatch(quickstart, /--enable-historic-sql/); - assert.doesNotMatch(quickstart, /press d<\\/kbd> to detach/); - assert.doesNotMatch(primarySources, /historicSql/); - assert.doesNotMatch(primarySources, /Historic SQL/); - assert.doesNotMatch(examplesIndex, /ktx ingest run --project-dir/); - assert.doesNotMatch(localWarehouseReadme, /ktx ingest run --project-dir/); - - assert.match(rootReadme, /raw-sources\//); - assert.doesNotMatch(rootReadme, new RegExp(`${['live', 'database'].join('-')}/`)); - assert.doesNotMatch(rootReadme, /ktx scan/); - assert.doesNotMatch(rootReadme, /Run a local ingest smoke test/); - assert.doesNotMatch(rootReadme, /ktx ingest run --project-dir/); - assert.doesNotMatch(rootReadme, /ktx ingest status --project-dir/); - }); -``` - -- [ ] **Step 2: Run the failing docs regression test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL with assertions matching the stale docs-site and example README -content. - -- [ ] **Step 3: Commit the failing test** - -```bash -git add scripts/examples-docs.test.mjs -git commit -m "test(docs): cover unified ingest public docs" -``` - -### Task 2: Rewrite the CLI reference surface - -**Files:** -- Modify: `docs-site/content/docs/cli-reference/ktx-ingest.mdx` -- Delete: `docs-site/content/docs/cli-reference/ktx-scan.mdx` -- Modify: `docs-site/content/docs/cli-reference/meta.json` -- Modify: `docs-site/content/docs/cli-reference/ktx-dev.mdx` - -- [ ] **Step 1: Rewrite `ktx-ingest.mdx`** - -Replace `docs-site/content/docs/cli-reference/ktx-ingest.mdx` with: - -````mdx ---- -title: "ktx ingest" -description: "Build, inspect, and replay KTX context ingest runs." ---- - -`ktx ingest` builds or refreshes KTX context from configured connections. -Database connections build schema context. Context-source connections ingest -metadata from tools such as dbt, Looker, Metabase, MetricFlow, LookML, and -Notion. - -## Command signature - -```bash -ktx ingest [options] [connectionId] -``` - -Use a connection id to build one configured connection. Use `--all` to build -every configured connection. Database connections run before context-source -connections when you use `--all`. - -## Build options - -| Flag | Description | Default | -|------|-------------|---------| -| `--all` | Build every configured connection | `false` | -| `--fast` | Use deterministic database schema ingest | Stored connection default, or `fast` | -| `--deep` | Use AI-enriched database ingest | Stored connection default, or `fast` | -| `--query-history` | Include database query-history usage patterns | Stored connection default | -| `--no-query-history` | Skip database query-history usage patterns for this run | Stored connection default | -| `--query-history-window-days ` | Query-history lookback window for this run | Stored connection default | -| `--plain` | Print plain text output | `true` | -| `--json` | Print JSON output | `false` | -| `--no-input` | Disable interactive terminal input | `false` | - -`--fast` and `--deep` are mutually exclusive. Depth flags apply only to -database connections. Query-history flags apply only to database connections -that support query history. - -## Status and replay - -| Subcommand | Description | -|------------|-------------| -| `status [runId]` | Print status for the latest or selected stored ingest run or report file | -| `replay ` | Replay a stored ingest run or bundle report through memory-flow output | - -Both subcommands accept `--report-file `, `--plain`, `--json`, `--viz`, -and `--no-input`. - -## Examples - -```bash -ktx ingest warehouse -ktx ingest warehouse --fast -ktx ingest warehouse --deep -ktx ingest warehouse --deep --query-history -ktx ingest warehouse --query-history-window-days 30 -ktx ingest notion -ktx ingest --all -ktx ingest --all --deep - -ktx ingest status -ktx ingest status run-abc123 -ktx ingest status --json - -ktx ingest replay run-abc123 -ktx ingest replay run-abc123 --viz -ktx ingest replay run-abc123 --report-file /tmp/ingest-report.json -``` - -## Common errors - -| Error | Cause | Recovery | -|-------|-------|----------| -| Connection not configured | The connection id is not present in `ktx.yaml` | Add the connection with `ktx setup` or update `ktx.yaml` | -| Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` | -| Query history is unsupported | The selected database driver does not support query history | Run schema ingest without query-history flags | -| Latest run not found | No stored ingest report exists in this project | Run `ktx ingest ` first | -| Visual replay fails in a non-interactive shell | Visual report replay needs a terminal | Use `ktx ingest status --json` for agent and CI workflows | -```` - -- [ ] **Step 2: Remove the public scan page** - -Delete `docs-site/content/docs/cli-reference/ktx-scan.mdx`. - -- [ ] **Step 3: Remove `ktx-scan` from CLI metadata** - -In `docs-site/content/docs/cli-reference/meta.json`, replace the full file -with: - -```json -{ - "title": "CLI Reference", - "defaultOpen": true, - "pages": [ - "ktx-setup", - "ktx-connection", - "ktx-ingest", - "ktx-sl", - "ktx-wiki", - "ktx-status", - "ktx-dev" - ] -} -``` - -- [ ] **Step 4: Update the dev command reference** - -In `docs-site/content/docs/cli-reference/ktx-dev.mdx`, replace this paragraph: - -```mdx -`ktx dev` contains development-only project initialization and managed runtime commands. Scan and ingest commands live at the root as [`ktx scan`](/docs/cli-reference/ktx-scan) and [`ktx ingest`](/docs/cli-reference/ktx-ingest). -``` - -with: - -```mdx -`ktx dev` contains development-only project initialization and managed runtime commands. Context building lives at the root as [`ktx ingest`](/docs/cli-reference/ktx-ingest). -``` - -- [ ] **Step 5: Run the docs regression test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL only on the remaining guide, integration, quickstart, primary -source, and example README stale wording. - -- [ ] **Step 6: Commit CLI reference cleanup** - -```bash -git add docs-site/content/docs/cli-reference/ktx-ingest.mdx docs-site/content/docs/cli-reference/meta.json docs-site/content/docs/cli-reference/ktx-dev.mdx -git rm docs-site/content/docs/cli-reference/ktx-scan.mdx -git commit -m "docs: align ingest CLI reference with unified UX" -``` - -### Task 3: Update context-build guides - -**Files:** -- Modify: `docs-site/content/docs/guides/building-context.mdx` -- Modify: `docs-site/content/docs/integrations/context-sources.mdx` -- Modify: `docs-site/content/docs/concepts/context-as-code.mdx` - -- [ ] **Step 1: Update stored report guidance in `building-context.mdx`** - -In `docs-site/content/docs/guides/building-context.mdx`, replace the -`### Watching progress` section through the paragraph after it with: - -````mdx -### Inspecting stored reports - -```bash -# Check status of the latest ingest -ktx ingest status - -# Check a specific run -ktx ingest status - -# Replay a past ingest run -ktx ingest replay -``` - -`ktx ingest replay` opens the stored memory-flow output for a completed run. -Foreground context builds do not detach into background control sessions; if a -run is interrupted, rerun `ktx ingest ` or `ktx ingest --all`. -```` - -- [ ] **Step 2: Replace the adapter table in `building-context.mdx`** - -In the same file, replace the `### Available adapters` heading, table, and -following sentence with: - -```mdx -### Supported context sources - -| Driver | Source | What gets ingested | -|--------|--------|--------------------| -| `dbt` | dbt project | Model definitions, column descriptions, tests, tags | -| `metricflow` | MetricFlow semantic models | Metrics, dimensions, entities, semantic joins | -| `lookml` | LookML files | Views, explores, dimensions, measures, joins | -| `looker` | Looker API | Explores, looks, dashboard metadata | -| `metabase` | Metabase API | Questions, dashboards, table metadata | -| `notion` | Notion API | Database pages, knowledge articles | - -Query history is a database connection facet. Enable it with -`connections..context.queryHistory` or pass `--query-history` for a current -run. See [Context Sources](/docs/integrations/context-sources) for -driver-specific setup and auth configuration. -``` - -- [ ] **Step 3: Update context-source workflow commands** - -In `docs-site/content/docs/integrations/context-sources.mdx`, replace the -numbered workflow with: - -```mdx -Agents must configure and ingest context sources in this order: - -1. Add the context source connection in `ktx.yaml` or with `ktx setup`. -2. Store tokens as `env:NAME` or `file:/path/to/secret`. -3. Run `ktx ingest ` for one source or `ktx ingest --all` for - every configured source. -4. Check progress with `ktx ingest status --json`. -5. Review generated `semantic-layer/` YAML and `wiki/` Markdown files in git. -6. Validate changed semantic sources with `ktx sl validate`. -``` - -- [ ] **Step 4: Update scheduled ingest wording** - -In `docs-site/content/docs/concepts/context-as-code.mdx`, replace this -paragraph: - -```mdx -Teams usually run this on demand while setting up a source, then schedule it once the source is stable. A cron job or CI schedule can run `ktx ingest run --connection-id --adapter --no-input` overnight on an ingest branch so the latest dbt manifests, BI metadata, and documentation updates are ready for review each morning. -``` - -with: - -```mdx -Teams usually run this on demand while setting up a source, then schedule it -once the source is stable. A cron job or CI schedule can run `ktx ingest --all ---no-input` overnight on an ingest branch so the latest schema context, dbt -manifests, BI metadata, and documentation updates are ready for review each -morning. -``` - -- [ ] **Step 5: Run the docs regression test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL only on quickstart, primary source, and example README stale -wording. - -- [ ] **Step 6: Commit guide cleanup** - -```bash -git add docs-site/content/docs/guides/building-context.mdx docs-site/content/docs/integrations/context-sources.mdx docs-site/content/docs/concepts/context-as-code.mdx -git commit -m "docs: update context build guides for unified ingest" -``` - -### Task 4: Update setup and primary-source docs - -**Files:** -- Modify: `docs-site/content/docs/getting-started/quickstart.mdx` -- Modify: `docs-site/content/docs/integrations/primary-sources.mdx` - -- [ ] **Step 1: Update database setup copy in quickstart** - -In `docs-site/content/docs/getting-started/quickstart.mdx`, replace the first -paragraph under `## Step 3: Connect a database` with: - -```mdx -Select one or more databases for KTX to connect to. The wizard supports -SQLite, PostgreSQL, MySQL, ClickHouse, SQL Server, BigQuery, and Snowflake. -``` - -Replace this sentence: - -```mdx -After connecting, KTX automatically runs a connection test and a structural scan: -``` - -with: - -```mdx -After connecting, KTX automatically runs a connection test and builds fast -schema context: -``` - -Replace the example output block in Step 3 with: - -````mdx -``` -Testing postgres-warehouse - Connection test passed - Driver: PostgreSQL - Tables: 42 - -Building schema context for postgres-warehouse - Running fast database ingest - -Schema context complete for postgres-warehouse - Changes: 42 new tables - -Primary source ready - postgres-warehouse - PostgreSQL - schema context complete -``` -```` - -Replace this paragraph: - -```mdx -For Snowflake and BigQuery, the wizard offers **Historic SQL** configuration for query history views. For PostgreSQL, enable Historic SQL with `--enable-historic-sql` when `pg_stat_statements` is configured. -``` - -with: - -```mdx -For PostgreSQL, Snowflake, and BigQuery, the wizard can enable query-history -ingest when the warehouse history feature is available. Query history is stored -under `connections..context.queryHistory` in `ktx.yaml`. -``` - -- [ ] **Step 2: Update context-build copy in quickstart** - -In the same file, replace the first two paragraphs under -`## Step 5: Build context` with: - -```mdx -This is where KTX builds agent-ready context. It uses the database context -depth saved by setup and ingests metadata from any configured context sources. - -Fast database context builds deterministic schema grounding. Deep database -context also generates AI descriptions, embeddings, and relationship evidence -when those capabilities are configured. -``` - -Replace the paragraph and background example that starts with `For a small -database` and ends with the fenced context-build block with: - -````mdx -For a small database (under 50 tables), this can take a few minutes. Larger -warehouses can take longer. Context builds run in the foreground; press -Ctrl+C to stop the current run and rerun `ktx setup` or `ktx ingest` -when you are ready to try again. -```` - -Replace this output line in the completion example: - -```text - postgres-warehouse: enriched scan complete -``` - -with: - -```text - postgres-warehouse: deep context complete -``` - -Replace the next-steps bullet: - -```mdx -- **Build more context** - learn about [scanning](/docs/guides/building-context), relationship detection, and ingestion workflows in the Building Context guide. -``` - -with: - -```mdx -- **Build more context** - learn about [database ingest](/docs/guides/building-context), relationship detection, and source ingestion workflows in the Building Context guide. -``` - -- [ ] **Step 3: Update primary-source query-history config** - -In `docs-site/content/docs/integrations/primary-sources.mdx`, replace the -introductory paragraph and shared conventions with: - -```mdx -KTX connects to your data warehouse or database to build schema context, -discover relationships, and execute semantic layer queries. Each connection is -defined in `ktx.yaml` under the `connections` key. - -All connectors share these conventions: - -- Sensitive values support `env:VAR_NAME` (read from environment) and - `file:/path/to/secret` (read from file) references -- Connections are read-only; KTX never writes to your database -- Database ingest discovers tables, columns, types, and constraints - automatically -``` - -In the connection field reference table, replace the `historicSql` row with: - -```mdx -| `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it | -``` - -Replace every feature row label `Historic SQL` with `Query history`. - -Replace each `### Historic SQL` heading with `### Query history`. - -Replace the PostgreSQL query-history config block with: - -```yaml -context: - queryHistory: - enabled: true - minExecutions: 5 - filters: - dropTrivialProbes: true -``` - -Replace the Snowflake query-history config block with: - -```yaml -context: - queryHistory: - enabled: true - windowDays: 90 - minExecutions: 5 - filters: - dropTrivialProbes: true - serviceAccounts: - patterns: ['^svc_'] - mode: exclude - redactionPatterns: [] -``` - -Replace the BigQuery query-history config block with: - -```yaml -context: - queryHistory: - enabled: true - windowDays: 90 - minExecutions: 5 - filters: - dropTrivialProbes: true - serviceAccounts: - patterns: ['@bot\\.'] - mode: exclude - redactionPatterns: [] -``` - -Replace the common-errors row: - -```mdx -| Historic SQL is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun scan or setup | -``` - -with: - -```mdx -| Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest --query-history` or `ktx setup` | -``` - -Replace the common-errors row: - -```mdx -| Scan returns no tables | Schema/database/project filter is wrong or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | -``` - -with: - -```mdx -| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | -``` - -Replace the common-errors row: - -```mdx -| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on structural scan output | -``` - -with: - -```mdx -| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on fast schema context | -``` - -- [ ] **Step 4: Run targeted stale-term search** - -Run: - -```bash -rg -n "Historic SQL|historicSql|--enable-historic-sql|--historic-sql|ktx scan|ktx ingest watch|ktx ingest run --connection-id|--adapter |live-database" docs-site/content/docs/getting-started/quickstart.mdx docs-site/content/docs/integrations/primary-sources.mdx docs-site/content/docs/cli-reference docs-site/content/docs/guides/building-context.mdx docs-site/content/docs/integrations/context-sources.mdx docs-site/content/docs/concepts/context-as-code.mdx -``` - -Expected: no output. - -- [ ] **Step 5: Run the docs regression test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL only on example README stale adapter-command wording. - -- [ ] **Step 6: Commit setup and primary-source docs cleanup** - -```bash -git add docs-site/content/docs/getting-started/quickstart.mdx docs-site/content/docs/integrations/primary-sources.mdx -git commit -m "docs: update setup and primary source ingest wording" -``` - -### Task 5: Remove public fake-adapter example commands - -**Files:** -- Modify: `examples/README.md` -- Modify: `examples/local-warehouse/README.md` - -- [ ] **Step 1: Rewrite the local-warehouse section in `examples/README.md`** - -In `examples/README.md`, replace the `## local-warehouse` section with: - -````md -## local-warehouse - -`local-warehouse/` is a contributor fixture for local CLI smoke tests. It uses -the internal fake ingest adapter so tests can exercise memory-flow behavior -without a live database or external service. - -For normal context building, use the public connection-centric commands: - -```bash -ktx ingest -ktx ingest --all -``` - -The copied project initializes its own Git repository on first use. -```` - -- [ ] **Step 2: Rewrite `examples/local-warehouse/README.md`** - -Replace `examples/local-warehouse/README.md` with: - -````md -# local-warehouse fixture - -This directory is a contributor fixture for KTX CLI smoke tests. It uses the -internal fake ingest adapter so tests can run without a live database or -external service. - -Normal users should build context with connection-centric ingest: - -```bash -ktx ingest -ktx ingest --all -``` - -The public ingest workflow is documented in -`docs-site/content/docs/cli-reference/ktx-ingest.mdx` and -`docs-site/content/docs/guides/building-context.mdx`. -```` - -- [ ] **Step 3: Run the docs regression test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 4: Commit example docs cleanup** - -```bash -git add examples/README.md examples/local-warehouse/README.md -git commit -m "docs: stop advertising adapter-backed example ingest" -``` - -### Task 6: Final verification - -**Files:** -- Verify: `scripts/examples-docs.test.mjs` -- Verify: `docs-site/content/docs/**/*.mdx` -- Verify: `examples/**/*.md` - -- [ ] **Step 1: Run docs regression tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 2: Run docs-site build** - -Run: - -```bash -pnpm --filter ktx-docs run build -``` - -Expected: PASS. If the build fails because this workspace lacks external build -prerequisites, capture the error and run `pnpm --filter ktx-docs run test` as -the closest available docs-site check. - -- [ ] **Step 3: Run final stale public-surface search** - -Run: - -```bash -rg -n "ktx scan|ktx ingest run --connection-id|--adapter |ktx ingest watch|live-database|Historic SQL|historicSql|--enable-historic-sql|--historic-sql" docs-site/content/docs examples/README.md examples/local-warehouse/README.md -``` - -Expected: no output. - -- [ ] **Step 4: Inspect git status** - -Run: - -```bash -git status --short -``` - -Expected: only the files intentionally changed by this plan appear. - -- [ ] **Step 5: Commit verification updates if needed** - -If verification required small documentation or test fixes, commit them: - -```bash -git add scripts/examples-docs.test.mjs docs-site/content/docs examples/README.md examples/local-warehouse/README.md -git commit -m "docs: close unified ingest public docs gaps" -``` - -## Self-review - -- Spec coverage: This plan covers the remaining public documentation surfaces - that still contradicted the unified ingest UX spec. It intentionally does not - rename internal scan packages, internal adapter keys, raw artifact paths, or - developer-only test fixtures. -- Placeholder scan: No task contains open-ended placeholders. Each edit names - exact files and exact replacement text or commands. -- Type consistency: This is a documentation-only plan. Command names and config - keys match the implemented CLI and config code: `ktx ingest `, - `ktx ingest --all`, `ktx ingest status`, `ktx ingest replay`, and - `connections..context.queryHistory`. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-final-public-surface-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-final-public-surface-closure.md deleted file mode 100644 index f6a33580..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-final-public-surface-closure.md +++ /dev/null @@ -1,494 +0,0 @@ -# Unified Ingest V1 Final Public Surface Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking public-surface gaps in unified -`ktx ingest`. - -**Architecture:** Keep the current connection-centric ingest planner and hidden -legacy debug commands. Fix the public query-history execution path so it passes -the full canonical `connections..context.queryHistory` pull config to the -historic-SQL adapter, and filter hidden Commander commands from the -documentation command-tree script so docs/discovery output matches normal CLI -help. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages, -pnpm workspace scripts. - ---- - -## Current audit - -The implemented unified-ingest plan chain covers most of the original -`docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md` spec: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` route through `public-ingest.ts`. -- Database targets run before source targets. Public source ingest uses - `allowImplicitAdapter: true`, so `ingest.adapters` is no longer required for - inferred public adapters. -- Public database ingest maps `fast` to structural scan internals and `deep` to - enriched scan internals, honors `scan.relationships.enabled`, and isolates - deep-readiness failures per target under `--all`. -- Normal `ktx --help` hides `scan`; normal `ktx ingest --help` hides `run` and - `watch`; setup help exposes query-history flags instead of Historic SQL flags. -- Setup stores `connections..context.depth` and - `connections..context.queryHistory`, migrates legacy `historicSql`, and - uses foreground-only context-build state. -- Public docs-site CLI pages no longer document `ktx scan`, - `ktx ingest run --adapter`, or live `ktx ingest watch` as normal workflows. - -### V1-blocking gaps - -- Public query-history ingest drops configured pull fields. The lower-level - adapter path maps canonical `context.queryHistory` to the existing - `historicSqlUnifiedPullConfigSchema`, but `executePublicIngestTarget()` always - passes `historicSqlPullConfigOverride` with only `dialect` and sometimes - `windowDays`. Normal `ktx ingest warehouse --query-history` can therefore - ignore configured `minExecutions`, `filters`, `redactionPatterns`, - `concurrency`, and `staleArchiveAfterDays`. -- The documentation command-tree script still prints hidden commands. Running - `pnpm --filter @ktx/cli run docs:commands` currently prints top-level - `scan ` and `ktx ingest run` / `ktx ingest watch`, even though - the spec requires `ktx scan` and live `ingest watch` not to be presented as - normal public command surfaces. - -### Non-blocking gaps - -- Hidden debug commands remain callable: `ktx scan`, `ktx ingest run`, and - `ktx ingest watch`. The spec allows hidden/debug placement for old - implementation surfaces in v1. -- Internal adapter keys, package names, WorkUnit keys, raw artifact paths, and - JSON/debug output can continue to use `scan`, `live-database`, and - `historic-sql`. -- Developer-only scripts and tests can keep scan/live-database terminology when - they exercise internal connector or artifact behavior. -- Public docs still use "scan" as a generic noun in a few conceptual database - sections. They do not document `ktx scan` as the public command, so this is - wording cleanup, not v1-blocking behavior. - -## File structure - -- Modify `packages/cli/src/public-ingest.ts`: preserve the full canonical - query-history pull config in public ingest plans and pass that config to the - lower-level historic-SQL adapter run. -- Modify `packages/cli/src/public-ingest.test.ts`: add regression coverage for - configured query-history fields and current-run `windowDays` overrides. -- Modify `packages/cli/src/command-tree.ts`: filter Commander commands marked - hidden via Commander private `_hidden`, matching Commander help behavior. -- Modify `packages/cli/src/command-tree.test.ts`: cover hidden top-level and - nested command filtering in the pure walker. -- Modify `packages/cli/src/print-command-tree.test.ts`: lock the rendered KTX - docs command tree against hidden unified-ingest commands. - -## Tasks - -### Task 1: Preserve canonical query-history pull config in public ingest - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write the failing public-ingest query-history config test** - -In `packages/cli/src/public-ingest.test.ts`, add this test inside the -`runKtxPublicIngest` describe block, near the existing query-history execution -tests: - -```ts - it('preserves configured query-history pull fields while overriding the current-run window', async () => { - const io = makeIo(); - const project = deepReadyProject({ - warehouse: { - driver: 'postgres', - context: { - queryHistory: { - enabled: true, - windowDays: 90, - minExecutions: 7, - concurrency: 3, - staleArchiveAfterDays: 120, - filters: { - dropTrivialProbes: true, - serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, - orchestrators: { mode: 'mark-only' }, - dropFailedBelow: { errorRate: 0.5, executions: 3 }, - }, - redactionPatterns: ['(?i)secret'], - }, - }, - }, - }); - const runScan = vi.fn(async () => 0); - const runIngest = vi.fn(async () => 0); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - queryHistory: 'enabled', - queryHistoryWindowDays: 30, - }, - io.io, - { loadProject: vi.fn(async () => project), runScan, runIngest }, - ), - ).resolves.toBe(0); - - const ingestArgs = runIngest.mock.calls[0]?.[0]; - expect(ingestArgs).toMatchObject({ - command: 'run', - connectionId: 'warehouse', - adapter: 'historic-sql', - allowImplicitAdapter: true, - historicSqlPullConfigOverride: { - dialect: 'postgres', - windowDays: 30, - minExecutions: 7, - concurrency: 3, - staleArchiveAfterDays: 120, - filters: { - dropTrivialProbes: true, - serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, - orchestrators: { mode: 'mark-only' }, - dropFailedBelow: { errorRate: 0.5, executions: 3 }, - }, - redactionPatterns: ['(?i)secret'], - }, - }); - expect(ingestArgs?.historicSqlPullConfigOverride).not.toHaveProperty('enabled'); - }); -``` - -- [ ] **Step 2: Run the failing public-ingest test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts --testTimeout 30000 -``` - -Expected: FAIL. The new assertion sees `historicSqlPullConfigOverride` with -`dialect: 'postgres'` and `windowDays: 30`, but without `minExecutions`, -`filters`, `redactionPatterns`, `concurrency`, or -`staleArchiveAfterDays`. - -- [ ] **Step 3: Add the full query-history pull config to public plans** - -In `packages/cli/src/public-ingest.ts`, update the `queryHistory` field on -`KtxPublicIngestPlanTarget` to include a pull config for enabled query-history -runs: - -```ts - queryHistory?: { - enabled: boolean; - dialect?: HistoricSqlDialect; - windowDays?: number; - pullConfig?: Record; - unsupported?: boolean; - skippedStoredByFast?: boolean; - }; -``` - -Still in `packages/cli/src/public-ingest.ts`, add this helper below -`positiveInteger()`: - -```ts -function queryHistoryPullConfig(input: { - stored: Record; - dialect: HistoricSqlDialect; - windowDays?: number; -}): Record { - const { enabled: _enabled, dialect: _dialect, ...storedConfig } = input.stored; - return { - ...storedConfig, - dialect: input.dialect, - ...(input.windowDays !== undefined ? { windowDays: input.windowDays } : {}), - }; -} -``` - -Then replace the enabled-query-history return inside -`resolveDatabaseTargetOptions()` with this version: - -```ts - if (requestedQh && dialect) { - if (depth === 'fast') { - input.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`); - } - depth = 'deep'; - return { - databaseDepth: depth, - queryHistory: { - ...queryHistory, - enabled: true, - dialect, - pullConfig: queryHistoryPullConfig({ - stored: storedQh, - dialect, - windowDays: queryHistory.windowDays, - }), - }, - steps: ['database-schema', 'query-history'], - }; - } -``` - -- [ ] **Step 4: Pass the preserved pull config into the historic-SQL adapter** - -In `packages/cli/src/public-ingest.ts`, replace the -`historicSqlPullConfigOverride` construction in `executePublicIngestTarget()` -with: - -```ts - historicSqlPullConfigOverride: - target.queryHistory.pullConfig ?? { - dialect: target.queryHistory.dialect, - ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}), - }, -``` - -The surrounding `ingestArgs` object must still include: - -```ts - adapter: 'historic-sql', - outputMode: sourceIngestOutputMode(args, io), - inputMode: args.inputMode, - allowImplicitAdapter: true, -``` - -- [ ] **Step 5: Run the public-ingest tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts --testTimeout 30000 -``` - -Expected: PASS. The new regression test proves public ingest preserves stored -query-history fields while `--query-history-window-days 30` overrides only -`windowDays` for the current run. - -- [ ] **Step 6: Commit** - -Run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(cli): preserve query-history pull config in public ingest" -``` - -### Task 2: Hide debug commands from the docs command tree - -**Files:** -- Modify: `packages/cli/src/command-tree.ts` -- Test: `packages/cli/src/command-tree.test.ts` -- Test: `packages/cli/src/print-command-tree.test.ts` - -- [ ] **Step 1: Write the failing hidden-command walker test** - -In `packages/cli/src/command-tree.test.ts`, add this test inside the -`walkCommandTree` describe block: - -```ts - it('omits Commander hidden commands from the public tree', () => { - const root = new Command('ktx'); - root.command('scan', { hidden: true }).description('Run a standalone connection scan'); - const ingest = root.command('ingest').description('Build or inspect KTX context'); - ingest.command('run', { hidden: true }).description('Run local ingest by adapter'); - ingest.command('watch', { hidden: true }).description('Open a stored visual report'); - ingest.command('status').description('Print status'); - root.command('status').description('Check readiness'); - - const tree = walkCommandTree(root); - - expect(tree.children.map((child) => child.name)).toEqual(['ingest', 'status']); - expect(tree.children[0]).toMatchObject({ - name: 'ingest', - children: [{ name: 'status', description: 'Print status', aliases: [], arguments: [], children: [] }], - }); - }); -``` - -- [ ] **Step 2: Write the failing rendered KTX tree assertions** - -In `packages/cli/src/print-command-tree.test.ts`, add these assertions to the -first `renders an indented tree rooted at "ktx" with known top-level commands` -test after the existing `not.toContain()` assertions: - -```ts - expect(output).not.toContain('scan '); - expect(output).not.toContain('│ ├── run'); - expect(output).not.toContain('│ ├── watch'); - expect(output).not.toContain('│ └── watch'); -``` - -- [ ] **Step 3: Run the failing command-tree tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts src/print-command-tree.test.ts -``` - -Expected: FAIL. The walker includes hidden commands because it currently maps -over `command.commands` without filtering Commander `_hidden` entries. - -- [ ] **Step 4: Filter hidden Commander commands in the walker** - -In `packages/cli/src/command-tree.ts`, add this helper above -`walkCommandTree()`: - -```ts -function isHiddenCommand(command: CommandUnknownOpts): boolean { - return (command as CommandUnknownOpts & { _hidden?: boolean })._hidden === true; -} -``` - -Then replace the `children` field inside `walkCommandTree()` with: - -```ts - children: command.commands.filter((child) => !isHiddenCommand(child)).map((child) => walkCommandTree(child)), -``` - -The complete function should read: - -```ts -export function walkCommandTree(command: CommandUnknownOpts): CommandTreeNode { - return { - name: command.name(), - description: command.description(), - aliases: command.aliases(), - arguments: command.registeredArguments.map(formatArgumentDeclaration), - children: command.commands.filter((child) => !isHiddenCommand(child)).map((child) => walkCommandTree(child)), - }; -} -``` - -- [ ] **Step 5: Run the command-tree tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/command-tree.test.ts src/print-command-tree.test.ts -``` - -Expected: PASS. The pure walker omits hidden commands and the rendered KTX tree -no longer contains `scan `, `ingest run`, or `ingest watch`. - -- [ ] **Step 6: Verify the docs command output directly** - -Run: - -```bash -pnpm --filter @ktx/cli run docs:commands > /tmp/ktx-command-tree.txt -rg -n "scan |^[[:space:][:graph:]]*run[[:space:]]+Run local ingest|^[[:space:][:graph:]]*watch \\[runId\\]" /tmp/ktx-command-tree.txt -``` - -Expected: the first command succeeds and writes the command tree. The `rg` -command exits with status `1` and prints no matches. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add packages/cli/src/command-tree.ts packages/cli/src/command-tree.test.ts packages/cli/src/print-command-tree.test.ts -git commit -m "fix(cli): omit hidden commands from docs command tree" -``` - -### Task 3: Final verification - -**Files:** -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `packages/cli/src/command-tree.ts` -- Verify: `packages/cli/src/public-ingest.test.ts` -- Verify: `packages/cli/src/command-tree.test.ts` -- Verify: `packages/cli/src/print-command-tree.test.ts` - -- [ ] **Step 1: Run focused CLI regression tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/local-adapters.test.ts src/index.test.ts src/command-tree.test.ts src/print-command-tree.test.ts --testTimeout 30000 -``` - -Expected: PASS. This covers public ingest execution, adapter config mapping, -normal help routing, and docs command-tree rendering. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS with no TypeScript errors. - -- [ ] **Step 3: Run docs command-tree output check** - -Run: - -```bash -pnpm --filter @ktx/cli run docs:commands > /tmp/ktx-command-tree.txt -rg -n "scan |^[[:space:][:graph:]]*run[[:space:]]+Run local ingest|^[[:space:][:graph:]]*watch \\[runId\\]" /tmp/ktx-command-tree.txt -``` - -Expected: the `docs:commands` command succeeds. The `rg` command exits `1` -with no matches. - -- [ ] **Step 4: Run TypeScript dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If Knip reports unrelated existing findings, inspect them and -record the exact findings in the implementation notes before deciding whether -they are related to this plan. - -- [ ] **Step 5: Inspect the final diff** - -Run: - -```bash -git status --short -git diff -- packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/command-tree.ts packages/cli/src/command-tree.test.ts packages/cli/src/print-command-tree.test.ts -``` - -Expected: only the intended files are modified. The diff contains no generated -`dist/` output and no unrelated documentation changes. - -- [ ] **Step 6: Commit verification-only fixes if needed** - -If verification required expectation or type-only fixes, run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/command-tree.ts packages/cli/src/command-tree.test.ts packages/cli/src/print-command-tree.test.ts -git commit -m "test(cli): close unified ingest final public surface checks" -``` - -If no files changed during verification, do not create an empty commit. - -## Self-review - -- Spec coverage: This plan covers the remaining v1-blocking public query-history - config mapping and public command discovery output. It intentionally leaves - hidden debug command callability and internal scan/live-database/historic-sql - names as non-blocking because the original spec allows internal/debug names - in v1. -- Placeholder scan: No task uses deferred placeholders or unnamed edge-handling - steps. Each code step names the exact file, insertion point, and code shape. -- Type consistency: New `pullConfig` data stays under - `KtxPublicIngestPlanTarget.queryHistory` and flows unchanged into the - existing `KtxIngestArgs.historicSqlPullConfigOverride` field. Command-tree - filtering uses Commander `_hidden`, the same field Commander help uses. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-final-ux-labels.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-final-ux-labels.md deleted file mode 100644 index b2db0aa9..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-final-ux-labels.md +++ /dev/null @@ -1,802 +0,0 @@ -# Unified Ingest V1 Final UX Labels Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking public UX gaps in unified ingest warning aggregation and setup/status terminology. - -**Architecture:** Keep the implemented connection-centric ingest planner, hidden debug commands, and internal scan/live-database/historic-sql boundaries. Add one warning accumulator lane for unsupported database query-history targets, then update normal setup/status/docs copy so public database groups are called `Databases` rather than `Primary sources`. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, Node test runner, KTX CLI/context packages. - ---- - -## Current Audit - -Implemented unified-ingest plans already cover the original spec's main v1 behavior: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, `--query-history`, `--no-query-history`, and `--query-history-window-days` route through `packages/cli/src/public-ingest.ts`. -- Database targets are ordered before source targets, public source ingest bypasses `ingest.adapters`, and database depth maps to structural/enriched scan internals. -- Deep readiness is evaluated before target work starts, and `--all` isolates per-target failures. -- Setup stores `connections..context.depth` and `connections..context.queryHistory`, migrates legacy `historicSql`, and uses foreground-only context-build state. -- Normal help hides `ktx scan`, `ktx ingest run`, and live `ktx ingest watch`; docs no longer present those as normal public workflows. -- Foreground progress uses `Databases` and `Context sources`, and normal progress/failure output sanitizes scan/live-database/historic-sql internals. - -### V1-Blocking Gaps - -- `ktx ingest --all --query-history` does not aggregate unsupported database query-history warnings. Source depth/query-history warnings are aggregated, but unsupported database drivers currently add one warning per target from `resolveDatabaseTargetOptions()`, contrary to the original spec's `--all` warning aggregation rule for non-applicable query-history flags. -- Normal setup/status surfaces still use the old `Primary sources` public label for databases: - - `packages/cli/src/setup.ts` prints `Primary sources configured`. - - `packages/cli/src/setup-context.ts` prints a `Primary sources:` success group. - - `packages/cli/src/setup-ready-menu.ts` labels the database section `Primary sources`. - - `packages/cli/src/setup-databases.ts` uses `primary source` in normal interactive prompts, skip/failure messages, and success headings. - - `README.md`, `docs-site/content/docs/getting-started/quickstart.mdx`, and `docs-site/content/docs/cli-reference/ktx-setup.mdx` still mirror the old label. - -### Non-Blocking Gaps - -- Hidden debug commands can remain callable: `ktx scan`, `ktx ingest run`, and `ktx ingest watch`. -- Internal adapter keys, raw artifact paths, WorkUnit keys, package names, tests, and developer-only scripts can continue to use `scan`, `live-database`, and `historic-sql`. -- Public conceptual docs may still use `scan` as a generic noun where they are describing internal database metadata artifacts rather than documenting `ktx scan` as the public command. -- Internal readiness config names such as `scan.enrichment.mode` can remain because they are current `ktx.yaml` field names. - -## File Structure - -- Modify `packages/cli/src/public-ingest.ts`: aggregate unsupported database query-history warnings for `--all`. -- Modify `packages/cli/src/public-ingest.test.ts`: add regression tests for explicit and stored unsupported query-history aggregation. -- Modify `packages/cli/src/setup-ready-menu.ts`: change the ready-project database menu label to `Databases`. -- Modify `packages/cli/src/setup-ready-menu.test.ts`: update the ready-menu expected label. -- Modify `packages/cli/src/setup.ts`: change setup status output from `Primary sources configured` to `Databases configured`. -- Modify `packages/cli/src/setup.test.ts`: update status and empty-selection expectations. -- Modify `packages/cli/src/setup-context.ts`: change setup context success grouping from `Primary sources` to `Databases`. -- Modify `packages/cli/src/setup-context.test.ts`: assert the success output uses `Databases`. -- Modify `packages/cli/src/setup-databases.ts`: change normal database setup copy from `primary source(s)` / `knowledge sources` to `database(s)` / `context sources`. -- Modify `packages/cli/src/setup-databases.test.ts`: update expected prompt/output strings. -- Modify `README.md`: update the setup status example label. -- Modify `docs-site/content/docs/getting-started/quickstart.mdx`: update setup success/status examples. -- Modify `docs-site/content/docs/cli-reference/ktx-setup.mdx`: update setup status example. -- Modify `scripts/examples-docs.test.mjs`: add docs regression assertions for the old `Primary sources` label. - -## Tasks - -### Task 1: Aggregate Unsupported Query-History Warnings - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Add failing unsupported warning aggregation tests** - -In `packages/cli/src/public-ingest.test.ts`, add these tests after the existing test named `warns and skips query history for unsupported database drivers`: - -```ts - it('aggregates unsupported query-history warnings for all database targets', () => { - const plan = buildPublicIngestPlan( - deepReadyProject({ - local: { driver: 'sqlite' }, - mysql_warehouse: { driver: 'mysql' }, - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }), - { - projectDir: '/tmp/project', - all: true, - depth: 'deep', - queryHistory: 'enabled', - }, - ); - - expect(plan.targets).toEqual([ - expect.objectContaining({ - connectionId: 'local', - queryHistory: { enabled: false, unsupported: true }, - steps: ['database-schema'], - }), - expect.objectContaining({ - connectionId: 'mysql_warehouse', - queryHistory: { enabled: false, unsupported: true }, - steps: ['database-schema'], - }), - expect.objectContaining({ - connectionId: 'warehouse', - queryHistory: expect.objectContaining({ enabled: true, dialect: 'postgres' }), - steps: ['database-schema', 'query-history'], - }), - ]); - expect(plan.warnings).toEqual([ - '--query-history is not supported for 2 database connections (mysql, sqlite); running schema ingest for those connections.', - ]); - }); - - it('aggregates stored unsupported query-history config warnings for all database targets', () => { - const plan = buildPublicIngestPlan( - projectWithConnections({ - local: { driver: 'sqlite', context: { queryHistory: { enabled: true } } }, - mysql_warehouse: { driver: 'mysql', context: { queryHistory: { enabled: true } } }, - }), - { - projectDir: '/tmp/project', - all: true, - queryHistory: 'default', - }, - ); - - expect(plan.targets).toEqual([ - expect.objectContaining({ - connectionId: 'local', - queryHistory: { enabled: false, unsupported: true }, - steps: ['database-schema'], - }), - expect.objectContaining({ - connectionId: 'mysql_warehouse', - queryHistory: { enabled: false, unsupported: true }, - steps: ['database-schema'], - }), - ]); - expect(plan.warnings).toEqual([ - '2 database connections have query history enabled in ktx.yaml, but their drivers do not support it; running schema ingest for those connections.', - ]); - }); -``` - -- [ ] **Step 2: Run the failing public ingest tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "unsupported query-history" -``` - -Expected: FAIL because the new `--all` cases currently receive one warning per unsupported database target. - -- [ ] **Step 3: Add unsupported query-history warning accumulator state** - -In `packages/cli/src/public-ingest.ts`, replace the current warning accumulator interface and factory with: - -```ts -interface KtxUnsupportedQueryHistoryWarning { - connectionId: string; - driver: string; - reason: 'explicit' | 'stored'; -} - -interface KtxPublicIngestWarningAccumulator { - warnings: string[]; - ignoredDepthForSources: string[]; - ignoredQueryHistoryForSources: string[]; - unsupportedQueryHistoryForDatabases: KtxUnsupportedQueryHistoryWarning[]; -} - -function createWarningAccumulator(): KtxPublicIngestWarningAccumulator { - return { - warnings: [], - ignoredDepthForSources: [], - ignoredQueryHistoryForSources: [], - unsupportedQueryHistoryForDatabases: [], - }; -} -``` - -- [ ] **Step 4: Add unsupported database warning formatting** - -In `packages/cli/src/public-ingest.ts`, add these helpers after `sourceIgnoredWarning()`: - -```ts -function unsupportedDriverList(entries: KtxUnsupportedQueryHistoryWarning[]): string { - return [...new Set(entries.map((entry) => entry.driver))].sort((left, right) => left.localeCompare(right)).join(', '); -} - -function unsupportedQueryHistoryWarnings( - entries: KtxUnsupportedQueryHistoryWarning[], - all: boolean, -): string[] { - if (entries.length === 0) { - return []; - } - - const warnings: string[] = []; - const explicitEntries = entries.filter((entry) => entry.reason === 'explicit'); - const storedEntries = entries.filter((entry) => entry.reason === 'stored'); - - if (explicitEntries.length === 1 || (!all && explicitEntries.length > 0)) { - warnings.push( - ...explicitEntries.map( - (entry) => - `--query-history is not supported for ${entry.driver}; running schema ingest for ${entry.connectionId}.`, - ), - ); - } else if (explicitEntries.length > 1) { - warnings.push( - `--query-history is not supported for ${explicitEntries.length} database connections (${unsupportedDriverList( - explicitEntries, - )}); running schema ingest for those connections.`, - ); - } - - if (storedEntries.length === 1 || (!all && storedEntries.length > 0)) { - warnings.push( - ...storedEntries.map( - (entry) => - `${entry.connectionId} has query history enabled in ktx.yaml, but ${entry.driver} does not support it; running schema ingest.`, - ), - ); - } else if (storedEntries.length > 1) { - warnings.push( - `${storedEntries.length} database connections have query history enabled in ktx.yaml, but their drivers do not support it; running schema ingest for those connections.`, - ); - } - - return warnings; -} -``` - -- [ ] **Step 5: Use the accumulator in `finalizeWarnings()`** - -In `packages/cli/src/public-ingest.ts`, replace the start of `finalizeWarnings()` with: - -```ts - const warnings = [ - ...accumulator.warnings, - ...unsupportedQueryHistoryWarnings(accumulator.unsupportedQueryHistoryForDatabases, args.all), - ]; -``` - -Keep the existing source depth/query-history aggregation logic below that new `warnings` initialization. - -- [ ] **Step 6: Record unsupported database targets instead of pushing immediate warnings** - -In `packages/cli/src/public-ingest.ts`, change the `resolveDatabaseTargetOptions()` input type so `warnings` is the full accumulator: - -```ts - warnings: KtxPublicIngestWarningAccumulator; -``` - -Inside the unsupported query-history branch, replace the current `input.warnings.push(...)` block with: - -```ts - input.warnings.unsupportedQueryHistoryForDatabases.push({ - connectionId: input.connectionId, - driver: input.driver, - reason: explicitQueryHistory === 'enabled' || input.args.queryHistoryWindowDays !== undefined ? 'explicit' : 'stored', - }); -``` - -In the supported query-history branch, replace: - -```ts - input.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`); -``` - -with: - -```ts - input.warnings.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`); -``` - -In the stored query-history skipped-by-fast branch, replace: - -```ts - input.warnings.push( - `${input.connectionId} has query history enabled in ktx.yaml, but --fast skips query-history processing.`, - ); -``` - -with: - -```ts - input.warnings.warnings.push( - `${input.connectionId} has query history enabled in ktx.yaml, but --fast skips query-history processing.`, - ); -``` - -In `targetForConnection()`, replace the database resolver call with: - -```ts - const options = resolveDatabaseTargetOptions({ connectionId, driver, connection, args, warnings }); -``` - -- [ ] **Step 7: Verify unsupported warning aggregation passes** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "unsupported query-history" -``` - -Expected: PASS. The single-target warning tests keep the old exact messages, while `--all` unsupported database targets receive one aggregate warning per reason. - -- [ ] **Step 8: Commit unsupported warning aggregation** - -Run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(cli): aggregate unsupported query-history warnings" -``` - -### Task 2: Rename Public Setup Database Labels - -**Files:** -- Modify: `packages/cli/src/setup-ready-menu.ts` -- Modify: `packages/cli/src/setup.ts` -- Modify: `packages/cli/src/setup-context.ts` -- Modify: `packages/cli/src/setup-databases.ts` -- Test: `packages/cli/src/setup-ready-menu.test.ts` -- Test: `packages/cli/src/setup.test.ts` -- Test: `packages/cli/src/setup-context.test.ts` -- Test: `packages/cli/src/setup-databases.test.ts` -- Modify: `README.md` -- Modify: `docs-site/content/docs/getting-started/quickstart.mdx` -- Modify: `docs-site/content/docs/cli-reference/ktx-setup.mdx` -- Test: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Write failing CLI copy expectations** - -In `packages/cli/src/setup-ready-menu.test.ts`, change the expected database option to: - -```ts - { value: 'databases', label: 'Databases' }, -``` - -In `packages/cli/src/setup-context.test.ts`, add these assertions after each `expect(io.stdout()).toContain('KTX context is ready for agents.');` assertion in the successful build and existing-context tests: - -```ts - expect(io.stdout()).toContain('Databases:'); - expect(io.stdout()).not.toContain('Primary sources:'); -``` - -In `packages/cli/src/setup.test.ts`, change the empty database selection expectation to: - -```ts - expect(testIo.stdout()).toContain( - 'KTX cannot work without at least one database. Select a database or press Escape to go back.', - ); - expect(testIo.stderr()).not.toContain('No databases selected.'); -``` - -In `packages/cli/src/setup.test.ts`, in the existing-project status test, add: - -```ts - expect(rendered).toContain('Databases configured: no'); - expect(rendered).not.toContain('Primary sources configured'); -``` - -- [ ] **Step 2: Write failing setup database prompt expectations** - -In `packages/cli/src/setup-databases.test.ts`, update the old public copy expectations to the new database labels: - -```ts -expect(prompts.multiselect).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Which databases should KTX connect to?'), - }), -); -``` - -For configured database menu expectations, use: - -```ts -expect(prompts.select).toHaveBeenCalledWith({ - message: 'Databases already configured: warehouse\nWhat would you like to do?', - options: [ - { value: 'continue', label: 'Continue to context sources' }, - { value: 'add', label: 'Add another database' }, - ], -}); -``` - -For the `postgres-warehouse` configured menu expectations, use: - -```ts -expect(prompts.select).toHaveBeenCalledWith({ - message: 'Databases already configured: postgres-warehouse\nWhat would you like to do?', - options: [ - { value: 'continue', label: 'Continue to context sources' }, - { value: 'add', label: 'Add another database' }, - ], -}); -``` - -For empty-selection output expectations, use: - -```ts -expect(io.stdout()).not.toContain('KTX cannot work without at least one database'); -``` - -For successful initial scan/setup output, use: - -```ts -expect(io.stdout()).toContain('◇ Database ready'); -expect(io.stdout()).not.toContain('Primary source ready'); -``` - -Rename test descriptions that contain `primary source` or `primary sources` so they use `database` or `databases`. For example: - -```ts - it('shows every supported database in the interactive checklist', async () => { -``` - -```ts - it('shows a configured database menu instead of the type checklist when a database exists', async () => { -``` - -```ts - it('lets users add another database after completing the first one', async () => { -``` - -- [ ] **Step 3: Run failing setup label tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-ready-menu.test.ts src/setup.test.ts src/setup-context.test.ts src/setup-databases.test.ts -t "ready menu|readiness checklist|context is ready|database|primary source|configured" -``` - -Expected: FAIL because production copy still uses `Primary sources` and `primary source`. - -- [ ] **Step 4: Update the ready menu and status labels** - -In `packages/cli/src/setup-ready-menu.ts`, change: - -```ts - { value: 'databases', label: 'Primary sources' }, -``` - -to: - -```ts - { value: 'databases', label: 'Databases' }, -``` - -In `packages/cli/src/setup.ts`, change: - -```ts - `Primary sources configured: ${formatConnectionList(status.databases.map((database) => database.connectionId))}`, -``` - -to: - -```ts - `Databases configured: ${formatConnectionList(status.databases.map((database) => database.connectionId))}`, -``` - -In `packages/cli/src/setup-context.ts`, change: - -```ts - io.stdout.write('Primary sources:\n'); -``` - -to: - -```ts - io.stdout.write('Databases:\n'); -``` - -- [ ] **Step 5: Update setup database prompt and output copy** - -In `packages/cli/src/setup-databases.ts`, change: - -```ts - const backDestination = canReturnToDriverSelection ? 'primary source selection' : 'the previous setup step'; -``` - -to: - -```ts - const backDestination = canReturnToDriverSelection ? 'database selection' : 'the previous setup step'; -``` - -Replace the entire `configuredPrimarySourcesPrompt()` return value with: - -```ts - return { - message: `Databases already configured: ${connectionIds.join(', ')}\nWhat would you like to do?`, - options: [ - { value: 'continue', label: 'Continue to context sources' }, - { value: 'add', label: 'Add another database' }, - ], - }; -``` - -Change the successful database setup heading from: - -```ts - writeSetupSection(input.io, 'Primary source ready', [ -``` - -to: - -```ts - writeSetupSection(input.io, 'Database ready', [ -``` - -Change the non-interactive no-database error from: - -```ts - 'KTX cannot work without a primary source. Pass --database or --database-connection-id, or pass --skip-databases to leave setup incomplete.\n', -``` - -to: - -```ts - 'KTX cannot work without a database. Pass --database or --database-connection-id, or pass --skip-databases to leave setup incomplete.\n', -``` - -Change the driver multiselect message from: - -```ts - message: withMultiselectNavigation('Which primary sources should KTX connect to?'), -``` - -to: - -```ts - message: withMultiselectNavigation('Which databases should KTX connect to?'), -``` - -Change the empty-selection warning from: - -```ts - io.stdout.write('│ KTX cannot work without at least one primary source. Select a source or press Escape to go back.\n'); -``` - -to: - -```ts - io.stdout.write('│ KTX cannot work without at least one database. Select a database or press Escape to go back.\n'); -``` - -Change the skip output from: - -```ts - io.stdout.write('│ Primary source setup skipped. KTX cannot work until you add a primary source.\n'); -``` - -to: - -```ts - io.stdout.write('│ Database setup skipped. KTX cannot work until you add a database.\n'); -``` - -Change the no-completed-database output from: - -```ts - io.stdout.write('│ KTX cannot work without a primary source.\n'); -``` - -to: - -```ts - io.stdout.write('│ KTX cannot work without a database.\n'); -``` - -Change the retry prompt message and skip label from: - -```ts - message: `Primary source setup failed for ${connectionChoice.connectionId}`, -``` - -```ts - { value: 'skip', label: 'Skip this primary source' }, -``` - -to: - -```ts - message: `Database setup failed for ${connectionChoice.connectionId}`, -``` - -```ts - { value: 'skip', label: 'Skip this database' }, -``` - -Change the final failure line from: - -```ts - io.stderr.write('No primary source connections completed setup.\n'); -``` - -to: - -```ts - io.stderr.write('No database connections completed setup.\n'); -``` - -- [ ] **Step 6: Update public docs examples** - -In `README.md`, replace: - -```text -Primary sources configured: yes (postgres-warehouse) -``` - -with: - -```text -Databases configured: yes (postgres-warehouse) -``` - -In `docs-site/content/docs/getting-started/quickstart.mdx`, replace the database-ready heading line: - -```text -Primary source ready - postgres-warehouse - PostgreSQL - schema context complete -``` - -with: - -```text -Database ready - postgres-warehouse - PostgreSQL - schema context complete -``` - -In `docs-site/content/docs/getting-started/quickstart.mdx`, replace the setup success group: - -```text -Primary sources: - postgres-warehouse: deep context complete -``` - -with: - -```text -Databases: - postgres-warehouse: deep context complete -``` - -In `docs-site/content/docs/getting-started/quickstart.mdx`, replace: - -```text -Primary sources configured: yes (postgres-warehouse) -``` - -with: - -```text -Databases configured: yes (postgres-warehouse) -``` - -In `docs-site/content/docs/cli-reference/ktx-setup.mdx`, replace: - -```text -Primary sources configured: yes (postgres-warehouse) -``` - -with: - -```text -Databases configured: yes (postgres-warehouse) -``` - -- [ ] **Step 7: Add public docs regression assertions** - -In `scripts/examples-docs.test.mjs`, inside the test named `documents unified public ingest workflows in the docs site`, add: - -```js - const setupReference = await readText('docs-site/content/docs/cli-reference/ktx-setup.mdx'); -``` - -Then add these assertions near the existing `quickstart` and `rootReadme` assertions: - -```js - assert.match(rootReadme, /Databases configured: yes \(postgres-warehouse\)/); - assert.match(quickstart, /Databases:\n postgres-warehouse: deep context complete/); - assert.match(quickstart, /Databases configured: yes \(postgres-warehouse\)/); - assert.match(setupReference, /Databases configured: yes \(postgres-warehouse\)/); - assert.doesNotMatch(rootReadme, /Primary sources configured/); - assert.doesNotMatch(quickstart, /Primary sources/); - assert.doesNotMatch(setupReference, /Primary sources configured/); -``` - -- [ ] **Step 8: Verify setup label tests pass** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-ready-menu.test.ts src/setup.test.ts src/setup-context.test.ts src/setup-databases.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Verify docs examples pass** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 10: Scan for stale public labels** - -Run: - -```bash -rg -n "Primary sources?:|Primary sources? configured|Primary source ready|knowledge sources" packages/cli/src README.md docs-site/content/docs scripts/examples-docs.test.mjs -``` - -Expected: no matches in public CLI source, README/docs examples, or the docs regression test. - -- [ ] **Step 11: Commit public setup labels** - -Run: - -```bash -git add packages/cli/src/setup-ready-menu.ts packages/cli/src/setup-ready-menu.test.ts packages/cli/src/setup.ts packages/cli/src/setup.test.ts packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts README.md docs-site/content/docs/getting-started/quickstart.mdx docs-site/content/docs/cli-reference/ktx-setup.mdx scripts/examples-docs.test.mjs -git commit -m "fix(cli): align setup database labels" -``` - -### Task 3: Final V1 Verification - -**Files:** -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `packages/cli/src/setup-ready-menu.ts` -- Verify: `packages/cli/src/setup.ts` -- Verify: `packages/cli/src/setup-context.ts` -- Verify: `packages/cli/src/setup-databases.ts` -- Verify: `README.md` -- Verify: `docs-site/content/docs/getting-started/quickstart.mdx` -- Verify: `docs-site/content/docs/cli-reference/ktx-setup.mdx` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/context-build-view.test.ts src/setup-ready-menu.test.ts src/setup.test.ts src/setup-context.test.ts src/setup-databases.test.ts src/index.test.ts src/command-tree.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run docs regression tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 3: Run public unified-ingest stale-copy scans** - -Run: - -```bash -rg -n "Primary sources?:|Primary sources? configured|Primary source ready|knowledge sources" packages/cli/src README.md docs-site/content/docs scripts/examples-docs.test.mjs -``` - -Expected: no matches. - -Run: - -```bash -rg -n "ktx scan|ktx ingest run --connection-id|--adapter |ktx ingest watch|live-database|Historic SQL|historicSql" README.md docs-site/content/docs examples/README.md examples/local-warehouse/README.md -``` - -Expected: no matches. Matches in developer scripts, internal package names, tests, or artifact paths outside this public-docs command are non-blocking under the original spec. - -- [ ] **Step 4: Run package pre-commit on changed files** - -Run: - -```bash -uv run pre-commit run --files packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/setup-ready-menu.ts packages/cli/src/setup-ready-menu.test.ts packages/cli/src/setup.ts packages/cli/src/setup.test.ts packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts README.md docs-site/content/docs/getting-started/quickstart.mdx docs-site/content/docs/cli-reference/ktx-setup.mdx scripts/examples-docs.test.mjs -``` - -Expected: PASS. If pre-commit is unavailable because the local `uv` version or hook environment is missing, record the exact failure and run the focused Vitest and Node tests from Steps 1 and 2. - -- [ ] **Step 5: Commit final verification if needed** - -If Step 4 made formatting changes, run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/setup-ready-menu.ts packages/cli/src/setup-ready-menu.test.ts packages/cli/src/setup.ts packages/cli/src/setup.test.ts packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts README.md docs-site/content/docs/getting-started/quickstart.mdx docs-site/content/docs/cli-reference/ktx-setup.mdx scripts/examples-docs.test.mjs -git commit -m "test: verify unified ingest final ux labels" -``` - -If Step 4 made no changes, do not create an empty commit. - -## Self-Review - -- Spec coverage: This plan covers the remaining v1-blocking public gaps found in the audit: unsupported database query-history warning aggregation for `--all`, and old public `Primary sources` terminology in setup/status/docs where the spec's user-facing grouping is `Databases`. Core routing, depth, query-history execution, setup config, foreground-only state, hidden debug commands, public docs command shape, and output sanitization are already implemented by the prior plan chain. -- Placeholder scan: The plan contains exact files, exact tests, exact code snippets, exact commands, and expected outcomes. -- Type consistency: The new accumulator type is `KtxUnsupportedQueryHistoryWarning`; `resolveDatabaseTargetOptions()` receives `KtxPublicIngestWarningAccumulator`; warning strings used in tests match the implementation snippets exactly. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-foreground-and-retry-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-foreground-and-retry-closure.md deleted file mode 100644 index f665d6a1..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-foreground-and-retry-closure.md +++ /dev/null @@ -1,932 +0,0 @@ -# Unified Ingest V1 Foreground and Retry Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking public UX gaps in the unified -`ktx ingest` redesign. - -**Architecture:** Keep the implemented connection-centric ingest planner and -shared foreground context-build view. Add a small public messaging layer for -notices, warnings, and retry guidance so TTY, non-TTY, and setup next-step -surfaces all match the original spec without changing internal adapter names. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages, -Markdown plan documentation. - ---- - -## Current audit - -The implemented unified-ingest plans cover the main v1 behavior: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` route through the public ingest planner. -- Database targets run before source targets. Public source ingest bypasses - `ingest.adapters`. Fast and deep map to structural and enriched database - ingest, and deep readiness failures are isolated per target under `--all`. -- `ktx scan`, `ktx ingest run`, and `ktx ingest watch` are hidden from normal - help. Setup stores `connections..context.depth` and - `connections..context.queryHistory`. -- Setup context builds are foreground-only, legacy context-build states are - normalized to stale, and public docs no longer advertise `ktx scan` or - adapter-backed `ktx ingest run` as normal workflows. - -### V1-blocking gaps - -- Interactive foreground `ktx ingest` and setup context builds compute public - warnings but never render them. A TTY user can pass `--deep` for source - connections, `--query-history` for unsupported targets, or `--fast` with - stored query history and receive no warning in the foreground view. -- Explicit query-history runs do not state that database schema ingest runs - before query-history processing. The spec requires that message when a user - explicitly passes `--query-history`. -- Plain non-TTY failures report generic step failures such as - `warehouse failed at database-schema.` and a debug command, but they do not - include the retry guidance required by the error-handling section. -- Setup next-step output still describes the context-build action as - `Build or resume agent-ready context` through `ktx setup`, and it says the - build covers `primary-source scans and context-source ingests`. The public - model is `setup` configures, `ingest` builds or refreshes context, and status - explains readiness. -- The guided demo foreground replay still shows `scanning tables...` and - `tables scanned`, even though the normal foreground view must use - `reading schema` or `building schema context`. - -### Non-blocking gaps - -- Hidden debug commands can continue to call `ktx scan`, `ktx ingest run`, and - `ktx ingest watch`. -- Internal adapter keys, raw artifact paths, WorkUnit keys, package names, and - JSON or debug output can continue to use `scan`, `live-database`, and - `historic-sql`. -- Developer docs can continue to mention scan internals when they describe - connector implementation details. -- Existing `autoWatch`, `detached`, and `paused` type remnants in setup code - are not user-facing because setup context state is normalized before display. - -## File structure - -- Modify `packages/cli/src/public-ingest.ts`: add public plan notices, print - schema-before-query-history notices, and add retry guidance to plain - non-TTY failure details. -- Modify `packages/cli/src/public-ingest.test.ts`: cover explicit - query-history notices and retry guidance in plain output. -- Modify `packages/cli/src/context-build-view.ts`: render foreground notices - and warnings from `buildPublicIngestPlan`. -- Modify `packages/cli/src/context-build-view.test.ts`: cover warning and - notice rendering in the foreground view. -- Modify `packages/cli/src/next-steps.ts`: make the public build command - `ktx ingest --all` and remove resume/scan wording from setup next steps. -- Modify `packages/cli/src/next-steps.test.ts`: update public next-step - expectations. -- Modify `packages/cli/src/setup-demo-tour.ts`: replace demo replay scan copy - with schema-context copy. -- Modify `packages/cli/src/setup-demo-tour.test.ts`: lock the demo replay - wording against `scan` terms. - -## Tasks - -### Task 1: Render foreground notices and warnings - -**Files:** -- Modify: `packages/cli/src/context-build-view.ts` -- Test: `packages/cli/src/context-build-view.test.ts` - -- [ ] **Step 1: Write failing foreground-message tests** - -In `packages/cli/src/context-build-view.test.ts`, add these tests inside the -`renderContextBuildView` describe block, near the existing rendering tests: - -```ts - it('renders public warnings in the foreground view', () => { - const state = initViewState([ - { - connectionId: 'docs', - driver: 'notion', - operation: 'source-ingest', - adapter: 'notion', - debugCommand: 'ktx ingest docs --debug', - steps: ['source-ingest', 'memory-update'], - }, - ]); - - const rendered = renderContextBuildView(state, { - styled: false, - warnings: ['--deep affects database ingest only; ignoring it for docs.'], - }); - - expect(rendered).toContain('Warnings:'); - expect(rendered).toContain('--deep affects database ingest only; ignoring it for docs.'); - }); - - it('renders public notices in the foreground view before warnings', () => { - const state = initViewState([ - { - connectionId: 'warehouse', - driver: 'postgres', - operation: 'database-ingest', - debugCommand: 'ktx ingest warehouse --debug', - steps: ['database-schema', 'query-history'], - databaseDepth: 'deep', - detectRelationships: true, - queryHistory: { enabled: true, dialect: 'postgres' }, - }, - ]); - - const rendered = renderContextBuildView(state, { - styled: false, - notices: ['Schema ingest runs before query history for warehouse.'], - warnings: ['--query-history requires deep ingest; running warehouse with --deep.'], - }); - - expect(rendered.indexOf('Notices:')).toBeLessThan(rendered.indexOf('Warnings:')); - expect(rendered).toContain('Schema ingest runs before query history for warehouse.'); - expect(rendered).toContain('--query-history requires deep ingest; running warehouse with --deep.'); - }); -``` - -- [ ] **Step 2: Run the failing foreground-message tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts -t "renders public warnings|renders public notices" -``` - -Expected: FAIL because `renderContextBuildView` does not accept or render -`warnings` or `notices`. - -- [ ] **Step 3: Add render options for foreground messages** - -In `packages/cli/src/context-build-view.ts`, add this helper after -`renderTargetGroup`: - -```ts -function renderMessageGroup(label: string, messages: string[], styled: boolean): string[] { - if (messages.length === 0) return []; - const renderedMessages = messages.map((message) => ` - ${message}`); - return ['', ` ${label}:`, ...renderedMessages.map((line) => (styled ? dim(line) : line))]; -} -``` - -Then change the `renderContextBuildView` signature from: - -```ts -export function renderContextBuildView( - state: ContextBuildViewState, - options: { styled?: boolean; showHint?: boolean; hintText?: string; projectDir?: string } = {}, -): string { -``` - -to: - -```ts -export function renderContextBuildView( - state: ContextBuildViewState, - options: { - styled?: boolean; - showHint?: boolean; - hintText?: string; - projectDir?: string; - notices?: string[]; - warnings?: string[]; - } = {}, -): string { -``` - -In the `lines` array inside `renderContextBuildView`, insert the notice and -warning groups after the `Context sources` group: - -```ts - ...renderTargetGroup('Databases', state.primarySources, state.frame, styled, width), - ...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width), - ...renderMessageGroup('Notices', options.notices ?? [], styled), - ...renderMessageGroup('Warnings', options.warnings ?? [], styled), - '', -``` - -- [ ] **Step 4: Pass plan messages into foreground rendering** - -In `packages/cli/src/context-build-view.ts`, inside `runContextBuild`, change: - -```ts - const viewOpts = { styled: true, projectDir: args.projectDir }; -``` - -to: - -```ts - const viewOpts = { - styled: true, - projectDir: args.projectDir, - notices: plan.notices ?? [], - warnings: plan.warnings, - }; -``` - -This makes every call to `paint()` and the final non-TTY foreground fallback -render the same public messages. - -- [ ] **Step 5: Run the foreground-message tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts -t "renders public warnings|renders public notices" -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts -git commit -m "fix: render unified ingest foreground warnings" -``` - -### Task 2: State schema-before-query-history for explicit runs - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/context-build-view.ts` -- Test: `packages/cli/src/public-ingest.test.ts` -- Test: `packages/cli/src/context-build-view.test.ts` - -- [ ] **Step 1: Write failing explicit query-history notice tests** - -In `packages/cli/src/public-ingest.test.ts`, add this test inside -`describe('buildPublicIngestPlan', ...)` after the existing query-history -planning tests: - -```ts - it('adds a schema-first notice when query history is explicitly enabled', () => { - const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }); - - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - queryHistory: 'enabled', - }).notices, - ).toEqual(['Schema ingest runs before query history for warehouse.']); - }); -``` - -In `packages/cli/src/public-ingest.test.ts`, add this test inside -`describe('runKtxPublicIngest', ...)` after -`runs query history after schema ingest with current-run window override`: - -```ts - it('prints the schema-first notice for explicit query-history runs', async () => { - const io = makeIo(); - const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }); - const runScan = vi.fn(async () => 0); - const runIngest = vi.fn(async () => 0); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - queryHistory: 'enabled', - }, - io.io, - { loadProject: vi.fn(async () => project), runScan, runIngest }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Schema ingest runs before query history for warehouse.'); - }); -``` - -In `packages/cli/src/context-build-view.test.ts`, add this test near the -existing `runContextBuild` tests: - -```ts - it('passes schema-first notices from the plan into foreground output', async () => { - const io = makeIo(); - const project = { - ...projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }), - config: { - ...projectWithConnections({ warehouse: { driver: 'postgres' } }).config, - connections: { - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }, - llm: { - provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret - models: { default: 'gpt-test' }, - }, - scan: { - ...projectWithConnections({ warehouse: { driver: 'postgres' } }).config.scan, - enrichment: { - mode: 'llm', - embeddings: { - backend: 'openai', - model: 'text-embedding-3-small', - dimensions: 1536, - }, - }, - }, - }, - }; - const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation)); - - await expect( - runContextBuild( - project, - { - projectDir: '/tmp/project', - inputMode: 'disabled', - targetConnectionId: 'warehouse', - all: false, - queryHistory: 'enabled', - }, - io.io, - { executeTarget, now: () => 1000 }, - ), - ).resolves.toMatchObject({ exitCode: 0 }); - - expect(io.stdout()).toContain('Schema ingest runs before query history for warehouse.'); - }); -``` - -- [ ] **Step 2: Run the failing query-history notice tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/context-build-view.test.ts -t "schema-first notice|passes schema-first" -``` - -Expected: FAIL because plans do not include `notices`, and plain output does -not print schema-first text. - -- [ ] **Step 3: Add notices to the public ingest plan** - -In `packages/cli/src/public-ingest.ts`, update `KtxPublicIngestPlan`: - -```ts -export interface KtxPublicIngestPlan { - projectDir: string; - targets: KtxPublicIngestPlanTarget[]; - warnings: string[]; - notices?: string[]; -} -``` - -Add this helper after `finalizeWarnings`: - -```ts -function schemaFirstQueryHistoryNotice( - targets: KtxPublicIngestPlanTarget[], - args: { queryHistory?: KtxPublicIngestQueryHistoryFlag }, -): string | null { - if (args.queryHistory !== 'enabled') { - return null; - } - const queryHistoryTargets = targets.filter((target) => target.queryHistory?.enabled === true); - if (queryHistoryTargets.length === 0) { - return null; - } - if (queryHistoryTargets.length === 1) { - return `Schema ingest runs before query history for ${queryHistoryTargets[0].connectionId}.`; - } - return `Schema ingest runs before query history for ${queryHistoryTargets.length} database connections.`; -} -``` - -In `buildPublicIngestPlan`, replace the direct return with: - -```ts - const orderedTargets = [ - ...targets.filter((t) => t.operation === 'database-ingest'), - ...targets.filter((t) => t.operation === 'source-ingest'), - ]; - const notice = schemaFirstQueryHistoryNotice(orderedTargets, args); - return { - projectDir: args.projectDir, - targets: orderedTargets, - warnings: finalizeWarnings(warnings, args), - ...(notice ? { notices: [notice] } : {}), - }; -``` - -- [ ] **Step 4: Print notices in plain public ingest** - -In `packages/cli/src/public-ingest.ts`, inside `runKtxPublicIngest`, change: - -```ts - if (!args.json && plan.warnings.length > 0) { - for (const warning of plan.warnings) { - io.stderr.write(`Warning: ${warning}\n`); - } - } -``` - -to: - -```ts - if (!args.json) { - for (const notice of plan.notices ?? []) { - io.stdout.write(`${notice}\n`); - } - for (const warning of plan.warnings) { - io.stderr.write(`Warning: ${warning}\n`); - } - } -``` - -Task 1 already passes `plan.notices` into `runContextBuild`, so explicit -query-history foreground runs render the same notice in the view. - -- [ ] **Step 5: Run the query-history notice tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/context-build-view.test.ts -t "schema-first notice|passes schema-first" -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts -git commit -m "fix: explain query history schema order" -``` - -### Task 3: Add retry guidance to plain public failures - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write failing plain retry tests** - -In `packages/cli/src/public-ingest.test.ts`, replace these assertions in -`runs all independent targets and reports partial failures`: - -```ts - expect(io.stdout()).toContain('warehouse failed at database-schema.'); - expect(io.stdout()).toContain('Debug: ktx ingest warehouse --debug'); -``` - -with: - -```ts - expect(io.stdout()).toContain('warehouse failed at database-schema.'); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --fast'); - expect(io.stdout()).not.toContain('Debug: ktx ingest warehouse --debug'); -``` - -Then add this test after `runs all independent targets and reports partial -failures`: - -```ts - it('prints query-history retry guidance for query-history facet failures', async () => { - const io = makeIo(); - const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }); - const runScan = vi.fn(async () => 0); - const runIngest = vi.fn(async () => 1); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - queryHistory: 'enabled', - }, - io.io, - { loadProject: vi.fn(async () => project), runScan, runIngest }, - ), - ).resolves.toBe(1); - - expect(io.stdout()).toContain('warehouse failed at query-history.'); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history'); - expect(io.stdout()).not.toContain('historic-sql'); - }); -``` - -- [ ] **Step 2: Run the failing retry tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "partial failures|query-history retry" -``` - -Expected: FAIL because plain failures still print `Debug:` and lack retry -commands. - -- [ ] **Step 3: Add retry command formatting to public ingest** - -In `packages/cli/src/public-ingest.ts`, add these helpers before -`markTargetResult`: - -```ts -function retryCommandForTarget( - target: KtxPublicIngestPlanTarget, - args: Extract, -): string { - const projectPart = ` --project-dir ${args.projectDir}`; - const depthPart = target.databaseDepth ? ` --${target.databaseDepth}` : ''; - const queryHistoryPart = target.queryHistory?.enabled === true ? ' --query-history' : ''; - const windowPart = - target.queryHistory?.enabled === true && target.queryHistory.windowDays !== undefined - ? ` --query-history-window-days ${target.queryHistory.windowDays}` - : ''; - return `ktx ingest ${target.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`; -} - -function trimTrailingPeriod(value: string): string { - return value.endsWith('.') ? value.slice(0, -1) : value; -} - -function failureDetailWithRetry(input: { - target: KtxPublicIngestPlanTarget; - args: Extract; - failedOperation: KtxPublicIngestStepName; - failureDetail?: string; -}): string { - const detail = input.failureDetail?.trim(); - const base = - detail && detail.startsWith(`${input.target.connectionId} `) - ? detail - : detail - ? `${input.target.connectionId} failed: ${detail}` - : `${input.target.connectionId} failed at ${input.failedOperation}.`; - return `${trimTrailingPeriod(base)}. Retry: ${retryCommandForTarget(input.target, input.args)}`; -} -``` - -- [ ] **Step 4: Thread run args into failure detail construction** - -Change the `markTargetResult` signature in `packages/cli/src/public-ingest.ts` -from: - -```ts -function markTargetResult( - target: KtxPublicIngestPlanTarget, - status: 'done' | 'failed', - failedOperation?: KtxPublicIngestStepName, - failureDetail?: string, -): KtxPublicIngestTargetResult { -``` - -to: - -```ts -function markTargetResult( - target: KtxPublicIngestPlanTarget, - args: Extract, - status: 'done' | 'failed', - failedOperation?: KtxPublicIngestStepName, - failureDetail?: string, -): KtxPublicIngestTargetResult { -``` - -Inside the failed-step branch, replace: - -```ts - detail: failureDetail ?? `${target.connectionId} failed at ${selectedFailedOperation}.`, -``` - -with: - -```ts - detail: failureDetailWithRetry({ - target, - args, - failedOperation: selectedFailedOperation, - failureDetail, - }), -``` - -Update every `markTargetResult` call in `executePublicIngestTarget`: - -```ts - return markTargetResult( - target, - args, - 'failed', - 'database-schema', - capturedScanIo ? firstCapturedFailureLine(capturedScanIo.capturedOutput()) : undefined, - ); -``` - -```ts - return markTargetResult(target, args, 'failed', 'query-history'); -``` - -```ts - return markTargetResult(target, args, 'done'); -``` - -```ts - return markTargetResult(target, args, exitCode === 0 ? 'done' : 'failed'); -``` - -- [ ] **Step 5: Stop printing debug commands in plain failure summaries** - -In `renderPlainResults`, remove this block: - -```ts - if (failedStep.debugCommand) { - io.stdout.write(` Debug: ${failedStep.debugCommand}\n`); - } -``` - -Debug commands remain available through JSON and debug surfaces, but normal -plain output now focuses on the connection and retry action. - -- [ ] **Step 6: Run the retry tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "partial failures|query-history retry" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix: add public ingest retry guidance" -``` - -### Task 4: Replace setup next-step scan/resume wording - -**Files:** -- Modify: `packages/cli/src/next-steps.ts` -- Test: `packages/cli/src/next-steps.test.ts` - -- [ ] **Step 1: Write failing next-step copy tests** - -In `packages/cli/src/next-steps.test.ts`, replace the expected -`KTX_CONTEXT_BUILD_COMMANDS` value with: - -```ts - expect(KTX_CONTEXT_BUILD_COMMANDS).toEqual([ - { - command: 'ktx ingest --all', - description: 'Build or refresh agent-ready context from configured connections', - }, - { - command: 'ktx status', - description: 'Check setup and context readiness', - }, - ]); -``` - -In the test named `keeps setup next steps focused on building context when the -build is not ready`, replace: - -```ts - expect(rendered).toContain('primary-source scans and context-source ingests'); - expect(rendered).toContain('ktx setup'); -``` - -with: - -```ts - expect(rendered).toContain('Run ingest to build database schema context before context-source ingest.'); - expect(rendered).toContain('ktx ingest --all'); - expect(rendered).not.toContain('resume'); - expect(rendered).not.toContain('scan'); -``` - -- [ ] **Step 2: Run the failing next-step copy tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/next-steps.test.ts -``` - -Expected: FAIL because the current copy still recommends `ktx setup` for the -context-build action and uses resume/scan wording. - -- [ ] **Step 3: Update the next-step command constants** - -In `packages/cli/src/next-steps.ts`, change `KTX_CONTEXT_BUILD_COMMANDS` to: - -```ts -export const KTX_CONTEXT_BUILD_COMMANDS = [ - { - command: 'ktx ingest --all', - description: 'Build or refresh agent-ready context from configured connections', - }, - { - command: 'ktx status', - description: 'Check setup and context readiness', - }, -] as const; -``` - -In `formatSetupNextStepLines`, replace: - -```ts - `${indent}Preferred route: run the CLI build; it covers primary-source scans and context-source ingests.`, -``` - -with: - -```ts - `${indent}Run ingest to build database schema context before context-source ingest.`, -``` - -- [ ] **Step 4: Run the next-step copy tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/next-steps.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/next-steps.ts packages/cli/src/next-steps.test.ts -git commit -m "fix: align setup next steps with unified ingest" -``` - -### Task 5: Clean guided demo foreground scan wording - -**Files:** -- Modify: `packages/cli/src/setup-demo-tour.ts` -- Test: `packages/cli/src/setup-demo-tour.test.ts` - -- [ ] **Step 1: Write failing demo wording tests** - -In `packages/cli/src/setup-demo-tour.test.ts`, add this test inside -`describe('buildDemoReplayTimeline', ...)`: - -```ts - it('uses schema-context wording for database progress', () => { - const renderedTimeline = timeline - .map((event) => [event.detailLine, event.summaryText].filter(Boolean).join(' ')) - .join('\n'); - - expect(renderedTimeline).toContain('reading schema'); - expect(renderedTimeline).toContain('56 tables'); - expect(renderedTimeline).not.toContain('scanning'); - expect(renderedTimeline).not.toContain('scanned'); - }); -``` - -- [ ] **Step 2: Run the failing demo wording test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-demo-tour.test.ts -t "schema-context wording" -``` - -Expected: FAIL because the demo timeline still uses `scanning tables...` and -`tables scanned`. - -- [ ] **Step 3: Replace demo timeline database copy** - -In `packages/cli/src/setup-demo-tour.ts`, inside `buildDemoReplayTimeline`, -replace the first three events: - -```ts - // postgres-warehouse: scan - { delayMs: 0, connectionId: 'postgres-warehouse', status: 'running', detailLine: null, summaryText: null }, - { delayMs: 1200, connectionId: 'postgres-warehouse', status: 'running', detailLine: '[50%] scanning tables...', summaryText: null }, - { delayMs: 2400, connectionId: 'postgres-warehouse', status: 'done', detailLine: null, summaryText: '56 tables scanned' }, -``` - -with: - -```ts - // postgres-warehouse: database schema context - { delayMs: 0, connectionId: 'postgres-warehouse', status: 'running', detailLine: null, summaryText: null }, - { delayMs: 1200, connectionId: 'postgres-warehouse', status: 'running', detailLine: '[50%] reading schema...', summaryText: null }, - { delayMs: 2400, connectionId: 'postgres-warehouse', status: 'done', detailLine: null, summaryText: '56 tables' }, -``` - -- [ ] **Step 4: Run the demo wording test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-demo-tour.test.ts -t "schema-context wording" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "fix: remove scan wording from demo progress" -``` - -### Task 6: Final verification - -**Files:** -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `packages/cli/src/context-build-view.ts` -- Verify: `packages/cli/src/next-steps.ts` -- Verify: `packages/cli/src/setup-demo-tour.ts` -- Verify: relevant tests - -- [ ] **Step 1: Run focused Vitest coverage** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/context-build-view.test.ts src/next-steps.test.ts src/setup-demo-tour.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code check after TypeScript changes** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. - -- [ ] **Step 5: Search for stale public wording in touched surfaces** - -Run: - -```bash -rg -n "Build or resume agent-ready|primary-source scans|scanning tables|tables scanned|Debug: ktx ingest" packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/next-steps.ts packages/cli/src/next-steps.test.ts packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -``` - -Expected: no matches. - -- [ ] **Step 6: Commit verification fixes if any were needed** - -If verification required edits, run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/next-steps.ts packages/cli/src/next-steps.test.ts packages/cli/src/setup-demo-tour.ts packages/cli/src/setup-demo-tour.test.ts -git commit -m "test: verify unified ingest ux closure" -``` - -If no edits were needed, do not create an empty commit. - -## Self-review - -- Spec coverage: The plan covers the remaining v1-blocking warning, - schema-first query-history, retry-guidance, setup next-step, and foreground - demo wording gaps. Core command routing, depth policy, query-history config, - setup depth, docs-site command references, foreground-only state, and reserved - ids are already covered by earlier implemented plans. -- Placeholder scan: The plan contains exact file paths, concrete test code, - implementation snippets, commands, and expected results. No red-flag - placeholders are present. -- Type consistency: `notices` is added as an optional - `KtxPublicIngestPlan` property and threaded through `renderContextBuildView` - options. Retry helpers use existing `KtxPublicIngestPlanTarget`, - `KtxPublicIngestArgs`, and `KtxPublicIngestStepName` types. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-progress-copy-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-progress-copy-closure.md deleted file mode 100644 index edfb80cc..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-progress-copy-closure.md +++ /dev/null @@ -1,559 +0,0 @@ -# Unified Ingest V1 Progress Copy Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the remaining v1-blocking scan wording from normal public -unified-ingest progress, failure, and setup scope-selection output. - -**Architecture:** Keep the implemented connection-centric ingest planner, -hidden legacy commands, and foreground context-build view. Add a small shared -public-copy helper for lower-level database ingest and query-history messages, -then use it from foreground progress and direct public failure summarization. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages. - ---- - -## Current audit - -The implemented unified-ingest plan chain covers the original spec's main v1 -behavior: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` route through `public-ingest.ts`. -- Database targets run before source targets, inferred public adapters bypass - `ingest.adapters`, and `fast` or `deep` maps to structural or enriched - database ingest internals. -- Deep readiness is evaluated before target work starts, and `--all` isolates - per-target deep-readiness failures. -- Setup stores `connections..context.depth` and - `connections..context.queryHistory`, migrates legacy `historicSql`, and - uses foreground-only setup context state. -- Normal help hides `ktx scan`, `ktx ingest run`, and `ktx ingest watch`; docs - and command-tree output no longer present those as normal public workflows. - -### V1-blocking gaps - -- Foreground `ktx ingest` and setup context-build progress still pass database - ingest progress messages through from scan internals. A normal user can see - messages such as `Preparing scan`, even though the spec says the foreground - view must use `reading schema` or `building schema context` and must not show - `scan` in normal mode. -- Direct public database ingest failure summaries sanitize `live-database` and - `historic-sql`, but not scan-specific failure lines such as - `KTX scan enrichment failed after structural scan completed: ...`. -- Interactive database setup still asks for `PostgreSQL schemas to scan`, which - keeps scan wording in normal setup output after the public model changed to - database schema context. - -### Non-blocking gaps - -- Hidden debug commands can remain callable: `ktx scan`, `ktx ingest run`, and - `ktx ingest watch`. -- Internal adapter keys, raw artifact paths, WorkUnit keys, package names, - tests, and developer-only scripts can continue to use `scan`, - `live-database`, and `historic-sql`. -- README package taxonomy such as `Postgres scan connector` can remain because - it describes internal package ownership, not normal command usage. -- Internal readiness configuration names such as `scan.enrichment.mode` can - remain because they refer to existing `ktx.yaml` configuration fields. - -## File structure - -- Create `packages/cli/src/public-ingest-copy.ts`: shared copy sanitizer for - database ingest and query-history messages used by public output paths. -- Create `packages/cli/src/public-ingest-copy.test.ts`: unit coverage for the - sanitizer. -- Modify `packages/cli/src/context-build-view.ts`: sanitize foreground - database progress messages and reuse the shared query-history sanitizer. -- Modify `packages/cli/src/context-build-view.test.ts`: cover foreground - progress output with lower-level scan messages. -- Modify `packages/cli/src/public-ingest.ts`: use the shared public output-line - sanitizer for captured failure details. -- Modify `packages/cli/src/public-ingest.test.ts`: cover direct public failure - output for scan-enrichment failures. -- Modify `packages/cli/src/setup-databases.ts`: change the schema scope prompt - from `schemas to scan` to `schemas to include`. -- Modify `packages/cli/src/setup-databases.test.ts`: update the schema prompt - expectation and assert scan wording is absent. - -## Tasks - -### Task 1: Add shared public ingest copy sanitizers - -**Files:** -- Create: `packages/cli/src/public-ingest-copy.ts` -- Create: `packages/cli/src/public-ingest-copy.test.ts` - -- [ ] **Step 1: Write the public-copy tests** - -Create `packages/cli/src/public-ingest-copy.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { - publicDatabaseIngestMessage, - publicIngestOutputLine, - publicQueryHistoryMessage, -} from './public-ingest-copy.js'; - -describe('public ingest copy sanitizers', () => { - it('maps database scan progress into schema-context wording', () => { - expect(publicDatabaseIngestMessage('Preparing scan')).toBe('Preparing database ingest'); - expect(publicDatabaseIngestMessage('Inspecting database schema')).toBe('Reading database schema'); - expect(publicDatabaseIngestMessage('Writing schema artifacts')).toBe('Writing schema context'); - expect(publicDatabaseIngestMessage('Enriching schema metadata')).toBe('Building enriched schema context'); - }); - - it('maps database scan failure text into public database ingest wording', () => { - expect( - publicDatabaseIngestMessage( - 'KTX scan enrichment failed after structural scan completed: embedding service timed out', - ), - ).toBe('Database enrichment failed after schema context completed: embedding service timed out'); - expect(publicDatabaseIngestMessage('structural scan wrote partial artifacts')).toBe( - 'schema context wrote partial artifacts', - ); - expect(publicDatabaseIngestMessage('scan results may be less complete')).toBe( - 'database context may be less complete', - ); - }); - - it('maps query-history adapter progress into public wording', () => { - expect(publicQueryHistoryMessage('Fetching source files for warehouse/historic-sql', 'warehouse')).toBe( - 'Fetching query history for warehouse', - ); - expect(publicQueryHistoryMessage('Curating warehouse/historic-sql work units', 'warehouse')).toBe( - 'Curating warehouse query history work units', - ); - expect(publicQueryHistoryMessage('historic SQL local ingest failed', 'warehouse')).toBe( - 'query history local ingest failed', - ); - }); - - it('sanitizes captured public output lines across database and query-history internals', () => { - expect( - publicIngestOutputLine( - 'KTX scan enrichment failed after structural scan completed in raw-sources/warehouse/live-database/sync-1', - ), - ).toBe('Database enrichment failed after schema context completed in raw-sources/warehouse/database schema/sync-1'); - expect(publicIngestOutputLine('Historic SQL local ingest requires a configured reader')).toBe( - 'query history local ingest requires a configured reader', - ); - }); -}); -``` - -- [ ] **Step 2: Run the failing public-copy tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest-copy.test.ts -``` - -Expected: FAIL because `packages/cli/src/public-ingest-copy.ts` does not exist. - -- [ ] **Step 3: Implement the shared sanitizers** - -Create `packages/cli/src/public-ingest-copy.ts`: - -```ts -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -const DATABASE_INGEST_REPLACEMENTS: Array<[RegExp, string]> = [ - [/\bPreparing scan\b/gi, 'Preparing database ingest'], - [/\bInspecting database schema\b/gi, 'Reading database schema'], - [/\bWriting schema artifacts\b/gi, 'Writing schema context'], - [/\bEnriching schema metadata\b/gi, 'Building enriched schema context'], - [ - /\bKTX scan enrichment failed after structural scan completed\b/gi, - 'Database enrichment failed after schema context completed', - ], - [/\bstructural scan\b/gi, 'schema context'], - [/\benriched scan\b/gi, 'deep database ingest'], - [/\bscan results\b/gi, 'database context'], -]; - -export function publicDatabaseIngestMessage(message: string): string { - return DATABASE_INGEST_REPLACEMENTS.reduce( - (current, [pattern, replacement]) => current.replace(pattern, replacement), - message, - ); -} - -export function publicQueryHistoryMessage(message: string, connectionId?: string): string { - let current = message; - if (connectionId && connectionId.length > 0) { - const escapedConnectionId = escapeRegExp(connectionId); - current = current - .replace( - new RegExp(`Fetching source files for ${escapedConnectionId}/historic-sql`, 'i'), - `Fetching query history for ${connectionId}`, - ) - .replace(`${connectionId}/historic-sql`, `${connectionId} query history`); - } - return current.replace(/\bhistoric-sql\b/g, 'query history').replace(/\bhistoric SQL\b/gi, 'query history'); -} - -export function publicIngestOutputLine(line: string): string { - return publicQueryHistoryMessage(publicDatabaseIngestMessage(line)).replace(/\blive-database\b/g, 'database schema'); -} -``` - -- [ ] **Step 4: Run the public-copy tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest-copy.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the shared sanitizer** - -Run: - -```bash -git add packages/cli/src/public-ingest-copy.ts packages/cli/src/public-ingest-copy.test.ts -git commit -m "fix(cli): add public ingest copy sanitizers" -``` - -### Task 2: Sanitize foreground progress and captured public failures - -**Files:** -- Modify: `packages/cli/src/context-build-view.ts` -- Modify: `packages/cli/src/context-build-view.test.ts` -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/public-ingest.test.ts` -- Test: `packages/cli/src/public-ingest-copy.test.ts` - -- [ ] **Step 1: Write the failing foreground progress test** - -In `packages/cli/src/context-build-view.test.ts`, add this test inside the -`runContextBuild` describe block near the existing query-history progress test: - -```ts - it('renders database ingest progress without scan wording', async () => { - const io = makeIo(); - const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); - const executeTarget = vi.fn(async (target, _args, _targetIo, deps) => { - await deps.scanProgress?.update(0.05, 'Preparing scan'); - await deps.scanProgress?.update(0.15, 'Inspecting database schema'); - await deps.scanProgress?.update(0.7, 'Writing schema artifacts'); - return successResult(target.connectionId, target.driver, target.operation); - }); - - await expect( - runContextBuild( - project, - { - projectDir: '/tmp/project', - inputMode: 'disabled', - targetConnectionId: 'warehouse', - all: false, - }, - io.io, - { executeTarget, now: () => 1000, sourceProgressThrottleMs: 0 }, - ), - ).resolves.toMatchObject({ exitCode: 0 }); - - expect(io.stdout()).toContain('Preparing database ingest'); - expect(io.stdout()).toContain('Reading database schema'); - expect(io.stdout()).toContain('Writing schema context'); - expect(io.stdout()).not.toContain('Preparing scan'); - expect(io.stdout()).not.toMatch(/\bscan\b/i); - }); -``` - -- [ ] **Step 2: Write the failing direct public failure test** - -In `packages/cli/src/public-ingest.test.ts`, add this test inside the -`runKtxPublicIngest` describe block near -`suppresses internal scan output for public database ingest summaries`: - -```ts - it('sanitizes captured database scan failure details in direct public output', async () => { - const io = makeIo(); - const project = deepReadyProject({ warehouse: { driver: 'postgres', context: { depth: 'deep' } } }); - const runScan = vi.fn(async (_args, scanIo) => { - scanIo.stdout.write('KTX scan enrichment failed after structural scan completed: embedding service timed out\n'); - return 1; - }); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - depth: 'deep', - }, - io.io, - { loadProject: vi.fn(async () => project), runScan }, - ), - ).resolves.toBe(1); - - expect(io.stdout()).toContain( - 'warehouse failed: Database enrichment failed after schema context completed: embedding service timed out.', - ); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep'); - expect(io.stdout()).not.toContain('KTX scan enrichment failed'); - expect(io.stdout()).not.toContain('structural scan'); - }); -``` - -- [ ] **Step 3: Run the failing integration tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/public-ingest.test.ts -t "database ingest progress|captured database scan failure" --testTimeout 30000 -``` - -Expected: FAIL because foreground progress still prints `Preparing scan`, and -captured direct failures still print the lower-level scan failure text. - -- [ ] **Step 4: Use the shared sanitizer in foreground progress** - -In `packages/cli/src/context-build-view.ts`, add this import: - -```ts -import { publicDatabaseIngestMessage, publicQueryHistoryMessage } from './public-ingest-copy.js'; -``` - -Replace the existing `publicProgressMessage()` implementation: - -```ts -function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string { - if (!target.steps.includes('query-history')) { - return message; - } - return message - .replace( - new RegExp(`Fetching source files for ${target.connectionId}/historic-sql`, 'i'), - `Fetching query history for ${target.connectionId}`, - ) - .replace(`${target.connectionId}/historic-sql`, `${target.connectionId} query history`) - .replace(/\bhistoric-sql\b/g, 'query history') - .replace(/\bhistoric SQL\b/gi, 'query history'); -} -``` - -with: - -```ts -function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string { - if (target.operation === 'database-ingest') { - return publicDatabaseIngestMessage(message); - } - if (target.steps.includes('query-history')) { - return publicQueryHistoryMessage(message, target.connectionId); - } - return message; -} -``` - -- [ ] **Step 5: Use the shared sanitizer in public ingest failure capture** - -In `packages/cli/src/public-ingest.ts`, add this import: - -```ts -import { publicIngestOutputLine } from './public-ingest-copy.js'; -``` - -Delete the local `publicIngestOutputLine()` function: - -```ts -function publicIngestOutputLine(line: string): string { - return line - .replace(/\blive-database\b/g, 'database schema') - .replace(/\bhistoric-sql\b/g, 'query history') - .replace(/\bhistoric SQL\b/gi, 'query history'); -} -``` - -Leave `firstCapturedFailureLine()` calling `publicIngestOutputLine` unchanged; -the imported function now provides the broader public wording. - -- [ ] **Step 6: Run the integration tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest-copy.test.ts src/context-build-view.test.ts src/public-ingest.test.ts --testTimeout 30000 -``` - -Expected: PASS. - -- [ ] **Step 7: Commit foreground and failure sanitization** - -Run: - -```bash -git add packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/public-ingest-copy.ts packages/cli/src/public-ingest-copy.test.ts -git commit -m "fix(cli): sanitize public ingest progress copy" -``` - -### Task 3: Rename setup schema scope prompt - -**Files:** -- Modify: `packages/cli/src/setup-databases.ts` -- Modify: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Update the setup prompt expectation** - -In `packages/cli/src/setup-databases.test.ts`, in the test named -`prompts for discovered Postgres schemas before the first scan`, replace: - -```ts - message: expect.stringContaining('PostgreSQL schemas to scan'), -``` - -with: - -```ts - message: expect.stringContaining('PostgreSQL schemas to include'), -``` - -Add this assertion after the `toHaveBeenCalledWith` block: - -```ts - expect(String(prompts.multiselect.mock.calls[0]?.[0].message)).not.toContain('to scan'); -``` - -- [ ] **Step 2: Run the failing setup prompt test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts -t "prompts for discovered Postgres schemas before the first scan" --testTimeout 30000 -``` - -Expected: FAIL because the prompt still says `PostgreSQL schemas to scan`. - -- [ ] **Step 3: Rename the setup scope prompt** - -In `packages/cli/src/setup-databases.ts`, replace: - -```ts - `${spec.promptLabel} to scan\n` + - `KTX found multiple ${spec.nounPlural}. Select every ${spec.noun} agents should use.`, -``` - -with: - -```ts - `${spec.promptLabel} to include\n` + - `KTX found multiple ${spec.nounPlural}. Select every ${spec.noun} agents should use.`, -``` - -- [ ] **Step 4: Run the setup prompt test again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts -t "prompts for discovered Postgres schemas before the first scan" --testTimeout 30000 -``` - -Expected: PASS. - -- [ ] **Step 5: Commit setup prompt wording** - -Run: - -```bash -git add packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "fix(cli): rename setup schema scope prompt" -``` - -### Task 4: Final verification - -**Files:** -- Verify: `packages/cli/src/public-ingest-copy.ts` -- Verify: `packages/cli/src/context-build-view.ts` -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `packages/cli/src/setup-databases.ts` - -- [ ] **Step 1: Run targeted unified-ingest tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest-copy.test.ts src/context-build-view.test.ts src/public-ingest.test.ts src/setup-databases.test.ts --testTimeout 30000 -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Scan normal public files for the closed wording gaps** - -Run: - -```bash -rg -n "Preparing scan|KTX scan enrichment failed|structural scan completed|schemas to scan" packages/cli/src/context-build-view.ts packages/cli/src/public-ingest.ts packages/cli/src/setup-databases.ts packages/cli/src/*.test.ts -``` - -Expected: no matches except historical expectations in low-level `scan.test.ts` -or internal scan-specific tests that are not part of the command above. - -- [ ] **Step 4: Run workspace dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. - -- [ ] **Step 5: Commit final verification marker if needed** - -If the verification steps required only the commits above, no additional -commit is needed. If a verification fix changed files, run: - -```bash -git add packages/cli/src/public-ingest-copy.ts packages/cli/src/public-ingest-copy.test.ts packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "test(cli): verify unified ingest public progress copy" -``` - -## Self-review - -Spec coverage: this plan covers the remaining normal public output paths where -scan wording still leaks into unified ingest: - -- Foreground progress now maps database scan progress into schema-context copy. -- Captured direct public failure summaries now map scan-enrichment failures into - database ingest copy. -- Interactive setup schema scope selection now says `schemas to include`, not - `schemas to scan`. - -The plan intentionally leaves hidden debug commands, internal artifact paths, -developer scripts, low-level scan tests, and configuration field names alone. -Those are non-blocking under the original spec's implementation-detail -allowances. - -Placeholder scan: no task uses deferred code markers, unnamed edge handling, or -undefined helper names. Every changed helper, test, and command is named with -the file that owns it. - -Type consistency: the new helper exports -`publicDatabaseIngestMessage()`, `publicQueryHistoryMessage()`, and -`publicIngestOutputLine()`. Later tasks import those exact names from -`./public-ingest-copy.js`. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-public-output-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-public-output-closure.md deleted file mode 100644 index 47b88817..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-public-output-closure.md +++ /dev/null @@ -1,1224 +0,0 @@ -# Unified Ingest V1 Public Output Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking gaps where public `ktx ingest` and -setup still expose internal scan, adapter, or Historic SQL behavior. - -**Architecture:** Keep the current connection-centric `ktx ingest` planner and -depth policy. Tighten the public execution layer so inferred source adapters -bypass `ingest.adapters`, database ingest captures internal scan output, TTY -public ingest uses the shared foreground context-build view, and setup output -uses schema-context and query-history language. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages. - ---- - -## Audit - -The implemented unified-ingest plans cover the core command routing, depth -policy, setup depth defaults, foreground-only setup context build, canonical -`context.queryHistory` setup storage, reserved ingest connection ids, and -default config cleanup. - -### V1-blocking gaps - -- `ktx ingest ` still calls `runKtxIngest` without - `allowImplicitAdapter: true`, so public source ingest still requires - `ingest.adapters` entries. -- Direct public database ingest still streams internal `runKtxScan` output, - including `KTX scan completed`, `Mode: structural`, artifact paths, and - `live-database` path segments. -- Direct interactive public ingest does not use the shared foreground - context-build view; only setup uses that view. -- `--query-history-window-days` does not itself request query-history ingest, - so `ktx ingest warehouse --query-history-window-days 30` silently runs only - schema ingest when stored query history is disabled. -- `ktx ingest --all --deep` emits one ignored-depth warning per non-database - source instead of aggregating the warning. -- Setup database output still says `Scanning`, `structural scan`, prints - `live-database` report paths, and suggests `ktx scan` as a retry/debug - command. -- Setup help and prompts still expose `Historic SQL` flags and wording instead - of query-history wording. - -### Non-blocking gaps - -- Hidden debug surfaces can still call internal commands: `ktx scan`, - `ktx ingest run`, and `ktx ingest watch`. -- Internal package names, adapter keys, raw artifact paths, scan tests, and - scripts can continue to use `scan`, `live-database`, and `historic-sql`. -- README package descriptions such as `Postgres scan connector` are internal - package taxonomy, not normal CLI command guidance. -- `README.md` says rerunning setup resumes the wizard; that is setup-flow - language, not a context-build background resume path. - -## File structure - -- Modify `packages/cli/src/public-ingest.ts`: set implicit adapters for public - source ingest, treat query-history window overrides as query-history - requests, aggregate `--all` source warnings, capture database scan output for - plain public ingest, and delegate interactive TTY runs to the shared context - build view. -- Modify `packages/cli/src/public-ingest.test.ts`: cover adapter bypass, - quiet database ingest output, query-history window semantics, aggregated - warnings, and TTY foreground delegation. -- Modify `packages/cli/src/context-build-view.ts`: allow the foreground view - to run a single requested connection and pass through public ingest flags. -- Modify `packages/cli/src/context-build-view.test.ts`: cover single-target - foreground execution and flag passthrough. -- Modify `packages/cli/src/setup-databases.ts`: rename public setup wording to - schema context and query history, stop printing internal report paths in - normal setup output, and replace `ktx scan` retry/debug suggestions with - `ktx ingest --fast`. -- Modify `packages/cli/src/setup-databases.test.ts`: update setup output, - failure, and query-history expectations. -- Modify `packages/cli/src/commands/setup-commands.ts`: replace public - Historic SQL setup flags with query-history setup flags. -- Modify `packages/cli/src/index.test.ts`: update setup help and conflicting - query-history flag tests. - -## Tasks - -### Task 1: Bypass adapter allow-lists for public source ingest - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write the failing adapter-bypass test** - -Add this test inside the `runKtxPublicIngest` describe block in -`packages/cli/src/public-ingest.test.ts`: - -```ts - it('bypasses adapter allow-lists for connection-centric source ingest', async () => { - const runIngest = vi.fn(async () => 0); - const io = makeIo(); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/ktx', - targetConnectionId: 'docs', - all: false, - json: false, - inputMode: 'disabled', - }, - io.io, - { - loadProject: async () => - projectWithConnections({ - docs: { driver: 'notion' }, - }), - runIngest, - }, - ), - ).resolves.toBe(0); - - expect(runIngest).toHaveBeenCalledWith( - expect.objectContaining({ - command: 'run', - connectionId: 'docs', - adapter: 'notion', - allowImplicitAdapter: true, - }), - io.io, - ); - }); -``` - -- [ ] **Step 2: Run the failing adapter-bypass test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "adapter allow-lists" -``` - -Expected: FAIL because public source ingest does not pass -`allowImplicitAdapter: true`. - -- [ ] **Step 3: Add `allowImplicitAdapter` for inferred source adapters** - -In `packages/cli/src/public-ingest.ts`, update the source-ingest -`KtxIngestArgs` object in `executePublicIngestTarget`: - -```ts - const ingestArgs: KtxIngestArgs = { - command: 'run', - projectDir: args.projectDir, - connectionId: target.connectionId, - adapter: target.adapter ?? target.driver, - ...(target.sourceDir ? { sourceDir: target.sourceDir } : {}), - outputMode: sourceIngestOutputMode(args, io), - inputMode: args.inputMode, - allowImplicitAdapter: true, - }; -``` - -- [ ] **Step 4: Run the adapter-bypass test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "adapter allow-lists" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(ingest): bypass adapter allow-list for public source ingest" -``` - -### Task 2: Fix query-history window semantics and aggregate source warnings - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write failing query-history and warning tests** - -Add these tests inside the `buildPublicIngestPlan` describe block in -`packages/cli/src/public-ingest.test.ts`: - -```ts - it('treats query-history window override as current-run query-history enablement', () => { - const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { queryHistory: { enabled: false, windowDays: 90 } } }, - }); - - const plan = buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - queryHistory: 'default', - queryHistoryWindowDays: 30, - }); - - expect(plan.targets[0]).toMatchObject({ - connectionId: 'warehouse', - databaseDepth: 'deep', - queryHistory: { enabled: true, dialect: 'postgres', windowDays: 30 }, - steps: ['database-schema', 'query-history'], - }); - }); - - it('warns and skips query-history window override for unsupported database drivers', () => { - const plan = buildPublicIngestPlan( - projectWithConnections({ - local: { driver: 'sqlite' }, - }), - { - projectDir: '/tmp/project', - targetConnectionId: 'local', - all: false, - queryHistory: 'default', - queryHistoryWindowDays: 30, - }, - ); - - expect(plan.targets[0]).toMatchObject({ - connectionId: 'local', - databaseDepth: 'fast', - queryHistory: { enabled: false, windowDays: 30, unsupported: true }, - steps: ['database-schema'], - }); - expect(plan.warnings).toEqual(['--query-history is not supported for sqlite; running schema ingest for local.']); - }); - - it('aggregates ignored database-depth warnings for all source targets', () => { - const plan = buildPublicIngestPlan( - projectWithConnections({ - warehouse: { driver: 'postgres' }, - docs: { driver: 'notion' }, - dbt: { driver: 'dbt' }, - }), - { - projectDir: '/tmp/project', - all: true, - depth: 'deep', - queryHistory: 'default', - }, - ); - - expect(plan.warnings).toEqual(['--deep ignored for 2 non-database sources.']); - }); -``` - -- [ ] **Step 2: Run the failing public ingest planning tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "query-history window override|unsupported database drivers|aggregates ignored" -``` - -Expected: FAIL because window-days alone does not request query history and -source warnings are emitted per source. - -- [ ] **Step 3: Add a warning accumulator** - -In `packages/cli/src/public-ingest.ts`, add these types and helpers near -`queryHistoryDialectByDriver`: - -```ts -interface KtxPublicIngestWarningAccumulator { - warnings: string[]; - ignoredDepthForSources: string[]; - ignoredQueryHistoryForSources: string[]; -} - -function createWarningAccumulator(): KtxPublicIngestWarningAccumulator { - return { - warnings: [], - ignoredDepthForSources: [], - ignoredQueryHistoryForSources: [], - }; -} - -function sourceIgnoredWarning(option: string, connectionIds: string[], all: boolean): string | null { - if (connectionIds.length === 0) { - return null; - } - if (all) { - const sourceLabel = connectionIds.length === 1 ? '1 non-database source' : `${connectionIds.length} non-database sources`; - return `${option} ignored for ${sourceLabel}.`; - } - return `${option} affects database ingest only; ignoring it for ${connectionIds[0]}.`; -} - -function finalizeWarnings( - accumulator: KtxPublicIngestWarningAccumulator, - args: { - all: boolean; - depth?: KtxPublicIngestDepth; - queryHistory?: KtxPublicIngestQueryHistoryFlag; - queryHistoryWindowDays?: number; - }, -): string[] { - const warnings = [...accumulator.warnings]; - const depthOption = args.depth ? `--${args.depth}` : null; - if (depthOption) { - const warning = sourceIgnoredWarning(depthOption, accumulator.ignoredDepthForSources, args.all); - if (warning) warnings.push(warning); - } - if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) { - const warning = sourceIgnoredWarning('--query-history', accumulator.ignoredQueryHistoryForSources, args.all); - if (warning) warnings.push(warning); - } - return warnings; -} -``` - -- [ ] **Step 4: Use window-days as query-history intent** - -In `resolveDatabaseTargetOptions`, replace the current `requestedQh` line with: - -```ts - const windowOverrideRequested = input.args.queryHistoryWindowDays !== undefined; - const requestedQh = - explicitQueryHistory === 'enabled' || - (explicitQueryHistory !== 'disabled' && (windowOverrideRequested || storedEnabled)); -``` - -Leave the existing `--query-history requires deep ingest` warning in place so -`--fast --query-history-window-days 30` upgrades the run to deep with the same -warning as `--fast --query-history`. - -- [ ] **Step 5: Route source warnings through the accumulator** - -Change the `warnings` parameter in `targetForConnection` from `string[]` to -`KtxPublicIngestWarningAccumulator`. In the source-adapter branch, replace the -current warning pushes with: - -```ts - if (args.depth) { - warnings.ignoredDepthForSources.push(connectionId); - } - if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) { - warnings.ignoredQueryHistoryForSources.push(connectionId); - } -``` - -In the database branch, pass `warnings.warnings` into -`resolveDatabaseTargetOptions`: - -```ts - const options = resolveDatabaseTargetOptions({ - connectionId, - driver, - connection, - args, - warnings: warnings.warnings, - }); -``` - -In `buildPublicIngestPlan`, replace the `warnings` array construction with: - -```ts - const warnings = createWarningAccumulator(); - const targets = selected.map(([connectionId, connection]) => - targetForConnection(connectionId, connection, project.config, args, warnings), - ); - return { - projectDir: args.projectDir, - targets: [ - ...targets.filter((t) => t.operation === 'database-ingest'), - ...targets.filter((t) => t.operation === 'source-ingest'), - ], - warnings: finalizeWarnings(warnings, args), - }; -``` - -- [ ] **Step 6: Run the public ingest planning tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "query-history window override|unsupported database drivers|aggregates ignored" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(ingest): honor query history window intent" -``` - -### Task 3: Suppress internal scan output in public database ingest - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write the failing quiet-output test** - -Add this test inside the `runKtxPublicIngest` describe block in -`packages/cli/src/public-ingest.test.ts`: - -```ts - it('suppresses internal scan output for public database ingest summaries', async () => { - const io = makeIo(); - const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); - const runScan = vi.fn(async (_args, scanIo) => { - scanIo.stdout.write('KTX scan completed\n'); - scanIo.stdout.write('Mode: structural\n'); - scanIo.stdout.write('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json\n'); - scanIo.stdout.write('Raw sources: raw-sources/warehouse/live-database/sync-1\n'); - return 0; - }); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - }, - io.io, - { loadProject: vi.fn(async () => project), runScan }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Ingest finished\n'); - expect(io.stdout()).toContain('warehouse'); - expect(io.stdout()).not.toContain('KTX scan completed'); - expect(io.stdout()).not.toContain('Mode: structural'); - expect(io.stdout()).not.toContain('Report: raw-sources'); - expect(io.stdout()).not.toContain('live-database'); - }); -``` - -- [ ] **Step 2: Run the failing quiet-output test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "suppresses internal scan output" -``` - -Expected: FAIL because `executePublicIngestTarget` passes the public IO -directly to `runScan`. - -- [ ] **Step 3: Add captured public scan IO** - -In `packages/cli/src/public-ingest.ts`, add these helpers near -`sourceIngestOutputMode`: - -```ts -interface CapturedPublicIngestIo extends KtxCliIo { - capturedOutput(): string; -} - -function createCapturedPublicIngestIo(): CapturedPublicIngestIo { - let output = ''; - return { - stdout: { - isTTY: false, - write(chunk: string) { - output += chunk; - }, - }, - stderr: { - write(chunk: string) { - output += chunk; - }, - }, - capturedOutput() { - return output; - }, - }; -} - -function firstCapturedFailureLine(output: string): string | undefined { - return output - .split(/\r?\n/) - .map((line) => line.trim()) - .find((line) => line.length > 0 && !line.startsWith('KTX scan completed')); -} -``` - -Change `markTargetResult` to accept a detail override: - -```ts -function markTargetResult( - target: KtxPublicIngestPlanTarget, - status: 'done' | 'failed', - failedOperation?: KtxPublicIngestStepName, - failureDetail?: string, -): KtxPublicIngestTargetResult { - const selectedFailedOperation = - failedOperation ?? (target.operation === 'database-ingest' ? 'database-schema' : 'source-ingest'); - return { - connectionId: target.connectionId, - driver: target.driver, - steps: defaultSteps(target).map((step) => { - if (!target.steps.includes(step.operation)) { - return step; - } - if (status === 'done') { - return { ...step, status: 'done' }; - } - if (step.operation === selectedFailedOperation) { - return { - ...step, - status: 'failed', - detail: failureDetail ?? `${target.connectionId} failed at ${selectedFailedOperation}.`, - }; - } - return { ...step, status: 'not-run' }; - }), - }; -} -``` - -In the database-ingest branch of `executePublicIngestTarget`, replace the direct -`runScan` call block with: - -```ts - const runScan = deps.runScan ?? runKtxScan; - const capturedScanIo = deps.scanProgress ? null : createCapturedPublicIngestIo(); - const scanIo = capturedScanIo ?? io; - const scanExitCode = deps.scanProgress - ? await runScan(scanArgs, scanIo, { progress: deps.scanProgress }) - : await runScan(scanArgs, scanIo); - if (scanExitCode !== 0) { - return markTargetResult( - target, - 'failed', - 'database-schema', - capturedScanIo ? firstCapturedFailureLine(capturedScanIo.capturedOutput()) : undefined, - ); - } -``` - -- [ ] **Step 4: Run the quiet-output test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts -t "suppresses internal scan output" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(ingest): hide scan internals from public database ingest" -``` - -### Task 4: Use the shared foreground view for interactive public ingest - -**Files:** -- Modify: `packages/cli/src/context-build-view.ts` -- Modify: `packages/cli/src/public-ingest.ts` -- Test: `packages/cli/src/context-build-view.test.ts` -- Test: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write failing foreground-view tests** - -In `packages/cli/src/context-build-view.test.ts`, add this test inside the -`runContextBuild` describe block: - -```ts - it('runs only the requested connection when foreground build receives a target', async () => { - const io = makeIo(); - const project = projectWithConnections({ - warehouse: { driver: 'postgres' }, - docs: { driver: 'notion' }, - }); - const executeTarget = vi.fn(async (target) => - successResult(target.connectionId, target.driver, target.operation), - ); - - await expect( - runContextBuild( - project, - { - projectDir: '/tmp/project', - inputMode: 'disabled', - targetConnectionId: 'warehouse', - all: false, - depth: 'fast', - queryHistory: 'default', - }, - io.io, - { executeTarget, now: () => 1000 }, - ), - ).resolves.toMatchObject({ exitCode: 0 }); - - expect(executeTarget).toHaveBeenCalledTimes(1); - expect(executeTarget.mock.calls[0]?.[0]).toMatchObject({ - connectionId: 'warehouse', - operation: 'database-ingest', - databaseDepth: 'fast', - }); - expect(io.stdout()).toContain('Databases:'); - expect(io.stdout()).toContain('warehouse'); - expect(io.stdout()).not.toContain('docs'); - }); -``` - -In `packages/cli/src/public-ingest.test.ts`, update `makeIo` to accept -interactive stdin: - -```ts -function makeIo(options: { isTTY?: boolean; interactive?: boolean } = {}) { - let stdout = ''; - let stderr = ''; - return { - io: { - ...(options.interactive - ? { - stdin: { - isTTY: true, - setRawMode: vi.fn(), - }, - } - : {}), - stdout: { - isTTY: options.isTTY, - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} -``` - -Then add this test inside the `runKtxPublicIngest` describe block: - -```ts - it('delegates interactive TTY public ingest to the foreground context-build view', async () => { - const io = makeIo({ isTTY: true, interactive: true }); - const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); - const runContextBuild = vi.fn(async () => ({ exitCode: 0 })); - const runScan = vi.fn(async () => 0); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'auto', - depth: 'fast', - queryHistory: 'default', - }, - io.io, - { loadProject: vi.fn(async () => project), runContextBuild, runScan }, - ), - ).resolves.toBe(0); - - expect(runContextBuild).toHaveBeenCalledWith( - project, - expect.objectContaining({ - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - depth: 'fast', - queryHistory: 'default', - }), - io.io, - ); - expect(runScan).not.toHaveBeenCalled(); - }); -``` - -- [ ] **Step 2: Run the failing foreground-view tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/public-ingest.test.ts -t "requested connection|foreground context-build view" -``` - -Expected: FAIL because `runContextBuild` always plans `--all`, and -`runKtxPublicIngest` does not delegate interactive TTY runs. - -- [ ] **Step 3: Extend foreground context-build args** - -In `packages/cli/src/context-build-view.ts`, replace `ContextBuildArgs` with: - -```ts -export interface ContextBuildArgs { - projectDir: string; - inputMode: 'auto' | 'disabled'; - targetConnectionId?: string; - all?: boolean; - depth?: Extract['depth']; - queryHistory?: Extract['queryHistory']; - queryHistoryWindowDays?: number; - scanMode?: 'structural' | 'enriched'; - detectRelationships?: boolean; -} -``` - -In `runContextBuild`, replace the hard-coded plan call with: - -```ts - const plan = buildPublicIngestPlan(project, { - projectDir: args.projectDir, - ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}), - all: args.all ?? true, - ...(args.depth ? { depth: args.depth } : {}), - ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), - ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), - ...(args.scanMode ? { scanMode: args.scanMode } : {}), - }); -``` - -Replace the `runArgs` construction with: - -```ts - const runArgs: Extract = { - command: 'run', - projectDir: args.projectDir, - ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}), - all: args.all ?? true, - json: false, - inputMode: args.inputMode, - ...(args.depth ? { depth: args.depth } : {}), - ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), - ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), - ...(args.scanMode ? { scanMode: args.scanMode } : {}), - ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}), - }; -``` - -- [ ] **Step 4: Add a foreground-build dependency to public ingest** - -In `packages/cli/src/public-ingest.ts`, add this interface near -`KtxPublicIngestDeps`: - -```ts -interface KtxPublicContextBuildArgs { - projectDir: string; - inputMode: 'auto' | 'disabled'; - targetConnectionId?: string; - all?: boolean; - depth?: KtxPublicIngestDepth; - queryHistory?: KtxPublicIngestQueryHistoryFlag; - queryHistoryWindowDays?: number; - scanMode?: Extract['mode']; - detectRelationships?: boolean; -} -``` - -Add this optional dependency to `KtxPublicIngestDeps`: - -```ts - runContextBuild?: ( - project: KtxPublicIngestProject, - args: KtxPublicContextBuildArgs, - io: KtxCliIo, - ) => Promise<{ exitCode: number }>; -``` - -Add this helper near `sourceIngestOutputMode`: - -```ts -function shouldUseForegroundContextBuildView( - args: Extract, - io: KtxCliIo, -): boolean { - return args.inputMode === 'auto' && args.json !== true && io.stdout.isTTY === true && hasInteractiveInput(io); -} -``` - -In `runKtxPublicIngest`, after loading `project` and before rendering warnings -or executing targets, add: - -```ts - if (shouldUseForegroundContextBuildView(args, io)) { - const { runContextBuild } = await import('./context-build-view.js'); - const contextBuild = deps.runContextBuild ?? runContextBuild; - const result = await contextBuild( - project, - { - projectDir: args.projectDir, - ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}), - all: args.all, - inputMode: args.inputMode, - ...(args.depth ? { depth: args.depth } : {}), - ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), - ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), - ...(args.scanMode ? { scanMode: args.scanMode } : {}), - ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}), - }, - io, - ); - return result.exitCode; - } -``` - -- [ ] **Step 5: Run the foreground-view tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/public-ingest.test.ts -t "requested connection|foreground context-build view" -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "feat(ingest): use foreground view for interactive public ingest" -``` - -### Task 5: Clean setup database output and query-history setup wording - -**Files:** -- Modify: `packages/cli/src/setup-databases.ts` -- Modify: `packages/cli/src/setup-databases.test.ts` -- Modify: `packages/cli/src/commands/setup-commands.ts` -- Modify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing setup output expectations** - -In `packages/cli/src/setup-databases.test.ts`, update the test named -`summarizes connection test and structural scan output during setup` so the -final output expectation is: - -```ts - expect(io.stdout()).toContain('◇ Building schema context for postgres-warehouse'); - expect(io.stdout()).toContain('│ Running fast database ingest…'); - expect(io.stdout()).toContain('◇ Schema context complete for postgres-warehouse'); - expect(io.stdout()).toContain('│ Changes: 3 changed tables'); - expect(io.stdout()).toContain('◇ Primary source ready'); - expect(io.stdout()).toContain('│ postgres-warehouse · PostgreSQL · schema context complete'); - expect(io.stdout()).not.toContain('Scanning postgres-warehouse'); - expect(io.stdout()).not.toContain('Scan complete for postgres-warehouse'); - expect(io.stdout()).not.toContain('structural scan complete'); - expect(io.stdout()).not.toContain('Report: raw-sources'); - expect(io.stdout()).not.toContain('live-database'); -``` - -In the setup scan-failure test that currently expects `ktx scan`, replace the -expectation with: - -```ts - expect(io.stderr()).toContain(`Retry: ktx ingest warehouse --project-dir ${tempDir} --fast`); - expect(io.stderr()).not.toContain('ktx scan'); -``` - -In the test named `writes Postgres Historic SQL config with minExecutions and -ignores window/redaction output`, replace the output expectation with: - -```ts - expect(io.stdout()).toContain('Query history probe...'); - expect(io.stdout()).not.toContain('Historic SQL probe...'); -``` - -In the test named `prints a non-blocking Postgres Historic SQL probe failure -after connection test succeeds`, replace the output expectation with: - -```ts - expect(io.stdout()).toContain('Query history probe...'); - expect(io.stdout()).not.toContain('Historic SQL probe...'); -``` - -- [ ] **Step 2: Write failing setup help expectations** - -In `packages/cli/src/index.test.ts`, update the setup help assertion that -currently checks Historic SQL flags to: - -```ts - for (const expected of [ - '--enable-query-history', - '--disable-query-history', - '--query-history-window-days', - '--query-history-min-executions', - '--query-history-service-account-pattern', - '--query-history-redaction-pattern', - ]) { - expect(testIo.stdout()).toContain(expected); - } - expect(testIo.stdout()).not.toContain('--enable-historic-sql'); - expect(testIo.stdout()).not.toContain('--historic-sql-window-days'); -``` - -Replace the conflicting Historic SQL setup flags test with: - -```ts - it('rejects conflicting query-history setup flags', async () => { - const tempDir = await makeTempProject(); - const setupIo = makeIo(); - - await expect( - runKtxCli(['--project-dir', tempDir, 'setup', '--enable-query-history', '--disable-query-history'], setupIo.io, { - setup: vi.fn(async () => 0), - }), - ).resolves.toBe(1); - - expect(setupIo.stderr()).toContain( - 'Choose only one query-history action: --enable-query-history or --disable-query-history.', - ); - }); -``` - -- [ ] **Step 3: Run the failing setup tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts src/index.test.ts -t "structural scan output|query-history setup flags|conflicting query-history|Postgres Historic SQL|non-blocking Postgres" -``` - -Expected: FAIL because setup still uses scan and Historic SQL wording. - -- [ ] **Step 4: Rename setup database args to query history** - -In `packages/cli/src/setup-databases.ts`, replace the Historic SQL argument -fields in `KtxSetupDatabasesArgs` with query-history fields: - -```ts - enableQueryHistory?: boolean; - disableQueryHistory?: boolean; - queryHistoryWindowDays?: number; - queryHistoryMinExecutions?: number; - queryHistoryServiceAccountPatterns?: string[]; - queryHistoryRedactionPatterns?: string[]; -``` - -Update references in `maybeApplyHistoricSqlConfig`: - -```ts - if (!dialect) { - if (input.args.enableQueryHistory === true) { - throw new Error( - `Query history setup is only supported for Snowflake, BigQuery, and Postgres, not ${driverLabel(input.driver)}.`, - ); - } - return input.connection; - } - - let enabled = input.args.enableQueryHistory === true; - if (input.args.disableQueryHistory === true) { - enabled = false; - } else if (input.args.inputMode !== 'disabled' && input.args.enableQueryHistory !== true && dialect !== 'postgres') { - const choice = await input.prompts.select({ - message: `Enable query-history ingest for this ${driverLabel(input.driver)} connection?`, - options: [ - { value: 'yes', label: 'Enable query history' }, - { value: 'no', label: 'Do not enable query history' }, - { value: 'back', label: 'Back' }, - ], - }); - if (choice === 'back') return 'back'; - enabled = choice === 'yes'; - } - - if (dialect === 'postgres' && input.args.enableQueryHistory !== true && input.args.disableQueryHistory !== true) { - return input.connection; - } -``` - -Update the query-history config construction: - -```ts - const common: Record = { - ...existing, - enabled: true, - filters: historicSqlFiltersForSetup(input.args.queryHistoryServiceAccountPatterns), - }; - - if (dialect === 'postgres') { - return withQueryHistoryConfig(input.connection, { - ...common, - minExecutions: input.args.queryHistoryMinExecutions ?? 5, - }); - } - - return withQueryHistoryConfig(input.connection, { - ...common, - windowDays: input.args.queryHistoryWindowDays ?? 90, - redactionPatterns: input.args.queryHistoryRedactionPatterns ?? [], - }); -``` - -Update both calls to `maybeApplyHistoricSqlConfig` and -`applyHistoricSqlConfigToExistingConnection` by using the renamed args fields; -the function name can remain internal for this task because the source adapter -key is still `historic-sql`. - -- [ ] **Step 5: Replace setup scan wording and command suggestions** - -In `packages/cli/src/setup-databases.ts`, delete `shortenScanReportPath`. -Then replace the scan output block in `validateAndScanConnection` with: - -```ts - writeSetupSection(input.io, `Building schema context for ${input.connectionId}`, [ - 'Running fast database ingest…', - ]); -``` - -Replace the Native SQLite retry failure lines with: - -```ts - [ - rebuildCode === 0 - ? `Fast database ingest still failed for ${input.connectionId} after rebuilding Native SQLite.` - : `Native SQLite rebuild failed for ${input.connectionId}.`, - 'Fix: pnpm run native:rebuild', - `Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast`, - ].join('\n'), -``` - -Replace the non-ABI failure lines with: - -```ts - [ - `Fast database ingest failed for ${input.connectionId}.`, - `Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast --debug`, - ].join('\n'), -``` - -Replace the success section with: - -```ts - const scanOutput = scanIo.stdoutText(); - writeSetupSection( - input.io, - `Schema context complete for ${input.connectionId}`, - [`Changes: ${summarizeScanChanges(scanOutput)}`], - ); - writeSetupSection(input.io, 'Primary source ready', [ - `${input.connectionId} · ${driverDisplay} · schema context complete`, - ]); -``` - -Replace the probe label in `maybeRunHistoricSqlSetupProbe`: - -```ts - input.io.stdout.write('│ Query history probe...\n'); -``` - -- [ ] **Step 6: Replace public setup flags** - -In `packages/cli/src/commands/setup-commands.ts`, replace the Historic SQL -options with: - -```ts - .option('--enable-query-history', 'Enable query history when the selected database supports it', false) - .option('--disable-query-history', 'Disable query history for the selected database', false) - .option('--query-history-window-days ', 'Query-history lookback window', positiveInteger) - .option('--query-history-min-executions ', 'Minimum executions for a query-history template', positiveInteger) - .option( - '--query-history-service-account-pattern ', - 'Query-history service-account regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option( - '--query-history-redaction-pattern ', - 'Query-history SQL-literal redaction regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) -``` - -Replace the conflict check with: - -```ts - if (options.enableQueryHistory && options.disableQueryHistory) { - context.io.stderr.write( - 'Choose only one query-history action: --enable-query-history or --disable-query-history.\n', - ); - context.setExitCode(1); - return; - } -``` - -Replace the setup arg mapping with: - -```ts - ...(options.enableQueryHistory ? { enableQueryHistory: true } : {}), - ...(options.disableQueryHistory ? { disableQueryHistory: true } : {}), - ...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}), - ...(options.queryHistoryMinExecutions !== undefined - ? { queryHistoryMinExecutions: options.queryHistoryMinExecutions } - : {}), - ...(options.queryHistoryServiceAccountPattern.length > 0 - ? { queryHistoryServiceAccountPatterns: options.queryHistoryServiceAccountPattern } - : {}), - ...(options.queryHistoryRedactionPattern.length > 0 - ? { queryHistoryRedactionPatterns: options.queryHistoryRedactionPattern } - : {}), -``` - -- [ ] **Step 7: Update setup database tests to renamed args** - -In `packages/cli/src/setup-databases.test.ts`, replace test input property -names as follows: - -```ts -enableHistoricSql -> enableQueryHistory -disableHistoricSql -> disableQueryHistory -historicSqlWindowDays -> queryHistoryWindowDays -historicSqlMinExecutions -> queryHistoryMinExecutions -historicSqlServiceAccountPatterns -> queryHistoryServiceAccountPatterns -historicSqlRedactionPatterns -> queryHistoryRedactionPatterns -``` - -Also rename test names that include `Historic SQL` to use `query history`. -Keep assertions that `configText` does not contain `historic-sql`. - -- [ ] **Step 8: Run the setup tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts src/index.test.ts -t "schema context|query history|query-history setup flags|conflicting query-history" -``` - -Expected: PASS. - -- [ ] **Step 9: Commit** - -```bash -git add packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/index.test.ts -git commit -m "fix(setup): use schema context and query history wording" -``` - -### Task 6: Final verification - -**Files:** -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `packages/cli/src/context-build-view.ts` -- Verify: `packages/cli/src/setup-databases.ts` -- Verify: `packages/cli/src/commands/setup-commands.ts` - -- [ ] **Step 1: Run targeted CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/context-build-view.test.ts src/setup-databases.test.ts src/index.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run the CLI test suite** - -Run: - -```bash -pnpm --filter @ktx/cli run test 2>&1 | tee /tmp/ktx-cli-unified-ingest-public-output.log -``` - -Expected: PASS. If it fails, inspect -`/tmp/ktx-cli-unified-ingest-public-output.log`, fix the failing assertion or -implementation, and rerun this command. - -- [ ] **Step 4: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If Knip reports public exports or dynamic CLI entrypoints, -verify each report before deleting code. - -- [ ] **Step 5: Commit verification fixes** - -If Step 1 through Step 4 required any changes, commit them: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/index.test.ts -git commit -m "test(cli): verify unified ingest public output" -``` - -If no files changed during verification, do not create an empty commit. - -## Self-review - -- Spec coverage: This plan covers the remaining public v1 gaps: adapter - allow-list bypass, quiet public database ingest output, TTY foreground view, - query-history window overrides, aggregated `--all` source warnings, setup - schema-context wording, setup query-history wording, and `ktx scan` retry - removal from normal setup output. -- Placeholder scan: The plan contains no placeholder markers, deferred tasks, - or "write tests later" steps. -- Type consistency: The plan keeps public ingest fields aligned with - `KtxPublicIngestArgs`, uses `allowImplicitAdapter` consistently with - `runKtxIngest`, and renames setup query-history args consistently from the - Commander layer through `runKtxSetupDatabasesStep`. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-public-plain-output-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-public-plain-output-closure.md deleted file mode 100644 index f9b9c956..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-public-plain-output-closure.md +++ /dev/null @@ -1,598 +0,0 @@ -# Unified Ingest V1 Public Plain Output Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the last v1-blocking adapter-centric and internal source-key leaks from normal public `ktx ingest` plain output. - -**Architecture:** Keep the current connection-centric public ingest planner and hidden debug commands. Sanitize low-level ingest report labels in `ingest.ts`, and capture low-level source/query-history output in `public-ingest.ts` so public plain `ktx ingest ` renders only the unified result table, warnings, notices, and retry guidance. JSON output and hidden debug commands may continue to expose raw `sourceKey` values for troubleshooting. - -**Tech Stack:** TypeScript, Commander, Vitest, pnpm workspace scripts. - ---- - -## Current audit - -The unified ingest plan chain has implemented the main v1 behavior: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` route through `public-ingest.ts`. -- Database targets run before source targets, deep readiness is target-local - for `--all`, and inferred public adapters bypass `ingest.adapters`. -- Normal command help hides `ktx scan`, `ktx ingest run`, and - `ktx ingest watch`; docs-site command references no longer publish those - as normal workflows. -- Setup stores `connections..context.depth` and - `connections..context.queryHistory`, migrates legacy `historicSql`, and - uses foreground-only context-build state. - -### V1-blocking gaps - -- Direct public non-TTY or `--no-input` source ingest still delegates to - `runKtxIngest()` with the real CLI IO. The lower-level reporter prints - `Adapter: ` and routine report details before the public result - table. For query history this can print `Adapter: historic-sql`, violating - the spec requirement that normal output use query-history wording and keep - internal adapter names out of routine output. -- `ktx ingest status` and `ktx ingest replay` plain output call the same - lower-level report formatter. Stored database reports can therefore print - `Adapter: live-database`, and stored query-history reports can print - `Adapter: historic-sql`, even though `status` and `replay` are public - report-viewing surfaces. - -### Non-blocking gaps - -- Hidden debug commands remain callable: `ktx scan`, `ktx ingest run`, and - `ktx ingest watch`. -- JSON output, debug output, tests, internal artifact paths, WorkUnit keys, - adapter package names, and developer scripts can continue to use - `scan`, `live-database`, and `historic-sql`. -- Public docs still use "scan" as a generic implementation noun in a few - contributor or concept pages. They do not present `ktx scan` as the normal - public command, so that is later wording cleanup. - -## File structure - -- Modify `packages/cli/src/ingest.ts`: replace the plain report `Adapter:` - label with public source labels, while leaving JSON report payloads intact. -- Modify `packages/cli/src/public-ingest.ts`: capture lower-level source and - query-history plain output for direct public ingest, sanitize failure detail - lines, and render only the public summary table. -- Modify `packages/cli/src/ingest.test.ts`: update existing report label - expectations and add regressions for `live-database` and `historic-sql` - stored-report labels. -- Modify `packages/cli/src/public-ingest.test.ts`: add regressions proving - direct public source and query-history runs do not leak lower-level adapter - report output. - -## Tasks - -### Task 1: Use public source labels in stored report output - -**Files:** -- Modify: `packages/cli/src/ingest.ts` -- Modify: `packages/cli/src/ingest.test.ts` - -- [ ] **Step 1: Add failing stored-report label tests** - -Add these tests inside the existing `describe('runKtxIngest', () => { ... })` -block in `packages/cli/src/ingest.test.ts`, near the existing -`runs local ingest and reads status` test: - -```typescript - it('labels internal database reports without adapter names in plain status output', async () => { - const projectDir = join(tempDir, 'project'); - await writeWarehouseConfig(projectDir); - const report = localFakeBundleReport('scan-job-1', { - id: 'report-scan-1', - runId: 'run-scan-1', - connectionId: 'warehouse', - sourceKey: 'live-database', - }); - const io = makeIo(); - - await expect( - runKtxIngest( - { - command: 'status', - projectDir, - reportFile: '/tmp/scan-report.json', - outputMode: 'plain', - }, - io.io, - { - readReportFile: vi.fn(async () => report), - }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Source: Database schema\n'); - expect(io.stdout()).not.toContain('Adapter:'); - expect(io.stdout()).not.toContain('live-database'); - expect(io.stderr()).toBe(''); - }); - - it('labels internal query-history reports without adapter names in plain status output', async () => { - const projectDir = join(tempDir, 'project'); - await writeWarehouseConfig(projectDir); - const report = localFakeBundleReport('query-history-job-1', { - id: 'report-query-history-1', - runId: 'run-query-history-1', - connectionId: 'warehouse', - sourceKey: 'historic-sql', - }); - const io = makeIo(); - - await expect( - runKtxIngest( - { - command: 'status', - projectDir, - reportFile: '/tmp/query-history-report.json', - outputMode: 'plain', - }, - io.io, - { - readReportFile: vi.fn(async () => report), - }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Source: Query history\n'); - expect(io.stdout()).not.toContain('Adapter:'); - expect(io.stdout()).not.toContain('historic-sql'); - expect(io.stderr()).toBe(''); - }); -``` - -- [ ] **Step 2: Run the failing stored-report tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts --testNamePattern "labels internal" -``` - -Expected: FAIL. The output still contains `Adapter: live-database` or -`Adapter: historic-sql`, and it does not contain the new public `Source:` -labels. - -- [ ] **Step 3: Add public report source labels** - -In `packages/cli/src/ingest.ts`, add these helpers above -`function writeReportStatus(...)`: - -```typescript -const REPORT_SOURCE_LABELS = new Map([ - ['live-database', 'Database schema'], - ['historic-sql', 'Query history'], - ['dbt', 'dbt'], - ['metricflow', 'MetricFlow'], - ['lookml', 'LookML'], - ['looker', 'Looker'], - ['metabase', 'Metabase'], - ['notion', 'Notion'], -]); - -function reportSourceLabel(sourceKey: string): string { - const label = REPORT_SOURCE_LABELS.get(sourceKey); - if (label) { - return label; - } - return sourceKey - .split(/[-_]+/) - .filter((part) => part.length > 0) - .map((part) => `${part[0]?.toUpperCase() ?? ''}${part.slice(1)}`) - .join(' '); -} -``` - -Then replace the `Adapter:` line in `writeReportStatus()`: - -```typescript - io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`); -``` - -The full function should keep the remaining fields unchanged: - -```typescript -function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void { - const counts = savedMemoryCountsForReport(report); - io.stdout.write(`Report: ${report.id}\n`); - io.stdout.write(`Run: ${report.runId}\n`); - io.stdout.write(`Job: ${report.jobId}\n`); - io.stdout.write(`Status: ${reportStatus(report)}\n`); - io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`); - io.stdout.write(`Connection: ${report.connectionId}\n`); - io.stdout.write(`Sync: ${report.body.syncId}\n`); - io.stdout.write( - `Diff: +${report.body.diffSummary.added}/~${report.body.diffSummary.modified}/-${report.body.diffSummary.deleted}/=${report.body.diffSummary.unchanged}\n`, - ); - io.stdout.write(`Work units: ${report.body.workUnits.length}\n`); - io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`); - io.stdout.write(`Provenance rows: ${report.body.provenanceRows.length}\n`); -} -``` - -- [ ] **Step 4: Update existing report label expectations** - -In `packages/cli/src/ingest.test.ts`, update the existing assertions that -still expect the old `Adapter:` label: - -```typescript -expect(statusIo.stdout()).toContain('Source: Metabase'); -``` - -```typescript -expect(io.stdout()).toContain('Source: Query history\n'); -``` - -```typescript -expect(io.stdout()).toContain('Source: Looker'); -``` - -```typescript -expect(statusIo.stdout()).toContain('Source: Looker'); -``` - -Remove the corresponding `Adapter: metabase`, `Adapter: historic-sql`, and -`Adapter: looker` expectations. - -- [ ] **Step 5: Run the stored-report tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts --testNamePattern "labels internal|runs public Metabase|historic-sql projection|Looker" -``` - -Expected: PASS. Plain report output uses `Source:` labels and does not print -`Adapter:` for the covered status and run summaries. - -- [ ] **Step 6: Commit stored-report label cleanup** - -Run: - -```bash -git add packages/cli/src/ingest.ts packages/cli/src/ingest.test.ts -git commit -m "fix(cli): use public source labels in ingest reports" -``` - -### Task 2: Capture low-level output during public source ingest - -**Files:** -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Add failing public source-output tests** - -Add these tests to `packages/cli/src/public-ingest.test.ts` near the existing -public output tests for captured scan output and query-history retry guidance: - -```typescript - it('suppresses lower-level source report output during direct public source ingest', async () => { - const io = makeIo(); - const project = projectWithConnections({ - docs: { driver: 'notion' }, - }); - const runIngest = vi.fn(async (_args, ingestIo) => { - ingestIo.stdout.write('Report: report-docs-1\n'); - ingestIo.stdout.write('Adapter: notion\n'); - ingestIo.stdout.write('Saved memory: 2 wiki, 0 SL\n'); - return 0; - }); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'docs', - all: false, - json: false, - inputMode: 'disabled', - }, - io.io, - { loadProject: vi.fn(async () => project), runIngest }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Ingest finished'); - expect(io.stdout()).toContain('docs'); - expect(io.stdout()).toContain('source-ingest'); - expect(io.stdout()).not.toContain('Report: report-docs-1'); - expect(io.stdout()).not.toContain('Adapter:'); - expect(io.stdout()).not.toContain('notion\n'); - expect(io.stderr()).toBe(''); - }); - - it('suppresses historic-sql report output during direct public query-history ingest', async () => { - const io = makeIo(); - const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, - }); - const runScan = vi.fn(async () => 0); - const runIngest = vi.fn(async (_args, ingestIo) => { - ingestIo.stdout.write('Report: report-query-history-1\n'); - ingestIo.stdout.write('Adapter: historic-sql\n'); - ingestIo.stdout.write('Saved memory: 1 wiki, 1 SL\n'); - return 0; - }); - - await expect( - runKtxPublicIngest( - { - command: 'run', - projectDir: '/tmp/project', - targetConnectionId: 'warehouse', - all: false, - json: false, - inputMode: 'disabled', - queryHistory: 'enabled', - }, - io.io, - { loadProject: vi.fn(async () => project), runScan, runIngest }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Schema ingest runs before query history for warehouse.'); - expect(io.stdout()).toContain('Ingest finished'); - expect(io.stdout()).toContain('warehouse'); - expect(io.stdout()).toContain('done'); - expect(io.stdout()).not.toContain('Report: report-query-history-1'); - expect(io.stdout()).not.toContain('Adapter:'); - expect(io.stdout()).not.toContain('historic-sql'); - expect(io.stderr()).toBe(''); - }); -``` - -- [ ] **Step 2: Run the failing public source-output tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts --testNamePattern "suppresses" -``` - -Expected: FAIL. The direct public run writes lower-level `Report:` and -`Adapter:` lines into normal public stdout. - -- [ ] **Step 3: Add captured ingest output helpers** - -In `packages/cli/src/public-ingest.ts`, keep the existing -`createCapturedPublicIngestIo()` helper and replace -`firstCapturedFailureLine()` with these helpers: - -```typescript -const INTERNAL_STATUS_LINE_RE = - /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Work units|Saved memory|Provenance rows):\s*/; - -function publicIngestOutputLine(line: string): string { - return line - .replace(/\blive-database\b/g, 'database schema') - .replace(/\bhistoric-sql\b/g, 'query history') - .replace(/\bhistoric SQL\b/gi, 'query history'); -} - -function firstCapturedFailureLine(output: string): string | undefined { - return output - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .filter((line) => !line.startsWith('KTX scan completed')) - .filter((line) => !INTERNAL_STATUS_LINE_RE.test(line)) - .map(publicIngestOutputLine) - .find((line) => line.length > 0); -} -``` - -- [ ] **Step 4: Capture query-history ingest output** - -In `executePublicIngestTarget()`, replace the query-history branch with this -captured-output flow: - -```typescript - if (target.queryHistory?.enabled === true) { - const { runKtxIngest } = await import('./ingest.js'); - const runIngest = deps.runIngest ?? runKtxIngest; - const ingestArgs: KtxIngestArgs = { - command: 'run', - projectDir: args.projectDir, - connectionId: target.connectionId, - adapter: 'historic-sql', - outputMode: sourceIngestOutputMode(args, io), - inputMode: args.inputMode, - allowImplicitAdapter: true, - historicSqlPullConfigOverride: - target.queryHistory.pullConfig ?? { - dialect: target.queryHistory.dialect, - ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}), - }, - }; - const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo(); - const ingestIo = capturedIngestIo ?? io; - const qhExitCode = deps.ingestProgress - ? await runIngest(ingestArgs, ingestIo, { progress: deps.ingestProgress }) - : await runIngest(ingestArgs, ingestIo); - if (qhExitCode !== 0) { - return markTargetResult( - target, - args, - 'failed', - 'query-history', - capturedIngestIo ? firstCapturedFailureLine(capturedIngestIo.capturedOutput()) : undefined, - ); - } - } -``` - -This keeps foreground progress working because `runContextBuild()` supplies -`deps.ingestProgress` and already passes a captured IO object into -`executePublicIngestTarget()`. - -- [ ] **Step 5: Capture source ingest output** - -In the source-ingest branch of `executePublicIngestTarget()`, replace the -direct `runIngest(..., io, ...)` call with this captured-output flow: - -```typescript - const runIngest = deps.runIngest ?? runKtxIngest; - const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo(); - const ingestIo = capturedIngestIo ?? io; - const exitCode = deps.ingestProgress - ? await runIngest(ingestArgs, ingestIo, { progress: deps.ingestProgress }) - : await runIngest(ingestArgs, ingestIo); - return markTargetResult( - target, - args, - exitCode === 0 ? 'done' : 'failed', - 'source-ingest', - capturedIngestIo ? firstCapturedFailureLine(capturedIngestIo.capturedOutput()) : undefined, - ); -``` - -Keep the existing `ingestArgs` object unchanged: - -```typescript - const ingestArgs: KtxIngestArgs = { - command: 'run', - projectDir: args.projectDir, - connectionId: target.connectionId, - adapter: target.adapter ?? target.driver, - ...(target.sourceDir ? { sourceDir: target.sourceDir } : {}), - outputMode: sourceIngestOutputMode(args, io), - inputMode: args.inputMode, - allowImplicitAdapter: true, - }; -``` - -- [ ] **Step 6: Run the public source-output tests again** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts --testNamePattern "suppresses|retry guidance|foreground" -``` - -Expected: PASS. Direct public source and query-history runs no longer print -low-level `Report:`, `Adapter:`, `live-database`, or `historic-sql` lines in -plain stdout, while existing foreground and retry guidance tests still pass. - -- [ ] **Step 7: Commit public source-output capture** - -Run: - -```bash -git add packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(cli): suppress low-level public ingest output" -``` - -### Task 3: Final verification - -**Files:** -- Verify: `packages/cli/src/ingest.ts` -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `packages/cli/src/ingest.test.ts` -- Verify: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run \ - src/public-ingest.test.ts \ - src/context-build-view.test.ts \ - src/ingest.test.ts \ - src/ingest-viz.test.ts \ - src/command-tree.test.ts \ - src/print-command-tree.test.ts -``` - -Expected: PASS. These tests cover direct public ingest, foreground context -builds, stored report rendering, visual report rendering, and hidden command -tree filtering. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS with no TypeScript errors. - -- [ ] **Step 3: Verify generated command tree still hides debug commands** - -Run: - -```bash -pnpm --filter @ktx/cli run docs:commands >/tmp/ktx-command-tree.txt -rg "scan |ingest run|ingest watch" /tmp/ktx-command-tree.txt -``` - -Expected: the `docs:commands` command succeeds. The `rg` command exits `1` -with no matches. - -- [ ] **Step 4: Search public docs and normal CLI surfaces for old public command guidance** - -Run: - -```bash -rg -n "ktx scan|ktx ingest run|ktx ingest watch|--enable-historic-sql|--historic-sql|historicSql|Historic SQL|live-database" \ - README.md docs-site/content examples/README.md examples/local-warehouse/README.md examples/postgres-historic/README.md -``` - -Expected: no v1-blocking matches. Matches that refer only to internal raw -artifact paths such as `raw-sources/warehouse/historic-sql` are allowed only in -the Postgres query-history smoke README. - -- [ ] **Step 5: Run dead-code checks after TypeScript changes** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If Knip reports unrelated existing findings, inspect them and -record the unrelated findings before finishing. - -- [ ] **Step 6: Inspect final diff** - -Run: - -```bash -git status --short -git diff -- packages/cli/src/ingest.ts packages/cli/src/public-ingest.ts packages/cli/src/ingest.test.ts packages/cli/src/public-ingest.test.ts -``` - -Expected: only the intended TypeScript source and test files are modified. -The diff contains no generated `dist/` files and no docs changes beyond this -plan. - -- [ ] **Step 7: Commit verification-only fixes if needed** - -Run only if verification required small expectation or formatting fixes: - -```bash -git add packages/cli/src/ingest.ts packages/cli/src/public-ingest.ts packages/cli/src/ingest.test.ts packages/cli/src/public-ingest.test.ts -git commit -m "test(cli): verify unified ingest public plain output" -``` - -Expected: no commit is needed when all checks pass after Tasks 1 and 2. - -## Self-review - -- Spec coverage: This plan closes the remaining v1-blocking normal-output - leaks for direct public source ingest, public query-history ingest, and - public stored-report status/replay output. It intentionally leaves hidden - debug commands, JSON payloads, internal artifact paths, and developer tests - untouched. -- Placeholder scan: The plan contains concrete file paths, exact test code, - exact implementation snippets, commands, and expected results. -- Type consistency: The snippets use existing local types and helpers: - `KtxIngestArgs`, `createCapturedPublicIngestIo()`, - `firstCapturedFailureLine()`, `sourceIngestOutputMode()`, - `markTargetResult()`, `localFakeBundleReport()`, and `makeIo()`. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-query-history-status-cleanup.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-query-history-status-cleanup.md deleted file mode 100644 index e9d03ec8..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-query-history-status-cleanup.md +++ /dev/null @@ -1,1339 +0,0 @@ -# Unified Ingest V1 Query History Status Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking public UX gaps in the unified -`ktx ingest` redesign. - -**Architecture:** Keep the implemented connection-centric ingest planner and -foreground context-build view. Patch the public setup, status, doctor, retry, -and example surfaces so canonical `context.queryHistory` and -`ktx ingest ` are the only normal user-facing paths. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, KTX CLI/context packages, -Markdown examples, shell smoke scripts. - ---- - -## Current audit - -The three implemented unified-ingest plans cover most of the original spec: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, - `--query-history`, `--no-query-history`, and - `--query-history-window-days` are routed through `public-ingest.ts`. -- Database targets run before source targets, inferred public adapters bypass - `ingest.adapters`, fast/deep map to structural/enriched scan internals, and - deep readiness failures are per-target failures under `--all`. -- `ktx scan`, `ktx ingest run`, and `ktx ingest watch` are hidden from normal - help. -- Setup stores `connections..context.depth`, config parsing rejects - reserved ingest subcommand ids, generated default config omits normal - adapter allow-list entries, and setup context builds are foreground-only. -- Public database ingest suppresses normal internal scan output, source ingest - passes `allowImplicitAdapter: true`, query-history window overrides enable - query history for the current run, and TTY public ingest delegates to the - shared foreground view. - -### V1-blocking gaps - -- `packages/cli/src/setup.ts` still exposes and forwards - `enableHistoricSql`, `disableHistoricSql`, and `historicSql*` args into the - database setup step. Public Commander flags now produce `enableQueryHistory` - and `queryHistory*`, so full `ktx setup --enable-query-history ...` does not - reach `runKtxSetupDatabasesStep`. -- Interactive Postgres setup does not ask whether to enable query history when - no query-history flag is provided, even though Postgres is a supported v1 - query-history driver. -- `ktx status`/project doctor still reads legacy - `connections..historicSql`, ignores canonical - `connections..context.queryHistory`, and prints public - `Postgres Historic SQL` labels. -- `ktx ingest status` with no stored reports still suggests - `ktx ingest run --connection-id --adapter `, which the spec - explicitly removes from normal guidance. -- Public query-history failures can surface `Historic SQL local ingest...` - messages from `local-adapters.ts`. -- The shared foreground view always formats retry guidance as `ktx setup`, - even when it is running direct public `ktx ingest `. -- Query-history foreground progress can show raw `historic-sql` adapter text - from lower-level ingest progress messages. -- Public examples still document old query-history and adapter surfaces: - `examples/postgres-historic/README.md`, - `examples/postgres-historic/scripts/smoke.sh`, and - `examples/README.md` still use `Historic SQL`, `--enable-historic-sql`, - `--historic-sql-*`, and `ktx ingest run --adapter historic-sql`. -- Checked-in example project configs still contain normal - `ingest.adapters: [live-database]`, contrary to the v1 config model. - -### Non-blocking gaps - -- Hidden debug commands can continue to call `ktx scan`, `ktx ingest run`, and - `ktx ingest watch`. -- Internal adapter keys, package names, raw artifact paths, WorkUnit keys, - skill names, and JSON/debug output can continue to use `scan`, - `live-database`, and `historic-sql`. -- Internal scripts such as relationship verification and artifact packaging can - keep standalone scan/live-database terminology when they are explicitly - developer-only. -- `setup.ts` still has dead `detached`/`paused`/`autoWatch` type remnants. - They are not currently user-facing because setup context state is normalized - and background watch flows have been removed. -- README package taxonomy such as `Postgres scan connector` can remain because - it describes internal package ownership, not public command usage. - -## File structure - -- Modify `packages/cli/src/setup.ts`: rename setup args and database-step - forwarding from historic-SQL names to query-history names. -- Modify `packages/cli/src/setup.test.ts`: cover full setup forwarding of - query-history flags into the database setup runner. -- Modify `packages/cli/src/setup-databases.ts`: ask the query-history prompt - for Postgres when interactive and no explicit query-history flag is supplied. -- Modify `packages/cli/src/setup-databases.test.ts`: cover interactive - Postgres query-history enablement through the canonical - `context.queryHistory` shape. -- Modify `packages/cli/src/historic-sql-doctor.ts`: read canonical - query-history config, keep legacy fallback for pre-migration configs, and - rename public doctor labels/messages to query history. -- Modify `packages/cli/src/historic-sql-doctor.test.ts`: update doctor unit - expectations for canonical config and public wording. -- Modify `packages/cli/src/doctor.test.ts`: update project doctor integration - expectations. -- Modify `packages/cli/src/ingest.ts`: replace stale no-report status guidance - with `ktx ingest ` wording. -- Modify `packages/cli/src/ingest-viz.test.ts`: cover the no-report status - guidance. -- Modify `packages/cli/src/local-adapters.ts`: change public-facing - query-history capability errors away from `Historic SQL`. -- Modify `packages/cli/src/local-adapters.test.ts`: cover at least one - query-history capability error message. -- Modify `packages/cli/src/context-build-view.ts`: accept an entrypoint for - retry text and sanitize public query-history progress messages. -- Modify `packages/cli/src/context-build-view.test.ts`: cover direct ingest - retry guidance and sanitized query-history progress. -- Modify `packages/cli/src/public-ingest.ts`: pass `entrypoint: 'ingest'` to - the foreground context-build view. -- Modify `packages/cli/src/public-ingest.test.ts`: cover public foreground - delegation with the entrypoint. -- Modify `examples/postgres-historic/README.md`: rename public query-history - wording and commands. -- Modify `examples/postgres-historic/scripts/smoke.sh`: use new setup flags. -- Modify `examples/README.md`: remove old Historic SQL public wording. -- Modify `examples/local-warehouse/ktx.yaml` and - `examples/orbit-relationship-verification/ktx.yaml`: remove - `live-database` from normal checked-in `ingest.adapters`. -- Modify `scripts/examples-docs.test.mjs`: assert the public examples no - longer advertise old flags or adapter commands. - -## Tasks - -### Task 1: Fix full setup query-history argument plumbing - -**Files:** -- Modify: `packages/cli/src/setup.ts` -- Test: `packages/cli/src/setup.test.ts` - -- [ ] **Step 1: Write the failing setup forwarding test** - -In `packages/cli/src/setup.test.ts`, add query-history fields to the existing -test named `runs database setup after embeddings succeed`: - -```ts - enableQueryHistory: true, - queryHistoryWindowDays: 30, - queryHistoryMinExecutions: 12, - queryHistoryServiceAccountPatterns: ['^svc_'], - queryHistoryRedactionPatterns: ['(?i)secret'], -``` - -The full args object in that test should include: - -```ts - databaseDrivers: ['postgres'], - databaseConnectionId: 'warehouse', - databaseUrl: 'env:DATABASE_URL', - databaseSchemas: ['public'], - enableQueryHistory: true, - queryHistoryWindowDays: 30, - queryHistoryMinExecutions: 12, - queryHistoryServiceAccountPatterns: ['^svc_'], - queryHistoryRedactionPatterns: ['(?i)secret'], - skipDatabases: false, -``` - -Extend the `expect(databases).toHaveBeenCalledWith(...)` assertion in the same -test: - -```ts - enableQueryHistory: true, - queryHistoryWindowDays: 30, - queryHistoryMinExecutions: 12, - queryHistoryServiceAccountPatterns: ['^svc_'], - queryHistoryRedactionPatterns: ['(?i)secret'], -``` - -- [ ] **Step 2: Run the failing setup test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup.test.ts -t "runs database setup after embeddings succeed" -``` - -Expected: FAIL because `runKtxSetup` still forwards the old -`enableHistoricSql` and `historicSql*` fields. - -- [ ] **Step 3: Rename setup args and forwarding** - -In `packages/cli/src/setup.ts`, replace the query-history section of -`KtxSetupArgs`: - -```ts - enableHistoricSql?: boolean; - disableHistoricSql?: boolean; - historicSqlWindowDays?: number; - historicSqlMinExecutions?: number; - historicSqlServiceAccountPatterns?: string[]; - historicSqlRedactionPatterns?: string[]; -``` - -with: - -```ts - enableQueryHistory?: boolean; - disableQueryHistory?: boolean; - queryHistoryWindowDays?: number; - queryHistoryMinExecutions?: number; - queryHistoryServiceAccountPatterns?: string[]; - queryHistoryRedactionPatterns?: string[]; -``` - -In the database-step call in `runKtxSetupInner`, replace: - -```ts - ...(args.enableHistoricSql !== undefined ? { enableHistoricSql: args.enableHistoricSql } : {}), - ...(args.disableHistoricSql !== undefined ? { disableHistoricSql: args.disableHistoricSql } : {}), - ...(args.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: args.historicSqlWindowDays } : {}), - ...(args.historicSqlMinExecutions !== undefined - ? { historicSqlMinExecutions: args.historicSqlMinExecutions } - : {}), - ...(args.historicSqlServiceAccountPatterns - ? { historicSqlServiceAccountPatterns: args.historicSqlServiceAccountPatterns } - : {}), - ...(args.historicSqlRedactionPatterns - ? { historicSqlRedactionPatterns: args.historicSqlRedactionPatterns } - : {}), -``` - -with: - -```ts - ...(args.enableQueryHistory !== undefined ? { enableQueryHistory: args.enableQueryHistory } : {}), - ...(args.disableQueryHistory !== undefined ? { disableQueryHistory: args.disableQueryHistory } : {}), - ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), - ...(args.queryHistoryMinExecutions !== undefined - ? { queryHistoryMinExecutions: args.queryHistoryMinExecutions } - : {}), - ...(args.queryHistoryServiceAccountPatterns - ? { queryHistoryServiceAccountPatterns: args.queryHistoryServiceAccountPatterns } - : {}), - ...(args.queryHistoryRedactionPatterns - ? { queryHistoryRedactionPatterns: args.queryHistoryRedactionPatterns } - : {}), -``` - -- [ ] **Step 4: Run the setup forwarding test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup.test.ts -t "runs database setup after embeddings succeed" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup.ts packages/cli/src/setup.test.ts -git commit -m "fix(setup): forward query history flags" -``` - -### Task 2: Ask Postgres query-history setup interactively - -**Files:** -- Modify: `packages/cli/src/setup-databases.ts` -- Test: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Write the failing Postgres prompt test** - -In `packages/cli/src/setup-databases.test.ts`, add this test after -`writes Postgres query history config with minExecutions and ignores window/redaction output`: - -```ts - it('asks interactive Postgres setup whether to enable query history', async () => { - await writeFile( - join(tempDir, 'ktx.yaml'), - [ - 'project: warehouse', - 'connections:', - ' warehouse:', - ' driver: postgres', - ' url: env:DATABASE_URL', - ' readonly: true', - '', - ].join('\n'), - 'utf-8', - ); - const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['yes'] }); - const historicSqlProbe = vi.fn(async () => ({ ok: true, lines: [] })); - - const result = await runKtxSetupDatabasesStep( - { - projectDir: tempDir, - inputMode: 'auto', - databaseConnectionIds: ['warehouse'], - databaseSchemas: [], - skipDatabases: false, - }, - io.io, - { - prompts, - testConnection: vi.fn(async () => 0), - scanConnection: vi.fn(async () => 0), - historicSqlProbe, - }, - ); - - expect(result.status).toBe('ready'); - expect(prompts.select).toHaveBeenCalledWith({ - message: 'Enable query-history ingest for this PostgreSQL connection?', - options: [ - { value: 'yes', label: 'Enable query history' }, - { value: 'no', label: 'Do not enable query history' }, - { value: 'back', label: 'Back' }, - ], - }); - expect(historicSqlProbe).toHaveBeenCalledWith({ - projectDir: tempDir, - connectionId: 'warehouse', - dialect: 'postgres', - }); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse).toMatchObject({ - context: { - queryHistory: { - enabled: true, - minExecutions: 5, - filters: { dropTrivialProbes: true }, - }, - }, - }); - }); -``` - -- [ ] **Step 2: Run the failing Postgres prompt test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts -t "asks interactive Postgres setup" -``` - -Expected: FAIL because Postgres currently returns without asking when no -explicit query-history flag is supplied. - -- [ ] **Step 3: Prompt for all supported query-history drivers** - -In `packages/cli/src/setup-databases.ts`, replace this branch in -`maybeApplyHistoricSqlConfig`: - -```ts - } else if (input.args.inputMode !== 'disabled' && input.args.enableQueryHistory !== true && dialect !== 'postgres') { -``` - -with: - -```ts - } else if (input.args.inputMode !== 'disabled' && input.args.enableQueryHistory !== true) { -``` - -Then delete this early return: - -```ts - if (dialect === 'postgres' && input.args.enableQueryHistory !== true && input.args.disableQueryHistory !== true) { - return input.connection; - } -``` - -- [ ] **Step 4: Run the Postgres prompt test** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-databases.test.ts -t "asks interactive Postgres setup|writes Postgres query history config" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/setup-databases.ts packages/cli/src/setup-databases.test.ts -git commit -m "fix(setup): prompt for postgres query history" -``` - -### Task 3: Rename status/doctor query-history readiness output - -**Files:** -- Modify: `packages/cli/src/historic-sql-doctor.ts` -- Modify: `packages/cli/src/historic-sql-doctor.test.ts` -- Modify: `packages/cli/src/doctor.test.ts` - -- [ ] **Step 1: Write failing canonical doctor expectations** - -In `packages/cli/src/historic-sql-doctor.test.ts`, update the first test name -and expected object: - -```ts - it('passes when no Postgres query-history connections are enabled', async () => { -``` - -```ts - expect(checks).toEqual([ - { - id: 'query-history-postgres', - label: 'Postgres query history', - status: 'pass', - detail: 'No enabled Postgres query-history connections', - }, - ]); -``` - -In the success test, replace the configured connection with canonical -query-history config: - -```ts - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, - context: { queryHistory: { enabled: true } }, - }, -``` - -Update the probe assertion to match the same connection shape, and update the -expected check: - -```ts - { - id: 'query-history-postgres-warehouse', - label: 'Postgres query history (warehouse)', - status: 'pass', - detail: 'pg_stat_statements ready (PostgreSQL 16.4)', - }, -``` - -Update the warning and capability-error tests to expect -`query-history-postgres-warehouse` and -`Postgres query history (warehouse)`. - -Add this legacy fallback test before the non-Postgres-driver failure test: - -```ts - it('still checks legacy historicSql blocks before setup migration', async () => { - const probe = vi.fn(async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - })); - - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { postgresHistoricSqlProbe: probe }, - ); - - expect(checks).toEqual([ - { - id: 'query-history-postgres-warehouse', - label: 'Postgres query history (warehouse)', - status: 'pass', - detail: 'pg_stat_statements ready (PostgreSQL 16.4)', - }, - ]); - }); -``` - -Update the non-Postgres-driver failure expected object: - -```ts - { - id: 'query-history-postgres-warehouse', - label: 'Postgres query history (warehouse)', - status: 'fail', - detail: 'connections.warehouse.context.queryHistory is enabled but driver is mysql', - fix: 'Set connections.warehouse.driver to postgres or disable query history for this connection', - }, -``` - -In `packages/cli/src/doctor.test.ts`, rename the test to -`includes Postgres query-history readiness in project doctor output`, write -canonical config, and update the injected check: - -```ts - ' context:', - ' queryHistory:', - ' enabled: true', -``` - -```ts - id: 'query-history-postgres-warehouse', - label: 'Postgres query history (warehouse)', -``` - -Update the output assertion: - -```ts - expect(testIo.stdout()).toContain('PASS Postgres query history (warehouse): pg_stat_statements ready'); -``` - -- [ ] **Step 2: Run the failing doctor tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/historic-sql-doctor.test.ts src/doctor.test.ts -t "query-history|historicSql blocks" -``` - -Expected: FAIL because the doctor still reads `historicSql` only and prints -`Postgres Historic SQL`. - -- [ ] **Step 3: Read canonical query-history config in the doctor** - -In `packages/cli/src/historic-sql-doctor.ts`, replace `historicSqlRecord` and -`isEnabledPostgresHistoricSql` with: - -```ts -function recordValue(value: unknown): Record | null { - return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : null; -} - -function queryHistoryRecord(connection: KtxProjectConnectionConfig): Record | null { - const context = recordValue(connection.context); - return recordValue(context?.queryHistory); -} - -function legacyHistoricSqlRecord(connection: KtxProjectConnectionConfig): Record | null { - return recordValue(connection.historicSql); -} - -function isEnabledPostgresQueryHistory(connection: KtxProjectConnectionConfig): boolean { - const queryHistory = queryHistoryRecord(connection); - if (queryHistory) { - return queryHistory.enabled === true; - } - const legacy = legacyHistoricSqlRecord(connection); - return legacy?.enabled === true && legacy.dialect === 'postgres'; -} -``` - -Rename `checkId`: - -```ts -function checkId(connectionId: string): string { - return `query-history-postgres-${connectionId.replace(/[^a-z0-9_-]+/gi, '-')}`; -} -``` - -Update `capabilityFailureFix`: - -```ts - if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') { - return 'Use PostgreSQL 14 or newer, or disable query history for this connection'; - } - return `Fix connections.${connectionId} Postgres settings, then rerun \`ktx status --project-dir ${projectDir}\``; -``` - -Update `runPostgresHistoricSqlDoctorChecks` target selection and no-target -result: - -```ts - const targets = Object.entries(project.config.connections) - .filter(([, connection]) => isEnabledPostgresQueryHistory(connection)) - .sort(([left], [right]) => left.localeCompare(right)); - - if (targets.length === 0) { - return [ - check('pass', 'query-history-postgres', 'Postgres query history', 'No enabled Postgres query-history connections'), - ]; - } -``` - -Update the per-target label and non-Postgres failure: - -```ts - const label = `Postgres query history (${connectionId})`; - if (!isPostgresDriver(connection)) { - checks.push( - check( - 'fail', - checkId(connectionId), - label, - `connections.${connectionId}.context.queryHistory is enabled but driver is ${String(connection.driver)}`, - `Set connections.${connectionId}.driver to postgres or disable query history for this connection`, - ), - ); - continue; - } -``` - -- [ ] **Step 4: Run the doctor tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/historic-sql-doctor.test.ts src/doctor.test.ts -t "query-history|historicSql blocks" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/historic-sql-doctor.ts packages/cli/src/historic-sql-doctor.test.ts packages/cli/src/doctor.test.ts -git commit -m "fix(status): report query history readiness" -``` - -### Task 4: Remove stale adapter-command suggestions and public Historic SQL errors - -**Files:** -- Modify: `packages/cli/src/ingest.ts` -- Modify: `packages/cli/src/ingest-viz.test.ts` -- Modify: `packages/cli/src/local-adapters.ts` -- Modify: `packages/cli/src/local-adapters.test.ts` - -- [ ] **Step 1: Write failing no-report status guidance test** - -In `packages/cli/src/ingest-viz.test.ts`, add this test after -`returns an error code for missing status`: - -```ts - it('suggests public ingest when status has no stored reports', async () => { - const projectDir = join(tempDir, 'project'); - await writeWarehouseConfig(projectDir); - const io = makeIo(); - - await expect(runKtxIngest({ command: 'status', projectDir, outputMode: 'plain' }, io.io)).resolves.toBe(1); - - expect(io.stderr()).toContain('No local ingest reports were found. Run `ktx ingest ` first.'); - expect(io.stderr()).not.toContain('ktx ingest run --connection-id'); - expect(io.stderr()).not.toContain('--adapter'); - }); -``` - -- [ ] **Step 2: Write failing query-history error wording test** - -In `packages/cli/src/local-adapters.test.ts`, add this test before the closing -`describe` block: - -```ts - it('uses query-history wording for public BigQuery capability errors', async () => { - await writeProject( - tempDir, - [ - 'project: warehouse', - 'connections:', - ' bq:', - ' driver: bigquery', - ' readonly: true', - ' dataset_id: analytics', - ' credentials_json: "{}"', - ' context:', - ' queryHistory:', - ' enabled: true', - 'ingest:', - ' adapters:', - ' - historic-sql', - '', - ].join('\n'), - ); - const project = await loadKtxProject({ projectDir: tempDir }); - - expect(() => - createKtxCliLocalIngestAdapters(project, { - historicSqlConnectionId: 'bq', - sqlAnalysis: sqlAnalysisStub(), - }), - ).toThrow('Query history BigQuery connection requires credentials_json.project_id'); - }); -``` - -- [ ] **Step 3: Run the failing output tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest-viz.test.ts src/local-adapters.test.ts -t "public ingest when status|query-history wording" -``` - -Expected: FAIL because current output still mentions `ktx ingest run` and -`Historic SQL`. - -- [ ] **Step 4: Replace stale status guidance** - -In `packages/cli/src/ingest.ts`, replace: - -```ts - : 'No local ingest reports were found. Run `ktx ingest run --connection-id --adapter ` first.', -``` - -with: - -```ts - : 'No local ingest reports were found. Run `ktx ingest ` first.', -``` - -- [ ] **Step 5: Rename public query-history capability errors** - -In `packages/cli/src/local-adapters.ts`, replace user-facing error strings: - -```ts -`Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}` -`Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}` -`Historic SQL local ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}` -'Historic SQL BigQuery connection requires credentials_json.project_id' -``` - -with: - -```ts -`Query history ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}` -`Query history ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}` -`Query history ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}` -'Query history BigQuery connection requires credentials_json.project_id' -``` - -- [ ] **Step 6: Run the output tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest-viz.test.ts src/local-adapters.test.ts -t "public ingest when status|query-history wording" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/ingest.ts packages/cli/src/ingest-viz.test.ts packages/cli/src/local-adapters.ts packages/cli/src/local-adapters.test.ts -git commit -m "fix(ingest): remove legacy public guidance" -``` - -### Task 5: Fix foreground retry guidance and query-history progress copy - -**Files:** -- Modify: `packages/cli/src/context-build-view.ts` -- Modify: `packages/cli/src/context-build-view.test.ts` -- Modify: `packages/cli/src/public-ingest.ts` -- Modify: `packages/cli/src/public-ingest.test.ts` - -- [ ] **Step 1: Write failing foreground view tests** - -In `packages/cli/src/context-build-view.test.ts`, add this test in the -`runContextBuild` describe block: - -```ts - it('uses direct ingest retry guidance for public ingest failures', async () => { - const io = makeIo(); - const project = projectWithConnections({ - warehouse: { driver: 'postgres' }, - }); - const executeTarget = vi.fn(async (target) => failedResult(target.connectionId, target.driver, target.operation)); - - await runContextBuild( - project, - { projectDir: '/tmp/project', inputMode: 'disabled', targetConnectionId: 'warehouse', all: false, entrypoint: 'ingest' }, - io.io, - { executeTarget, now: () => 1000 }, - ); - - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project'); - expect(io.stdout()).not.toContain('Retry: ktx setup'); - }); -``` - -Add this progress-copy test in the same describe block: - -```ts - it('renders query-history progress without the historic-sql adapter key', async () => { - const io = makeIo(); - const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true } } }, - }); - const executeTarget = vi.fn(async (target, _args, _targetIo, deps) => { - deps.ingestProgress?.({ percent: 5, message: 'Fetching source files for warehouse/historic-sql' }); - return successResult(target.connectionId, target.driver, target.operation); - }); - - await runContextBuild( - project, - { projectDir: '/tmp/project', inputMode: 'disabled', targetConnectionId: 'warehouse', all: false, entrypoint: 'ingest' }, - io.io, - { executeTarget, now: () => 1000, sourceProgressThrottleMs: 0 }, - ); - - expect(io.stdout()).toContain('Fetching query history for warehouse'); - expect(io.stdout()).not.toContain('historic-sql'); - }); -``` - -In `packages/cli/src/public-ingest.test.ts`, update the test named -`delegates interactive TTY public ingest to the foreground context-build view` -so the `runContextBuild` assertion includes: - -```ts - entrypoint: 'ingest', -``` - -- [ ] **Step 2: Run the failing foreground tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/public-ingest.test.ts -t "direct ingest retry|query-history progress|foreground context-build view" -``` - -Expected: FAIL because `ContextBuildArgs` has no entrypoint and progress text -is not sanitized. - -- [ ] **Step 3: Add entrypoint-aware retry commands** - -In `packages/cli/src/context-build-view.ts`, extend `ContextBuildArgs`: - -```ts - entrypoint?: 'setup' | 'ingest'; -``` - -Replace `resumeCommand` with: - -```ts -function retryCommand(input: { - projectDir?: string; - entrypoint?: 'setup' | 'ingest'; - connectionId?: string; - depth?: 'fast' | 'deep'; -}): string { - const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : ''; - if (input.entrypoint === 'ingest' && input.connectionId) { - const depthPart = input.depth ? ` --${input.depth}` : ''; - return `ktx ingest ${input.connectionId}${projectPart}${depthPart}`; - } - return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup'; -} -``` - -Update `failureTextForTarget` to accept `entrypoint` and pass the target depth: - -```ts - entrypoint?: 'setup' | 'ingest'; -``` - -Replace the network retry line: - -```ts - `Retry: ${resumeCommand(input.projectDir)}`, -``` - -with: - -```ts - `Retry: ${retryCommand({ - projectDir: input.projectDir, - entrypoint: input.entrypoint, - connectionId: input.target.connectionId, - depth: input.target.databaseDepth, - })}`, -``` - -For non-network failures, append retry text when `entrypoint === 'ingest'`: - -```ts - const fallback = input.fallback ?? `${input.target.connectionId} failed.`; - if (input.entrypoint === 'ingest') { - return `${fallback} Retry: ${retryCommand({ - projectDir: input.projectDir, - entrypoint: input.entrypoint, - connectionId: input.target.connectionId, - depth: input.target.databaseDepth, - })}`; - } - return fallback; -``` - -Pass `entrypoint: args.entrypoint` where `failureTextForTarget` is called. - -- [ ] **Step 4: Sanitize public query-history progress text** - -In `packages/cli/src/context-build-view.ts`, add: - -```ts -function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string { - if (target.steps.includes('query-history')) { - return message - .replace(`${target.connectionId}/historic-sql`, `${target.connectionId} query history`) - .replace(/\bhistoric-sql\b/g, 'query history') - .replace(/\bhistoric SQL\b/gi, 'query history'); - } - return message; -} -``` - -Change `formatProgressDetail` to accept the target: - -```ts -function formatProgressDetail( - update: Pick, - target: KtxPublicIngestPlanTarget, -): string { - const percent = Math.max(0, Math.min(100, Math.round(update.percent))); - return `[${percent}%] ${publicProgressMessage(update.message, target)}`; -} -``` - -Update the `updateTargetProgress` call site: - -```ts - targetState.detailLine = formatProgressDetail(update, targetState.target); -``` - -Update the capture progress callback: - -```ts - targetState.detailLine = publicProgressMessage(message, targetState.target); -``` - -- [ ] **Step 5: Pass foreground entrypoint from public ingest** - -In `packages/cli/src/public-ingest.ts`, add this field to the -`contextBuild(...)` args object: - -```ts - entrypoint: 'ingest', -``` - -- [ ] **Step 6: Run the foreground tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/context-build-view.test.ts src/public-ingest.test.ts -t "direct ingest retry|query-history progress|foreground context-build view" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/context-build-view.ts packages/cli/src/context-build-view.test.ts packages/cli/src/public-ingest.ts packages/cli/src/public-ingest.test.ts -git commit -m "fix(ingest): polish foreground retry copy" -``` - -### Task 6: Update public examples and checked-in example configs - -**Files:** -- Modify: `examples/postgres-historic/README.md` -- Modify: `examples/postgres-historic/scripts/smoke.sh` -- Modify: `examples/README.md` -- Modify: `examples/local-warehouse/ktx.yaml` -- Modify: `examples/orbit-relationship-verification/ktx.yaml` -- Modify: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Write failing examples-docs assertions** - -In `scripts/examples-docs.test.mjs`, replace the historic-SQL assertions with: - -```js - assert.doesNotMatch(examples, /Historic SQL/); - assert.doesNotMatch(examples, /historic-SQL/); - assert.match(examples, /query-history ingest via `pg_stat_statements`/); - assert.doesNotMatch(readme, /--enable-historic-sql/); - assert.doesNotMatch(readme, /--historic-sql-min-executions/); - assert.doesNotMatch(readme, /ktx ingest run --project-dir/); - assert.doesNotMatch(readme, /--adapter historic-sql/); - assert.match(readme, /--enable-query-history/); - assert.match(readme, /--query-history-min-executions 2/); - assert.match(readme, /Postgres query history/); -``` - -Add assertions for checked-in example configs: - -```js - test('checked-in example configs do not include public live-database adapters', async () => { - const localWarehouseConfig = await readFile('examples/local-warehouse/ktx.yaml', 'utf8'); - const orbitConfig = await readFile('examples/orbit-relationship-verification/ktx.yaml', 'utf8'); - - assert.doesNotMatch(localWarehouseConfig, /live-database/); - assert.doesNotMatch(orbitConfig, /live-database/); - }); -``` - -- [ ] **Step 2: Run the failing examples-docs test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: FAIL because examples still document old flags and configs still -contain `live-database`. - -- [ ] **Step 3: Update Postgres query-history example docs** - -In `examples/postgres-historic/README.md`, replace the title: - -```md -# Postgres Historic SQL Example -``` - -with: - -```md -# Postgres Query History Example -``` - -Replace the opening paragraph: - -```md -This example is a manual smoke for the redesigned Postgres historic-SQL ingest -path through `pg_stat_statements`. It starts Postgres 14 with the extension -preloaded, generates query workload under separate users, runs `ktx setup` with -`--enable-historic-sql`, and verifies the unified staged artifacts: -``` - -with: - -```md -This example is a manual smoke for Postgres query-history ingest through -`pg_stat_statements`. It starts Postgres 14 with the extension preloaded, -generates query workload under separate users, runs `ktx setup` with -`--enable-query-history`, and verifies the staged query-history artifacts: -``` - -Replace setup flags: - -```bash - --enable-historic-sql \ - --historic-sql-min-executions 2 \ -``` - -with: - -```bash - --enable-query-history \ - --query-history-min-executions 2 \ -``` - -Replace the manual ingest command: - -```bash -pnpm run ktx -- ingest run --project-dir /tmp/ktx-postgres-historic \ - --connection-id warehouse \ - --adapter historic-sql \ - --plain \ - --yes \ - --no-input -``` - -with: - -```bash -pnpm run ktx -- ingest warehouse --project-dir /tmp/ktx-postgres-historic \ - --query-history \ - --no-input -``` - -Apply these exact prose replacements in `examples/postgres-historic/README.md`: - -```md -Postgres Historic SQL Example -``` - -becomes: - -```md -Postgres Query History Example -``` - -```md -The smoke validates the historic-SQL raw snapshot path without requiring LLM -credentials. It uses KTX's local stage-only ingest API after `ktx setup`, so the -deterministic reader, batch SQL parser, stable artifact writer, and diff-based -WorkUnit planning are checked independently from curation. -``` - -becomes: - -```md -The smoke validates the query-history raw snapshot path without requiring LLM -credentials. It uses KTX's local stage-only ingest API after `ktx setup`, so the -deterministic reader, batch SQL parser, stable artifact writer, and diff-based -WorkUnit planning are checked independently from curation. -``` - -```md -Create a project and enable historic SQL: -``` - -becomes: - -```md -Create a project and enable query history: -``` - -```md -Expected output includes `PASS Postgres Historic SQL (warehouse)` when -`pg_stat_statements` is installed, `pg_read_all_stats` is granted, and tracking -is enabled. -``` - -becomes: - -```md -Expected output includes `PASS Postgres query history (warehouse)` when -`pg_stat_statements` is installed, `pg_read_all_stats` is granted, and tracking -is enabled. -``` - -```md -Run local historic-SQL ingest: -``` - -becomes: - -```md -Run query-history ingest: -``` - -```md -The full `ingest run` path also runs curation WorkUnits, so it requires a -configured LLM provider. -``` - -becomes: - -```md -The public query-history ingest path also runs curation WorkUnits, so it -requires a configured LLM provider. -``` - -Keep literal `source: "historic-sql"`, raw -`raw-sources/.../historic-sql` paths, and WorkUnit key examples only in the -artifact inspection section where they describe internal artifacts. - -Replace the troubleshooting bullet: - -```md -- Empty snapshot: rerun `scripts/generate-workload.sh base` and keep - `--historic-sql-min-executions 2` for the smoke. -``` - -with: - -```md -- Empty snapshot: rerun `scripts/generate-workload.sh base` and keep - `--query-history-min-executions 2` for the smoke. -``` - -- [ ] **Step 4: Update the smoke setup flags** - -In `examples/postgres-historic/scripts/smoke.sh`, replace: - -```bash - --enable-historic-sql \ - --historic-sql-min-executions 2 \ -``` - -with: - -```bash - --enable-query-history \ - --query-history-min-executions 2 \ -``` - -- [ ] **Step 5: Update example index wording** - -In `examples/README.md`, replace: - -```md -`postgres-historic/` is a manual Docker-backed smoke for Postgres historic-SQL -ingest via `pg_stat_statements`. It verifies setup, unified Historic SQL artifacts, -managed daemon batch SQL analysis, bounded pattern WorkUnit shards, and -no-WorkUnit idempotency for unchanged bucketed table inputs and pattern shards. -``` - -with: - -```md -`postgres-historic/` is a manual Docker-backed smoke for Postgres query-history -ingest via `pg_stat_statements`. It verifies setup, staged query-history -artifacts, managed daemon batch SQL analysis, bounded pattern WorkUnit shards, -and no-WorkUnit idempotency for unchanged bucketed table inputs and pattern -shards. -``` - -- [ ] **Step 6: Remove live-database from example configs** - -In `examples/local-warehouse/ktx.yaml`, replace: - -```yaml -ingest: - adapters: - - fake - - live-database -``` - -with: - -```yaml -ingest: - adapters: - - fake -``` - -In `examples/orbit-relationship-verification/ktx.yaml`, replace: - -```yaml -ingest: - adapters: - - live-database -``` - -with: - -```yaml -ingest: - adapters: [] -``` - -- [ ] **Step 7: Run examples-docs tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add examples/postgres-historic/README.md examples/postgres-historic/scripts/smoke.sh examples/README.md examples/local-warehouse/ktx.yaml examples/orbit-relationship-verification/ktx.yaml scripts/examples-docs.test.mjs -git commit -m "docs(examples): use unified query history wording" -``` - -### Task 7: Final verification - -**Files:** -- Verify: `packages/cli/src/setup.ts` -- Verify: `packages/cli/src/setup-databases.ts` -- Verify: `packages/cli/src/historic-sql-doctor.ts` -- Verify: `packages/cli/src/ingest.ts` -- Verify: `packages/cli/src/local-adapters.ts` -- Verify: `packages/cli/src/context-build-view.ts` -- Verify: `packages/cli/src/public-ingest.ts` -- Verify: `examples/` -- Verify: `scripts/examples-docs.test.mjs` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run \ - src/setup.test.ts \ - src/setup-databases.test.ts \ - src/historic-sql-doctor.test.ts \ - src/doctor.test.ts \ - src/ingest-viz.test.ts \ - src/local-adapters.test.ts \ - src/context-build-view.test.ts \ - src/public-ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run examples docs test** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code check for TypeScript changes** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only known unrelated Knip findings. Investigate and fix -new findings introduced by this plan. - -- [ ] **Step 5: Check remaining public old-surface references** - -Run: - -```bash -rg -n "ktx ingest run --connection-id|--enable-historic-sql|--historic-sql|Postgres Historic SQL|Historic SQL local ingest|live-database" README.md examples packages/cli/src scripts/examples-docs.test.mjs -``` - -Expected: no matches in public docs, setup/status/ingest public output, or -example configs. Matches in hidden-command tests, internal adapter tests, -debug-only scripts, and low-level scan tests are acceptable only when the file -is explicitly exercising internal behavior. - -- [ ] **Step 6: Commit verification-only fixes if needed** - -If Step 4 or Step 5 required edits, commit them: - -```bash -git add -git commit -m "chore(ingest): finish public query history cleanup" -``` - -Expected: no commit is needed when all checks pass without further edits. - -## Self-review - -- Spec coverage: this plan covers the remaining public setup query-history - path, canonical status readiness, stale command guidance, public foreground - retry/progress copy, public examples, and generated/example config cleanup. -- Placeholder scan: no task uses placeholder implementation language. -- Type consistency: all new public fields use `queryHistory*`; internal file - names and adapter keys can remain `historic-sql` where they are not normal - public UX. diff --git a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-verification-copy-closure.md b/docs/superpowers/plans/2026-05-13-unified-ingest-v1-verification-copy-closure.md deleted file mode 100644 index ce3e7eba..00000000 --- a/docs/superpowers/plans/2026-05-13-unified-ingest-v1-verification-copy-closure.md +++ /dev/null @@ -1,326 +0,0 @@ -# Unified Ingest V1 Verification Copy Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking verification and setup-copy gaps in the unified `ktx ingest` UX. - -**Architecture:** Keep the implemented connection-centric ingest planner unchanged. Fix the test-only TypeScript error that currently blocks `@ktx/cli` type-check, then replace the remaining normal setup help/output references to old "primary source" terminology with database-oriented copy. - -**Tech Stack:** TypeScript ESM, Commander, Vitest, pnpm workspace scripts, uv pre-commit. - ---- - -## Current Audit - -Implemented unified-ingest plans already cover the original spec's main v1 behavior: - -- `ktx ingest [connectionId]`, `ktx ingest --all`, `--fast`, `--deep`, `--query-history`, `--no-query-history`, and `--query-history-window-days` route through `packages/cli/src/public-ingest.ts`. -- Database targets are ordered before source targets, public source ingest bypasses `ingest.adapters`, and database depth maps to structural/enriched scan internals. -- Deep readiness is evaluated per target before target work starts, and `--all` isolates eligible targets from independent failures. -- Setup stores `connections..context.depth` and `connections..context.queryHistory`, migrates legacy `historicSql`, and uses foreground-only context-build state. -- Normal `ktx` and `ktx ingest` help hide `ktx scan`, `ktx ingest run`, and live `ktx ingest watch`. -- Foreground progress and normal public output sanitize scan/live-database/historic-sql internals. - -### V1-Blocking Gaps - -- `pnpm --filter @ktx/cli run type-check` fails: - -```text -src/setup-databases.test.ts(1078,39): error TS2339: Property 'mock' does not exist on type '(options: { message: string; options: KtxSetupPromptOption[]; required?: boolean | undefined; initialValues?: string[] | undefined; }) => Promise'. -``` - -- Normal setup help/output still exposes the old database category as "primary source": - - `packages/cli/src/commands/setup-commands.ts` documents `--skip-databases` as `KTX cannot work until a primary source is added`. - - `packages/cli/src/setup-sources.ts` prints `Connect a primary source before adding context sources.` - - `packages/cli/src/setup-context.ts` prints `No primary or context sources are configured for a KTX context build.` - -### Non-Blocking Gaps - -- Hidden debug commands remain callable: `ktx scan`, `ktx ingest run`, and `ktx ingest watch`. -- Internal adapter keys, artifact paths, WorkUnit keys, package names, tests, and developer-only scripts can continue to use `scan`, `live-database`, `historic-sql`, and internal `primarySource*` identifiers. -- Public docs still have a `Primary Sources` integration page and a quickstart sentence about BI metadata mapping to primary source connections. That is broader documentation information architecture cleanup, not a v1 blocker for the normal command/help/output behavior in this spec. - -## File Structure - -- Modify `packages/cli/src/setup-databases.test.ts`: use Vitest's typed mock helper for the existing `prompts.multiselect` assertion. -- Modify `packages/cli/src/setup-sources.ts`: change the normal missing-database message before context source setup. -- Modify `packages/cli/src/setup-sources.test.ts`: update the missing-database regression. -- Modify `packages/cli/src/setup-context.ts`: change the normal no-target context-build error. -- Modify `packages/cli/src/setup-context.test.ts`: update the no-target context-build regression. -- Modify `packages/cli/src/commands/setup-commands.ts`: change the public `--skip-databases` help copy. -- Modify `packages/cli/src/index.test.ts`: assert setup help no longer contains public "primary source" wording. - -## Tasks - -### Task 1: Repair Setup Database Test Type-Check - -**Files:** -- Modify: `packages/cli/src/setup-databases.test.ts` - -- [ ] **Step 1: Replace the untyped mock access** - -In `packages/cli/src/setup-databases.test.ts`, in the test named `prompts for discovered Postgres schemas before the first scan`, replace: - -```ts - expect(String(prompts.multiselect.mock.calls[0]?.[0].message)).not.toContain('to scan'); -``` - -with: - -```ts - expect(String(vi.mocked(prompts.multiselect).mock.calls[0]?.[0].message)).not.toContain('to scan'); -``` - -- [ ] **Step 2: Run the setup database type-check regression** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected before the fix: FAIL with `TS2339: Property 'mock' does not exist`. - -Expected after the fix: PASS. - -- [ ] **Step 3: Commit the type-check repair** - -Run: - -```bash -git add packages/cli/src/setup-databases.test.ts -git commit -m "test(cli): fix setup database test type-check" -``` - -### Task 2: Replace Remaining Normal Setup Primary-Source Copy - -**Files:** -- Modify: `packages/cli/src/setup-sources.ts` -- Modify: `packages/cli/src/setup-sources.test.ts` -- Modify: `packages/cli/src/setup-context.ts` -- Modify: `packages/cli/src/setup-context.test.ts` -- Modify: `packages/cli/src/commands/setup-commands.ts` -- Modify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Update setup source missing-database expectations** - -In `packages/cli/src/setup-sources.test.ts`, replace the test name and output expectation: - -```ts - it('does not offer context sources until a primary source exists', async () => { -``` - -with: - -```ts - it('does not offer context sources until a database exists', async () => { -``` - -and replace: - -```ts - expect(io.stdout()).toContain('Connect a primary source before adding context sources.'); -``` - -with: - -```ts - expect(io.stdout()).toContain('Connect a database before adding context sources.'); -``` - -- [ ] **Step 2: Update setup context no-target expectations** - -In `packages/cli/src/setup-context.test.ts`, replace: - -```ts - expect(io.stderr()).toContain('No primary or context sources are configured for a KTX context build.'); -``` - -with: - -```ts - expect(io.stderr()).toContain('No databases or context sources are configured for a KTX context build.'); -``` - -- [ ] **Step 3: Add setup help regression coverage** - -In `packages/cli/src/index.test.ts`, in the test named `documents setup as a bare command without subcommands`, add these assertions after the existing query-history flag assertions and before the historic-SQL assertions: - -```ts - expect(testIo.stdout()).toContain('KTX cannot work until a database is added'); - expect(testIo.stdout()).not.toContain('primary source'); - expect(testIo.stdout()).not.toContain('primary sources'); -``` - -- [ ] **Step 4: Run the failing setup-copy tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-sources.test.ts src/setup-context.test.ts src/index.test.ts -t "context sources until a database exists|No databases or context sources|documents setup as a bare command" -``` - -Expected: FAIL because implementation still prints `primary source` in setup source/context output and setup help. - -- [ ] **Step 5: Update setup source output** - -In `packages/cli/src/setup-sources.ts`, replace: - -```ts - const message = 'Connect a primary source before adding context sources.'; -``` - -with: - -```ts - const message = 'Connect a database before adding context sources.'; -``` - -- [ ] **Step 6: Update setup context output** - -In `packages/cli/src/setup-context.ts`, replace: - -```ts - io.stderr.write('No primary or context sources are configured for a KTX context build.\n'); -``` - -with: - -```ts - io.stderr.write('No databases or context sources are configured for a KTX context build.\n'); -``` - -- [ ] **Step 7: Update public setup help output** - -In `packages/cli/src/commands/setup-commands.ts`, replace: - -```ts - .option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added', false) -``` - -with: - -```ts - .option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a database is added', false) -``` - -- [ ] **Step 8: Run the setup-copy tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-sources.test.ts src/setup-context.test.ts src/index.test.ts -t "context sources until a database exists|No databases or context sources|documents setup as a bare command" -``` - -Expected: PASS. - -- [ ] **Step 9: Commit the setup-copy repair** - -Run: - -```bash -git add packages/cli/src/setup-sources.ts packages/cli/src/setup-sources.test.ts packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/index.test.ts -git commit -m "fix(cli): remove primary-source wording from setup output" -``` - -### Task 3: Final V1 Verification - -**Files:** -- Verify: `packages/cli/src/setup-databases.test.ts` -- Verify: `packages/cli/src/setup-sources.ts` -- Verify: `packages/cli/src/setup-sources.test.ts` -- Verify: `packages/cli/src/setup-context.ts` -- Verify: `packages/cli/src/setup-context.test.ts` -- Verify: `packages/cli/src/commands/setup-commands.ts` -- Verify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Run focused unified ingest tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/public-ingest.test.ts src/context-build-view.test.ts src/setup-ready-menu.test.ts src/setup.test.ts src/setup-context.test.ts src/setup-databases.test.ts src/setup-sources.test.ts src/index.test.ts src/command-tree.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run docs regression tests** - -Run: - -```bash -node --test scripts/examples-docs.test.mjs -``` - -Expected: PASS. - -- [ ] **Step 3: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Check the normal setup public-copy surface** - -Run: - -```bash -rg -n "primary source|primary sources|Primary Sources|primary-source" \ - packages/cli/src/commands/setup-commands.ts \ - packages/cli/src/setup-sources.ts \ - packages/cli/src/setup-context.ts \ - packages/cli/src/index.test.ts \ - packages/cli/src/setup-sources.test.ts \ - packages/cli/src/setup-context.test.ts -``` - -Expected: no matches. - -- [ ] **Step 5: Check the unified ingest public command surface** - -Run: - -```bash -node packages/cli/dist/bin.js ingest --help -node packages/cli/dist/bin.js --help -``` - -Expected: normal help lists `ktx ingest [connectionId]`, `--all`, `--fast`, `--deep`, `--query-history`, `status`, and `replay`; it does not list `ktx scan`, `ktx ingest run`, or `ktx ingest watch`. - -- [ ] **Step 6: Run pre-commit on changed files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/cli/src/setup-databases.test.ts \ - packages/cli/src/setup-sources.ts \ - packages/cli/src/setup-sources.test.ts \ - packages/cli/src/setup-context.ts \ - packages/cli/src/setup-context.test.ts \ - packages/cli/src/commands/setup-commands.ts \ - packages/cli/src/index.test.ts -``` - -Expected: PASS. If pre-commit cannot run because the local hook environment or pinned tool version is unavailable, record the exact failure and keep the focused Vitest, docs, and type-check results from Steps 1-3. - -- [ ] **Step 7: Commit verification formatting if needed** - -If Step 6 changes files, run: - -```bash -git add packages/cli/src/setup-databases.test.ts packages/cli/src/setup-sources.ts packages/cli/src/setup-sources.test.ts packages/cli/src/setup-context.ts packages/cli/src/setup-context.test.ts packages/cli/src/commands/setup-commands.ts packages/cli/src/index.test.ts -git commit -m "test(cli): verify unified ingest setup closure" -``` - -If Step 6 makes no changes, do not create an empty commit. - -## Self-Review - -- Spec coverage: This plan covers the remaining v1-blocking issues found in the audit: package type-check is currently red, and normal setup help/output still exposes the old public database category as `primary source` instead of database-oriented copy. Core ingest routing, depth behavior, query-history behavior, foreground-only state, warning aggregation, public command help, and scan/live-database/historic-sql output sanitization are already implemented by prior plans. -- Placeholder scan: The plan contains concrete file paths, exact replacement snippets, exact commands, and expected outcomes. -- Type consistency: The only test typing change uses the existing Vitest pattern already used elsewhere in `packages/cli/src/setup-databases.test.ts`: `vi.mocked(prompts.multiselect).mock.calls`. diff --git a/docs/superpowers/plans/2026-05-13-warehouse-verification-prompt-shape-closure.md b/docs/superpowers/plans/2026-05-13-warehouse-verification-prompt-shape-closure.md deleted file mode 100644 index 05223b93..00000000 --- a/docs/superpowers/plans/2026-05-13-warehouse-verification-prompt-shape-closure.md +++ /dev/null @@ -1,345 +0,0 @@ -# Warehouse Verification Prompt Shape Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make every warehouse-verification prompt use KTX's shipped -`sql_execution` input shape so ingest agents include `connectionName` when they -probe warehouse identifiers. - -**Architecture:** Keep the warehouse verification tool code unchanged. Add -prompt-asset tests that reject Kaelio's old session-only SQL examples, then -update the shared identifier protocol and the three remaining per-skill SQL -probe examples that still show the legacy shape. - -**Tech Stack:** Markdown skill prompts, TypeScript, Vitest, pnpm workspace -commands. - ---- - -## Audit Summary - -The warehouse verification tools, runner wiring, adapter target fan-out, and -focused tests are present. Focused verification passed: - -```bash -pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/ingest/local-adapters.test.ts src/ingest/adapters/notion/notion.adapter.test.ts src/ingest/adapters/lookml/lookml.adapter.test.ts src/ingest/adapters/metricflow/metricflow.adapter.test.ts -pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "supplies a scan-connector query executor" -``` - -Remaining v1-blocking gap: - -- `packages/context/skills/lookml_ingest/SKILL.md`, - `packages/context/skills/metricflow_ingest/SKILL.md`, and - `packages/context/skills/sl_capture/SKILL.md` still contain - `sql_execution({ sql ... })` / "session shape" guidance inherited from - Kaelio. KTX's tool contract is - `sql_execution({connectionName, sql, rowLimit?})`, so these examples can make - agents call the shipped tool with invalid input. - -Non-blocking gaps remain out of scope for this v1 plan: - -- Full DDL-style `entity_details` formatting with FK profile summaries. -- AST-backed SQL validation for data-modifying CTE bodies. -- Search over generated `enrichment/descriptions.json`. -- Per-WorkUnit reuse of a single `WarehouseCatalogService` instance for cache - hits across separate tool calls. -- A deterministic fake-LLM end-to-end Notion hallucination regression. Prompt - guards and tool contract tests cover the v1 contract; a broader behavior - regression can land as follow-up. - -## File Structure - -Modify these files: - -- `packages/context/src/memory/memory-runtime-assets.test.ts`: add a prompt - guard that rejects the legacy session-only `sql_execution` shape. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts`: strengthen the - shared prompt asset assertion for the KTX `connectionName` SQL shape. -- `packages/context/skills/_shared/identifier-verification.md`: make both SQL - probe instructions show the KTX `connectionName` argument. -- `packages/context/skills/notion_synthesize/SKILL.md`: inline the updated - protocol block. -- `packages/context/skills/dbt_ingest/SKILL.md`: inline the updated protocol - block. -- `packages/context/skills/lookml_ingest/SKILL.md`: inline the updated protocol - block and fix the legacy SQL fallback example. -- `packages/context/skills/looker_ingest/SKILL.md`: inline the updated - protocol block. -- `packages/context/skills/metabase_ingest/SKILL.md`: inline the updated - protocol block. -- `packages/context/skills/metricflow_ingest/SKILL.md`: inline the updated - protocol block and fix the legacy SQL fallback example. -- `packages/context/skills/live_database_ingest/SKILL.md`: inline the updated - protocol block. -- `packages/context/skills/historic_sql_table_digest/SKILL.md`: inline the - updated protocol block. -- `packages/context/skills/historic_sql_patterns/SKILL.md`: inline the updated - protocol block. -- `packages/context/skills/knowledge_capture/SKILL.md`: inline the updated - protocol block. -- `packages/context/skills/sl_capture/SKILL.md`: inline the updated protocol - block and fix the join-discovery SQL example. - -### Task 1: Add Prompt Guards For The KTX SQL Tool Shape - -**Files:** -- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` -- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Add the failing memory asset guard** - -In `packages/context/src/memory/memory-runtime-assets.test.ts`, add this test -after `does not ship stale warehouse verification tool names or fictional -identifiers`: - -```ts - it('ships only the KTX connectionName sql_execution call shape in writer guidance', async () => { - const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); - - expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); - expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); - - for (const skillName of verificationWriterSkills) { - const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); - expect(body).toContain('sql_execution({connectionName'); - expect(body).not.toContain('sql_execution({ sql'); - expect(body).not.toContain('session shape'); - expect(body).not.toContain('connection is already pinned by the ingest session'); - } - }); -``` - -- [ ] **Step 2: Strengthen the shared ingest asset guard** - -In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, update -`packages identifier verification prompt assets` so the final assertions are: - -```ts - expect(shared).toContain('discover_data'); - expect(shared).toContain('entity_details'); - expect(shared).toContain('sql_execution'); - expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); - expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); -``` - -- [ ] **Step 3: Run the failing prompt guards** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: FAIL. The failure must mention at least one current legacy string: -`sql_execution({ sql`, `session shape`, or missing -`sql_execution({connectionName`. - -### Task 2: Update The Shared Identifier Verification Protocol - -**Files:** -- Modify: `packages/context/skills/_shared/identifier-verification.md` -- Modify: `packages/context/skills/notion_synthesize/SKILL.md` -- Modify: `packages/context/skills/dbt_ingest/SKILL.md` -- Modify: `packages/context/skills/lookml_ingest/SKILL.md` -- Modify: `packages/context/skills/looker_ingest/SKILL.md` -- Modify: `packages/context/skills/metabase_ingest/SKILL.md` -- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` -- Modify: `packages/context/skills/live_database_ingest/SKILL.md` -- Modify: `packages/context/skills/historic_sql_table_digest/SKILL.md` -- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md` -- Modify: `packages/context/skills/knowledge_capture/SKILL.md` -- Modify: `packages/context/skills/sl_capture/SKILL.md` - -- [ ] **Step 1: Replace the shared protocol text** - -Replace the full `## Identifier Verification Protocol` block in -`packages/context/skills/_shared/identifier-verification.md` with: - -```md -## Identifier Verification Protocol - -Before writing a wiki page or SL source on any topic: - -1. `discover_data({query: ""})` - see what wikis, SL sources, and raw - tables already exist. Prefer updating existing pages over creating new ones. - -Before emitting any `schema.table` or `schema.table.column` into a wiki body, -SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: - -2. `entity_details({connectionName, targets: [{display: ""}]})` - - confirm the identifier resolves; inspect native types, FK/PK, and - sampleValues. -3. For literal values from the source, such as status codes or plan tiers, - check whether they appear in `entity_details` sampleValues for the relevant - column. If sampleValues is short or the sample may have missed real values, - run a `sql_execution` probe with the same warehouse connection name: - `sql_execution({connectionName, sql: "SELECT DISTINCT FROM LIMIT 50"})`. -4. If the candidate identifier still does not resolve, do one of: - - Use `sql_execution({connectionName, sql: "SELECT 1 FROM LIMIT 0"})`. - If it errors, the identifier is fictional. - - Wrap the identifier in `[unverified - from ]` in the wiki body, - citing the exact raw path that mentioned it. - - When recording `emit_unmapped_fallback` with `no_physical_table`, include - the failing probe error in `clarification`. -5. Never copy `.
    ` placeholder strings from these instructions - into output. -``` - -- [ ] **Step 2: Inline the same protocol in every writer skill** - -Replace the existing `## Identifier Verification Protocol` block in each writer -skill with the exact block from Step 1: - -```bash -packages/context/skills/notion_synthesize/SKILL.md -packages/context/skills/dbt_ingest/SKILL.md -packages/context/skills/lookml_ingest/SKILL.md -packages/context/skills/looker_ingest/SKILL.md -packages/context/skills/metabase_ingest/SKILL.md -packages/context/skills/metricflow_ingest/SKILL.md -packages/context/skills/live_database_ingest/SKILL.md -packages/context/skills/historic_sql_table_digest/SKILL.md -packages/context/skills/historic_sql_patterns/SKILL.md -packages/context/skills/knowledge_capture/SKILL.md -packages/context/skills/sl_capture/SKILL.md -``` - -- [ ] **Step 3: Run the shared prompt asset tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: still FAIL because the per-skill legacy SQL examples in LookML, -MetricFlow, and `sl_capture` have not been fixed yet. - -### Task 3: Fix Legacy Per-Skill SQL Examples - -**Files:** -- Modify: `packages/context/skills/lookml_ingest/SKILL.md` -- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` -- Modify: `packages/context/skills/sl_capture/SKILL.md` - -- [ ] **Step 1: Fix the LookML fallback probe example** - -In `packages/context/skills/lookml_ingest/SKILL.md`, replace the current -Required flow item 2 with: - -```md -2. If the table isn't in the manifest, use the warehouse `connectionName` - returned by `discover_data` or the target connection chosen from - `sl_discover`, then call a dialect-appropriate SQL probe with that - connection name, for example: - `sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. - Replace `warehouse`, `analytics`, and `orders` with the verified connection, - schema or dataset, and table from the WorkUnit evidence. -``` - -- [ ] **Step 2: Fix the MetricFlow fallback probe example** - -In `packages/context/skills/metricflow_ingest/SKILL.md`, replace the paragraph -that begins `If \`sl_discover\` errors` with: - -```md -If `sl_discover` errors because no such table exists, use `discover_data` and -`entity_details` to find the warehouse target. If a SQL probe is still needed, -call `sql_execution` with the same warehouse connection name, for example: -`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`. -**Never invent column names** - every column in `columns:`, `grain:`, and -`sql:` must be sourced from raw files, `entity_details`, or a successful SQL -probe. -``` - -- [ ] **Step 3: Fix the `sl_capture` join probe example** - -In `packages/context/skills/sl_capture/SKILL.md`, replace Tool sequence item 6 -with: - -```md -6. For join discovery: use `sql_execution({connectionName: "warehouse", sql: "SELECT count(*) FROM public.orders o JOIN public.customers c ON c.id = o.customer_id LIMIT 20"})` with the target warehouse connection name and dialect-correct table names to verify the join key exists in both tables and assess cardinality before declaring the join. -``` - -- [ ] **Step 4: Run the prompt asset tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. The tests must report 2 files passed. - -### Task 4: Final Verification - -**Files:** -- No new files. - -- [ ] **Step 1: Run focused warehouse prompt and tool tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Inspect final diff** - -Run: - -```bash -git diff -- packages/context/src/memory/memory-runtime-assets.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/skills/_shared/identifier-verification.md packages/context/skills/notion_synthesize/SKILL.md packages/context/skills/dbt_ingest/SKILL.md packages/context/skills/lookml_ingest/SKILL.md packages/context/skills/looker_ingest/SKILL.md packages/context/skills/metabase_ingest/SKILL.md packages/context/skills/metricflow_ingest/SKILL.md packages/context/skills/live_database_ingest/SKILL.md packages/context/skills/historic_sql_table_digest/SKILL.md packages/context/skills/historic_sql_patterns/SKILL.md packages/context/skills/knowledge_capture/SKILL.md packages/context/skills/sl_capture/SKILL.md -``` - -Expected: only prompt wording and prompt-asset guards changed. No tool -implementation files changed. - -- [ ] **Step 4: Commit** - -Run: - -```bash -git add packages/context/src/memory/memory-runtime-assets.test.ts packages/context/src/ingest/ingest-runtime-assets.test.ts packages/context/skills/_shared/identifier-verification.md packages/context/skills/notion_synthesize/SKILL.md packages/context/skills/dbt_ingest/SKILL.md packages/context/skills/lookml_ingest/SKILL.md packages/context/skills/looker_ingest/SKILL.md packages/context/skills/metabase_ingest/SKILL.md packages/context/skills/metricflow_ingest/SKILL.md packages/context/skills/live_database_ingest/SKILL.md packages/context/skills/historic_sql_table_digest/SKILL.md packages/context/skills/historic_sql_patterns/SKILL.md packages/context/skills/knowledge_capture/SKILL.md packages/context/skills/sl_capture/SKILL.md -git commit -m "fix(context): align warehouse sql probe prompt shape" -``` - -Expected: one focused commit. - -## Self-Review - -Spec coverage: - -- The original spec requires `sql_execution` inputs to include - `connectionName`; this plan removes contradictory session-only examples from - all active writer guidance. -- The shared protocol remains in `_shared` and inlined in every synthesis - writer skill named by the original spec. -- The tool implementation remains unchanged because the shipped schema already - enforces the v1 contract. - -Placeholder scan: - -- The plan has no deferred implementation markers. -- Prompt examples use concrete `warehouse`, `analytics`, and `orders` example - names only to demonstrate JSON shape, and each example tells the worker to - replace them with discovered evidence. - -Type consistency: - -- Tests assert the exact KTX tool call shape: - `sql_execution({connectionName, sql: ...})`. -- Prompt wording consistently uses `connectionName`, matching - `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts`. diff --git a/docs/superpowers/plans/2026-05-13-warehouse-verification-sql-example-closure.md b/docs/superpowers/plans/2026-05-13-warehouse-verification-sql-example-closure.md deleted file mode 100644 index 2d1b1779..00000000 --- a/docs/superpowers/plans/2026-05-13-warehouse-verification-sql-example-closure.md +++ /dev/null @@ -1,215 +0,0 @@ -# Warehouse Verification SQL Example Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the last connectionless `sql_execution` prompt example so -warehouse-verification writer guidance always matches KTX's shipped tool -contract. - -**Architecture:** Keep the warehouse verification tool code unchanged. Tighten -the prompt asset guard so multiline `sql_execution({ sql: ... })` examples -fail tests, then update the stale `sl_capture` worked example to pass -`connectionName` explicitly. - -**Tech Stack:** Markdown skill prompts, TypeScript, Vitest, pnpm workspace -commands. - ---- - -## Audit summary - -The warehouse verification tools, runner wiring, source-adapter target fan-out, -CLI query executor, and focused tests are present. Focused verification passed: - -```bash -pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/ingest/local-adapters.test.ts src/ingest/adapters/notion/notion.adapter.test.ts src/ingest/adapters/lookml/lookml.adapter.test.ts src/ingest/adapters/metricflow/metricflow.adapter.test.ts -pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "supplies a scan-connector query executor" -``` - -Remaining v1-blocking gap: - -- `packages/context/skills/sl_capture/SKILL.md` still contains a worked example - with a multiline `sql_execution({ sql: ... })` call. KTX's tool contract is - `sql_execution({connectionName, sql, rowLimit?})`, so this example can teach - agents to call the shipped tool with invalid input. - -Non-blocking gaps remain out of scope for this v1 plan: - -- Full DDL-style `entity_details` formatting with FK profile summaries. -- AST-backed SQL validation for data-modifying CTE bodies. -- Search over generated `enrichment/descriptions.json`. -- Per-WorkUnit reuse of a single `WarehouseCatalogService` instance for cache - hits across separate tool calls. -- A deterministic fake-LLM end-to-end Notion hallucination regression. -- Tokenized or embedding-backed raw schema search ranking in `discover_data`. - -## File structure - -Modify these files: - -- `packages/context/src/memory/memory-runtime-assets.test.ts`: add a prompt - guard that catches multiline `sql_execution` calls without `connectionName`. -- `packages/context/skills/sl_capture/SKILL.md`: update the stale worked - example to include the target warehouse `connectionName`. - -### Task 1: Add a multiline SQL prompt guard - -**Files:** -- Modify: `packages/context/src/memory/memory-runtime-assets.test.ts` - -- [ ] **Step 1: Add a helper that extracts `sql_execution` call examples** - -In `packages/context/src/memory/memory-runtime-assets.test.ts`, add this helper -after `forbiddenProductPattern()`: - -```ts -function sqlExecutionCallBlocks(body: string): string[] { - const blocks: string[] = []; - const marker = 'sql_execution({'; - let offset = 0; - - while (offset < body.length) { - const start = body.indexOf(marker, offset); - if (start === -1) { - break; - } - const end = body.indexOf('})', start + marker.length); - blocks.push(body.slice(start, end === -1 ? start + marker.length : end + 2)); - offset = start + marker.length; - } - - return blocks; -} -``` - -- [ ] **Step 2: Strengthen the existing SQL-shape test** - -Replace the body of -`ships only the KTX connectionName sql_execution call shape in writer guidance` -with: - -```ts - const shared = await readFile(join(skillsDir, '_shared', 'identifier-verification.md'), 'utf-8'); - const bodies = [{ name: '_shared/identifier-verification.md', body: shared }]; - - expect(shared).toContain('sql_execution({connectionName, sql: "SELECT DISTINCT'); - expect(shared).toContain('sql_execution({connectionName, sql: "SELECT 1 FROM'); - - for (const skillName of verificationWriterSkills) { - const body = await readFile(join(skillsDir, skillName, 'SKILL.md'), 'utf-8'); - bodies.push({ name: `${skillName}/SKILL.md`, body }); - expect(body).toContain('sql_execution({connectionName'); - expect(body).not.toContain('sql_execution({ sql'); - expect(body).not.toContain('session shape'); - expect(body).not.toContain('connection is already pinned by the ingest session'); - } - - for (const { name, body } of bodies) { - const calls = sqlExecutionCallBlocks(body); - expect(calls.length, `${name} should contain sql_execution guidance`).toBeGreaterThan(0); - expect( - calls.filter((call) => !call.includes('connectionName')), - `${name} has sql_execution calls without connectionName`, - ).toEqual([]); - expect(body, `${name} has a connectionless multiline sql_execution call`).not.toMatch( - /sql_execution\(\{\s*sql\s*:/, - ); - } -``` - -- [ ] **Step 3: Run the failing prompt guard** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts -t "connectionName sql_execution" -``` - -Expected: FAIL. The failure must identify -`sl_capture/SKILL.md` as having a `sql_execution` call without -`connectionName` or a connectionless multiline `sql_execution` call. - -- [ ] **Step 4: Commit the failing guard** - -Run: - -```bash -git add packages/context/src/memory/memory-runtime-assets.test.ts -git commit -m "test(context): catch connectionless sql execution prompt examples" -``` - -### Task 2: Fix the stale `sl_capture` SQL example - -**Files:** -- Modify: `packages/context/skills/sl_capture/SKILL.md` -- Test: `packages/context/src/memory/memory-runtime-assets.test.ts` -- Test: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Update the worked example** - -In `packages/context/skills/sl_capture/SKILL.md`, replace the `sql_execution` -block in "Worked example - new join" with: - -```md -sql_execution({ - connectionName: "warehouse", - sql: "SELECT COUNT(*), COUNT(DISTINCT a.admin_user_id) FROM public.fct_orders a JOIN public.fct_mau_multiprotocol b ON a.admin_user_id = b.admin_user_id LIMIT 1" -}) -``` - -- [ ] **Step 2: Run the prompt guards** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runtime-assets.test.ts src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run a direct stale-shape scan** - -Run: - -```bash -rg -n -U "sql_execution\\(\\{\\s*\\n\\s*sql:" packages/context/skills packages/context/prompts -``` - -Expected: no matches and exit code 1. - -- [ ] **Step 4: Run the context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the prompt fix** - -Run: - -```bash -git add packages/context/skills/sl_capture/SKILL.md -git commit -m "fix(context): include connection name in sl capture sql example" -``` - -## Self-review - -Spec coverage: - -- The only remaining v1-blocking prompt-shape gap has a failing test and a - direct prompt edit. -- Tool implementation, runner wiring, adapter scoping, and CLI execution - remain covered by the focused suites listed in the audit summary. - -Placeholder scan: - -- This plan contains no deferred implementation placeholders. - -Type consistency: - -- The plan uses the shipped KTX tool shape: - `sql_execution({connectionName, sql, rowLimit?})`. diff --git a/docs/superpowers/plans/2026-05-13-warehouse-verification-structured-target-miss-closure.md b/docs/superpowers/plans/2026-05-13-warehouse-verification-structured-target-miss-closure.md deleted file mode 100644 index 48983c4a..00000000 --- a/docs/superpowers/plans/2026-05-13-warehouse-verification-structured-target-miss-closure.md +++ /dev/null @@ -1,236 +0,0 @@ -# Warehouse Verification Structured Target Miss Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `entity_details` return model-visible not-found evidence for every documented target shape, including structured `{catalog, db, name, column?}` targets. - -**Architecture:** Keep the existing warehouse verification module. Add focused tests for missing structured table and column targets, then route structured target labels through the same candidate lookup used by display targets while preserving exact structured resolution. - -**Tech Stack:** TypeScript, Node 22, Vitest, AI SDK v6 tools, Zod, KTX ingest tools. - ---- - -## Audit Summary - -The implemented plans have landed the warehouse verification tools, ingest -runner wiring, adapter warehouse target fan-out, CLI read-only query executor, -and prompt-shape closures. Focused verification passed on May 13, 2026: - -```bash -pnpm --filter @ktx/context exec vitest run src/connections/dialects.test.ts src/connections/read-only-sql.test.ts src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts src/ingest/ingest-prompts.test.ts src/ingest/ingest-runtime-assets.test.ts src/memory/memory-runtime-assets.test.ts src/ingest/local-adapters.test.ts src/ingest/adapters/notion/notion.adapter.test.ts src/ingest/adapters/lookml/lookml.adapter.test.ts src/ingest/adapters/metricflow/metricflow.adapter.test.ts -pnpm --filter @ktx/cli exec vitest run src/ingest-query-executor.test.ts src/ingest.test.ts -t "supplies a scan-connector query executor" -rg -n -U "sql_execution\\(\\{\\s*\\n\\s*sql:" packages/context/skills packages/context/prompts -rg -n "wiki_sl_search|sl_describe_table|orbit_analytics\\.customer" packages/context/skills packages/context/prompts packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts packages/context/src/sl/tools/sl-warehouse-validation.ts -``` - -Remaining v1-blocking gap: - -- `entity_details` accepts structured targets, but if a structured table target - does not exist, it records `structured.missing` and emits no markdown. Tool - outputs are sent to the model as markdown only, so the synthesis agent gets - an empty response instead of the required "Not found in scan" verification - signal. - -Non-blocking gaps remain out of scope for this v1 plan: - -- Full DDL-style `entity_details` formatting with FK and profile summaries. -- AST-backed SQL validation for data-modifying CTE bodies. -- Dialect-specific row-limit wrapping for SQL Server probes. -- Search over generated `enrichment/descriptions.json`. -- Per-WorkUnit reuse of a single `WarehouseCatalogService` instance for cache - hits across separate tool calls. -- A deterministic fake-LLM end-to-end Notion hallucination regression. -- Cleanup of legacy demo Orbit wiki fixtures that still mention - `orbit_analytics.customer`. - -## File Structure - -Modify these files: - -- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`: add failing coverage for missing structured targets. -- `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`: render missing structured targets into markdown and reuse candidate lookup. - -### Task 1: Report Structured Target Misses In `entity_details` - -**Files:** -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` - -- [ ] **Step 1: Add failing structured miss tests** - -In `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts`, add these tests after `reports missing explicit columns instead of returning an empty column list`: - -```ts - it('reports missing structured table targets in model-visible markdown', async () => { - const result = await tool.call( - { - connectionName: 'warehouse', - targets: [{ catalog: null, db: 'public', name: 'orderz' }], - }, - context, - ); - - expect(result.markdown).toContain('Not found in scan: public.orderz'); - expect(result.markdown).toContain('Closest matches: orders'); - expect(result.structured.resolved).toHaveLength(0); - expect(result.structured.missing).toHaveLength(1); - }); - - it('reports missing structured column targets in model-visible markdown', async () => { - const result = await tool.call( - { - connectionName: 'warehouse', - targets: [{ catalog: null, db: 'public', name: 'orders', column: 'plan_tier' }], - }, - context, - ); - - expect(result.markdown).toContain('Column not found in scan: public.orders.plan_tier'); - expect(result.markdown).toContain('Available columns: id, status'); - expect(result.structured.resolved).toHaveLength(0); - expect(result.structured.missing).toHaveLength(1); - }); -``` - -- [ ] **Step 2: Run the failing focused test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -t "structured" -``` - -Expected: FAIL. The first new test must fail because `result.markdown` does not contain `Not found in scan: public.orderz`. - -- [ ] **Step 3: Add structured target labels and candidate lookup** - -In `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts`, add this type alias after `type EntityDetailsInput = z.infer;`: - -```ts -type EntityDetailsTarget = EntityDetailsInput['targets'][number]; -``` - -Add these helpers after `function allowedConnectionNames(context: ToolContext): ReadonlySet | null { ... }`: - -```ts -function targetLabel(target: EntityDetailsTarget): string { - if ('display' in target) { - return target.display; - } - return [target.catalog, target.db, target.name, target.column].filter((part): part is string => !!part).join('.'); -} - -function appendMissingTargetMarkdown(parts: string[], target: EntityDetailsTarget, candidates: KtxTableRef[]): void { - parts.push(`Not found in scan: ${targetLabel(target)}`); - if (candidates.length > 0) { - parts.push(`Closest matches: ${candidates.map((candidate) => candidate.name).join(', ')}`); - } -} - -async function resolveTarget( - catalog: WarehouseCatalogService, - connectionName: string, - target: EntityDetailsTarget, -): Promise<{ resolved: (KtxTableRef & { column?: string }) | null; candidates: KtxTableRef[] }> { - if ('display' in target) { - return catalog.resolveDisplayTarget(connectionName, target.display); - } - - const candidateResolution = await catalog.resolveDisplayTarget(connectionName, targetLabel(target)); - return { - resolved: { - catalog: target.catalog, - db: target.db, - name: target.name, - column: target.column, - }, - candidates: candidateResolution.candidates, - }; -} -``` - -Then replace the `const resolution = ...` block inside the `for (const target of input.targets)` loop with: - -```ts - const resolution = await resolveTarget(catalog, input.connectionName, target); -``` - -Replace the missing-resolution block with: - -```ts - if (!resolution.resolved) { - missing.push({ target, candidates: resolution.candidates }); - appendMissingTargetMarkdown(parts, target, resolution.candidates); - continue; - } -``` - -Replace the missing-detail block with: - -```ts - if (!detail) { - missing.push({ target, candidates: resolution.candidates }); - appendMissingTargetMarkdown(parts, target, resolution.candidates); - continue; - } -``` - -- [ ] **Step 4: Run the focused entity-details tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Run warehouse verification regression tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts src/ingest/tools/warehouse-verification/entity-details.tool.test.ts src/ingest/tools/warehouse-verification/discover-data.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts \ - packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts -git commit -m "fix(context): report structured entity detail misses" -``` - -## Self-review - -Spec coverage: - -- The original `entity_details` contract says structured and display targets - are mixed shapes and unresolved targets must produce `Not found in scan` with - candidates. This plan adds that model-visible behavior for structured table - misses and preserves the existing column-miss behavior. - -Placeholder scan: - -- This plan contains no deferred implementation placeholders. - -Type consistency: - -- The plan uses the existing `WarehouseCatalogService`, `KtxTableRef`, - `EntityDetailsStructured`, and `ToolOutput` types without adding public API - compatibility wrappers. diff --git a/docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md b/docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md deleted file mode 100644 index eb8e812d..00000000 --- a/docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md +++ /dev/null @@ -1,808 +0,0 @@ -# Connection Driver Discriminated Union Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the loose `connectionSchema` in `packages/context/src/project/config.ts` with a Zod 4 discriminated union keyed on `driver`, so that every driver's documented connection fields — including the `mappings` block — appear in the JSON schema emitted by `ktx dev schema`. - -**Architecture:** Add a new module `packages/context/src/project/driver-schemas.ts` that defines one `z.looseObject({ driver: z.literal('x'), ... })` per supported driver and combines them with `z.discriminatedUnion('driver', [...])`. Reuse the existing Metabase/Looker/LookML mapping shapes from `mappings-yaml-schema.ts` by exporting them. Wire the union into `config.ts`. Each per-driver shape stays `looseObject` so today's existing yaml configs with extra fields keep parsing. - -**Tech Stack:** TypeScript (Node 22+, ESM, `NodeNext`), Zod 4 (`^4.4.3`), Vitest, pnpm workspace. - ---- - -## File Structure - -**Create:** -- `packages/context/src/project/driver-schemas.ts` — per-driver Zod schemas + the discriminated union and exported types. -- `packages/context/src/project/driver-schemas.test.ts` — unit tests for each driver schema and the union. - -**Modify:** -- `packages/context/src/project/mappings-yaml-schema.ts` — export the three mapping shapes (`metabaseMappingsSchema`, `lookerMappingsSchema`, `lookmlMappingsSchema`) with `.describe()` annotations and a small description on each field so they surface meaningfully in JSON Schema. -- `packages/context/src/project/config.ts:209-214` — replace `connectionSchema` with the discriminated union imported from `driver-schemas.ts`. Update `KtxProjectConnectionConfig` (line `272`) to be `z.infer` — still works because `connectionSchema` is the union name we keep. -- `packages/context/src/project/index.ts` — re-export `KtxConnectionConfig` per-driver type aliases if useful (optional; only if tests need them). -- `packages/context/src/project/config.test.ts` — add a test that the JSON schema now describes `mappings` for metabase/looker/lookml. - -**No changes needed:** -- `packages/context/src/project/mappings-yaml-schema.ts` parsing helpers (`parseMetabaseMappingBootstrap`, etc.) keep working because `KtxProjectConnectionConfig` still has loose-object semantics per driver. -- Doc files in `docs-site/` already show the `mappings` blocks correctly. - ---- - -## Drivers In Scope - -The discriminated union enumerates the drivers actually used in code, fixtures, and docs (no `fake`/test-only driver — none exist in fixtures, verified via `grep "driver:\s*fake"`). - -Warehouse drivers (read `driver`, `url`; nothing else schema-modeled — kept `looseObject` so warehouse-specific overrides like `historicSql`/`context.queryHistory` pass through): -- `postgres`, `postgresql` (separate literals; KTX normalizes `postgresql` → `postgres` at runtime, but ktx.yaml accepts both) -- `mysql` -- `snowflake` -- `bigquery` -- `sqlite` -- `clickhouse` -- `sqlserver` - -Context-source drivers (model documented fields): -- `metabase` — `api_url`, `api_key`, `api_key_ref`, `network_proxy`/`networkProxy`, `mappings` (metabaseMappingsSchema). -- `looker` — `base_url`, `client_id`, `client_secret`, `client_secret_ref`, `mappings` (lookerMappingsSchema). -- `lookml` — `repoUrl` (camelCase intentional — matches code at `setup-sources.ts:1466`), `branch`, `path`, `auth_token_ref`, `mappings` (lookmlMappingsSchema). -- `notion` — `auth_token`, `auth_token_ref`, `crawl_mode` (`'selected_roots' | 'all_accessible'`), `root_page_ids`, `root_database_ids`, `root_data_source_ids`, `max_pages_per_run`, `max_knowledge_creates_per_run`, `max_knowledge_updates_per_run`. -- `dbt` — `source_dir`, `repo_url`, `branch`, `path`, `auth_token_ref`, `profiles_path`, `target`, `project_name`. -- `metricflow` — `metricflow` (nested object: `repoUrl`, `branch`, `path`, `auth_token_ref`). - -Why not strict-object: existing warehouse connections may carry `historicSql` / `context.queryHistory` blocks and other driver-tunable fields not modeled here. `looseObject` preserves the current pass-through behavior while still surfacing the documented fields in JSON Schema. - ---- - -## Task 1: Export and describe mapping shapes - -Make the three existing mapping schemas reusable and documented. - -**Files:** -- Modify: `packages/context/src/project/mappings-yaml-schema.ts:4-31` -- Test: `packages/context/src/project/mappings-yaml-schema.test.ts` (no behavior change — existing tests must still pass) - -- [ ] **Step 1: Add a failing test that imports the new exports** - -Append to `packages/context/src/project/mappings-yaml-schema.test.ts` (inside the existing `describe` block): - -```typescript -import { - metabaseMappingsSchema, - lookerMappingsSchema, - lookmlMappingsSchema, -} from './mappings-yaml-schema.js'; - -// ...inside describe(...) - -it('exports mapping shapes that parse documented examples', () => { - expect(metabaseMappingsSchema.parse({ databaseMappings: { '1': 'wh' } })).toMatchObject({ - databaseMappings: { '1': 'wh' }, - syncMode: 'ALL', - }); - expect(lookerMappingsSchema.parse({ connectionMappings: { x: 'wh' } })).toEqual({ - connectionMappings: { x: 'wh' }, - }); - expect(lookmlMappingsSchema.parse({ expectedLookerConnectionName: 'x' })).toEqual({ - expectedLookerConnectionName: 'x', - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/mappings-yaml-schema.test.ts` -Expected: FAIL with `metabaseMappingsSchema is not exported` or equivalent module-resolution error. - -- [ ] **Step 3: Add `export` and `.describe()` to the three schemas** - -In `packages/context/src/project/mappings-yaml-schema.ts`, change the three internal `const` declarations: - -```typescript -export const metabaseMappingsSchema = z - .object({ - databaseMappings: z - .record(z.string(), stringTargetSchema) - .default({}) - .describe('Map of Metabase database ID (positive integer string) to KTX connection ID. Use null to explicitly unmap.'), - syncEnabled: z - .record(z.string(), z.boolean()) - .default({}) - .describe('Per-Metabase-database sync toggle, keyed by Metabase database ID string.'), - syncMode: metabaseSyncModeSchema - .default('ALL') - .describe('Sync scope: ALL ingests every mapped DB; ONLY restricts to syncEnabled=true; EXCEPT excludes syncEnabled=true.'), - selections: metabaseSelectionsSchema - .default({ collections: [], items: [] }) - .describe('Optional Metabase collection and item IDs to scope ingest.'), - defaultTagNames: z - .array(z.string().min(1)) - .default([]) - .describe('Default tag names applied to ingested Metabase artifacts.'), - }) - .describe('Metabase database-to-warehouse mapping and sync configuration.'); - -export const lookerMappingsSchema = z - .object({ - connectionMappings: z - .record(z.string().min(1), stringTargetSchema) - .default({}) - .describe('Map of Looker connection name to KTX connection ID. Use null to explicitly unmap.'), - }) - .describe('Looker connection-to-warehouse mapping configuration.'); - -export const lookmlMappingsSchema = z - .object({ - expectedLookerConnectionName: z - .string() - .min(1) - .nullable() - .default(null) - .describe('Looker connection name that LookML models must declare; mismatches block sl_write_source at ingest time.'), - }) - .describe('LookML connection-name expectation for ingest gating.'); -``` - -Leave `metabaseSyncModeSchema`, `metabaseSelectionsSchema`, `stringTargetSchema`, and `positiveIntegerValueSchema` private (no need to export). Leave all parsing helpers (`parseMetabaseMappingBootstrap` etc.) unchanged — they keep working because `.describe()` does not change runtime behavior. - -- [ ] **Step 4: Run test to verify it passes and existing tests still pass** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/mappings-yaml-schema.test.ts` -Expected: PASS for all tests including the new one. - -- [ ] **Step 5: Type-check the package** - -Run: `pnpm --filter @ktx/context run type-check` -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/project/mappings-yaml-schema.ts packages/context/src/project/mappings-yaml-schema.test.ts -git commit -m "refactor(context): export and describe mapping shape schemas" -``` - ---- - -## Task 2: Create the driver-schemas module — warehouse drivers - -Add the new module with the seven warehouse driver schemas first. Smaller surface, easier to validate. - -**Files:** -- Create: `packages/context/src/project/driver-schemas.ts` -- Test: `packages/context/src/project/driver-schemas.test.ts` - -- [ ] **Step 1: Write failing tests for warehouse driver schemas** - -Create `packages/context/src/project/driver-schemas.test.ts`: - -```typescript -import { describe, expect, it } from 'vitest'; -import { connectionConfigSchema } from './driver-schemas.js'; - -describe('connectionConfigSchema (driver discriminated union)', () => { - it.each([ - ['postgres', 'postgres://user:pass@host:5432/db'], // pragma: allowlist secret - ['postgresql', 'postgresql://user:pass@host:5432/db'], // pragma: allowlist secret - ['mysql', 'mysql://user:pass@host:3306/db'], // pragma: allowlist secret - ['snowflake', 'snowflake://account/db'], - ['bigquery', 'bigquery://project/dataset'], - ['sqlite', 'sqlite:///tmp/db.sqlite'], - ['clickhouse', 'clickhouse://host:8123/db'], - ['sqlserver', 'sqlserver://host:1433;database=db'], - ])('parses %s warehouse connection', (driver, url) => { - expect(connectionConfigSchema.parse({ driver, url })).toMatchObject({ driver, url }); - }); - - it('preserves unknown warehouse fields via looseObject passthrough', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'postgres', - url: 'postgres://x', - historicSql: { enabled: true }, - context: { queryHistory: { enabled: false } }, - }); - expect(parsed).toMatchObject({ - driver: 'postgres', - historicSql: { enabled: true }, - context: { queryHistory: { enabled: false } }, - }); - }); - - it('rejects an unknown driver', () => { - expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow(); - }); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` -Expected: FAIL — `driver-schemas.js` not found. - -- [ ] **Step 3: Create `driver-schemas.ts` with warehouse drivers only** - -Create `packages/context/src/project/driver-schemas.ts`: - -```typescript -import * as z from 'zod'; - -const warehouseDrivers = [ - 'postgres', - 'postgresql', - 'mysql', - 'snowflake', - 'bigquery', - 'sqlite', - 'clickhouse', - 'sqlserver', -] as const; - -function warehouseConnectionSchema(driver: (typeof warehouseDrivers)[number]) { - return z - .looseObject({ - driver: z.literal(driver), - url: z - .string() - .min(1) - .optional() - .describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'), - }) - .describe(`${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`); -} - -export const connectionConfigSchema = z.discriminatedUnion( - 'driver', - warehouseDrivers.map(warehouseConnectionSchema), -); - -export type KtxConnectionConfig = z.infer; -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` -Expected: PASS for all eight warehouse drivers + passthrough + unknown-driver rejection. - -- [ ] **Step 5: Type-check** - -Run: `pnpm --filter @ktx/context run type-check` -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts -git commit -m "feat(context): add driver-schemas module with warehouse drivers" -``` - ---- - -## Task 3: Add Metabase, Looker, LookML driver schemas (the mapping-bearing ones) - -These are the most important drivers — they're why we're doing this refactor. - -**Files:** -- Modify: `packages/context/src/project/driver-schemas.ts` -- Modify: `packages/context/src/project/driver-schemas.test.ts` - -- [ ] **Step 1: Write failing tests** - -Append to `packages/context/src/project/driver-schemas.test.ts`: - -```typescript -describe('connectionConfigSchema — context source drivers with mappings', () => { - it('parses a metabase connection with mappings', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'metabase', - api_url: 'https://metabase.example.com', - api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret - mappings: { - databaseMappings: { '3': 'prod-warehouse' }, - syncEnabled: { '3': true }, - syncMode: 'ONLY', - }, - }); - expect(parsed).toMatchObject({ - driver: 'metabase', - api_url: 'https://metabase.example.com', - mappings: { - databaseMappings: { '3': 'prod-warehouse' }, - syncMode: 'ONLY', - }, - }); - }); - - it('parses a looker connection with connectionMappings', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'looker', - base_url: 'https://looker.example.com', - client_id: 'abc', - client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret - mappings: { connectionMappings: { bigquery_prod: 'wh' } }, - }); - expect(parsed.mappings).toEqual({ connectionMappings: { bigquery_prod: 'wh' } }); - }); - - it('parses a lookml connection with expectedLookerConnectionName', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'lookml', - repoUrl: 'https://github.com/acme/looker.git', - branch: 'main', - mappings: { expectedLookerConnectionName: 'bigquery_prod' }, - }); - expect(parsed.mappings).toEqual({ expectedLookerConnectionName: 'bigquery_prod' }); - }); - - it('rejects metabase mapping with non-integer database key', () => { - expect(() => - connectionConfigSchema.parse({ - driver: 'metabase', - api_url: 'https://x', - mappings: { databaseMappings: { 'abc': 'wh' } }, - }), - ).toThrow(); - }); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` -Expected: FAIL — `driver: 'metabase'` is not in the discriminated union. - -- [ ] **Step 3: Extend `driver-schemas.ts` with metabase/looker/lookml schemas** - -Edit `packages/context/src/project/driver-schemas.ts` — add imports and the three new schemas, and include them in the union: - -```typescript -import * as z from 'zod'; -import { - lookerMappingsSchema, - lookmlMappingsSchema, - metabaseMappingsSchema, -} from './mappings-yaml-schema.js'; - -// ... (warehouseDrivers + warehouseConnectionSchema stay as-is) ... - -const positiveIntKeyMessage = (field: string) => - `${field} keys must be positive-integer strings (e.g. "1", "42")`; - -const positiveIntKeyRegex = /^[1-9]\d*$/; - -const metabaseMappingsStrictSchema = metabaseMappingsSchema.superRefine((value, ctx) => { - for (const key of Object.keys(value.databaseMappings ?? {})) { - if (!positiveIntKeyRegex.test(key)) { - ctx.addIssue({ code: 'custom', path: ['databaseMappings', key], message: positiveIntKeyMessage('databaseMappings') }); - } - } - for (const key of Object.keys(value.syncEnabled ?? {})) { - if (!positiveIntKeyRegex.test(key)) { - ctx.addIssue({ code: 'custom', path: ['syncEnabled', key], message: positiveIntKeyMessage('syncEnabled') }); - } - } -}); - -const metabaseConnectionSchema = z - .looseObject({ - driver: z.literal('metabase'), - api_url: z.string().url().describe('Metabase instance API URL (e.g. https://metabase.example.com).'), - api_key: z.string().min(1).optional().describe('Literal Metabase API key. Prefer api_key_ref for safety.'), - api_key_ref: z - .string() - .min(1) - .optional() - .describe('Reference to Metabase API key (e.g. env:METABASE_API_KEY or file:/path).'), - network_proxy: z - .looseObject({}) - .optional() - .describe('Optional network proxy configuration (snake_case form).'), - networkProxy: z - .looseObject({}) - .optional() - .describe('Optional network proxy configuration (camelCase form).'), - mappings: metabaseMappingsStrictSchema.optional().describe('Metabase database-to-warehouse mappings and sync configuration.'), - }) - .describe('Metabase context-source connection.'); - -const lookerConnectionSchema = z - .looseObject({ - driver: z.literal('looker'), - base_url: z.string().url().describe('Looker instance base URL (e.g. https://looker.example.com).'), - client_id: z.string().min(1).describe('Looker OAuth client ID.'), - client_secret: z.string().min(1).optional().describe('Literal Looker OAuth client secret. Prefer client_secret_ref.'), - client_secret_ref: z - .string() - .min(1) - .optional() - .describe('Reference to Looker OAuth client secret (e.g. env:LOOKER_CLIENT_SECRET).'), - mappings: lookerMappingsSchema.optional().describe('Looker connection-name to KTX warehouse mappings.'), - }) - .describe('Looker context-source connection.'); - -const lookmlConnectionSchema = z - .looseObject({ - driver: z.literal('lookml'), - repoUrl: z - .string() - .min(1) - .describe('Git URL of the LookML project (https, ssh, or file:). Field is camelCase by convention.'), - branch: z.string().min(1).optional().describe('Git branch (default "main" downstream).'), - path: z.string().optional().describe('Subdirectory within the repo when the LookML project lives in a monorepo.'), - auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos (e.g. env:GITHUB_TOKEN).'), - mappings: lookmlMappingsSchema.optional().describe('LookML expected-connection mapping for ingest gating.'), - }) - .describe('LookML context-source connection.'); - -export const connectionConfigSchema = z.discriminatedUnion( - 'driver', - [ - ...warehouseDrivers.map(warehouseConnectionSchema), - metabaseConnectionSchema, - lookerConnectionSchema, - lookmlConnectionSchema, - ], -); -``` - -Important: the existing `parseMetabaseMappingBootstrap` in `mappings-yaml-schema.ts` already enforces positive-integer keys via `assertPositiveIntegerKeys`. Adding `metabaseMappingsStrictSchema` here gives the same guarantee at the top-level config parse, so a malformed ktx.yaml fails fast at `parseKtxProjectConfig` time rather than at ingest time. - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` -Expected: PASS. - -- [ ] **Step 5: Type-check** - -Run: `pnpm --filter @ktx/context run type-check` -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts -git commit -m "feat(context): add metabase, looker, lookml driver schemas with mappings" -``` - ---- - -## Task 4: Add Notion, dbt, MetricFlow driver schemas - -The remaining context-source drivers; no `mappings` for these, but plenty of driver-specific fields. - -**Files:** -- Modify: `packages/context/src/project/driver-schemas.ts` -- Modify: `packages/context/src/project/driver-schemas.test.ts` - -- [ ] **Step 1: Write failing tests** - -Append to `packages/context/src/project/driver-schemas.test.ts`: - -```typescript -describe('connectionConfigSchema — notion / dbt / metricflow', () => { - it('parses a notion connection with selected_roots crawl', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'notion', - auth_token_ref: 'env:NOTION_TOKEN', - crawl_mode: 'selected_roots', - root_page_ids: ['abc', 'def'], - max_pages_per_run: 500, - }); - expect(parsed).toMatchObject({ - driver: 'notion', - crawl_mode: 'selected_roots', - root_page_ids: ['abc', 'def'], - max_pages_per_run: 500, - }); - }); - - it('rejects notion with unknown crawl_mode', () => { - expect(() => - connectionConfigSchema.parse({ - driver: 'notion', - auth_token_ref: 'env:NOTION_TOKEN', - crawl_mode: 'everything', - }), - ).toThrow(); - }); - - it('parses a dbt connection from a local source_dir', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'dbt', - source_dir: '/tmp/dbt-project', - target: 'dev', - }); - expect(parsed).toMatchObject({ driver: 'dbt', source_dir: '/tmp/dbt-project', target: 'dev' }); - }); - - it('parses a metricflow connection with nested config', () => { - const parsed = connectionConfigSchema.parse({ - driver: 'metricflow', - metricflow: { - repoUrl: 'https://github.com/acme/sl.git', - branch: 'main', - }, - }); - expect(parsed).toMatchObject({ - driver: 'metricflow', - metricflow: { repoUrl: 'https://github.com/acme/sl.git' }, - }); - }); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` -Expected: FAIL — `driver: 'notion'` etc. not in union. - -- [ ] **Step 3: Extend `driver-schemas.ts`** - -Add to `packages/context/src/project/driver-schemas.ts` before the final `connectionConfigSchema` export: - -```typescript -const notionConnectionSchema = z - .looseObject({ - driver: z.literal('notion'), - auth_token: z.string().min(1).optional().describe('Literal Notion integration token. Prefer auth_token_ref.'), - auth_token_ref: z - .string() - .min(1) - .optional() - .describe('Reference to Notion integration token (e.g. env:NOTION_TOKEN).'), - crawl_mode: z - .enum(['selected_roots', 'all_accessible']) - .optional() - .describe('Crawl scope. "selected_roots" requires at least one of root_page_ids, root_database_ids, root_data_source_ids.'), - root_page_ids: z.array(z.string().min(1)).optional().describe('Notion page IDs to crawl when crawl_mode is selected_roots.'), - root_database_ids: z.array(z.string().min(1)).optional().describe('Notion database IDs to crawl when crawl_mode is selected_roots.'), - root_data_source_ids: z - .array(z.string().min(1)) - .optional() - .describe('Notion data source IDs to crawl when crawl_mode is selected_roots.'), - max_pages_per_run: z - .number() - .int() - .min(1) - .max(10000) - .optional() - .describe('Maximum Notion pages fetched in a single ingest run.'), - max_knowledge_creates_per_run: z - .number() - .int() - .min(0) - .max(25) - .optional() - .describe('Maximum new wiki pages created per run.'), - max_knowledge_updates_per_run: z - .number() - .int() - .min(0) - .max(100) - .optional() - .describe('Maximum existing wiki pages updated per run.'), - }) - .describe('Notion context-source connection.'); - -const dbtConnectionSchema = z - .looseObject({ - driver: z.literal('dbt'), - source_dir: z.string().min(1).optional().describe('Absolute or project-relative path to a local dbt project.'), - repo_url: z.string().min(1).optional().describe('Git URL of the dbt project (https, ssh, or file:).'), - branch: z.string().min(1).optional().describe('Git branch when using repo_url.'), - path: z.string().optional().describe('Subdirectory within the repo when the dbt project lives in a monorepo.'), - auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'), - profiles_path: z.string().optional().describe('Override path to dbt profiles.yml.'), - target: z.string().min(1).optional().describe('dbt target name (e.g. dev, prod).'), - project_name: z.string().min(1).optional().describe('Override auto-detected dbt project name.'), - }) - .describe('dbt context-source connection.'); - -const metricflowConnectionSchema = z - .looseObject({ - driver: z.literal('metricflow'), - metricflow: z - .looseObject({ - repoUrl: z.string().min(1).describe('Git URL of the MetricFlow / SL project.'), - branch: z.string().min(1).optional().describe('Git branch (default "main").'), - path: z.string().optional().describe('Subdirectory within the repo when the SL config lives in a monorepo.'), - auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'), - }) - .describe('Nested MetricFlow configuration block.'), - }) - .describe('MetricFlow / SL context-source connection.'); -``` - -Then update the final union: - -```typescript -export const connectionConfigSchema = z.discriminatedUnion( - 'driver', - [ - ...warehouseDrivers.map(warehouseConnectionSchema), - metabaseConnectionSchema, - lookerConnectionSchema, - lookmlConnectionSchema, - notionConnectionSchema, - dbtConnectionSchema, - metricflowConnectionSchema, - ], -); -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/driver-schemas.test.ts` -Expected: PASS. - -- [ ] **Step 5: Type-check** - -Run: `pnpm --filter @ktx/context run type-check` -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/project/driver-schemas.ts packages/context/src/project/driver-schemas.test.ts -git commit -m "feat(context): add notion, dbt, metricflow driver schemas" -``` - ---- - -## Task 5: Wire the discriminated union into `config.ts` - -Now switch the top-level `connectionSchema` to the new union. This is the change that flips JSON-schema output. - -**Files:** -- Modify: `packages/context/src/project/config.ts:209-214, 272` -- Test: `packages/context/src/project/config.test.ts` — add a JSON-schema assertion. - -- [ ] **Step 1: Write a failing test for the JSON schema output** - -Append to `packages/context/src/project/config.test.ts`: - -```typescript -import { generateKtxProjectConfigJsonSchema } from './config.js'; - -describe('generateKtxProjectConfigJsonSchema', () => { - it('emits the metabase mappings shape under connections', () => { - const schema = generateKtxProjectConfigJsonSchema(); - const serialized = JSON.stringify(schema); - expect(serialized).toContain('databaseMappings'); - expect(serialized).toContain('connectionMappings'); - expect(serialized).toContain('expectedLookerConnectionName'); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/config.test.ts` -Expected: FAIL — the strings are not in the emitted schema yet because `connectionSchema` is still loose. - -- [ ] **Step 3: Replace `connectionSchema` in `config.ts`** - -In `packages/context/src/project/config.ts`, delete lines `209-214`: - -```typescript -const connectionSchema = z - .looseObject({ - driver: z.string().min(1).optional().describe('Connector driver identifier (e.g. "postgres", "bigquery", "snowflake").'), - url: z.string().optional().describe('Connection URL or DSN. Format depends on the driver; may contain environment-variable references.'), - }) - .describe('A single database/connector connection entry. Additional driver-specific fields are accepted and passed through.'); -``` - -Replace with an import + re-bind at the top of the file (after the existing imports): - -```typescript -import { connectionConfigSchema } from './driver-schemas.js'; - -const connectionSchema = connectionConfigSchema; -``` - -(Re-binding to the local name `connectionSchema` keeps the rest of the file unchanged, including the export of `KtxProjectConnectionConfig` at line `272`.) - -- [ ] **Step 4: Run the new test plus existing config tests** - -Run: `pnpm --filter @ktx/context exec vitest run src/project/` -Expected: PASS for all tests. - -If any existing test fails (e.g. a fixture used an undocumented driver string), update the fixture or expand the union — do not loosen the union. - -- [ ] **Step 5: Run the full context test suite to catch downstream regressions** - -Run: `pnpm --filter @ktx/context run test` -Expected: PASS. - -- [ ] **Step 6: Type-check the workspace** - -Run: `pnpm run type-check` -Expected: PASS. `KtxProjectConnectionConfig` is now a union; any consumer that destructured fields not present on every driver branch will surface here. - -If type-check fails in a consumer, the fix is usually `if (connection.driver === 'metabase')` style narrowing — or, for code that already does this dynamically (e.g. `String(connection.driver).toLowerCase() === 'metabase'`), an explicit cast at the call site is acceptable. Do not add `as any`; prefer narrowing. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/src/project/config.ts packages/context/src/project/config.test.ts -git commit -m "refactor(context): make connectionSchema a driver-discriminated union" -``` - ---- - -## Task 6: Verify the user-visible result and CLI smoke - -Confirm the original bug is fixed and the CLI behavior is unchanged. - -**Files:** none modified in this task. - -- [ ] **Step 1: Build the CLI** - -Run: `pnpm run build` -Expected: PASS. - -- [ ] **Step 2: Confirm `ktx dev schema | rg -i mapping` now returns hits** - -Run: `node scripts/run-ktx.mjs -- dev schema | rg -i mapping` -Expected: multiple lines, including the `databaseMappings`, `connectionMappings`, `expectedLookerConnectionName` keys and their descriptions. - -- [ ] **Step 3: Run the CLI smoke** - -Run: `pnpm --filter @ktx/cli run smoke` -Expected: PASS. - -- [ ] **Step 4: Run the broader workspace test suite** - -Run: `pnpm run test 2>&1 | tee /tmp/ktx-test-output.log` -Expected: PASS. Inspect `/tmp/ktx-test-output.log` if anything fails. - -- [ ] **Step 5: Run pre-commit on changed files** - -Run: `pnpm run check` -Expected: PASS. - -- [ ] **Step 6: Knip dead-code sweep (in case we introduced unused exports)** - -Run: `pnpm run dead-code` -Expected: PASS — or, if Knip flags `KtxConnectionConfig` as unused, decide whether to export it from `packages/context/src/project/index.ts` (preferred — it documents intent) or drop the export. - -If exporting: add to `packages/context/src/project/index.ts`: - -```typescript -export type { KtxConnectionConfig } from './driver-schemas.js'; -``` - -- [ ] **Step 7: Final commit if any docs / index changes were made** - -```bash -git status --short -# If only docs/index were touched in step 6: -git add packages/context/src/project/index.ts -git commit -m "chore(context): re-export KtxConnectionConfig from project package" -``` - ---- - -## Self-Review - -**1. Spec coverage:** Original request was "I need to be able to see full schema" with chosen approach option 1 (discriminated union). Task 5 step 2 verifies that `ktx dev schema | rg -i mapping` now returns hits. Task 6 step 2 is the explicit end-to-end check. All catalogued drivers (warehouse + metabase + looker + lookml + notion + dbt + metricflow) have a schema and a test. ✅ - -**2. Placeholder scan:** No "TBD", "add validation", "similar to Task N", or skipped code. Every step has the actual code or command. ✅ - -**3. Type consistency:** -- `connectionConfigSchema` is defined in Task 2 and extended (not renamed) in Tasks 3–4. ✅ -- `KtxConnectionConfig` (new type) appears only in `driver-schemas.ts` and the optional re-export in Task 6. `KtxProjectConnectionConfig` (existing type at `config.ts:272`) keeps its name. ✅ -- `metabaseMappingsSchema`, `lookerMappingsSchema`, `lookmlMappingsSchema` — Task 1 exports them; Task 3 imports them by the same names. ✅ -- `metabaseMappingsStrictSchema` is defined and used in Task 3 only. ✅ -- The `warehouseDrivers` array and `warehouseConnectionSchema` helper are introduced in Task 2 and reused unchanged in Task 4's union extension. ✅ - ---- - -## Execution Handoff - -Plan complete and saved to `docs/superpowers/plans/2026-05-14-connection-driver-discriminated-union.md`. Two execution options: - -**1. Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration. - -**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints. - -Which approach? diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-dictionary-search.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-dictionary-search.md deleted file mode 100644 index 63663937..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-dictionary-search.md +++ /dev/null @@ -1,939 +0,0 @@ -# Research Agent MCP Dictionary Search Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the MCP-shaped `dictionary_search` tool so external research agents can resolve user-mentioned literal values to profile-sampled warehouse columns. - -**Architecture:** Reuse the existing relationship-profile dictionary extraction as the source of truth, add a focused local dictionary-search service that reports coverage and non-authoritative misses per connection, then register the service through the MCP context tool surface and local project ports. The service re-reads the latest profile artifact on each call instead of keeping a long-lived cache, so scan freshness is correct for the MCP daemon v1. - -**Tech Stack:** TypeScript, Vitest, Zod, KTX local file store, relationship-profile artifacts, KTX MCP context ports. - ---- - -## Current Audit - -Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md` - -Implemented v1 slices: - -- `docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md` is implemented. Current source has sqlglot read-only validation in `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`, `SqlAnalysisPort.validateReadOnly()` in `packages/context/src/sql-analysis/ports.ts`, MCP `sql_execution` registration in `packages/context/src/mcp/context-tools.ts`, and local connector execution gated by validation in `packages/context/src/mcp/local-project-ports.ts`. -- `docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md` is implemented. Current source has `packages/context/src/scan/entity-details.ts`, MCP `entity_details` registration in `packages/context/src/mcp/context-tools.ts`, and local project wiring in `packages/context/src/mcp/local-project-ports.ts`. - -V1-blocking gaps remaining against the original spec: - -- `dictionary_search` is not registered on the MCP surface and `KtxMcpContextPorts` has no dictionary-search port. -- `discover_data` is not registered on the MCP surface and the unified ranked result shape is not implemented. -- The ingest-side warehouse-verification tools still use `connectionName` / `targets` / `rowLimit` contracts and have not been fully converged with shared MCP-shaped services. -- `ktx mcp start|stop|status|logs` and the HTTP Streamable MCP daemon do not exist. -- `ktx setup-agents` does not install MCP client config entries or the `ktx-research` skill. - -This plan covers only the next focused blocker: MCP `dictionary_search`. Later plans still need to cover `discover_data`, ingest contract convergence, the HTTP daemon, and setup-agent/research-skill installation. - -Non-blocking or explicitly out-of-scope gaps: - -- Python code execution over MCP. -- Stdio MCP transport. -- OS-level auto-start. -- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing. -- Streaming SQL results. - -## File Structure - -Create: - -- `packages/context/src/sl/dictionary-search.ts` - - Reads the latest `relationship-profile.json` per searched connection. - - Uses `loadLatestSlDictionaryEntries()` for dictionary entries. - - Returns spec-shaped `searched` coverage records, matches, and per-value miss reasons. - - Re-reads artifacts per call rather than caching, satisfying MCP freshness for v1. -- `packages/context/src/sl/dictionary-search.test.ts` - - Covers matches, non-authoritative misses, missing profile artifacts, no candidate columns, case-insensitive substring matching, and connection scoping. - -Modify: - -- `packages/context/src/sl/index.ts` - - Export the new service and response types. -- `packages/context/src/mcp/types.ts` - - Add `KtxDictionarySearchMcpPort` and include `dictionarySearch` in `KtxMcpContextPorts`. -- `packages/context/src/mcp/context-tools.ts` - - Add the `dictionary_search` Zod schema and registration. -- `packages/context/src/mcp/server.test.ts` - - Assert MCP registration and structured output for `dictionary_search`. -- `packages/context/src/mcp/local-project-ports.ts` - - Wire local project dictionary search to the new service. -- `packages/context/src/mcp/local-project-ports.test.ts` - - Cover local-port `dictionary_search` success and missing-profile behavior. -- `packages/context/src/mcp/index.ts` - - Export the new MCP port type if it is not already covered by existing barrel exports. - -## Task 1: Add The Dictionary Search Service - -**Files:** -- Create: `packages/context/src/sl/dictionary-search.test.ts` -- Create: `packages/context/src/sl/dictionary-search.ts` -- Modify: `packages/context/src/sl/index.ts` - -- [ ] **Step 1: Write failing service tests** - -Create `packages/context/src/sl/dictionary-search.test.ts`: - -```typescript -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; -import { createKtxDictionarySearchService } from './dictionary-search.js'; - -describe('createKtxDictionarySearchService', () => { - let tempDir: string; - let project: KtxLocalProject; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-dictionary-search-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL' }; - project.config.connections.billing = { driver: 'postgres', url: 'env:BILLING_DATABASE_URL' }; - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - async function seedProfile(input: { - connectionId: string; - syncId: string; - columns: Record; - }): Promise { - await project.fileStore.writeFile( - `raw-sources/${input.connectionId}/live-database/${input.syncId}/enrichment/relationship-profile.json`, - `${JSON.stringify( - { - connectionId: input.connectionId, - driver: 'postgres', - sqlAvailable: true, - queryCount: 4, - tables: [], - columns: input.columns, - warnings: [], - }, - null, - 2, - )}\n`, - 'ktx', - 'ktx@example.com', - 'Seed relationship profile', - ); - } - - it('returns matches and non-authoritative misses across configured connections', async () => { - await seedProfile({ - connectionId: 'warehouse', - syncId: 'sync-1', - columns: { - 'orders.status': { - table: { catalog: null, db: 'public', name: 'orders' }, - column: 'status', - nativeType: 'text', - normalizedType: 'string', - distinctCount: 3, - sampleValues: ['paid', 'refunded', 'pending'], - }, - }, - }); - await seedProfile({ - connectionId: 'billing', - syncId: 'sync-2', - columns: { - 'customers.name': { - table: { catalog: null, db: 'public', name: 'customers' }, - column: 'name', - nativeType: 'text', - normalizedType: 'string', - distinctCount: 4, - sampleValues: ['Acme Corp', 'Globex'], - }, - }, - }); - const service = createKtxDictionarySearchService(project); - - await expect(service.search({ values: ['PAID', 'missing'] })).resolves.toEqual({ - searched: [ - { - connectionId: 'billing', - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 1, - syncId: 'sync-2', - profiledAt: null, - }, - status: 'ready', - }, - { - connectionId: 'warehouse', - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 1, - syncId: 'sync-1', - profiledAt: null, - }, - status: 'ready', - }, - ], - results: [ - { - value: 'PAID', - matches: [ - { - connectionId: 'warehouse', - sourceName: 'orders', - columnName: 'status', - matchedValue: 'paid', - cardinality: 3, - }, - ], - misses: [{ connectionId: 'billing', reason: 'value_not_in_sample' }], - }, - { - value: 'missing', - matches: [], - misses: [ - { connectionId: 'billing', reason: 'value_not_in_sample' }, - { connectionId: 'warehouse', reason: 'value_not_in_sample' }, - ], - }, - ], - }); - }); - - it('distinguishes missing profile artifacts from profiles with no candidate columns', async () => { - await seedProfile({ - connectionId: 'billing', - syncId: 'sync-empty', - columns: { - 'events.id': { - table: { catalog: null, db: 'public', name: 'events' }, - column: 'id', - nativeType: 'integer', - normalizedType: 'integer', - distinctCount: 100, - sampleValues: [1, 2, 3], - }, - }, - }); - const service = createKtxDictionarySearchService(project); - - await expect(service.search({ values: ['Acme'] })).resolves.toEqual({ - searched: [ - { - connectionId: 'billing', - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 0, - syncId: 'sync-empty', - profiledAt: null, - }, - status: 'no_candidate_columns', - }, - { - connectionId: 'warehouse', - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 0, - syncId: null, - profiledAt: null, - }, - status: 'no_profile_artifact', - }, - ], - results: [ - { - value: 'Acme', - matches: [], - misses: [ - { connectionId: 'billing', reason: 'no_candidate_columns' }, - { connectionId: 'warehouse', reason: 'no_profile_artifact' }, - ], - }, - ], - }); - }); - - it('scopes search to the requested connection', async () => { - await seedProfile({ - connectionId: 'warehouse', - syncId: 'sync-1', - columns: { - 'orders.status': { - table: { catalog: null, db: 'public', name: 'orders' }, - column: 'status', - nativeType: 'text', - normalizedType: 'string', - distinctCount: 3, - sampleValues: ['paid'], - }, - }, - }); - await seedProfile({ - connectionId: 'billing', - syncId: 'sync-2', - columns: { - 'invoices.status': { - table: { catalog: null, db: 'public', name: 'invoices' }, - column: 'status', - nativeType: 'text', - normalizedType: 'string', - distinctCount: 2, - sampleValues: ['paid'], - }, - }, - }); - const service = createKtxDictionarySearchService(project); - - await expect(service.search({ connectionId: 'billing', values: ['paid'] })).resolves.toMatchObject({ - searched: [{ connectionId: 'billing', status: 'ready' }], - results: [ - { - value: 'paid', - matches: [{ connectionId: 'billing', sourceName: 'invoices', columnName: 'status', matchedValue: 'paid' }], - misses: [], - }, - ], - }); - }); -}); -``` - -- [ ] **Step 2: Run service tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/dictionary-search.test.ts -``` - -Expected: FAIL with `Cannot find module './dictionary-search.js'`. - -- [ ] **Step 3: Implement the dictionary search service** - -Create `packages/context/src/sl/dictionary-search.ts`: - -```typescript -import type { KtxLocalProject } from '../project/index.js'; -import { loadLatestSlDictionaryEntries, type SlDictionaryEntry } from './sl-dictionary-profile.js'; - -export type KtxDictionarySearchStatus = 'ready' | 'no_profile_artifact' | 'no_candidate_columns'; -export type KtxDictionarySearchMissReason = 'no_profile_artifact' | 'no_candidate_columns' | 'value_not_in_sample'; - -export interface KtxDictionarySearchInput { - values: string[]; - connectionId?: string; -} - -export interface KtxDictionarySearchCoverage { - sampledRows: number | null; - valuesPerColumn: number | null; - profiledColumns: number; - syncId: string | null; - profiledAt: string | null; -} - -export interface KtxDictionarySearchSearchedConnection { - connectionId: string; - coverage: KtxDictionarySearchCoverage; - status: KtxDictionarySearchStatus; -} - -export interface KtxDictionarySearchMatch { - connectionId: string; - sourceName: string; - columnName: string; - matchedValue: string; - cardinality: number | null; -} - -export interface KtxDictionarySearchMiss { - connectionId: string; - reason: KtxDictionarySearchMissReason; -} - -export interface KtxDictionarySearchValueResult { - value: string; - matches: KtxDictionarySearchMatch[]; - misses: KtxDictionarySearchMiss[]; -} - -export interface KtxDictionarySearchResponse { - searched: KtxDictionarySearchSearchedConnection[]; - results: KtxDictionarySearchValueResult[]; -} - -interface RelationshipProfileArtifact { - connectionId?: string; - profileSampleRows?: unknown; - sampleValuesPerColumn?: unknown; - profiledAt?: unknown; - extractedAt?: unknown; -} - -function uniqueSorted(values: Iterable): string[] { - return [...new Set([...values].filter((value) => value.trim().length > 0))].sort((left, right) => - left.localeCompare(right), - ); -} - -function latestProfileSyncId(path: string): string | null { - const parts = path.split('/'); - return parts.at(-3) ?? null; -} - -function optionalNumber(value: unknown): number | null { - return typeof value === 'number' && Number.isFinite(value) ? value : null; -} - -function optionalString(value: unknown): string | null { - return typeof value === 'string' && value.trim().length > 0 ? value : null; -} - -async function latestProfilePath(project: KtxLocalProject, connectionId: string): Promise { - const root = `raw-sources/${connectionId}/live-database`; - let files: string[]; - try { - files = (await project.fileStore.listFiles(root)).files; - } catch { - return null; - } - return files - .filter((path) => path.endsWith('/enrichment/relationship-profile.json')) - .sort((left, right) => left.localeCompare(right)) - .at(-1) ?? null; -} - -async function readProfile(project: KtxLocalProject, path: string): Promise { - const raw = await project.fileStore.readFile(path); - const parsed = JSON.parse(raw.content) as unknown; - return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) - ? (parsed as RelationshipProfileArtifact) - : {}; -} - -function profiledColumnCount(entries: readonly SlDictionaryEntry[]): number { - return new Set(entries.map((entry) => `${entry.sourceName}\u001f${entry.columnName}`)).size; -} - -async function searchedConnection( - project: KtxLocalProject, - connectionId: string, - entries: readonly SlDictionaryEntry[], -): Promise { - const path = await latestProfilePath(project, connectionId); - if (!path) { - return { - connectionId, - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 0, - syncId: null, - profiledAt: null, - }, - status: 'no_profile_artifact', - }; - } - - const profile = await readProfile(project, path); - const count = profiledColumnCount(entries); - return { - connectionId, - coverage: { - sampledRows: optionalNumber(profile.profileSampleRows), - valuesPerColumn: optionalNumber(profile.sampleValuesPerColumn), - profiledColumns: count, - syncId: latestProfileSyncId(path), - profiledAt: optionalString(profile.profiledAt) ?? optionalString(profile.extractedAt), - }, - status: count > 0 ? 'ready' : 'no_candidate_columns', - }; -} - -function entryMatchesValue(entry: SlDictionaryEntry, value: string): boolean { - return entry.value.toLowerCase().includes(value.toLowerCase()); -} - -function toMatch(entry: SlDictionaryEntry): KtxDictionarySearchMatch { - return { - connectionId: entry.connectionId, - sourceName: entry.sourceName, - columnName: entry.columnName, - matchedValue: entry.value, - cardinality: entry.cardinality, - }; -} - -function sortMatches(matches: KtxDictionarySearchMatch[]): KtxDictionarySearchMatch[] { - return matches.sort( - (left, right) => - left.connectionId.localeCompare(right.connectionId) || - left.sourceName.localeCompare(right.sourceName) || - left.columnName.localeCompare(right.columnName) || - left.matchedValue.localeCompare(right.matchedValue), - ); -} - -function missReason(status: KtxDictionarySearchStatus): KtxDictionarySearchMissReason { - return status === 'ready' ? 'value_not_in_sample' : status; -} - -export function createKtxDictionarySearchService(project: KtxLocalProject) { - return { - async search(input: KtxDictionarySearchInput): Promise { - const connectionIds = input.connectionId ? [input.connectionId] : uniqueSorted(Object.keys(project.config.connections)); - const entries = await loadLatestSlDictionaryEntries(project, connectionIds); - const entriesByConnection = new Map(); - for (const connectionId of connectionIds) { - entriesByConnection.set( - connectionId, - entries.filter((entry) => entry.connectionId === connectionId), - ); - } - - const searched = ( - await Promise.all( - connectionIds.map((connectionId) => - searchedConnection(project, connectionId, entriesByConnection.get(connectionId) ?? []), - ), - ) - ).sort((left, right) => left.connectionId.localeCompare(right.connectionId)); - const searchedByConnection = new Map(searched.map((connection) => [connection.connectionId, connection])); - - return { - searched, - results: input.values.map((value) => { - const matches = sortMatches(entries.filter((entry) => entryMatchesValue(entry, value)).map(toMatch)); - const matchedConnections = new Set(matches.map((match) => match.connectionId)); - return { - value, - matches, - misses: searched - .filter((connection) => !matchedConnections.has(connection.connectionId)) - .map((connection) => ({ - connectionId: connection.connectionId, - reason: missReason(searchedByConnection.get(connection.connectionId)?.status ?? 'no_profile_artifact'), - })), - }; - }), - }; - }, - }; -} -``` - -- [ ] **Step 4: Export the service** - -In `packages/context/src/sl/index.ts`, add: - -```typescript -export { - createKtxDictionarySearchService, -} from './dictionary-search.js'; -export type { - KtxDictionarySearchCoverage, - KtxDictionarySearchInput, - KtxDictionarySearchMatch, - KtxDictionarySearchMiss, - KtxDictionarySearchMissReason, - KtxDictionarySearchResponse, - KtxDictionarySearchSearchedConnection, - KtxDictionarySearchStatus, - KtxDictionarySearchValueResult, -} from './dictionary-search.js'; -``` - -- [ ] **Step 5: Run service tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/dictionary-search.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the service slice** - -Run: - -```bash -git add packages/context/src/sl/dictionary-search.ts packages/context/src/sl/dictionary-search.test.ts packages/context/src/sl/index.ts -git commit -m "feat(context): add dictionary search service" -``` - -## Task 2: Register The MCP `dictionary_search` Tool - -**Files:** -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/context-tools.ts` -- Modify: `packages/context/src/mcp/server.test.ts` -- Modify: `packages/context/src/mcp/index.ts` - -- [ ] **Step 1: Add MCP port types** - -In `packages/context/src/mcp/types.ts`, extend the imports: - -```typescript -import type { KtxDictionarySearchInput, KtxDictionarySearchResponse } from '../sl/index.js'; -``` - -Add this interface near the other MCP port interfaces: - -```typescript -export interface KtxDictionarySearchMcpPort { - search(input: KtxDictionarySearchInput): Promise; -} -``` - -Add the new optional port to `KtxMcpContextPorts`: - -```typescript -export interface KtxMcpContextPorts { - connections?: KtxConnectionsMcpPort; - knowledge?: KtxKnowledgeMcpPort; - semanticLayer?: KtxSemanticLayerMcpPort; - entityDetails?: KtxEntityDetailsMcpPort; - dictionarySearch?: KtxDictionarySearchMcpPort; - sqlExecution?: KtxSqlExecutionMcpPort; - ingest?: KtxIngestMcpPort; - scan?: KtxScanMcpPort; -} -``` - -- [ ] **Step 2: Write failing MCP registration test** - -In `packages/context/src/mcp/server.test.ts`, update the type import list to include: - -```typescript -KtxDictionarySearchMcpPort, -``` - -Add this test after the `entity_details` registration test: - -```typescript - it('registers dictionary_search when the host provides a dictionary-search port', async () => { - const fake = makeFakeServer(); - const dictionarySearch: KtxDictionarySearchMcpPort = { - search: vi.fn().mockResolvedValue({ - searched: [ - { - connectionId: 'warehouse', - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 1, - syncId: 'sync-1', - profiledAt: null, - }, - status: 'ready', - }, - ], - results: [ - { - value: 'paid', - matches: [ - { - connectionId: 'warehouse', - sourceName: 'orders', - columnName: 'status', - matchedValue: 'paid', - cardinality: 3, - }, - ], - misses: [], - }, - ], - }), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { dictionarySearch }, - }); - - expect(fake.tools.map((tool) => tool.name)).toEqual(['dictionary_search']); - await expect( - getTool(fake.tools, 'dictionary_search').handler({ - connectionId: 'warehouse', - values: ['paid'], - }), - ).resolves.toMatchObject({ - structuredContent: { - searched: [{ connectionId: 'warehouse', status: 'ready' }], - results: [ - { - value: 'paid', - matches: [{ connectionId: 'warehouse', sourceName: 'orders', columnName: 'status' }], - misses: [], - }, - ], - }, - }); - expect(dictionarySearch.search).toHaveBeenCalledWith({ - connectionId: 'warehouse', - values: ['paid'], - }); - }); -``` - -- [ ] **Step 3: Run failing MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "dictionary_search" -``` - -Expected: FAIL because `dictionary_search` is not registered. - -- [ ] **Step 4: Add the MCP schema and registration** - -In `packages/context/src/mcp/context-tools.ts`, add the input schema near the other research schemas: - -```typescript -const dictionarySearchSchema = z.object({ - values: z.array(z.string().min(1)).min(1).max(20), - connectionId: connectionIdSchema.optional(), -}); -``` - -Add this registration block after `entity_details` and before `sql_execution`: - -```typescript - if (ports.dictionarySearch) { - const dictionarySearch = ports.dictionarySearch; - registerParsedTool( - server, - 'dictionary_search', - { - title: 'Dictionary Search', - description: - 'Search profile-sampled warehouse values and report matching connection/source/column locations plus non-authoritative miss reasons.', - inputSchema: dictionarySearchSchema.shape, - }, - dictionarySearchSchema, - async (input) => jsonToolResult(await dictionarySearch.search(input)), - ); - } -``` - -- [ ] **Step 5: Confirm MCP barrel exports** - -Open `packages/context/src/mcp/index.ts`. If it exports from `./types.js`, no change is needed. If it lists named type exports, add `KtxDictionarySearchMcpPort` to that list. - -- [ ] **Step 6: Run MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "dictionary_search" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit MCP registration** - -Run: - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/index.ts -git commit -m "feat(context): register MCP dictionary search tool" -``` - -## Task 3: Wire Local Project MCP Ports - -**Files:** -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write failing local-port tests** - -In `packages/context/src/mcp/local-project-ports.test.ts`, add this test after the entity-details local-port tests: - -```typescript - it('exposes local dictionary search through MCP ports', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - await project.fileStore.writeFile( - 'raw-sources/warehouse/live-database/sync-1/enrichment/relationship-profile.json', - `${JSON.stringify( - { - connectionId: 'warehouse', - driver: 'postgres', - sqlAvailable: true, - queryCount: 4, - tables: [], - columns: { - 'orders.status': { - table: { catalog: null, db: 'public', name: 'orders' }, - column: 'status', - nativeType: 'text', - normalizedType: 'string', - distinctCount: 2, - sampleValues: ['paid', 'refunded'], - }, - }, - warnings: [], - }, - null, - 2, - )}\n`, - 'ktx', - 'ktx@example.com', - 'Seed dictionary profile', - ); - - const ports = createLocalProjectMcpContextPorts(project); - - await expect(ports.dictionarySearch?.search({ values: ['paid'] })).resolves.toMatchObject({ - searched: [{ connectionId: 'warehouse', status: 'ready' }], - results: [ - { - value: 'paid', - matches: [{ connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', matchedValue: 'paid' }], - misses: [], - }, - ], - }); - }); - - it('reports missing local dictionary profiles through MCP ports', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - - const ports = createLocalProjectMcpContextPorts(project); - - await expect(ports.dictionarySearch?.search({ values: ['paid'] })).resolves.toEqual({ - searched: [ - { - connectionId: 'warehouse', - coverage: { - sampledRows: null, - valuesPerColumn: null, - profiledColumns: 0, - syncId: null, - profiledAt: null, - }, - status: 'no_profile_artifact', - }, - ], - results: [ - { - value: 'paid', - matches: [], - misses: [{ connectionId: 'warehouse', reason: 'no_profile_artifact' }], - }, - ], - }); - }); -``` - -- [ ] **Step 2: Run failing local-port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "dictionary" -``` - -Expected: FAIL because `ports.dictionarySearch` is undefined. - -- [ ] **Step 3: Wire the local port** - -In `packages/context/src/mcp/local-project-ports.ts`, update the SL import block to include: - -```typescript -createKtxDictionarySearchService, -``` - -Add this port to the `ports` object returned by `createLocalProjectMcpContextPorts()` near `entityDetails`: - -```typescript - dictionarySearch: { - async search(input) { - return createKtxDictionarySearchService(project).search(input); - }, - }, -``` - -- [ ] **Step 4: Run local-port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "dictionary" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit local-port wiring** - -Run: - -```bash -git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat(context): expose local MCP dictionary search" -``` - -## Task 4: Final Verification - -**Files:** -- Verify all files changed in Tasks 1-3. - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/dictionary-search.test.ts src/mcp/server.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: PASS for dictionary-search service, MCP registration, and local-port coverage. - -- [ ] **Step 2: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Inspect diff** - -Run: - -```bash -git status --short -git diff --stat HEAD -``` - -Expected: only the dictionary-search service, MCP type/registration, tests, and exports changed. - -- [ ] **Step 4: Commit verification note if needed** - -If the previous tasks already committed all source changes, do not create an empty commit. If a small follow-up fix was required during verification, commit only those files: - -```bash -git add packages/context/src/sl/dictionary-search.ts packages/context/src/sl/dictionary-search.test.ts packages/context/src/sl/index.ts packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/index.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "test(context): cover MCP dictionary search" -``` diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-discover-data.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-discover-data.md deleted file mode 100644 index 351aed67..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-discover-data.md +++ /dev/null @@ -1,1317 +0,0 @@ -# Research Agent MCP Discover Data Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the MCP-shaped `discover_data` tool so external research agents get one ranked discovery view across wiki pages, semantic-layer sources/measures/dimensions, and raw warehouse schema. - -**Architecture:** Create a focused local discovery service in `packages/context/src/search/discover.ts` that builds deterministic per-kind refs from existing wiki, semantic-layer, and latest scan artifacts, fuses the wiki/SL/raw sub-searches with the existing RRF core, and re-reads local artifacts on every call for MCP daemon freshness. Register the service through the MCP context port and local project MCP ports without changing the existing ingest-only `discover_data` adapter yet. - -**Tech Stack:** TypeScript, Vitest, Zod, KTX local file store, KTX wiki/SL/scan services, KTX MCP context ports, existing `HybridSearchCore`/RRF search utilities. - ---- - -## Audit Summary - -Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md` - -Implemented v1 slices confirmed in current source: - -- Existing in-process MCP semantic runtime exists in `packages/context/src/mcp/server.ts`, `packages/context/src/mcp/context-tools.ts`, and `packages/context/src/mcp/local-project-ports.ts`. -- Ingest-only warehouse verification tools exist under `packages/context/src/ingest/tools/warehouse-verification/`. -- MCP `sql_execution` is implemented and parser-gated: `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` has `validate_read_only_sql_response`, `python/ktx-daemon/src/ktx_daemon/app.py` exposes `POST /sql/validate-read-only`, `packages/context/src/sql-analysis/ports.ts` has `validateReadOnly()`, and `packages/context/src/mcp/context-tools.ts` registers `sql_execution`. -- MCP `entity_details` is implemented: `packages/context/src/scan/entity-details.ts`, `KtxEntityDetailsMcpPort`, context-tool registration, and local project wiring all exist. -- MCP `dictionary_search` is implemented: `packages/context/src/sl/dictionary-search.ts`, `KtxDictionarySearchMcpPort`, context-tool registration, and local project wiring all exist. - -V1-blocking gaps still open: - -- `discover_data` is not implemented on the MCP surface. There is no `packages/context/src/search/discover.ts`, no `KtxDiscoverDataMcpPort`, no `ports.discover`, no MCP registration, and no local project wiring. -- `ktx mcp start|stop|status|logs` and the HTTP Streamable MCP daemon do not exist. There is no `packages/cli/src/commands/mcp-commands.ts`, no `packages/cli/src/managed-mcp-daemon.ts`, and `packages/cli/src/cli-program.ts` does not register an `mcp` command subtree. -- `ktx setup-agents` does not install `ktx-research`, write Claude Code/Cursor MCP JSON entries, or print Codex/opencode snippets. `plannedKtxAgentFiles()` still installs only the existing `ktx` skill/rule files. -- Ingest-side warehouse verification tools still use `connectionName`, `targets`, and `rowLimit` contracts. The original spec says these should converge on `connectionId` naming, but that cleanup can be planned after the MCP research surface is complete because this plan adds a separate MCP adapter with the required shape. - -Non-blocking or explicitly out-of-scope gaps: - -- Python code execution via MCP. -- Stdio MCP transport. -- OS-level auto-start. -- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing. -- Streaming SQL results. - -This plan covers only the next dependency-ordered v1 blocker: MCP `discover_data`. Later v1 plans still need to cover the HTTP daemon and setup-agent/research-skill installation. - -## File Structure - -Create: - -- `packages/context/src/search/discover.ts` - - Defines MCP-shaped `discover_data` input, ref, and response types. - - Searches wiki pages through `searchLocalKnowledgePages()` and `readLocalKnowledgePage()`. - - Searches semantic-layer records through `loadLocalSlSourceRecords()`. - - Searches raw schema by reading the latest `raw-sources//live-database/` scan artifacts directly. - - Fuses wiki, SL, and raw-schema candidates with `HybridSearchCore` using equal lane weights and normalizes final scores to `0..1`. - - Re-reads artifacts on every call; no long-lived cache. -- `packages/context/src/search/discover.test.ts` - - Covers unified result shape, kind filtering, connection scoping, score normalization, snippet cap, raw table refs, and freshness after a newer scan appears. - -Modify: - -- `packages/context/src/search/index.ts` - - Export `createKtxDiscoverDataService` and discover types. -- `packages/context/src/mcp/types.ts` - - Add `KtxDiscoverDataMcpPort` and `discover?: KtxDiscoverDataMcpPort` to `KtxMcpContextPorts`. -- `packages/context/src/mcp/context-tools.ts` - - Add the `discover_data` Zod schema and tool registration. -- `packages/context/src/mcp/server.test.ts` - - Assert `discover_data` registration and structured array output. -- `packages/context/src/mcp/local-project-ports.ts` - - Wire local project `discover.search()` to `createKtxDiscoverDataService()`. -- `packages/context/src/mcp/local-project-ports.test.ts` - - Cover local-port `discover_data` across wiki, SL, and raw schema. -- `packages/context/src/mcp/index.ts` - - Export the new MCP port type if it is not already covered by existing barrel exports. - -## Task 1: Add The Local Discover Data Service - -**Files:** -- Create: `packages/context/src/search/discover.test.ts` -- Create: `packages/context/src/search/discover.ts` -- Modify: `packages/context/src/search/index.ts` - -- [ ] **Step 1: Write failing service tests** - -Create `packages/context/src/search/discover.test.ts`: - -```typescript -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; -import { writeLocalKnowledgePage } from '../wiki/local-knowledge.js'; -import { createKtxDiscoverDataService } from './discover.js'; - -describe('createKtxDiscoverDataService', () => { - let tempDir: string; - let project: KtxLocalProject; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-discover-data-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL' }; - project.config.connections.billing = { driver: 'postgres', url: 'env:BILLING_DATABASE_URL' }; - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - async function seedWiki(): Promise { - await writeLocalKnowledgePage(project, { - key: 'orders-playbook', - scope: 'GLOBAL', - summary: 'Paid order operations', - content: 'Use paid orders and order_count to inspect monthly customer activity for Acme Corp.', - tags: ['orders'], - }); - } - - async function seedSl(): Promise { - await project.fileStore.writeFile( - 'semantic-layer/warehouse/orders.yaml', - [ - 'name: orders', - 'descriptions:', - ' user: Paid order facts', - 'table: public.orders', - 'grain: [id]', - 'columns:', - ' - name: status', - ' type: string', - ' descriptions:', - ' user: Payment status for the order', - ' - name: ordered_at', - ' type: time', - 'measures:', - ' - name: order_count', - ' expr: count(*)', - ' description: Number of paid orders', - '', - ].join('\n'), - 'ktx', - 'ktx@example.com', - 'seed sl source', - ); - } - - async function seedScan(input: { - connectionId?: string; - syncId: string; - tableName?: string; - comment?: string; - sampleValues?: string[]; - }): Promise { - const connectionId = input.connectionId ?? 'warehouse'; - const root = `raw-sources/${connectionId}/live-database/${input.syncId}`; - const tableName = input.tableName ?? 'orders'; - await project.fileStore.writeFile( - `${root}/connection.json`, - JSON.stringify( - { - connectionId, - driver: 'postgres', - extractedAt: `2026-05-14T09:00:00.000Z`, - scope: { schemas: ['public'] }, - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed scan connection', - ); - await project.fileStore.writeFile( - `${root}/tables/public-${tableName}.json`, - JSON.stringify( - { - catalog: null, - db: 'public', - name: tableName, - kind: 'table', - comment: input.comment ?? 'Orders table from warehouse', - estimatedRows: 123, - descriptions: { db: input.comment ?? 'Orders table from warehouse' }, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: 'Order id', - }, - { - name: 'status', - nativeType: 'text', - normalizedType: 'text', - dimensionType: 'string', - nullable: false, - primaryKey: false, - comment: 'Order status', - sampleValues: input.sampleValues ?? ['paid', 'pending'], - }, - ], - foreignKeys: [], - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed table', - ); - await project.fileStore.writeFile( - `${root}/scan-report.json`, - JSON.stringify( - { - connectionId, - driver: 'postgres', - syncId: input.syncId, - runId: `scan-${input.syncId}`, - trigger: 'mcp', - mode: 'enriched', - dryRun: false, - artifactPaths: { - rawSourcesDir: root, - reportPath: `${root}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts: [], - }, - diffSummary: { - tablesAdded: 1, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 0, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 0, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'completed', - tableDescriptions: 'completed', - columnDescriptions: 'completed', - embeddings: 'skipped', - deterministicRelationships: 'skipped', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'skipped', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 }, - enrichmentState: { resumedStages: [], completedStages: [], failedStages: [] }, - createdAt: '2026-05-14T09:00:00.000Z', - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed scan report', - ); - } - - it('returns unified ranked refs across wiki, semantic-layer, and raw schema', async () => { - await seedWiki(); - await seedSl(); - await seedScan({ syncId: 'sync-1', sampleValues: ['paid', 'refunded'] }); - const service = createKtxDiscoverDataService(project, { userId: 'local-user' }); - - const results = await service.search({ query: 'paid orders', connectionId: 'warehouse', limit: 10 }); - - expect(results.map((result) => result.kind)).toEqual( - expect.arrayContaining(['wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column']), - ); - expect(results.every((result) => result.score >= 0 && result.score <= 1)).toBe(true); - expect(results.every((result) => result.snippet === null || result.snippet.length <= 200)).toBe(true); - expect(results).toContainEqual( - expect.objectContaining({ - kind: 'table', - id: 'public.orders', - connectionId: 'warehouse', - tableRef: { catalog: null, db: 'public', name: 'orders' }, - matchedOn: expect.stringMatching(/name|description|comment|display/), - }), - ); - expect(results).toContainEqual( - expect.objectContaining({ - kind: 'column', - id: 'public.orders.status', - connectionId: 'warehouse', - columnName: 'status', - matchedOn: expect.stringMatching(/name|comment|description|sample_value/), - }), - ); - expect(results).toContainEqual( - expect.objectContaining({ - kind: 'sl_measure', - id: 'orders.order_count', - connectionId: 'warehouse', - summary: 'Number of paid orders', - snippet: 'count(*)', - matchedOn: expect.stringMatching(/name|description|expr/), - }), - ); - }); - - it('honors kind filters and connection scope', async () => { - await seedWiki(); - await seedSl(); - await seedScan({ syncId: 'sync-1', connectionId: 'warehouse', tableName: 'orders' }); - await seedScan({ syncId: 'sync-2', connectionId: 'billing', tableName: 'invoices', comment: 'Billing invoices' }); - const service = createKtxDiscoverDataService(project); - - const results = await service.search({ - query: 'orders', - connectionId: 'warehouse', - kinds: ['table', 'column'], - limit: 10, - }); - - expect(results.every((result) => result.kind === 'table' || result.kind === 'column')).toBe(true); - expect(results.every((result) => result.connectionId === 'warehouse')).toBe(true); - expect(results.some((result) => result.id.includes('invoices'))).toBe(false); - expect(results.some((result) => result.kind === 'wiki')).toBe(false); - }); - - it('re-reads the latest scan artifacts on each call', async () => { - await seedScan({ syncId: 'sync-1', tableName: 'orders', comment: 'Old orders table' }); - const service = createKtxDiscoverDataService(project); - await expect(service.search({ query: 'orders', connectionId: 'warehouse', kinds: ['table'], limit: 10 })).resolves.toEqual( - expect.arrayContaining([expect.objectContaining({ id: 'public.orders' })]), - ); - - await seedScan({ syncId: 'sync-2', tableName: 'invoices', comment: 'Invoice facts' }); - const fresh = await service.search({ query: 'invoice', connectionId: 'warehouse', kinds: ['table'], limit: 10 }); - - expect(fresh).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'public.invoices' })])); - expect(fresh.some((result) => result.id === 'public.orders')).toBe(false); - }); -}); -``` - -- [ ] **Step 2: Run the failing service tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/search/discover.test.ts -``` - -Expected: FAIL with `Cannot find module './discover.js'`. - -- [ ] **Step 3: Implement the discover service** - -Create `packages/context/src/search/discover.ts`: - -```typescript -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import type { KtxScanReport, KtxSchemaColumn, KtxSchemaTable, KtxTableRef } from '../scan/index.js'; -import { DEFAULT_PRIORITY, loadLocalSlSourceRecords, resolveDescription } from '../sl/index.js'; -import type { SemanticLayerSource } from '../sl/index.js'; -import { readLocalKnowledgePage, searchLocalKnowledgePages } from '../wiki/local-knowledge.js'; -import { HybridSearchCore, type FusedSearchCandidate, type SearchCandidateGenerator } from './index.js'; - -export type KtxDiscoverDataKind = 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column'; -export type KtxDiscoverDataMatchedOn = - | 'name' - | 'display' - | 'description' - | 'comment' - | 'expr' - | 'sample_value' - | 'body'; - -export interface KtxDiscoverDataInput { - query: string; - connectionId?: string; - kinds?: KtxDiscoverDataKind[]; - limit?: number; -} - -export interface KtxDiscoverDataRef { - kind: KtxDiscoverDataKind; - id: string; - score: number; - summary: string | null; - snippet: string | null; - matchedOn: KtxDiscoverDataMatchedOn; - connectionId?: string; - tableRef?: KtxTableRef; - columnName?: string; -} - -export type KtxDiscoverDataResponse = KtxDiscoverDataRef[]; - -export interface KtxDiscoverDataServiceOptions { - userId?: string; - embeddingService?: KtxEmbeddingPort | null; -} - -interface CandidateRecord { - ref: Omit; - rankScore: number; -} - -type RawTable = KtxSchemaTable & { - descriptions?: Record; - columns: Array; sampleValues?: unknown[] }>; -}; - -interface LatestScan { - report: KtxScanReport; - rawSourcesDir: string; - tables: RawTable[]; -} - -const ALL_KINDS: KtxDiscoverDataKind[] = ['wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column']; - -function normalize(value: string | null | undefined): string { - return (value ?? '').toLowerCase(); -} - -function queryTerms(query: string): string[] { - return query - .toLowerCase() - .split(/[^a-z0-9_]+/u) - .map((term) => term.trim()) - .filter(Boolean); -} - -function hasKind(kinds: ReadonlySet, kind: KtxDiscoverDataKind): boolean { - return kinds.has(kind); -} - -function cap200(value: string | null | undefined): string | null { - if (!value) { - return null; - } - const compact = value.replace(/\s+/g, ' ').trim(); - return compact.length > 200 ? compact.slice(0, 200) : compact; -} - -function snippetAround(text: string | null | undefined, terms: readonly string[]): string | null { - if (!text) { - return null; - } - const lower = text.toLowerCase(); - const index = terms.map((term) => lower.indexOf(term)).filter((position) => position >= 0).sort((a, b) => a - b)[0] ?? 0; - return cap200(text.slice(Math.max(0, index - 60), index + 140)); -} - -function textScore(value: string | null | undefined, terms: readonly string[]): number { - const haystack = normalize(value); - if (!haystack || terms.length === 0) { - return 0; - } - const matched = terms.filter((term) => haystack.includes(term)).length; - return matched / terms.length; -} - -function bestField( - fields: Array<{ matchedOn: KtxDiscoverDataMatchedOn; text: string | null | undefined; weight: number }>, - terms: readonly string[], -): { matchedOn: KtxDiscoverDataMatchedOn; score: number; text: string | null } | null { - const scored = fields - .map((field) => ({ - matchedOn: field.matchedOn, - score: textScore(field.text, terms) * field.weight, - text: field.text ?? null, - })) - .filter((field) => field.score > 0) - .sort((left, right) => right.score - left.score || left.matchedOn.localeCompare(right.matchedOn)); - return scored[0] ?? null; -} - -function displayForTable(table: KtxTableRef): string { - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - -function tableRef(table: KtxSchemaTable): KtxTableRef { - return { catalog: table.catalog, db: table.db, name: table.name }; -} - -async function readJson(project: KtxLocalProject, path: string): Promise { - return JSON.parse((await project.fileStore.readFile(path)).content) as T; -} - -async function latestScan(project: KtxLocalProject, connectionId: string): Promise { - const root = `raw-sources/${connectionId}/live-database`; - let files: string[]; - try { - files = (await project.fileStore.listFiles(root)).files; - } catch { - return null; - } - - const reportPath = files.filter((path) => path.endsWith('/scan-report.json')).sort().at(-1); - if (!reportPath) { - return null; - } - const report = await readJson(project, reportPath); - const rawSourcesDir = report.artifactPaths.rawSourcesDir ?? reportPath.slice(0, -'/scan-report.json'.length); - const listedTables = await project.fileStore.listFiles(`${rawSourcesDir}/tables`); - const tables: RawTable[] = []; - for (const path of listedTables.files.filter((file) => file.endsWith('.json')).sort()) { - tables.push(await readJson(project, path)); - } - return { report, rawSourcesDir, tables }; -} - -function configuredConnectionIds(project: KtxLocalProject, connectionId?: string): string[] { - return connectionId ? [connectionId] : Object.keys(project.config.connections).sort(); -} - -async function wikiCandidates( - project: KtxLocalProject, - input: KtxDiscoverDataInput, - options: KtxDiscoverDataServiceOptions, - terms: readonly string[], -): Promise { - const searchResults = await searchLocalKnowledgePages(project, { - query: input.query, - userId: options.userId, - embeddingService: options.embeddingService ?? null, - limit: Math.max(input.limit ?? 15, 25), - }); - const records: CandidateRecord[] = []; - for (const result of searchResults) { - const page = await readLocalKnowledgePage(project, { key: result.key, userId: options.userId }); - const content = page?.content ?? ''; - const matched = bestField( - [ - { matchedOn: 'name', text: result.key, weight: 1.1 }, - { matchedOn: 'description', text: result.summary, weight: 1 }, - { matchedOn: 'body', text: content, weight: 0.8 }, - ], - terms, - ); - records.push({ - rankScore: result.score + (matched?.score ?? 0), - ref: { - kind: 'wiki', - id: result.key, - summary: result.summary || null, - snippet: snippetAround(content, terms), - matchedOn: matched?.matchedOn ?? 'body', - }, - }); - } - return records.sort((left, right) => right.rankScore - left.rankScore || left.ref.id.localeCompare(right.ref.id)); -} - -async function slCandidates( - project: KtxLocalProject, - input: KtxDiscoverDataInput, - kinds: ReadonlySet, - terms: readonly string[], -): Promise { - const records: CandidateRecord[] = []; - for (const connectionId of configuredConnectionIds(project, input.connectionId)) { - const sources = await loadLocalSlSourceRecords(project, { connectionId }).catch(() => []); - for (const sourceRecord of sources) { - const source = sourceRecord.source; - if (hasKind(kinds, 'sl_source')) { - const description = resolveDescription(source.descriptions, { priority: DEFAULT_PRIORITY }); - const matched = bestField( - [ - { matchedOn: 'name', text: source.name, weight: 1.2 }, - { matchedOn: 'description', text: description, weight: 1 }, - { matchedOn: 'display', text: source.table ?? source.sql ?? null, weight: 0.8 }, - ], - terms, - ); - if (matched) { - records.push({ - rankScore: matched.score, - ref: { - kind: 'sl_source', - id: source.name, - connectionId, - summary: description, - snippet: - matched.matchedOn === 'description' - ? snippetAround(description, terms) - : cap200(`${source.name}: ${[...source.measures.map((measure) => measure.name), ...source.columns.map((column) => column.name)].slice(0, 3).join(', ')}`), - matchedOn: matched.matchedOn, - }, - }); - } - } - - if (hasKind(kinds, 'sl_measure')) { - for (const measure of source.measures) { - const matched = bestField( - [ - { matchedOn: 'name', text: measure.name, weight: 1.2 }, - { matchedOn: 'description', text: measure.description, weight: 1 }, - { matchedOn: 'expr', text: measure.expr, weight: 0.9 }, - ], - terms, - ); - if (matched) { - records.push({ - rankScore: matched.score, - ref: { - kind: 'sl_measure', - id: `${source.name}.${measure.name}`, - connectionId, - summary: measure.description ?? null, - snippet: cap200(measure.expr), - matchedOn: matched.matchedOn, - }, - }); - } - } - } - - if (hasKind(kinds, 'sl_dimension')) { - for (const column of source.columns) { - const description = resolveDescription(column.descriptions, { priority: DEFAULT_PRIORITY }); - const matched = bestField( - [ - { matchedOn: 'name', text: column.name, weight: 1.2 }, - { matchedOn: 'description', text: description, weight: 1 }, - { matchedOn: 'expr', text: column.expr, weight: 0.9 }, - ], - terms, - ); - if (matched) { - records.push({ - rankScore: matched.score, - ref: { - kind: 'sl_dimension', - id: `${source.name}.${column.name}`, - connectionId, - summary: description, - snippet: cap200(`${column.name} (${column.type})`), - matchedOn: matched.matchedOn, - }, - }); - } - } - } - } - } - return records.sort((left, right) => right.rankScore - left.rankScore || left.ref.id.localeCompare(right.ref.id)); -} - -async function rawCandidates( - project: KtxLocalProject, - input: KtxDiscoverDataInput, - kinds: ReadonlySet, - terms: readonly string[], -): Promise { - const records: CandidateRecord[] = []; - for (const connectionId of configuredConnectionIds(project, input.connectionId)) { - const scan = await latestScan(project, connectionId); - if (!scan) { - continue; - } - for (const table of scan.tables) { - const ref = tableRef(table); - const display = displayForTable(ref); - const tableDescription = resolveDescription(table.descriptions, { priority: DEFAULT_PRIORITY }) ?? table.comment; - if (hasKind(kinds, 'table')) { - const matched = bestField( - [ - { matchedOn: 'name', text: table.name, weight: 1.2 }, - { matchedOn: 'display', text: display, weight: 1.1 }, - { matchedOn: 'description', text: tableDescription, weight: 1 }, - { matchedOn: 'comment', text: table.comment, weight: 1 }, - ], - terms, - ); - if (matched) { - records.push({ - rankScore: matched.score, - ref: { - kind: 'table', - id: display, - connectionId, - tableRef: ref, - summary: tableDescription, - snippet: - matched.matchedOn === 'description' || matched.matchedOn === 'comment' - ? snippetAround(matched.text, terms) - : cap200(table.columns.slice(0, 5).map((column) => column.name).join(', ')), - matchedOn: matched.matchedOn, - }, - }); - } - } - - if (hasKind(kinds, 'column')) { - for (const column of table.columns) { - const columnDescription = resolveDescription(column.descriptions, { priority: DEFAULT_PRIORITY }) ?? column.comment; - const samples = (column.sampleValues ?? []).map((value) => String(value)).slice(0, 5); - const matched = bestField( - [ - { matchedOn: 'name', text: column.name, weight: 1.2 }, - { matchedOn: 'display', text: `${display}.${column.name}`, weight: 1.1 }, - { matchedOn: 'description', text: columnDescription, weight: 1 }, - { matchedOn: 'comment', text: column.comment, weight: 1 }, - { matchedOn: 'sample_value', text: samples.join(' '), weight: 0.9 }, - ], - terms, - ); - if (matched) { - records.push({ - rankScore: matched.score, - ref: { - kind: 'column', - id: `${display}.${column.name}`, - connectionId, - tableRef: ref, - columnName: column.name, - summary: columnDescription, - snippet: - matched.matchedOn === 'sample_value' - ? cap200(`${column.nativeType} - samples: ${samples.join(', ')}`) - : matched.matchedOn === 'description' || matched.matchedOn === 'comment' - ? snippetAround(matched.text, terms) - : cap200(column.nativeType), - matchedOn: matched.matchedOn, - }, - }); - } - } - } - } - } - return records.sort((left, right) => right.rankScore - left.rankScore || left.ref.id.localeCompare(right.ref.id)); -} - -function generator(name: string, candidates: CandidateRecord[], refsByKey: Map>): SearchCandidateGenerator { - candidates.forEach((candidate) => refsByKey.set(`${candidate.ref.kind}:${candidate.ref.connectionId ?? ''}:${candidate.ref.id}`, candidate.ref)); - return { - lane: name, - weight: 1, - async generate() { - return { - candidates: candidates.map((candidate, index) => ({ - id: `${candidate.ref.kind}:${candidate.ref.connectionId ?? ''}:${candidate.ref.id}`, - rank: index + 1, - rawScore: candidate.rankScore, - })), - }; - }, - }; -} - -function hydrate(fused: FusedSearchCandidate[], refsByKey: Map>): KtxDiscoverDataRef[] { - const maxScore = Math.max(...fused.map((candidate) => candidate.score), 0); - return fused - .map((candidate) => { - const ref = refsByKey.get(candidate.id); - if (!ref) { - return null; - } - return { - ...ref, - score: maxScore > 0 ? Number((candidate.score / maxScore).toFixed(6)) : 0, - }; - }) - .filter((result): result is KtxDiscoverDataRef => result !== null); -} - -export function createKtxDiscoverDataService( - project: KtxLocalProject, - options: KtxDiscoverDataServiceOptions = {}, -): { search(input: KtxDiscoverDataInput): Promise } { - return { - async search(input) { - const limit = Math.max(1, Math.min(input.limit ?? 15, 50)); - const query = input.query.trim(); - if (!query) { - return []; - } - const kinds = new Set(input.kinds ?? ALL_KINDS); - const terms = queryTerms(query); - const refsByKey = new Map>(); - const generators: SearchCandidateGenerator[] = []; - - if (hasKind(kinds, 'wiki')) { - generators.push(generator('wiki', await wikiCandidates(project, { ...input, limit }, options, terms), refsByKey)); - } - if (hasKind(kinds, 'sl_source') || hasKind(kinds, 'sl_measure') || hasKind(kinds, 'sl_dimension')) { - generators.push(generator('semantic_layer', await slCandidates(project, { ...input, limit }, kinds, terms), refsByKey)); - } - if (hasKind(kinds, 'table') || hasKind(kinds, 'column')) { - generators.push(generator('raw_schema', await rawCandidates(project, { ...input, limit }, kinds, terms), refsByKey)); - } - if (generators.length === 0) { - return []; - } - - const result = await new HybridSearchCore().search({ - queryText: query, - limit, - generators, - laneWeights: { wiki: 1, semantic_layer: 1, raw_schema: 1 }, - }); - return hydrate(result.results, refsByKey); - }, - }; -} -``` - -- [ ] **Step 4: Export the service** - -In `packages/context/src/search/index.ts`, add: - -```typescript -export { createKtxDiscoverDataService } from './discover.js'; -export type { - KtxDiscoverDataInput, - KtxDiscoverDataKind, - KtxDiscoverDataMatchedOn, - KtxDiscoverDataRef, - KtxDiscoverDataResponse, - KtxDiscoverDataServiceOptions, -} from './discover.js'; -``` - -- [ ] **Step 5: Run service tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/search/discover.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the service** - -Run: - -```bash -git add packages/context/src/search/discover.ts packages/context/src/search/discover.test.ts packages/context/src/search/index.ts -git commit -m "feat: add MCP discover data service" -``` - -Expected: commit succeeds. - -## Task 2: Register `discover_data` In The MCP Tool Surface - -**Files:** -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/context-tools.ts` -- Modify: `packages/context/src/mcp/server.test.ts` -- Modify: `packages/context/src/mcp/index.ts` - -- [ ] **Step 1: Write failing MCP registration test** - -In `packages/context/src/mcp/server.test.ts`, extend the import from `./types.js` to include: - -```typescript - KtxDiscoverDataMcpPort, -``` - -Add this test after the `dictionary_search` registration test: - -```typescript - it('registers discover_data when the host provides a discover port', async () => { - const fake = makeFakeServer(); - const discover: KtxDiscoverDataMcpPort = { - search: vi.fn().mockResolvedValue([ - { - kind: 'table', - id: 'public.orders', - score: 1, - summary: 'Orders table', - snippet: 'id, status', - matchedOn: 'name', - connectionId: 'warehouse', - tableRef: { catalog: null, db: 'public', name: 'orders' }, - }, - ]), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { discover }, - }); - - expect(fake.tools.map((tool) => tool.name)).toEqual(['discover_data']); - await expect( - getTool(fake.tools, 'discover_data').handler({ - query: 'orders', - connectionId: 'warehouse', - kinds: ['table'], - limit: 5, - }), - ).resolves.toMatchObject({ - structuredContent: { - refs: [ - { - kind: 'table', - id: 'public.orders', - connectionId: 'warehouse', - tableRef: { catalog: null, db: 'public', name: 'orders' }, - }, - ], - }, - }); - expect(discover.search).toHaveBeenCalledWith({ - query: 'orders', - connectionId: 'warehouse', - kinds: ['table'], - limit: 5, - }); - }); -``` - -- [ ] **Step 2: Run the failing MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "discover_data" -``` - -Expected: FAIL with an import or type error for `KtxDiscoverDataMcpPort`. - -- [ ] **Step 3: Add MCP discover port types** - -In `packages/context/src/mcp/types.ts`, add this import near the other search/scan imports: - -```typescript -import type { KtxDiscoverDataInput, KtxDiscoverDataResponse } from '../search/index.js'; -``` - -Add this interface after `KtxDictionarySearchMcpPort`: - -```typescript -export interface KtxDiscoverDataMcpPort { - search(input: KtxDiscoverDataInput): Promise; -} -``` - -Add this optional port to `KtxMcpContextPorts`: - -```typescript - discover?: KtxDiscoverDataMcpPort; -``` - -- [ ] **Step 4: Add the Zod schema and registration** - -In `packages/context/src/mcp/context-tools.ts`, add this schema after `dictionarySearchSchema`: - -```typescript -const discoverDataKindSchema = z.enum(['wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column']); - -const discoverDataSchema = z.object({ - query: z.string().min(1), - connectionId: connectionIdSchema.optional(), - kinds: z.array(discoverDataKindSchema).optional(), - limit: z.number().int().min(1).max(50).default(15).optional(), -}); -``` - -Add this registration block after the `dictionary_search` registration block and before `sql_execution`: - -```typescript - if (ports.discover) { - const discover = ports.discover; - registerParsedTool( - server, - 'discover_data', - { - title: 'Discover Data', - description: - 'Search across KTX wiki pages, semantic-layer sources/measures/dimensions, and raw warehouse schema refs.', - inputSchema: discoverDataSchema.shape, - }, - discoverDataSchema, - async (input) => jsonToolResult(await discover.search(input)), - ); - } -``` - -- [ ] **Step 5: Export MCP port types** - -Check `packages/context/src/mcp/index.ts`. If it already exports all types from `./types.js`, leave it unchanged. If it lists individual type exports, add: - -```typescript -export type { KtxDiscoverDataMcpPort } from './types.js'; -``` - -- [ ] **Step 6: Run MCP registration tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "discover_data" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit MCP registration** - -Run: - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/index.ts -git commit -m "feat: expose discover data MCP tool" -``` - -Expected: commit succeeds. - -## Task 3: Wire Local Project MCP Ports - -**Files:** -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write failing local-port test** - -In `packages/context/src/mcp/local-project-ports.test.ts`, add this test inside the existing `describe('createLocalProjectMcpContextPorts', ...)` block: - -```typescript - it('exposes local project discover_data across wiki, semantic-layer, and raw schema', async () => { - await project.fileStore.writeFile( - 'wiki/global/orders-playbook.md', - [ - '---', - 'summary: Paid order operations', - 'tags: [orders]', - 'refs: []', - 'sl_refs: []', - 'usage_mode: auto', - '---', - '', - 'Paid orders are used for customer activity analysis.', - '', - ].join('\n'), - 'ktx', - 'ktx@example.com', - 'seed wiki', - ); - await project.fileStore.writeFile( - 'semantic-layer/warehouse/orders.yaml', - [ - 'name: orders', - 'descriptions:', - ' user: Paid order facts', - 'table: public.orders', - 'grain: [id]', - 'columns:', - ' - name: status', - ' type: string', - ' descriptions:', - ' user: Payment status', - 'measures:', - ' - name: order_count', - ' expr: count(*)', - ' description: Number of paid orders', - '', - ].join('\n'), - 'ktx', - 'ktx@example.com', - 'seed sl', - ); - await project.fileStore.writeFile( - 'raw-sources/warehouse/live-database/sync-1/connection.json', - JSON.stringify({ connectionId: 'warehouse', driver: 'postgres', extractedAt: '2026-05-14T09:00:00.000Z' }, null, 2), - 'ktx', - 'ktx@example.com', - 'seed connection', - ); - await project.fileStore.writeFile( - 'raw-sources/warehouse/live-database/sync-1/tables/public-orders.json', - JSON.stringify( - { - catalog: null, - db: 'public', - name: 'orders', - kind: 'table', - comment: 'Orders table', - estimatedRows: 10, - columns: [ - { - name: 'status', - nativeType: 'text', - normalizedType: 'text', - dimensionType: 'string', - nullable: false, - primaryKey: false, - comment: 'Order status', - sampleValues: ['paid'], - }, - ], - foreignKeys: [], - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed table', - ); - await project.fileStore.writeFile( - 'raw-sources/warehouse/live-database/sync-1/scan-report.json', - JSON.stringify( - { - connectionId: 'warehouse', - driver: 'postgres', - syncId: 'sync-1', - runId: 'scan-1', - trigger: 'mcp', - mode: 'enriched', - dryRun: false, - artifactPaths: { - rawSourcesDir: 'raw-sources/warehouse/live-database/sync-1', - reportPath: 'raw-sources/warehouse/live-database/sync-1/scan-report.json', - manifestShards: [], - enrichmentArtifacts: [], - }, - diffSummary: { - tablesAdded: 1, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 0, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 0, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'completed', - tableDescriptions: 'completed', - columnDescriptions: 'completed', - embeddings: 'skipped', - deterministicRelationships: 'skipped', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'skipped', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 }, - enrichmentState: { resumedStages: [], completedStages: [], failedStages: [] }, - createdAt: '2026-05-14T09:00:00.000Z', - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed scan report', - ); - - const ports = createLocalProjectMcpContextPorts(project); - const results = await ports.discover?.search({ query: 'paid orders', connectionId: 'warehouse', limit: 10 }); - - expect(results).toEqual( - expect.arrayContaining([ - expect.objectContaining({ kind: 'wiki', id: 'orders-playbook' }), - expect.objectContaining({ kind: 'sl_source', id: 'orders', connectionId: 'warehouse' }), - expect.objectContaining({ kind: 'table', id: 'public.orders', connectionId: 'warehouse' }), - ]), - ); - }); -``` - -- [ ] **Step 2: Run the failing local-port test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "discover_data" -``` - -Expected: FAIL because `ports.discover` is undefined. - -- [ ] **Step 3: Wire the local port** - -In `packages/context/src/mcp/local-project-ports.ts`, add `createKtxDiscoverDataService` to the search import block: - -```typescript -import { createKtxDiscoverDataService } from '../search/index.js'; -``` - -Add this port in the `ports` object after `dictionarySearch`: - -```typescript - discover: { - async search(input) { - return createKtxDiscoverDataService(project, { userId: 'local', embeddingService }).search(input); - }, - }, -``` - -- [ ] **Step 4: Run local-port test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "discover_data" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit local-port wiring** - -Run: - -```bash -git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat: wire local discover data MCP port" -``` - -Expected: commit succeeds. - -## Task 4: Verify The Discover Slice - -**Files:** -- Verify: `packages/context/src/search/discover.ts` -- Verify: `packages/context/src/mcp/context-tools.ts` -- Verify: `packages/context/src/mcp/local-project-ports.ts` - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/search/discover.test.ts src/mcp/server.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run context test suite** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. - -- [ ] **Step 4: Check diff hygiene** - -Run: - -```bash -git diff --check -``` - -Expected: no output and exit code 0. - -- [ ] **Step 5: Document remaining v1 blockers in handoff** - -Run: - -```bash -test -e packages/context/src/search/discover.ts; printf 'discover:%s\n' "$?" -test -e packages/cli/src/commands/mcp-commands.ts; printf 'mcp-commands:%s\n' "$?" -test -e packages/cli/src/managed-mcp-daemon.ts; printf 'managed-mcp:%s\n' "$?" -test -e packages/cli/src/skills/research/SKILL.md; printf 'research-skill:%s\n' "$?" -``` - -Expected after this plan is implemented: - -```text -discover:0 -mcp-commands:1 -managed-mcp:1 -research-skill:1 -``` - -- [ ] **Step 6: Commit verification notes if code changed during verification** - -If verification required code fixes, run: - -```bash -git status --short -git add packages/context/src/search/discover.ts packages/context/src/search/discover.test.ts packages/context/src/search/index.ts packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts packages/context/src/mcp/index.ts -git commit -m "test: verify MCP discover data" -``` - -Expected: commit succeeds only when there are verification fixes to commit. If `git status --short` is empty, skip this commit. - -## Self-Review - -- Spec coverage: this plan covers the MCP-shaped `discover_data` input/output contract, kind filtering, optional `connectionId`, RRF fusion across wiki/SL/raw lanes, deterministic summary/snippet provenance, raw `tableRef` and `columnName`, score normalization, local project MCP registration, and freshness by re-reading artifacts on every call. -- Remaining v1-blocking spec coverage after this slice: HTTP Streamable MCP daemon, `ktx mcp` CLI lifecycle commands, setup-agent MCP config writers/snippet printers, `ktx-research` skill installation, and ingest-side `connectionName` contract convergence. -- Placeholder scan: no placeholder or deferred-work wording remains in this plan. -- Type consistency: `KtxDiscoverDataInput`, `KtxDiscoverDataRef`, `KtxDiscoverDataResponse`, and `KtxDiscoverDataMcpPort` are defined before use and match the MCP/local-port registration snippets. diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md deleted file mode 100644 index db165c02..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md +++ /dev/null @@ -1,1175 +0,0 @@ -# Research Agent MCP Entity Details Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the MCP-shaped `entity_details` tool so external research agents can inspect raw table and column metadata from the latest scan snapshot. - -**Architecture:** Build a focused scan service over persisted `raw-sources//live-database/` artifacts, using `scan-report.json` as the latest scan identity and `readLocalScanStructuralSnapshot()` as the schema reader. Register `entity_details` as an MCP context tool with pure structured output, then expose it through local project MCP ports. - -**Tech Stack:** TypeScript, Vitest, Zod, KTX local file store, KTX scan artifacts, KTX MCP context ports. - ---- - -## Current Audit - -Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md` - -Implemented v1 slice: - -- `docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md` is implemented. Evidence in current source: - - Python sqlglot validation exists at `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`. - - `POST /sql/validate-read-only` exists at `python/ktx-daemon/src/ktx_daemon/app.py`. - - `SqlAnalysisPort.validateReadOnly()` exists at `packages/context/src/sql-analysis/ports.ts`. - - MCP `sql_execution` registration exists at `packages/context/src/mcp/context-tools.ts`. - - Local MCP SQL execution validates through `SqlAnalysisPort` before connector execution in `packages/context/src/mcp/local-project-ports.ts`. - -V1-blocking gaps after that slice: - -- `entity_details` is not registered on the MCP surface. -- `discover_data` is not registered on the MCP surface. -- `dictionary_search` is not registered on the MCP surface. -- `ktx mcp start|stop|status|logs` and the HTTP Streamable MCP daemon do not exist. -- `ktx setup-agents` does not install MCP client config or a `ktx-research` skill. -- Ingest-side warehouse verification still uses `connectionName` contracts in places; the MCP surface must use `connectionId`. - -This plan covers only the next dependency-aware v1 blocker: MCP `entity_details`. Later plans still need to cover `dictionary_search`, `discover_data`, the HTTP daemon, and setup-agent/research-skill installation. - -## File Structure - -Create: - -- `packages/context/src/scan/entity-details.ts` - - Reads latest live-database scan artifacts for a connection. - - Resolves driver display strings or structured table refs. - - Returns structured table/column metadata and structured per-entity errors. -- `packages/context/src/scan/entity-details.test.ts` - - Covers latest-scan selection, display-string resolution, structured refs, column filtering, ambiguity, missing scan, and missing columns. - -Modify: - -- `packages/context/src/scan/index.ts` - - Export the new service and types. -- `packages/context/src/mcp/types.ts` - - Add `KtxEntityDetailsMcpPort` and response types to `KtxMcpContextPorts`. -- `packages/context/src/mcp/context-tools.ts` - - Add the `entity_details` input schema and registration. -- `packages/context/src/mcp/server.test.ts` - - Assert the MCP tool registration and structured output. -- `packages/context/src/mcp/local-project-ports.ts` - - Wire the local project port to the scan entity-details service. -- `packages/context/src/mcp/local-project-ports.test.ts` - - Cover local-port `entity_details` success and missing-scan behavior. -- `packages/context/src/mcp/index.ts` - - Export the new MCP port/response types. - -## Task 1: Add The Scan Entity Details Service - -**Files:** -- Create: `packages/context/src/scan/entity-details.test.ts` -- Create: `packages/context/src/scan/entity-details.ts` -- Modify: `packages/context/src/scan/index.ts` - -- [ ] **Step 1: Write failing service tests** - -Create `packages/context/src/scan/entity-details.test.ts`: - -```typescript -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; -import { createKtxEntityDetailsService } from './entity-details.js'; -import type { KtxConnectionDriver, KtxScanReport, KtxSchemaTable } from './types.js'; - -describe('createKtxEntityDetailsService', () => { - let tempDir: string; - let project: KtxLocalProject; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-entity-details-service-')); - project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - function scanReport(input: { - connectionId: string; - syncId: string; - runId: string; - driver?: KtxConnectionDriver; - createdAt?: string; - }): KtxScanReport { - const rawSourcesDir = `raw-sources/${input.connectionId}/live-database/${input.syncId}`; - return { - connectionId: input.connectionId, - driver: input.driver ?? 'postgres', - syncId: input.syncId, - runId: input.runId, - trigger: 'mcp', - mode: 'structural', - dryRun: false, - artifactPaths: { - rawSourcesDir, - reportPath: `${rawSourcesDir}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts: [], - }, - diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 1 }, - manifestShardsWritten: 0, - structuralSyncStats: { tablesWritten: 1, tablesDeleted: 0, foreignKeysWritten: 0 }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'skipped', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'skipped', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 }, - enrichmentState: { resumedStages: [], completedStages: [], failedStages: [] }, - createdAt: input.createdAt ?? '2026-05-14T09:00:00.000Z', - }; - } - - function ordersTable(input: { db?: string | null; estimatedRows?: number | null } = {}): KtxSchemaTable { - return { - catalog: null, - db: input.db ?? 'public', - name: 'orders', - kind: 'table', - comment: 'Customer orders', - estimatedRows: input.estimatedRows ?? 12, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: 'Order id', - }, - { - name: 'status', - nativeType: 'text', - normalizedType: 'text', - dimensionType: 'string', - nullable: false, - primaryKey: false, - comment: 'Order status', - }, - ], - foreignKeys: [ - { - fromColumn: 'customer_id', - toCatalog: null, - toDb: 'public', - toTable: 'customers', - toColumn: 'id', - constraintName: 'orders_customer_id_fkey', - }, - ], - }; - } - - async function seedScan(input: { - connectionId?: string; - syncId: string; - runId: string; - driver?: KtxConnectionDriver; - extractedAt?: string; - tables?: KtxSchemaTable[]; - }): Promise { - const connectionId = input.connectionId ?? 'warehouse'; - const report = scanReport({ - connectionId, - syncId: input.syncId, - runId: input.runId, - driver: input.driver, - createdAt: input.extractedAt, - }); - const root = report.artifactPaths.rawSourcesDir; - await project.fileStore.writeFile( - `${root}/connection.json`, - JSON.stringify( - { - connectionId, - driver: report.driver, - extractedAt: input.extractedAt ?? report.createdAt, - scope: { schemas: ['public'] }, - }, - null, - 2, - ), - 'ktx', - 'ktx@example.com', - 'seed connection', - ); - for (const table of input.tables ?? [ordersTable()]) { - await project.fileStore.writeFile( - `${root}/tables/${table.db ?? 'default'}-${table.name}.json`, - JSON.stringify(table, null, 2), - 'ktx', - 'ktx@example.com', - `seed ${table.name}`, - ); - } - await project.fileStore.writeFile( - `${root}/scan-report.json`, - JSON.stringify(report, null, 2), - 'ktx', - 'ktx@example.com', - 'seed scan report', - ); - } - - it('returns the latest scan snapshot table details for a display string', async () => { - await seedScan({ syncId: 'sync-1', runId: 'scan-old', extractedAt: '2026-05-14T08:00:00.000Z' }); - await seedScan({ - syncId: 'sync-2', - runId: 'scan-new', - extractedAt: '2026-05-14T09:00:00.000Z', - tables: [ordersTable({ estimatedRows: 99 })], - }); - const service = createKtxEntityDetailsService(project); - - const result = await service.read({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders' }], - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - ok: true, - connectionId: 'warehouse', - display: 'public.orders', - estimatedRows: 99, - snapshot: { - syncId: 'sync-2', - scanRunId: 'scan-new', - extractedAt: '2026-05-14T09:00:00.000Z', - }, - columns: [ - { name: 'id', nativeType: 'integer', primaryKey: true }, - { name: 'status', nativeType: 'text', nullable: false }, - ], - }); - }); - - it('filters requested columns while keeping full-table foreign keys', async () => { - await seedScan({ syncId: 'sync-1', runId: 'scan-1' }); - const service = createKtxEntityDetailsService(project); - - const result = await service.read({ - connectionId: 'warehouse', - entities: [{ table: { catalog: null, db: 'public', name: 'orders' }, columns: ['status'] }], - }); - - expect(result.results[0]).toMatchObject({ - ok: true, - columns: [{ name: 'status' }], - foreignKeys: [ - { - fromColumn: 'customer_id', - toDb: 'public', - toTable: 'customers', - toColumn: 'id', - }, - ], - }); - }); - - it('returns a structured missing-scan error', async () => { - const service = createKtxEntityDetailsService(project); - - const result = await service.read({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders' }], - }); - - expect(result.results).toEqual([ - { - ok: false, - connectionId: 'warehouse', - table: 'public.orders', - error: { - code: 'scan_missing', - message: 'No live-database scan found for connection "warehouse"; run `ktx ingest warehouse` or `ktx scan warehouse`.', - }, - }, - ]); - }); - - it('reports ambiguous bare table names across schemas', async () => { - await seedScan({ - syncId: 'sync-1', - runId: 'scan-1', - tables: [ordersTable({ db: 'public' }), ordersTable({ db: 'archive' })], - }); - const service = createKtxEntityDetailsService(project); - - const result = await service.read({ - connectionId: 'warehouse', - entities: [{ table: 'orders' }], - }); - - expect(result.results[0]).toMatchObject({ - ok: false, - error: { - code: 'ambiguous_table', - candidates: [ - { tableRef: { catalog: null, db: 'archive', name: 'orders' }, display: 'archive.orders' }, - { tableRef: { catalog: null, db: 'public', name: 'orders' }, display: 'public.orders' }, - ], - }, - }); - }); - - it('reports missing requested columns with available column candidates', async () => { - await seedScan({ syncId: 'sync-1', runId: 'scan-1' }); - const service = createKtxEntityDetailsService(project); - - const result = await service.read({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders', columns: ['status', 'plan_tier'] }], - }); - - expect(result.results[0]).toMatchObject({ - ok: false, - error: { - code: 'column_not_found', - message: 'Column(s) not found on public.orders: plan_tier', - candidates: ['id', 'status'], - }, - }); - }); -}); -``` - -- [ ] **Step 2: Run failing service tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/scan/entity-details.test.ts -``` - -Expected: FAIL because `packages/context/src/scan/entity-details.ts` does not exist. - -- [ ] **Step 3: Implement the service** - -Create `packages/context/src/scan/entity-details.ts`: - -```typescript -import type { KtxLocalProject } from '../project/index.js'; -import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; -import type { - KtxConnectionDriver, - KtxScanReport, - KtxSchemaColumn, - KtxSchemaSnapshot, - KtxSchemaTable, - KtxTableRef, -} from './types.js'; - -export type KtxEntityDetailsTableInput = string | KtxTableRef; - -export interface KtxEntityDetailsInput { - connectionId: string; - entities: Array<{ - table: KtxEntityDetailsTableInput; - columns?: string[]; - }>; -} - -export interface KtxEntityDetailsSnapshotInfo { - syncId: string; - extractedAt: string; - scanRunId: string | null; -} - -export interface KtxEntityDetailsColumn { - name: string; - nativeType: string; - normalizedType: string; - dimensionType: KtxSchemaColumn['dimensionType']; - nullable: boolean; - primaryKey: boolean; - comment: string | null; -} - -export interface KtxEntityDetailsRecord { - ok: true; - connectionId: string; - tableRef: KtxTableRef; - display: string; - kind: KtxSchemaTable['kind']; - comment: string | null; - estimatedRows: number | null; - columns: KtxEntityDetailsColumn[]; - foreignKeys: KtxSchemaTable['foreignKeys']; - snapshot: KtxEntityDetailsSnapshotInfo; -} - -export type KtxEntityDetailsErrorCode = 'scan_missing' | 'table_not_found' | 'ambiguous_table' | 'column_not_found'; - -export interface KtxEntityDetailsErrorResult { - ok: false; - connectionId: string; - table: KtxEntityDetailsTableInput; - snapshot?: KtxEntityDetailsSnapshotInfo; - error: { - code: KtxEntityDetailsErrorCode; - message: string; - candidates?: Array<{ tableRef: KtxTableRef; display: string }> | string[]; - }; -} - -export interface KtxEntityDetailsResponse { - results: Array; -} - -interface LatestScan { - report: KtxScanReport; - snapshot: KtxSchemaSnapshot; -} - -interface ResolveResult { - table: KtxSchemaTable | null; - error?: Omit & { message: string }; -} - -function normalize(value: string | null | undefined): string { - return (value ?? '').toLowerCase(); -} - -function refsEqual(left: KtxTableRef, right: KtxTableRef): boolean { - return ( - normalize(left.catalog) === normalize(right.catalog) && - normalize(left.db) === normalize(right.db) && - normalize(left.name) === normalize(right.name) - ); -} - -function cleanIdentifierPart(part: string): string { - return part.trim().replace(/^["'`\[]|["'`\]]$/g, ''); -} - -function splitDisplay(display: string): string[] { - return display - .trim() - .split('.') - .map(cleanIdentifierPart) - .filter(Boolean); -} - -function displayForTable(driver: KtxConnectionDriver, table: KtxTableRef): string { - if (driver === 'sqlite') { - return table.name; - } - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - -function tableRef(table: KtxSchemaTable): KtxTableRef { - return { catalog: table.catalog, db: table.db, name: table.name }; -} - -function candidateList(driver: KtxConnectionDriver, tables: KtxSchemaTable[]): Array<{ tableRef: KtxTableRef; display: string }> { - return tables - .map((table) => ({ - tableRef: tableRef(table), - display: displayForTable(driver, table), - })) - .sort((left, right) => left.display.localeCompare(right.display)); -} - -function parseDisplayRef(driver: KtxConnectionDriver, display: string): KtxTableRef | null { - const parts = splitDisplay(display); - if (driver === 'sqlite') { - return parts.length === 1 ? { catalog: null, db: null, name: parts[0]! } : null; - } - if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { - return parts.length === 3 ? { catalog: parts[0]!, db: parts[1]!, name: parts[2]! } : null; - } - if (parts.length === 2) { - return { catalog: null, db: parts[0]!, name: parts[1]! }; - } - if (parts.length === 3) { - return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; - } - return null; -} - -function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableInput): ResolveResult { - if (typeof input !== 'string') { - const table = snapshot.tables.find((candidate) => refsEqual(candidate, input)) ?? null; - return table - ? { table } - : { - table: null, - error: { - code: 'table_not_found', - message: `Table not found in latest scan: ${displayForTable(snapshot.driver, input)}`, - candidates: candidateList(snapshot.driver, snapshot.tables), - }, - }; - } - - const parsed = parseDisplayRef(snapshot.driver, input); - if (parsed) { - const table = snapshot.tables.find((candidate) => refsEqual(candidate, parsed)) ?? null; - return table - ? { table } - : { - table: null, - error: { - code: 'table_not_found', - message: `Table not found in latest scan: ${input}`, - candidates: candidateList(snapshot.driver, snapshot.tables), - }, - }; - } - - const byName = snapshot.tables.filter((candidate) => normalize(candidate.name) === normalize(input)); - if (byName.length === 1) { - return { table: byName[0]! }; - } - if (byName.length > 1) { - return { - table: null, - error: { - code: 'ambiguous_table', - message: `Table name "${input}" is ambiguous across schemas/catalogs; pass a structured table ref.`, - candidates: candidateList(snapshot.driver, byName), - }, - }; - } - return { - table: null, - error: { - code: 'table_not_found', - message: `Table not found in latest scan: ${input}`, - candidates: candidateList(snapshot.driver, snapshot.tables), - }, - }; -} - -function toColumn(column: KtxSchemaColumn): KtxEntityDetailsColumn { - return { - name: column.name, - nativeType: column.nativeType, - normalizedType: column.normalizedType, - dimensionType: column.dimensionType, - nullable: column.nullable, - primaryKey: column.primaryKey, - comment: column.comment, - }; -} - -function snapshotInfo(report: KtxScanReport, snapshot: KtxSchemaSnapshot): KtxEntityDetailsSnapshotInfo { - return { - syncId: report.syncId, - extractedAt: snapshot.extractedAt, - scanRunId: report.runId ?? null, - }; -} - -async function readJson(project: KtxLocalProject, path: string): Promise { - return JSON.parse((await project.fileStore.readFile(path)).content) as T; -} - -async function latestScan(project: KtxLocalProject, connectionId: string): Promise { - const root = `raw-sources/${connectionId}/live-database`; - let listed; - try { - listed = await project.fileStore.listFiles(root); - } catch { - return null; - } - const reportPath = listed.files.filter((path) => path.endsWith('/scan-report.json')).sort().at(-1); - if (!reportPath) { - return null; - } - const report = await readJson(project, reportPath); - const rawSourcesDir = report.artifactPaths.rawSourcesDir ?? reportPath.slice(0, -'/scan-report.json'.length); - const snapshot = await readLocalScanStructuralSnapshot({ - project, - connectionId, - driver: report.driver, - rawSourcesDir, - extractedAtFallback: report.createdAt, - }); - return { report, snapshot }; -} - -export function createKtxEntityDetailsService(project: KtxLocalProject) { - return { - async read(input: KtxEntityDetailsInput): Promise { - const scan = await latestScan(project, input.connectionId); - if (!scan) { - return { - results: input.entities.map((entity) => ({ - ok: false, - connectionId: input.connectionId, - table: entity.table, - error: { - code: 'scan_missing', - message: `No live-database scan found for connection "${input.connectionId}"; run \`ktx ingest ${input.connectionId}\` or \`ktx scan ${input.connectionId}\`.`, - }, - })), - }; - } - - const info = snapshotInfo(scan.report, scan.snapshot); - const results: KtxEntityDetailsResponse['results'] = []; - for (const entity of input.entities) { - const resolved = resolveTable(scan.snapshot, entity.table); - if (!resolved.table) { - results.push({ - ok: false, - connectionId: input.connectionId, - table: entity.table, - snapshot: info, - error: resolved.error!, - }); - continue; - } - - const requested = new Set((entity.columns ?? []).map((column) => normalize(column))); - const columns = requested.size - ? resolved.table.columns.filter((column) => requested.has(normalize(column.name))) - : resolved.table.columns; - if (requested.size && columns.length !== requested.size) { - const found = new Set(columns.map((column) => normalize(column.name))); - const missing = [...requested].filter((column) => !found.has(column)); - results.push({ - ok: false, - connectionId: input.connectionId, - table: entity.table, - snapshot: info, - error: { - code: 'column_not_found', - message: `Column(s) not found on ${displayForTable(scan.snapshot.driver, resolved.table)}: ${missing.join(', ')}`, - candidates: resolved.table.columns.map((column) => column.name), - }, - }); - continue; - } - - results.push({ - ok: true, - connectionId: input.connectionId, - tableRef: tableRef(resolved.table), - display: displayForTable(scan.snapshot.driver, resolved.table), - kind: resolved.table.kind, - comment: resolved.table.comment, - estimatedRows: resolved.table.estimatedRows, - columns: columns.map(toColumn), - foreignKeys: resolved.table.foreignKeys, - snapshot: info, - }); - } - return { results }; - }, - }; -} -``` - -In `packages/context/src/scan/index.ts`, add these exports near the other scan-service exports: - -```typescript -export type { - KtxEntityDetailsColumn, - KtxEntityDetailsErrorCode, - KtxEntityDetailsErrorResult, - KtxEntityDetailsInput, - KtxEntityDetailsRecord, - KtxEntityDetailsResponse, - KtxEntityDetailsSnapshotInfo, - KtxEntityDetailsTableInput, -} from './entity-details.js'; -export { createKtxEntityDetailsService } from './entity-details.js'; -``` - -- [ ] **Step 4: Run service tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/scan/entity-details.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the scan service** - -Run: - -```bash -git add packages/context/src/scan/entity-details.ts packages/context/src/scan/entity-details.test.ts packages/context/src/scan/index.ts -git commit -m "feat(context): add scan-backed entity details service" -``` - -## Task 2: Register The MCP `entity_details` Tool - -**Files:** -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/context-tools.ts` -- Modify: `packages/context/src/mcp/server.test.ts` -- Modify: `packages/context/src/mcp/index.ts` - -- [ ] **Step 1: Add MCP port types** - -In `packages/context/src/mcp/types.ts`, add this import near the other type imports: - -```typescript -import type { KtxEntityDetailsInput, KtxEntityDetailsResponse } from '../scan/entity-details.js'; -``` - -Add this interface immediately before `KtxSqlExecutionResponse`: - -```typescript -export interface KtxEntityDetailsMcpPort { - read(input: KtxEntityDetailsInput): Promise; -} -``` - -Add this optional port to `KtxMcpContextPorts`: - -```typescript - entityDetails?: KtxEntityDetailsMcpPort; -``` - -- [ ] **Step 2: Write failing MCP registration test** - -In `packages/context/src/mcp/server.test.ts`, update the `./types.js` import to include `KtxEntityDetailsMcpPort`. - -Add this test after the `sql_execution` registration test: - -```typescript - it('registers entity_details when the host provides an entity-details port', async () => { - const fake = makeFakeServer(); - const entityDetails: KtxEntityDetailsMcpPort = { - read: vi.fn().mockResolvedValue({ - results: [ - { - ok: true, - connectionId: 'warehouse', - tableRef: { catalog: null, db: 'public', name: 'orders' }, - display: 'public.orders', - kind: 'table', - comment: 'Customer orders', - estimatedRows: 12, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: null, - }, - ], - foreignKeys: [], - snapshot: { - syncId: 'sync-1', - extractedAt: '2026-05-14T09:00:00.000Z', - scanRunId: 'scan-1', - }, - }, - ], - }), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { entityDetails }, - }); - - expect(fake.tools.map((tool) => tool.name)).toEqual(['entity_details']); - await expect( - getTool(fake.tools, 'entity_details').handler({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders', columns: ['id'] }], - }), - ).resolves.toMatchObject({ - structuredContent: { - results: [ - { - ok: true, - connectionId: 'warehouse', - display: 'public.orders', - columns: [{ name: 'id' }], - }, - ], - }, - }); - expect(entityDetails.read).toHaveBeenCalledWith({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders', columns: ['id'] }], - }); - }); -``` - -- [ ] **Step 3: Run failing MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t entity_details -``` - -Expected: FAIL because `entity_details` is not registered. - -- [ ] **Step 4: Add schema and registration** - -In `packages/context/src/mcp/context-tools.ts`, add this schema after `scanArtifactReadSchema` and before `sqlExecutionSchema`: - -```typescript -const entityDetailsTableRefSchema = z.object({ - catalog: z.string().nullable(), - db: z.string().nullable(), - name: z.string().min(1), -}); - -const entityDetailsSchema = z.object({ - connectionId: connectionIdSchema, - entities: z - .array( - z.object({ - table: z.union([z.string().min(1), entityDetailsTableRefSchema]), - columns: z.array(z.string().min(1)).optional(), - }), - ) - .min(1) - .max(20), -}); -``` - -Add this registration block in `registerKtxContextTools`, after the semantic-layer block and before the `sqlExecution` block: - -```typescript - if (ports.entityDetails) { - const entityDetails = ports.entityDetails; - registerParsedTool( - server, - 'entity_details', - { - title: 'Entity Details', - description: - 'Read raw table and column metadata from the latest KTX live-database scan snapshot.', - inputSchema: entityDetailsSchema.shape, - }, - entityDetailsSchema, - async (input) => jsonToolResult(await entityDetails.read(input)), - ); - } -``` - -In `packages/context/src/mcp/index.ts`, add `KtxEntityDetailsMcpPort` to the exported type list. - -- [ ] **Step 5: Run MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t entity_details -``` - -Expected: PASS. - -- [ ] **Step 6: Commit MCP registration** - -Run: - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/index.ts -git commit -m "feat(context): register MCP entity details tool" -``` - -## Task 3: Wire Local Project MCP Ports - -**Files:** -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write failing local-port tests** - -In `packages/context/src/mcp/local-project-ports.test.ts`, add this helper after `testConnector`: - -```typescript - async function seedScanReport(projectDir: string, syncId = 'sync-1'): Promise { - const root = `raw-sources/warehouse/live-database/${syncId}`; - await mkdir(join(projectDir, root, 'tables'), { recursive: true }); - await writeFile( - join(projectDir, root, 'connection.json'), - JSON.stringify( - { - connectionId: 'warehouse', - driver: 'postgres', - extractedAt: '2026-05-14T09:00:00.000Z', - scope: { schemas: ['public'] }, - }, - null, - 2, - ), - 'utf-8', - ); - await writeFile( - join(projectDir, root, 'tables', 'orders.json'), - JSON.stringify( - { - catalog: null, - db: 'public', - name: 'orders', - kind: 'table', - comment: 'Customer orders', - estimatedRows: 12, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: null, - }, - ], - foreignKeys: [], - }, - null, - 2, - ), - 'utf-8', - ); - await writeFile( - join(projectDir, root, 'scan-report.json'), - JSON.stringify( - { - connectionId: 'warehouse', - driver: 'postgres', - syncId, - runId: 'scan-1', - trigger: 'mcp', - mode: 'structural', - dryRun: false, - artifactPaths: { - rawSourcesDir: root, - reportPath: `${root}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts: [], - }, - diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 1 }, - manifestShardsWritten: 0, - structuralSyncStats: { tablesWritten: 1, tablesDeleted: 0, foreignKeysWritten: 0 }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'skipped', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'skipped', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 }, - enrichmentState: { resumedStages: [], completedStages: [], failedStages: [] }, - createdAt: '2026-05-14T09:00:00.000Z', - }, - null, - 2, - ), - 'utf-8', - ); - } -``` - -Add these tests after the MCP SQL tests: - -```typescript - it('exposes local scan entity details through MCP ports', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - await seedScanReport(project.projectDir); - const ports = createLocalProjectMcpContextPorts(project); - - await expect( - ports.entityDetails?.read({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders', columns: ['id'] }], - }), - ).resolves.toMatchObject({ - results: [ - { - ok: true, - connectionId: 'warehouse', - display: 'public.orders', - columns: [{ name: 'id', nativeType: 'integer' }], - snapshot: { syncId: 'sync-1', scanRunId: 'scan-1' }, - }, - ], - }); - }); - - it('returns a structured local entity-details error when no scan exists', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - const ports = createLocalProjectMcpContextPorts(project); - - await expect( - ports.entityDetails?.read({ - connectionId: 'warehouse', - entities: [{ table: 'public.orders' }], - }), - ).resolves.toMatchObject({ - results: [ - { - ok: false, - connectionId: 'warehouse', - error: { code: 'scan_missing' }, - }, - ], - }); - }); -``` - -- [ ] **Step 2: Run failing local-port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "entity details" -``` - -Expected: FAIL because `ports.entityDetails` is undefined. - -- [ ] **Step 3: Wire the service into local ports** - -In `packages/context/src/mcp/local-project-ports.ts`, update the scan import block to include `createKtxEntityDetailsService`: - -```typescript - createKtxEntityDetailsService, -``` - -In the initial `ports` object returned by `createLocalProjectMcpContextPorts`, add this sibling after `semanticLayer` and before the closing `};`: - -```typescript - entityDetails: { - async read(input) { - return createKtxEntityDetailsService(project).read(input); - }, - }, -``` - -- [ ] **Step 4: Run local-port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "entity details" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit local-port wiring** - -Run: - -```bash -git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat(context): expose local MCP entity details" -``` - -## Task 4: Verification - -**Files:** -- Verify: all files changed in Tasks 1-3 - -- [ ] **Step 1: Run focused context tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/scan/entity-details.test.ts src/mcp/server.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run dead-code check for new exports** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If Knip reports unrelated pre-existing findings, record the exact unrelated findings and do not broaden this entity-details slice. - -- [ ] **Step 4: Confirm remaining v1 blockers still need later plans** - -Run: - -```bash -test -e packages/context/src/sl/dictionary-search.ts; printf 'dictionary-search:%s\n' "$?" -test -e packages/context/src/search/discover.ts; printf 'discover:%s\n' "$?" -test -e packages/cli/src/commands/mcp-commands.ts; printf 'mcp-commands:%s\n' "$?" -test -e packages/cli/src/skills/research/SKILL.md; printf 'research-skill:%s\n' "$?" -``` - -Expected: - -```text -dictionary-search:1 -discover:1 -mcp-commands:1 -research-skill:1 -``` - -These markers mean this plan landed `entity_details` only and did not claim the remaining research-agent v1 work. - -- [ ] **Step 5: Commit verification-only doc changes if any** - -Run: - -```bash -git status --short -``` - -Expected: no uncommitted source changes after the task commits. If verification updates this plan document, commit only the plan document with: - -```bash -git add docs/superpowers/plans/2026-05-14-research-agent-mcp-entity-details.md -git commit -m "docs: record research MCP entity details plan" -``` - -## Self-Review - -- Spec coverage for this slice: covers MCP `entity_details`, latest scan freshness by reading `scan-report.json` on each call, structured table refs, driver display strings, column filtering, FK preservation, snapshot freshness, and structured errors. -- Remaining spec coverage after this slice: `dictionary_search`, `discover_data`, `ktx mcp` HTTP daemon, setup-agent MCP config, and `ktx-research` skill are still v1-blocking and need later plans. -- Type consistency: `KtxEntityDetailsInput` is reused by the scan service, MCP port, schema parser, and local project port. diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-http-daemon.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-http-daemon.md deleted file mode 100644 index 00645dec..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-http-daemon.md +++ /dev/null @@ -1,1561 +0,0 @@ -# Research Agent MCP HTTP Daemon Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the HTTP-only `ktx mcp start|stop|status|logs` daemon so external MCP clients can reach the already implemented KTX research tools. - -**Architecture:** Keep the MCP tool contracts in `@ktx/context` and add CLI-owned HTTP hosting/lifecycle code. The public `ktx mcp start` command either runs a foreground HTTP server or spawns a hidden foreground child command, persists daemon state to `.ktx/mcp.json`, and writes logs to `.ktx/logs/mcp.log`; the HTTP server uses stateful `StreamableHTTPServerTransport` sessions with explicit host/origin/token checks. - -**Tech Stack:** TypeScript, Node 22 `node:http`, Commander, `@modelcontextprotocol/sdk@1.29.0`, Zod, Vitest, KTX managed Python daemon helpers. - ---- - -## Audit Summary - -Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md` - -Implemented v1 slices confirmed in current source: - -- MCP `sql_execution` is implemented and parser-gated: `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` validates SQL with sqlglot, `python/ktx-daemon/src/ktx_daemon/app.py` exposes `/sql/validate-read-only`, `packages/context/src/mcp/context-tools.ts` registers `sql_execution`, and `packages/context/src/mcp/local-project-ports.ts` only exposes it when both SQL analysis and local scan connector creation are available. -- MCP `entity_details` is implemented: `packages/context/src/scan/entity-details.ts`, `KtxEntityDetailsMcpPort`, context-tool registration, and local project wiring all exist. -- MCP `dictionary_search` is implemented: `packages/context/src/sl/dictionary-search.ts`, `KtxDictionarySearchMcpPort`, context-tool registration, and local project wiring all exist. -- MCP `discover_data` is implemented: `packages/context/src/search/discover.ts`, `KtxDiscoverDataMcpPort`, context-tool registration, and local project wiring all exist. - -Remaining v1-blocking gaps: - -- `ktx mcp start|stop|status|logs` and the HTTP Streamable MCP daemon are missing. There is no `packages/cli/src/commands/mcp-commands.ts`, no `packages/cli/src/managed-mcp-daemon.ts`, and `packages/cli/src/cli-program.ts` does not register an `mcp` subtree. -- `ktx setup-agents` does not install MCP client config entries or the `ktx-research` skill. `plannedKtxAgentFiles()` still installs only the existing `ktx` skill/rules. -- Ingest-side warehouse verification tools still use `connectionName`, not the spec-required `connectionId`, and `WarehouseCatalogService` still exposes `connectionName` in its service contract. - -Non-blocking gaps: - -- TLS, audit logging, rate limiting, per-tool authorization, OS-level autostart, stdio MCP transport, and multi-project switching remain explicitly out of scope for v1. - -This plan covers only the next dependency-aware blocker: the HTTP Streamable MCP daemon and `ktx mcp` lifecycle command subtree. After this plan lands, the remaining v1 plans are setup-agent/research-skill installation and ingest warehouse-verification contract convergence. - -## Documentation Notes - -- Context7 was checked for current MCP TypeScript SDK Streamable HTTP examples. -- The local `@modelcontextprotocol/sdk@1.29.0` package metadata was checked with `pnpm view`; its exported import path supports `@modelcontextprotocol/sdk/server/streamableHttp.js`. -- The 1.29.0 tarball types show `StreamableHTTPServerTransport` accepts `sessionIdGenerator`, `onsessioninitialized`, `onsessionclosed`, `allowedHosts`, `allowedOrigins`, and `enableDnsRebindingProtection`, and exposes `handleRequest(req, res, parsedBody?)`. - -## File Structure - -- Create `packages/cli/src/mcp-http-server.ts` - - Owns the foreground HTTP server. - - Validates Host, Origin, and bearer token policy before handing requests to the MCP SDK transport. - - Hosts `/health` and stateful `/mcp` `POST`/`GET`/`DELETE`. - - Builds a fresh `McpServer` per session with `createDefaultKtxMcpServer()`. -- Create `packages/cli/src/mcp-http-server.test.ts` - - Unit tests for host normalization, origin validation, token enforcement, `/health`, initialize session creation, unknown-session rejection, and DELETE cleanup. -- Create `packages/cli/src/managed-mcp-daemon.ts` - - Owns `.ktx/mcp.json`, `.ktx/logs/mcp.log`, background spawning, status probes, stop, and log reading. -- Create `packages/cli/src/managed-mcp-daemon.test.ts` - - Unit tests for state paths, start spawn arguments, token redaction from state/argv, status, stale state, stop, and log tailing. -- Create `packages/cli/src/commands/mcp-commands.ts` - - Registers public `start|stop|status|logs` and hidden `serve-internal`. -- Create `packages/cli/src/commands/mcp-commands.test.ts` - - Command-level tests for option parsing, non-loopback token requirement, state output, and hidden server command wiring. -- Modify `packages/cli/src/cli-program.ts` - - Add `mcp` to project-aware root commands. - - Register the MCP command subtree. -- Modify `packages/cli/package.json` - - Add `@modelcontextprotocol/sdk` as a direct dependency of `@ktx/cli`, because the CLI package will import the Streamable HTTP transport directly. - -## Task 1: Add MCP HTTP Security Helper Tests - -**Files:** -- Create: `packages/cli/src/mcp-http-server.test.ts` -- Create later: `packages/cli/src/mcp-http-server.ts` - -- [ ] **Step 1: Write the failing security helper tests** - -Create `packages/cli/src/mcp-http-server.test.ts` with: - -```typescript -import { describe, expect, it } from 'vitest'; -import { - buildMcpSecurityConfig, - isMcpRequestAuthorized, - normalizeHostHeader, -} from './mcp-http-server.js'; - -describe('normalizeHostHeader', () => { - it('normalizes host headers before allow-list comparison', () => { - expect(normalizeHostHeader('LOCALHOST:7878')).toBe('localhost'); - expect(normalizeHostHeader('127.0.0.1:7878')).toBe('127.0.0.1'); - expect(normalizeHostHeader('[::1]:7878')).toBe('::1'); - expect(normalizeHostHeader(' Example.COM ')).toBe('example.com'); - }); -}); - -describe('buildMcpSecurityConfig', () => { - it('allows loopback hosts without a token', () => { - const config = buildMcpSecurityConfig({ - host: '127.0.0.1', - port: 7878, - token: undefined, - allowedHosts: [], - allowedOrigins: [], - }); - - expect(config.token).toBeUndefined(); - expect(config.allowedHosts).toEqual(['localhost', '127.0.0.1', '::1']); - }); - - it('requires a token for non-loopback binding', () => { - expect(() => - buildMcpSecurityConfig({ - host: '0.0.0.0', - port: 7878, - token: undefined, - allowedHosts: [], - allowedOrigins: [], - }), - ).toThrow('Binding KTX MCP to 0.0.0.0 requires --token or KTX_MCP_TOKEN'); - }); - - it('validates allowed origins as full origins', () => { - expect(() => - buildMcpSecurityConfig({ - host: '127.0.0.1', - port: 7878, - token: undefined, - allowedHosts: [], - allowedOrigins: ['localhost:7878'], - }), - ).toThrow('Allowed origin must be a full origin URL'); - }); -}); - -describe('isMcpRequestAuthorized', () => { - const config = buildMcpSecurityConfig({ - host: '0.0.0.0', - port: 7878, - token: 'secret-token', - allowedHosts: ['mcp.example.test'], - allowedOrigins: ['https://mcp.example.test'], - }); - - it('accepts a valid host, origin, and bearer token', () => { - expect( - isMcpRequestAuthorized( - { - path: '/mcp', - headers: { - host: 'mcp.example.test:7878', - origin: 'https://mcp.example.test', - authorization: 'Bearer secret-token', - }, - }, - config, - ), - ).toEqual({ ok: true }); - }); - - it('rejects bad host headers before MCP handling', () => { - expect( - isMcpRequestAuthorized( - { path: '/health', headers: { host: 'evil.example.test' } }, - config, - ), - ).toEqual({ ok: false, status: 403, message: 'Host header is not allowed for KTX MCP.' }); - }); - - it('rejects browser origins unless explicitly allowed', () => { - expect( - isMcpRequestAuthorized( - { - path: '/health', - headers: { host: 'mcp.example.test', origin: 'https://evil.example.test' }, - }, - config, - ), - ).toEqual({ ok: false, status: 403, message: 'Origin header is not allowed for KTX MCP.' }); - }); - - it('requires bearer auth on /mcp when token auth is enabled', () => { - expect( - isMcpRequestAuthorized( - { path: '/mcp', headers: { host: 'mcp.example.test', authorization: 'Bearer wrong' } }, - config, - ), - ).toEqual({ ok: false, status: 401, message: 'Missing or invalid KTX MCP bearer token.' }); - }); - - it('does not require bearer auth on /health', () => { - expect(isMcpRequestAuthorized({ path: '/health', headers: { host: 'mcp.example.test' } }, config)).toEqual({ - ok: true, - }); - }); -}); -``` - -- [ ] **Step 2: Run the new tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/mcp-http-server.test.ts -``` - -Expected: FAIL because `./mcp-http-server.js` does not exist. - -- [ ] **Step 3: Implement the security helpers** - -Create `packages/cli/src/mcp-http-server.ts` with the helper surface first: - -```typescript -import type { IncomingHttpHeaders } from 'node:http'; - -const DEFAULT_ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1'] as const; - -export interface McpSecurityConfigInput { - host: string; - port: number; - token?: string; - allowedHosts: string[]; - allowedOrigins: string[]; -} - -export interface McpSecurityConfig { - host: string; - port: number; - token?: string; - allowedHosts: string[]; - allowedOrigins: string[]; -} - -export type McpAuthorizationResult = - | { ok: true } - | { ok: false; status: 401 | 403; message: string }; - -function isLoopbackHost(host: string): boolean { - const normalized = normalizeHostHeader(host); - return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '::1'; -} - -export function normalizeHostHeader(value: string): string { - const trimmed = value.trim().toLowerCase(); - if (trimmed.startsWith('[')) { - const close = trimmed.indexOf(']'); - return close >= 0 ? trimmed.slice(1, close) : trimmed.replace(/^\[/, ''); - } - const colon = trimmed.lastIndexOf(':'); - if (colon > -1 && trimmed.indexOf(':') === colon) { - return trimmed.slice(0, colon); - } - return trimmed; -} - -function fullOrigin(value: string): string { - let parsed: URL; - try { - parsed = new URL(value); - } catch { - throw new Error(`Allowed origin must be a full origin URL: ${value}`); - } - if (!parsed.protocol || !parsed.host || parsed.pathname !== '/' || parsed.search || parsed.hash) { - throw new Error(`Allowed origin must be a full origin URL: ${value}`); - } - return parsed.origin; -} - -export function buildMcpSecurityConfig(input: McpSecurityConfigInput): McpSecurityConfig { - if (!isLoopbackHost(input.host) && !input.token) { - throw new Error(`Binding KTX MCP to ${input.host} requires --token or KTX_MCP_TOKEN`); - } - const allowedHostSet = new Set(DEFAULT_ALLOWED_HOSTS); - if (!isLoopbackHost(input.host)) { - allowedHostSet.add(normalizeHostHeader(input.host)); - } - for (const host of input.allowedHosts) { - allowedHostSet.add(normalizeHostHeader(host)); - } - return { - host: input.host, - port: input.port, - ...(input.token ? { token: input.token } : {}), - allowedHosts: [...allowedHostSet], - allowedOrigins: input.allowedOrigins.map(fullOrigin), - }; -} - -function headerValue(headers: IncomingHttpHeaders | Record, name: string): string | undefined { - const value = headers[name.toLowerCase()]; - return Array.isArray(value) ? value[0] : value; -} - -export function isMcpRequestAuthorized( - request: { path: string; headers: IncomingHttpHeaders | Record }, - config: McpSecurityConfig, -): McpAuthorizationResult { - const host = headerValue(request.headers, 'host'); - if (!host || !config.allowedHosts.includes(normalizeHostHeader(host))) { - return { ok: false, status: 403, message: 'Host header is not allowed for KTX MCP.' }; - } - const origin = headerValue(request.headers, 'origin'); - if (origin && !config.allowedOrigins.includes(origin)) { - return { ok: false, status: 403, message: 'Origin header is not allowed for KTX MCP.' }; - } - if (request.path === '/mcp' && config.token) { - const auth = headerValue(request.headers, 'authorization'); - if (auth !== `Bearer ${config.token}`) { - return { ok: false, status: 401, message: 'Missing or invalid KTX MCP bearer token.' }; - } - } - return { ok: true }; -} -``` - -- [ ] **Step 4: Run the security helper tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/mcp-http-server.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/mcp-http-server.ts packages/cli/src/mcp-http-server.test.ts -git commit -m "feat(cli): add mcp http security helpers" -``` - -## Task 2: Add Foreground MCP HTTP Server - -**Files:** -- Modify: `packages/cli/src/mcp-http-server.ts` -- Modify: `packages/cli/src/mcp-http-server.test.ts` -- Modify: `packages/cli/package.json` - -- [ ] **Step 1: Add the direct SDK dependency to the CLI package** - -In `packages/cli/package.json`, add this dependency inside `"dependencies"`: - -```json -"@modelcontextprotocol/sdk": "^1.29.0" -``` - -Keep the dependency list alphabetized by package name. - -- [ ] **Step 2: Write failing HTTP server behavior tests** - -Append these imports to `packages/cli/src/mcp-http-server.test.ts`: - -```typescript -import { request } from 'node:http'; -import { AddressInfo } from 'node:net'; -import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; -import { runKtxMcpHttpServer } from './mcp-http-server.js'; -``` - -Append these helpers and tests: - -```typescript -function postJson(port: number, path: string, body: unknown, headers: Record = {}) { - return new Promise<{ status: number; headers: Record; body: string }>( - (resolve, reject) => { - const payload = JSON.stringify(body); - const req = request( - { - host: '127.0.0.1', - port, - path, - method: 'POST', - headers: { - host: `127.0.0.1:${port}`, - 'content-type': 'application/json', - 'content-length': Buffer.byteLength(payload), - ...headers, - }, - }, - (res) => { - const chunks: Buffer[] = []; - res.on('data', (chunk: Buffer) => chunks.push(chunk)); - res.on('end', () => - resolve({ - status: res.statusCode ?? 0, - headers: res.headers, - body: Buffer.concat(chunks).toString('utf8'), - }), - ); - }, - ); - req.on('error', reject); - req.end(payload); - }, - ); -} - -function get(port: number, path: string, headers: Record = {}) { - return new Promise<{ status: number; headers: Record; body: string }>( - (resolve, reject) => { - const req = request( - { - host: '127.0.0.1', - port, - path, - method: 'GET', - headers: { host: `127.0.0.1:${port}`, ...headers }, - }, - (res) => { - const chunks: Buffer[] = []; - res.on('data', (chunk: Buffer) => chunks.push(chunk)); - res.on('end', () => - resolve({ - status: res.statusCode ?? 0, - headers: res.headers, - body: Buffer.concat(chunks).toString('utf8'), - }), - ); - }, - ); - req.on('error', reject); - req.end(); - }, - ); -} - -function createTestMcpServer() { - return () => { - const server = new McpServer({ name: 'ktx-test', version: '0.0.0-test' }); - server.registerTool('ping', { inputSchema: {} }, async () => ({ - content: [{ type: 'text', text: 'pong' }], - })); - return server; - }; -} - -describe('runKtxMcpHttpServer', () => { - it('serves /health with project metadata', async () => { - const handle = await runKtxMcpHttpServer({ - projectDir: '/tmp/ktx-project', - host: '127.0.0.1', - port: 0, - allowedHosts: [], - allowedOrigins: [], - createMcpServer: createTestMcpServer(), - }); - try { - const port = (handle.server.address() as AddressInfo).port; - const response = await get(port, '/health'); - expect(response.status).toBe(200); - expect(JSON.parse(response.body)).toEqual({ - status: 'ok', - projectDir: '/tmp/ktx-project', - port, - }); - } finally { - await handle.close(); - } - }); - - it('allocates a stateful MCP session on initialize', async () => { - const handle = await runKtxMcpHttpServer({ - projectDir: '/tmp/ktx-project', - host: '127.0.0.1', - port: 0, - allowedHosts: [], - allowedOrigins: [], - createMcpServer: createTestMcpServer(), - }); - try { - const port = (handle.server.address() as AddressInfo).port; - const response = await postJson(port, '/mcp', { - jsonrpc: '2.0', - id: 1, - method: 'initialize', - params: { - protocolVersion: '2025-06-18', - capabilities: {}, - clientInfo: { name: 'vitest', version: '0.0.0' }, - }, - }); - - expect(response.status).toBe(200); - expect(response.headers['mcp-session-id']).toBeTruthy(); - } finally { - await handle.close(); - } - }); - - it('rejects unknown session ids with 404', async () => { - const handle = await runKtxMcpHttpServer({ - projectDir: '/tmp/ktx-project', - host: '127.0.0.1', - port: 0, - allowedHosts: [], - allowedOrigins: [], - createMcpServer: createTestMcpServer(), - }); - try { - const port = (handle.server.address() as AddressInfo).port; - const response = await postJson( - port, - '/mcp', - { jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }, - { 'mcp-session-id': 'missing-session' }, - ); - - expect(response.status).toBe(404); - expect(response.body).toContain('Unknown MCP session'); - } finally { - await handle.close(); - } - }); -}); -``` - -- [ ] **Step 3: Run the HTTP server tests to verify the new cases fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/mcp-http-server.test.ts -``` - -Expected: FAIL because `runKtxMcpHttpServer` is not implemented. - -- [ ] **Step 4: Implement the foreground server** - -Extend `packages/cli/src/mcp-http-server.ts` with: - -```typescript -import { randomUUID } from 'node:crypto'; -import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; -import { createDefaultKtxMcpServer } from '@ktx/context/mcp'; -import { createLocalProjectMcpContextPorts } from '@ktx/context/mcp'; -import { createLocalProjectMemoryCapture } from '@ktx/context/memory'; -import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; -import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js'; -import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; -import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import type { KtxCliIo } from './cli-runtime.js'; -import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; -import { createKtxCliScanConnector } from './local-scan-connectors.js'; -import { createManagedPythonSemanticLayerComputePort } from './managed-python-command.js'; -import { createManagedDaemonSqlAnalysisPort } from './managed-python-http.js'; - -export interface KtxMcpHttpServerHandle { - server: Server; - close(): Promise; -} - -export interface RunKtxMcpHttpServerOptions extends McpSecurityConfigInput { - projectDir: string; - cliVersion?: string; - io?: KtxCliIo; - createMcpServer?: () => McpServer; - loadProject?: typeof loadKtxProject; -} - -function writeJson(res: ServerResponse, status: number, body: object): void { - const payload = `${JSON.stringify(body)}\n`; - res.writeHead(status, { - 'content-type': 'application/json', - 'content-length': Buffer.byteLength(payload), - }); - res.end(payload); -} - -function writeText(res: ServerResponse, status: number, body: string): void { - res.writeHead(status, { 'content-type': 'text/plain; charset=utf-8' }); - res.end(body); -} - -function requestPath(req: IncomingMessage): string { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); - return url.pathname; -} - -async function readJsonBody(req: IncomingMessage): Promise { - const chunks: Buffer[] = []; - for await (const chunk of req) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - const raw = Buffer.concat(chunks).toString('utf8'); - return raw.trim().length === 0 ? undefined : (JSON.parse(raw) as unknown); -} - -async function defaultMcpServerFactory(input: { - project: KtxLocalProject; - projectDir: string; - cliVersion: string; - io?: KtxCliIo; -}): Promise<() => McpServer> { - const queryExecutor = createKtxCliIngestQueryExecutor(input.project); - const semanticLayerCompute = await createManagedPythonSemanticLayerComputePort({ - cliVersion: input.cliVersion, - installPolicy: 'auto', - io: input.io ?? { - stdout: { write() {} }, - stderr: { write() {} }, - }, - }); - const sqlAnalysis = createManagedDaemonSqlAnalysisPort({ - cliVersion: input.cliVersion, - projectDir: input.projectDir, - installPolicy: 'auto', - io: input.io ?? { - stdout: { write() {} }, - stderr: { write() {} }, - }, - }); - const contextTools = createLocalProjectMcpContextPorts(input.project, { - semanticLayerCompute, - queryExecutor, - sqlAnalysis, - localScan: { - createConnector: async (connectionId) => createKtxCliScanConnector(input.project, connectionId), - }, - localIngest: { - semanticLayerCompute, - queryExecutor, - }, - }); - let memoryCapture; - try { - memoryCapture = createLocalProjectMemoryCapture(input.project, { semanticLayerCompute, queryExecutor }); - } catch (error) { - input.io?.stderr.write(`KTX MCP memory_capture disabled: ${error instanceof Error ? error.message : String(error)}\n`); - } - - return () => - createDefaultKtxMcpServer({ - name: 'ktx', - version: input.cliVersion, - userContext: { userId: 'local' }, - contextTools, - memoryCapture, - }); -} - -export async function runKtxMcpHttpServer(options: RunKtxMcpHttpServerOptions): Promise { - const config = buildMcpSecurityConfig(options); - const project = - options.createMcpServer === undefined - ? await (options.loadProject ?? loadKtxProject)({ projectDir: options.projectDir }) - : undefined; - const createMcpServer = - options.createMcpServer ?? - (await defaultMcpServerFactory({ - project: project!, - projectDir: options.projectDir, - cliVersion: options.cliVersion ?? '0.0.0-private', - io: options.io, - })); - const sessions = new Map(); - - async function newTransport(): Promise { - let transport: StreamableHTTPServerTransport; - transport = new StreamableHTTPServerTransport({ - sessionIdGenerator: () => randomUUID(), - onsessioninitialized: (sessionId) => { - sessions.set(sessionId, transport); - }, - onsessionclosed: (sessionId) => { - sessions.delete(sessionId); - }, - allowedHosts: config.allowedHosts, - allowedOrigins: config.allowedOrigins, - enableDnsRebindingProtection: true, - }); - transport.onclose = () => { - if (transport.sessionId) { - sessions.delete(transport.sessionId); - } - }; - await createMcpServer().connect(transport); - return transport; - } - - const server = createServer(async (req, res) => { - const path = requestPath(req); - const auth = isMcpRequestAuthorized({ path, headers: req.headers }, config); - if (!auth.ok) { - writeText(res, auth.status, auth.message); - return; - } - - if (path === '/health' && req.method === 'GET') { - const address = server.address(); - const port = typeof address === 'object' && address ? address.port : config.port; - writeJson(res, 200, { status: 'ok', projectDir: options.projectDir, port }); - return; - } - - if (path !== '/mcp' || !['POST', 'GET', 'DELETE'].includes(req.method ?? '')) { - writeText(res, 404, 'Not found'); - return; - } - - const sessionId = req.headers['mcp-session-id']; - const normalizedSessionId = Array.isArray(sessionId) ? sessionId[0] : sessionId; - - if (req.method === 'POST') { - let body: unknown; - try { - body = await readJsonBody(req); - } catch (error) { - writeText(res, 400, `Invalid JSON body: ${error instanceof Error ? error.message : String(error)}`); - return; - } - const existing = normalizedSessionId ? sessions.get(normalizedSessionId) : undefined; - if (existing) { - await existing.handleRequest(req, res, body); - return; - } - if (normalizedSessionId) { - writeText(res, 404, `Unknown MCP session: ${normalizedSessionId}`); - return; - } - if (!isInitializeRequest(body)) { - writeText(res, 400, 'MCP initialize request is required before session traffic.'); - return; - } - await (await newTransport()).handleRequest(req, res, body); - return; - } - - if (!normalizedSessionId || !sessions.has(normalizedSessionId)) { - writeText(res, 404, normalizedSessionId ? `Unknown MCP session: ${normalizedSessionId}` : 'Missing MCP session id.'); - return; - } - await sessions.get(normalizedSessionId)!.handleRequest(req, res); - }); - - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(config.port, config.host, () => { - server.off('error', reject); - resolve(); - }); - }); - - return { - server, - async close() { - for (const transport of sessions.values()) { - await transport.close(); - } - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); - }, - }; -} -``` - -- [ ] **Step 5: Run the HTTP server tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/mcp-http-server.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/cli/package.json packages/cli/src/mcp-http-server.ts packages/cli/src/mcp-http-server.test.ts -git commit -m "feat(cli): host mcp over streamable http" -``` - -## Task 3: Add Managed MCP Daemon Lifecycle - -**Files:** -- Create: `packages/cli/src/managed-mcp-daemon.ts` -- Create: `packages/cli/src/managed-mcp-daemon.test.ts` - -- [ ] **Step 1: Write failing daemon lifecycle tests** - -Create `packages/cli/src/managed-mcp-daemon.test.ts` with: - -```typescript -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - mcpDaemonLayout, - readKtxMcpDaemonStatus, - startKtxMcpDaemon, - stopKtxMcpDaemon, - type KtxMcpDaemonChild, - type KtxMcpDaemonState, -} from './managed-mcp-daemon.js'; - -function child(pid = 4242): KtxMcpDaemonChild { - return { pid, unref: vi.fn() }; -} - -function state(projectDir: string, overrides: Partial = {}): KtxMcpDaemonState { - return { - schemaVersion: 1, - pid: 4242, - host: '127.0.0.1', - port: 7878, - tokenAuth: false, - projectDir, - startedAt: '2026-05-14T00:00:00.000Z', - logPath: join(projectDir, '.ktx/logs/mcp.log'), - ...overrides, - }; -} - -describe('managed MCP daemon lifecycle', () => { - let tempDir: string; - let projectDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-mcp-daemon-')); - projectDir = join(tempDir, 'project'); - await mkdir(projectDir, { recursive: true }); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('uses the spec state and log paths', () => { - expect(mcpDaemonLayout(projectDir)).toEqual({ - statePath: join(projectDir, '.ktx/mcp.json'), - logPath: join(projectDir, '.ktx/logs/mcp.log'), - }); - }); - - it('starts a detached child and writes state without the token value', async () => { - const spawnDaemon = vi.fn(() => child(5555)); - await startKtxMcpDaemon({ - projectDir, - cliVersion: '0.0.0-test', - host: '0.0.0.0', - port: 7879, - token: 'secret-token', - allowedHosts: ['mcp.example.test'], - allowedOrigins: ['https://mcp.example.test'], - binPath: '/repo/packages/cli/dist/bin.js', - spawnDaemon, - processAlive: vi.fn(() => false), - portAvailable: vi.fn(async () => true), - now: () => new Date('2026-05-14T00:00:00.000Z'), - }); - - expect(spawnDaemon).toHaveBeenCalledWith( - process.execPath, - [ - '/repo/packages/cli/dist/bin.js', - '--project-dir', - projectDir, - 'mcp', - 'serve-internal', - '--host', - '0.0.0.0', - '--port', - '7879', - '--allowed-host', - 'mcp.example.test', - '--allowed-origin', - 'https://mcp.example.test', - ], - expect.objectContaining({ - detached: true, - env: expect.objectContaining({ KTX_MCP_TOKEN: 'secret-token' }), - }), - ); - expect(JSON.stringify(JSON.parse(await readFile(join(projectDir, '.ktx/mcp.json'), 'utf8')))).not.toContain( - 'secret-token', - ); - }); - - it('reports running when the process is alive and health passes', async () => { - await mkdir(join(projectDir, '.ktx'), { recursive: true }); - await writeFile(join(projectDir, '.ktx/mcp.json'), `${JSON.stringify(state(projectDir), null, 2)}\n`); - - const status = await readKtxMcpDaemonStatus({ - projectDir, - processAlive: vi.fn(() => true), - fetchHealth: vi.fn(async () => ({ ok: true, body: { status: 'ok', projectDir, port: 7878 } })), - }); - - expect(status.kind).toBe('running'); - expect(status.url).toBe('http://127.0.0.1:7878/mcp'); - }); - - it('stops a recorded daemon and removes state', async () => { - await mkdir(join(projectDir, '.ktx'), { recursive: true }); - await writeFile(join(projectDir, '.ktx/mcp.json'), `${JSON.stringify(state(projectDir), null, 2)}\n`); - const alive = new Set([4242]); - const killProcess = vi.fn((pid: number) => alive.delete(pid)); - - await expect( - stopKtxMcpDaemon({ - projectDir, - processAlive: vi.fn((pid) => alive.has(pid)), - killProcess, - stopGraceMs: 1, - pollIntervalMs: 1, - }), - ).resolves.toEqual({ status: 'stopped' }); - - expect(killProcess).toHaveBeenCalledWith(4242, 'SIGTERM'); - await expect(readFile(join(projectDir, '.ktx/mcp.json'), 'utf8')).rejects.toThrow(); - }); -}); -``` - -- [ ] **Step 2: Run the lifecycle tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/managed-mcp-daemon.test.ts -``` - -Expected: FAIL because `./managed-mcp-daemon.js` does not exist. - -- [ ] **Step 3: Implement lifecycle state, start, status, and stop** - -Create `packages/cli/src/managed-mcp-daemon.ts` with: - -```typescript -import { spawn } from 'node:child_process'; -import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises'; -import { createServer } from 'node:net'; -import { dirname, join } from 'node:path'; -import { setTimeout as delay } from 'node:timers/promises'; -import { z } from 'zod'; - -export interface KtxMcpDaemonState { - schemaVersion: 1; - pid: number; - host: string; - port: number; - tokenAuth: boolean; - projectDir: string; - startedAt: string; - logPath: string; -} - -export interface KtxMcpDaemonChild { - pid?: number; - unref(): void; -} - -export type KtxMcpDaemonStatus = - | { kind: 'stopped'; detail: string } - | { kind: 'running'; detail: string; state: KtxMcpDaemonState; url: string } - | { kind: 'stale'; detail: string; state?: KtxMcpDaemonState }; - -const stateSchema = z.object({ - schemaVersion: z.literal(1), - pid: z.number().int().positive(), - host: z.string().min(1), - port: z.number().int().min(1).max(65535), - tokenAuth: z.boolean(), - projectDir: z.string().min(1), - startedAt: z.string().min(1), - logPath: z.string().min(1), -}); - -export function mcpDaemonLayout(projectDir: string): { statePath: string; logPath: string } { - return { - statePath: join(projectDir, '.ktx/mcp.json'), - logPath: join(projectDir, '.ktx/logs/mcp.log'), - }; -} - -function defaultProcessAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -function defaultKillProcess(pid: number, signal: NodeJS.Signals): void { - try { - process.kill(pid, signal); - } catch (error) { - if ((error as { code?: unknown }).code !== 'ESRCH') { - throw error; - } - } -} - -async function readState(projectDir: string): Promise { - try { - return stateSchema.parse(JSON.parse(await readFile(mcpDaemonLayout(projectDir).statePath, 'utf8')) as unknown); - } catch (error) { - if ((error as { code?: unknown }).code === 'ENOENT') { - return undefined; - } - throw error; - } -} - -async function writeState(projectDir: string, state: KtxMcpDaemonState): Promise { - const { statePath } = mcpDaemonLayout(projectDir); - await mkdir(dirname(statePath), { recursive: true }); - await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8'); -} - -async function defaultPortAvailable(host: string, port: number): Promise { - return await new Promise((resolve) => { - const server = createServer(); - server.once('error', () => resolve(false)); - server.listen(port, host, () => server.close(() => resolve(true))); - }); -} - -function defaultSpawnDaemon( - command: string, - args: string[], - options: { detached: boolean; stdio: ['ignore', number, number]; env: NodeJS.ProcessEnv }, -): KtxMcpDaemonChild { - return spawn(command, args, options); -} - -async function defaultFetchHealth(state: KtxMcpDaemonState): Promise<{ ok: boolean; body: unknown; detail?: string }> { - try { - const response = await fetch(`http://${state.host}:${state.port}/health`, { - headers: { host: `${state.host}:${state.port}` }, - }); - const body = await response.json(); - return { ok: response.ok, body, detail: response.ok ? undefined : `HTTP ${response.status}` }; - } catch (error) { - return { ok: false, body: null, detail: error instanceof Error ? error.message : String(error) }; - } -} - -export async function startKtxMcpDaemon(options: { - projectDir: string; - cliVersion: string; - host: string; - port: number; - token?: string; - allowedHosts: string[]; - allowedOrigins: string[]; - binPath: string; - processAlive?: (pid: number) => boolean; - portAvailable?: (host: string, port: number) => Promise; - spawnDaemon?: typeof defaultSpawnDaemon; - now?: () => Date; -}): Promise<{ status: 'started'; state: KtxMcpDaemonState; url: string }> { - const existing = await readState(options.projectDir).catch(() => undefined); - const processAlive = options.processAlive ?? defaultProcessAlive; - if (existing && processAlive(existing.pid)) { - throw new Error(`KTX MCP daemon is already recorded at http://${existing.host}:${existing.port}/mcp`); - } - const portAvailable = options.portAvailable ?? defaultPortAvailable; - if (!(await portAvailable(options.host, options.port))) { - throw new Error(`Port ${options.port} is already in use. Choose another port with --port .`); - } - - const { logPath } = mcpDaemonLayout(options.projectDir); - await mkdir(dirname(logPath), { recursive: true }); - const log = await open(logPath, 'a'); - const args = [ - options.binPath, - '--project-dir', - options.projectDir, - 'mcp', - 'serve-internal', - '--host', - options.host, - '--port', - String(options.port), - ...options.allowedHosts.flatMap((host) => ['--allowed-host', host]), - ...options.allowedOrigins.flatMap((origin) => ['--allowed-origin', origin]), - ]; - const child = (options.spawnDaemon ?? defaultSpawnDaemon)(process.execPath, args, { - detached: true, - stdio: ['ignore', log.fd, log.fd], - env: { - ...process.env, - KTX_CLI_VERSION: options.cliVersion, - ...(options.token ? { KTX_MCP_TOKEN: options.token } : {}), - }, - }); - if (!child.pid) { - throw new Error('Failed to start KTX MCP daemon: child process pid was not available.'); - } - child.unref(); - const state: KtxMcpDaemonState = { - schemaVersion: 1, - pid: child.pid, - host: options.host, - port: options.port, - tokenAuth: Boolean(options.token), - projectDir: options.projectDir, - startedAt: (options.now ?? (() => new Date()))().toISOString(), - logPath, - }; - await writeState(options.projectDir, state); - return { status: 'started', state, url: `http://${state.host}:${state.port}/mcp` }; -} - -export async function readKtxMcpDaemonStatus(options: { - projectDir: string; - processAlive?: (pid: number) => boolean; - fetchHealth?: (state: KtxMcpDaemonState) => Promise<{ ok: boolean; body: unknown; detail?: string }>; -}): Promise { - let state: KtxMcpDaemonState | undefined; - try { - state = await readState(options.projectDir); - } catch (error) { - return { kind: 'stale', detail: `MCP daemon state is invalid: ${error instanceof Error ? error.message : String(error)}` }; - } - if (!state) { - return { kind: 'stopped', detail: `No MCP daemon state at ${mcpDaemonLayout(options.projectDir).statePath}` }; - } - const processAlive = options.processAlive ?? defaultProcessAlive; - if (!processAlive(state.pid)) { - return { kind: 'stale', detail: `MCP daemon process ${state.pid} is not running`, state }; - } - const health = await (options.fetchHealth ?? defaultFetchHealth)(state); - if (!health.ok) { - return { kind: 'stale', detail: health.detail ?? 'MCP daemon health check failed', state }; - } - return { - kind: 'running', - detail: `KTX MCP daemon running at http://${state.host}:${state.port}/mcp`, - state, - url: `http://${state.host}:${state.port}/mcp`, - }; -} - -export async function stopKtxMcpDaemon(options: { - projectDir: string; - processAlive?: (pid: number) => boolean; - killProcess?: (pid: number, signal: NodeJS.Signals) => void; - stopGraceMs?: number; - pollIntervalMs?: number; -}): Promise<{ status: 'stopped' | 'already-stopped' }> { - const state = await readState(options.projectDir); - const { statePath } = mcpDaemonLayout(options.projectDir); - if (!state) { - return { status: 'already-stopped' }; - } - const processAlive = options.processAlive ?? defaultProcessAlive; - const killProcess = options.killProcess ?? defaultKillProcess; - if (processAlive(state.pid)) { - killProcess(state.pid, 'SIGTERM'); - const deadline = Date.now() + (options.stopGraceMs ?? 10_000); - while (Date.now() <= deadline && processAlive(state.pid)) { - await delay(options.pollIntervalMs ?? 100); - } - if (processAlive(state.pid)) { - killProcess(state.pid, 'SIGKILL'); - } - } - await rm(statePath, { force: true }); - return { status: 'stopped' }; -} -``` - -- [ ] **Step 4: Run the daemon lifecycle tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/managed-mcp-daemon.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/managed-mcp-daemon.ts packages/cli/src/managed-mcp-daemon.test.ts -git commit -m "feat(cli): manage mcp daemon lifecycle" -``` - -## Task 4: Register `ktx mcp` Commands - -**Files:** -- Create: `packages/cli/src/commands/mcp-commands.ts` -- Create: `packages/cli/src/commands/mcp-commands.test.ts` -- Modify: `packages/cli/src/cli-program.ts` - -- [ ] **Step 1: Write failing command tests** - -Create `packages/cli/src/commands/mcp-commands.test.ts` with: - -```typescript -import { Command } from '@commander-js/extra-typings'; -import { describe, expect, it, vi } from 'vitest'; -import type { KtxCliCommandContext } from '../cli-program.js'; -import { registerMcpCommands } from './mcp-commands.js'; - -function makeContext(overrides: Partial = {}): KtxCliCommandContext { - let exitCode = 0; - return { - io: { - stdout: { write: vi.fn() }, - stderr: { write: vi.fn() }, - }, - deps: {}, - packageInfo: { name: '@ktx/cli', version: '0.0.0-test' }, - setExitCode: (code) => { - exitCode = code; - }, - runInit: vi.fn(), - writeDebug: vi.fn(), - ...overrides, - get exitCode() { - return exitCode; - }, - } as KtxCliCommandContext; -} - -describe('registerMcpCommands', () => { - it('registers the public mcp lifecycle commands', () => { - const program = new Command().exitOverride(); - registerMcpCommands(program, makeContext()); - const mcp = program.commands.find((command) => command.name() === 'mcp'); - - expect(mcp?.commands.map((command) => command.name()).sort()).toEqual([ - 'logs', - 'serve-internal', - 'start', - 'status', - 'stop', - ]); - expect(mcp?.commands.find((command) => command.name() === 'serve-internal')?.hidden).toBe(true); - }); - - it('rejects non-loopback start without token before spawning', async () => { - const program = new Command().exitOverride(); - const startDaemon = vi.fn(); - const context = makeContext({ deps: { mcp: { startDaemon } } } as Partial); - registerMcpCommands(program, context); - - await expect(program.parseAsync(['mcp', 'start', '--host', '0.0.0.0'], { from: 'user' })).rejects.toThrow( - 'Binding KTX MCP to 0.0.0.0 requires --token or KTX_MCP_TOKEN', - ); - expect(startDaemon).not.toHaveBeenCalled(); - }); -}); -``` - -If `KtxCliDeps` does not yet include `mcp`, add this test helper shape in the test file: - -```typescript -type TestDeps = KtxCliCommandContext['deps'] & { - mcp?: { - startDaemon?: unknown; - stopDaemon?: unknown; - readStatus?: unknown; - runServer?: unknown; - }; -}; -``` - -Then cast `deps: { mcp: { startDaemon } } as TestDeps`. - -- [ ] **Step 2: Run the command tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/commands/mcp-commands.test.ts -``` - -Expected: FAIL because `./mcp-commands.js` does not exist. - -- [ ] **Step 3: Add MCP command dependency hooks** - -Find `KtxCliDeps` in `packages/cli/src/cli-runtime.ts` and add: - -```typescript - mcp?: { - startDaemon?: typeof import('./managed-mcp-daemon.js').startKtxMcpDaemon; - stopDaemon?: typeof import('./managed-mcp-daemon.js').stopKtxMcpDaemon; - readStatus?: typeof import('./managed-mcp-daemon.js').readKtxMcpDaemonStatus; - runServer?: typeof import('./mcp-http-server.js').runKtxMcpHttpServer; - }; -``` - -- [ ] **Step 4: Implement the MCP command subtree** - -Create `packages/cli/src/commands/mcp-commands.ts` with: - -```typescript -import { spawn } from 'node:child_process'; -import { readFile } from 'node:fs/promises'; -import { fileURLToPath } from 'node:url'; -import { Command } from '@commander-js/extra-typings'; -import { - buildMcpSecurityConfig, - runKtxMcpHttpServer, -} from '../mcp-http-server.js'; -import { - mcpDaemonLayout, - readKtxMcpDaemonStatus, - startKtxMcpDaemon, - stopKtxMcpDaemon, -} from '../managed-mcp-daemon.js'; -import { - collectOption, - parsePositiveIntegerOption, - resolveCommandProjectDir, - type KtxCliCommandContext, -} from '../cli-program.js'; - -function tokenFromOption(value: string | undefined): string | undefined { - return value ?? process.env.KTX_MCP_TOKEN; -} - -function binPath(): string { - return fileURLToPath(new URL('../bin.js', import.meta.url)); -} - -export function registerMcpCommands(program: Command, context: KtxCliCommandContext): void { - const mcp = program.command('mcp').description('Run the KTX MCP HTTP server'); - - mcp - .command('start') - .description('Start the KTX MCP HTTP server') - .option('--host ', 'Host to bind', '127.0.0.1') - .option('--port ', 'Port to bind', parsePositiveIntegerOption, 7878) - .option('--token ', 'Bearer token required for non-loopback binding') - .option('--foreground', 'Run in the foreground', false) - .option('--allowed-host ', 'Additional allowed Host header', collectOption, []) - .option('--allowed-origin ', 'Allowed browser Origin header', collectOption, []) - .action(async (options, command) => { - const projectDir = resolveCommandProjectDir(command); - const token = tokenFromOption(options.token); - buildMcpSecurityConfig({ - host: options.host, - port: options.port, - token, - allowedHosts: options.allowedHost, - allowedOrigins: options.allowedOrigin, - }); - if (options.foreground) { - await (context.deps.mcp?.runServer ?? runKtxMcpHttpServer)({ - projectDir, - cliVersion: context.packageInfo.version, - host: options.host, - port: options.port, - token, - allowedHosts: options.allowedHost, - allowedOrigins: options.allowedOrigin, - io: context.io, - }); - context.io.stdout.write(`KTX MCP server listening at http://${options.host}:${options.port}/mcp\n`); - return; - } - const result = await (context.deps.mcp?.startDaemon ?? startKtxMcpDaemon)({ - projectDir, - cliVersion: context.packageInfo.version, - host: options.host, - port: options.port, - token, - allowedHosts: options.allowedHost, - allowedOrigins: options.allowedOrigin, - binPath: binPath(), - }); - context.io.stdout.write(`KTX MCP daemon started: ${result.url}\n`); - }); - - mcp.command('stop').description('Stop the KTX MCP daemon').action(async (_options, command) => { - const result = await (context.deps.mcp?.stopDaemon ?? stopKtxMcpDaemon)({ - projectDir: resolveCommandProjectDir(command), - }); - context.io.stdout.write(result.status === 'stopped' ? 'KTX MCP daemon stopped.\n' : 'KTX MCP daemon is not running.\n'); - }); - - mcp.command('status').description('Show KTX MCP daemon status').action(async (_options, command) => { - const status = await (context.deps.mcp?.readStatus ?? readKtxMcpDaemonStatus)({ - projectDir: resolveCommandProjectDir(command), - }); - context.io.stdout.write(`${status.detail}\n`); - if (status.kind === 'running') { - context.io.stdout.write(`URL: ${status.url}\n`); - context.io.stdout.write(`PID: ${status.state.pid}\n`); - context.io.stdout.write(`Token auth: ${status.state.tokenAuth ? 'enabled' : 'disabled'}\n`); - context.io.stdout.write(`Project: ${status.state.projectDir}\n`); - } - }); - - mcp.command('logs').description('Print the KTX MCP daemon log').option('--follow', 'Follow log output', false).action(async (options, command) => { - const logPath = mcpDaemonLayout(resolveCommandProjectDir(command)).logPath; - if (options.follow) { - const child = spawn('tail', ['-f', logPath], { stdio: ['ignore', 'pipe', 'pipe'] }); - child.stdout?.on('data', (chunk: Buffer) => context.io.stdout.write(chunk.toString('utf8'))); - child.stderr?.on('data', (chunk: Buffer) => context.io.stderr.write(chunk.toString('utf8'))); - await new Promise((resolve) => child.on('close', resolve)); - return; - } - context.io.stdout.write(await readFile(logPath, 'utf8')); - }); - - mcp - .command('serve-internal', { hidden: true }) - .option('--host ', 'Host to bind', '127.0.0.1') - .requiredOption('--port ', 'Port to bind', parsePositiveIntegerOption) - .option('--allowed-host ', 'Additional allowed Host header', collectOption, []) - .option('--allowed-origin ', 'Allowed browser Origin header', collectOption, []) - .action(async (options, command) => { - await (context.deps.mcp?.runServer ?? runKtxMcpHttpServer)({ - projectDir: resolveCommandProjectDir(command), - cliVersion: context.packageInfo.version, - host: options.host, - port: options.port, - token: process.env.KTX_MCP_TOKEN, - allowedHosts: options.allowedHost, - allowedOrigins: options.allowedOrigin, - io: context.io, - }); - }); -} -``` - -- [ ] **Step 5: Wire the command into the root CLI** - -In `packages/cli/src/cli-program.ts`: - -Add the import: - -```typescript -import { registerMcpCommands } from './commands/mcp-commands.js'; -``` - -Change: - -```typescript -const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status']); -``` - -to: - -```typescript -const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'mcp']); -``` - -Add registration after `registerStatusCommands(program, context);`: - -```typescript - registerMcpCommands(program, context); -``` - -- [ ] **Step 6: Run command tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/commands/mcp-commands.test.ts src/cli-program.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add \ - packages/cli/src/commands/mcp-commands.ts \ - packages/cli/src/commands/mcp-commands.test.ts \ - packages/cli/src/cli-program.ts \ - packages/cli/src/cli-runtime.ts -git commit -m "feat(cli): add ktx mcp commands" -``` - -## Task 5: Final Verification And Handoff - -**Files:** -- Verify: `packages/cli/src/mcp-http-server.ts` -- Verify: `packages/cli/src/managed-mcp-daemon.ts` -- Verify: `packages/cli/src/commands/mcp-commands.ts` -- Verify: `packages/cli/package.json` - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run \ - src/mcp-http-server.test.ts \ - src/managed-mcp-daemon.test.ts \ - src/commands/mcp-commands.test.ts \ - src/cli-program.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run CLI package tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run workspace type-check** - -Run: - -```bash -pnpm run type-check -``` - -Expected: PASS. - -- [ ] **Step 5: Confirm remaining v1 blockers** - -Run: - -```bash -test -e packages/cli/src/skills/research/SKILL.md; printf 'research-skill:%s\n' "$?" -rg -n "connectionName" packages/context/src/ingest/tools/warehouse-verification -rg -n "mcpServers|mcp_servers|opencode|KTX_MCP_TOKEN" packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts -``` - -Expected after this plan is implemented: - -```text -research-skill:1 -``` - -Expected `rg "connectionName"`: matches remain under `packages/context/src/ingest/tools/warehouse-verification`, proving ingest contract convergence still needs a later v1 plan. - -Expected setup-agent `rg`: no complete MCP client config writer/snippet matrix yet, proving setup-agent/research-skill installation still needs a later v1 plan. - -- [ ] **Step 6: Commit final fixes if verification required any** - -If verification required changes, commit them: - -```bash -git add packages/cli/src packages/cli/package.json pnpm-lock.yaml -git commit -m "fix(cli): stabilize mcp daemon verification" -``` - -If no verification changes were needed, do not create an empty commit. - -## Self-Review - -- Spec coverage in this plan: covers `ktx mcp start|stop|status|logs`, foreground/background lifecycle, `.ktx/mcp.json`, `.ktx/logs/mcp.log`, HTTP-only `/mcp`, `/health`, stateful sessions, Host/Origin validation, non-loopback token requirement, and bearer checks on `/mcp`. -- Remaining v1-blocking spec coverage after this plan: setup-agent MCP client config installation, `ktx-research` skill installation, and ingest-side warehouse-verification `connectionName` to `connectionId` contract convergence. -- Placeholder scan: the plan contains no deferred work markers or vague implementation instructions. -- Type consistency: public names are consistent across tasks: `runKtxMcpHttpServer`, `buildMcpSecurityConfig`, `isMcpRequestAuthorized`, `mcpDaemonLayout`, `startKtxMcpDaemon`, `readKtxMcpDaemonStatus`, `stopKtxMcpDaemon`, and `registerMcpCommands`. diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-ingest-contract-convergence.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-ingest-contract-convergence.md deleted file mode 100644 index b57d72b9..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-ingest-contract-convergence.md +++ /dev/null @@ -1,804 +0,0 @@ -# Research Agent MCP Ingest Contract Convergence Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish the v1 research-agent MCP spec by converging the existing ingest warehouse-verification tools on `connectionId` terminology and a shared raw-schema catalog service. - -**Architecture:** Move the existing warehouse catalog reader out of the ingest-only tool folder into `packages/context/src/scan/warehouse-catalog.ts`, rename its public contract from `connectionName` to `connectionId`, and make the ingest adapters consume that shared service. Keep the ingest tools' ingest-specific output shape (`markdown` plus `structured`) and their existing `targets` / `rowLimit` controls; the v1 blocker is the divergent connection parameter and stale prompt guidance, not changing ingest output into the MCP pure-structured shape. - -**Tech Stack:** TypeScript, Zod, Vitest, existing KTX local file-store scan artifacts, existing ingest BaseTool framework. - ---- - -## Audit Summary - -Implemented and no longer v1-blocking: - -- MCP `sql_execution`, `entity_details`, `dictionary_search`, and `discover_data` are registered in `packages/context/src/mcp/context-tools.ts` and wired through local project ports. -- `sql_execution` is parser-gated through the Python sqlglot validator before reaching local scan connectors. -- The HTTP-only `ktx mcp` daemon exists with Streamable HTTP `POST`, `GET`, and `DELETE` handling, session tracking, host/origin checks, token checks for `/mcp`, lifecycle state, and CLI commands. -- `ktx setup-agents` installs the `ktx-research` skill, writes Claude/Cursor JSON MCP config entries, and prints Codex/opencode snippets. - -Remaining v1 blocker: - -- The ingest warehouse-verification tools still expose and teach `connectionName` while the spec requires `connectionId` across `warehouse-verification/*.tool.ts`, `WarehouseCatalogService`, callers, tests, and prompt assets. - -Non-blocking follow-ups not covered here: - -- `ktx mcp status` does not print `startedAt` as a separate line, although the state file records it. -- `ktx setup-agents` writes safe `${KTX_MCP_TOKEN}` references for shared project configs, but it does not offer the spec's optional skip prompt when token auth is active. -- `discover_data` sample-value snippets use ASCII `" - samples: "` instead of the spec prose's middle-dot separator. - -## File Structure - -- Move: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` to `packages/context/src/scan/warehouse-catalog.ts` - - Shared live-database scan catalog reader, display resolver, raw schema search, and table detail source of truth. -- Modify: `packages/context/src/scan/index.ts` - - Export the shared warehouse catalog service and public types. -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` - - Accept `connectionId`, call shared catalog service, and emit connectionId-shaped markdown and structured output. -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` - - Accept optional `connectionId`, search raw schema via shared catalog service, and teach follow-up calls with `connectionId`. -- Modify: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts` - - Accept `connectionId`, keep `rowLimit`, and pass `connectionId` to `SlConnectionCatalogPort.executeQuery`. -- Modify tests: - - `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` - - `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` - - `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts` - - `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` - - Rename the service test file to `packages/context/src/scan/warehouse-catalog.test.ts`. -- Modify prompt assets: - - `packages/context/skills/_shared/identifier-verification.md` - - `packages/context/skills/dbt_ingest/SKILL.md` - - `packages/context/skills/historic_sql_patterns/SKILL.md` - - `packages/context/skills/historic_sql_table_digest/SKILL.md` - - `packages/context/skills/live_database_ingest/SKILL.md` - - `packages/context/skills/looker_ingest/SKILL.md` - - `packages/context/skills/lookml_ingest/SKILL.md` - - `packages/context/skills/metabase_ingest/SKILL.md` - - `packages/context/skills/metricflow_ingest/SKILL.md` - - `packages/context/skills/notion_synthesize/SKILL.md` - - `packages/context/skills/sl_capture/SKILL.md` - - `packages/context/skills/wiki_capture/SKILL.md` - - Preserve Looker/LookML prose where `connectionName` refers to a Looker runtime field, not a KTX tool parameter. - -## Task 1: Add Failing Contract Tests - -**Files:** -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` -- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Add entity_details input-contract coverage** - -Add this test inside the existing `describe('EntityDetailsTool', ...)` block: - -```typescript -it('uses connectionId as the public input field', async () => { - expect( - tool.parseInput({ - connectionId: 'warehouse', - targets: [{ display: 'public.orders' }], - }), - ).toEqual({ - connectionId: 'warehouse', - targets: [{ display: 'public.orders' }], - }); - - expect(() => - tool.parseInput({ - connectionName: 'warehouse', - targets: [{ display: 'public.orders' }], - }), - ).toThrow(); -}); -``` - -Update the existing `tool.call(...)` inputs in the same test file from `connectionName` to `connectionId`. For example: - -```typescript -const result = await tool.call({ connectionId: 'warehouse', targets: [{ display: 'public.orders' }] }, context); -``` - -- [ ] **Step 2: Add sql_execution input-contract coverage** - -Add this test inside `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts`: - -```typescript -it('uses connectionId as the public input field', () => { - expect( - tool.parseInput({ - connectionId: 'warehouse', - sql: 'select 1', - rowLimit: 5, - }), - ).toEqual({ - connectionId: 'warehouse', - sql: 'select 1', - rowLimit: 5, - }); - - expect(() => - tool.parseInput({ - connectionName: 'warehouse', - sql: 'select 1', - rowLimit: 5, - }), - ).toThrow(); -}); -``` - -Update the existing `tool.call(...)` inputs in the same test file from `connectionName` to `connectionId`. - -- [ ] **Step 3: Add discover_data input and hint coverage** - -Update the existing discover tests so the first case calls: - -```typescript -const result = await tool.call({ query: 'orders', connectionId: 'warehouse', limit: 5 }, context); -``` - -Change the routing-hint assertions to: - -```typescript -expect(result.markdown).toContain('use `entity_details({connectionId, targets: [{display}]})`'); -``` - -In the multi-connection test, use a `connectionId` hit field and assert the follow-up call is connectionId-shaped: - -```typescript -catalog.searchByName.mockImplementation(async (connectionId: string, query: string) => [ - { - kind: 'table', - connectionId, - ref: { catalog: null, db: 'public', name: `${connectionId}_${query}` }, - display: `public.${connectionId}_${query}`, - matchedOn: 'name', - }, -]); - -const result = await tool.call({ query: 'orders', limit: 10 }, multiConnectionContext); - -expect(catalog.searchByName).toHaveBeenCalledWith('analytics', 'orders', 10); -expect(catalog.searchByName).toHaveBeenCalledWith('warehouse', 'orders', 10); -expect(result.markdown).toContain('connectionId=analytics'); -expect(result.markdown).toContain('connectionId=warehouse'); -expect(result.markdown).toContain( - 'entity_details({connectionId: "analytics", targets: [{display: "public.analytics_orders"}]})', -); -expect(result.structured.raw?.hits.map((hit) => hit.connectionId)).toEqual(['analytics', 'warehouse']); -``` - -Add a parse contract test: - -```typescript -it('uses connectionId as the optional connection filter', () => { - expect(tool.parseInput({ query: 'orders', connectionId: 'warehouse', limit: 5 })).toEqual({ - query: 'orders', - connectionId: 'warehouse', - limit: 5, - }); - - expect(() => tool.parseInput({ query: 'orders', connectionName: 'warehouse', limit: 5 })).toThrow(); -}); -``` - -- [ ] **Step 4: Add shared catalog output coverage** - -Rename `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.test.ts` to `packages/context/src/scan/warehouse-catalog.test.ts`. - -Update the import to: - -```typescript -import { WarehouseCatalogService } from './warehouse-catalog.js'; -``` - -Update the main detail assertion to use `connectionId`: - -```typescript -const detail = await catalog.getTable({ connectionId: 'warehouse', catalog: null, db: 'public', name: 'orders' }); - -expect(detail).toMatchObject({ - connectionId: 'warehouse', - display: 'public.orders', -}); -expect(detail).not.toHaveProperty('connectionName'); -``` - -Add raw hit coverage: - -```typescript -const hits = await catalog.searchByName('warehouse', 'orders', 5); -expect(hits[0]).toMatchObject({ - kind: 'table', - connectionId: 'warehouse', - display: 'public.orders', -}); -expect(hits[0]).not.toHaveProperty('connectionName'); -``` - -- [ ] **Step 5: Update prompt-asset test expectations first** - -In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, change the identifier verification expectations to: - -```typescript -expect(shared).toContain('sql_execution({connectionId, sql: "SELECT DISTINCT'); -expect(shared).toContain('sql_execution({connectionId, sql: "SELECT 1 FROM'); -expect(shared).not.toContain('entity_details({connectionName'); -expect(shared).not.toContain('sql_execution({connectionName'); -``` - -- [ ] **Step 6: Run focused tests and verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ - src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ - src/scan/warehouse-catalog.test.ts \ - src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: FAIL because schemas still require `connectionName`, the catalog service still returns `connectionName`, and the prompt asset still contains old tool-call examples. - -## Task 2: Move And Rename The Shared Warehouse Catalog Service - -**Files:** -- Move: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` to `packages/context/src/scan/warehouse-catalog.ts` -- Modify: `packages/context/src/scan/index.ts` -- Delete: `packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts` - -- [ ] **Step 1: Move the service into the scan package** - -Run: - -```bash -git mv packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts packages/context/src/scan/warehouse-catalog.ts -``` - -- [ ] **Step 2: Fix imports for the new location** - -In `packages/context/src/scan/warehouse-catalog.ts`, change the imports at the top to: - -```typescript -import { getDialectForDriver } from '../connections/index.js'; -import type { KtxFileStorePort } from '../core/index.js'; -import type { - KtxConnectionDriver, - KtxSchemaColumn, - KtxSchemaForeignKey, - KtxSchemaTable, - KtxTableRef, -} from './types.js'; -``` - -- [ ] **Step 3: Rename public catalog fields and method parameters** - -In `packages/context/src/scan/warehouse-catalog.ts`, rename the service's public contract to this shape: - -```typescript -export interface TableDetail { - connectionId: string; - catalog: string | null; - db: string | null; - name: string; - display: string; - kind: string; - comment: string | null; - description: string | null; - rowCount: number | null; - columns: WarehouseColumnDetail[]; - foreignKeys: KtxSchemaForeignKey[]; -} - -export type RawSchemaHit = - | { - kind: 'table'; - connectionId: string; - ref: KtxTableRef; - display: string; - matchedOn: 'name' | 'db' | 'comment' | 'description'; - } - | { - kind: 'column'; - connectionId: string; - ref: KtxTableRef & { column: string }; - display: string; - matchedOn: 'name' | 'comment' | 'description'; - }; - -interface ConnectionCatalog { - connectionId: string; - syncId: string; - driver: CatalogDriver; - tables: KtxSchemaTable[]; - profile: RelationshipProfileArtifact | null; -} -``` - -Update the method signatures to: - -```typescript -async hasScan(connectionId: string): Promise -async getLatestSyncId(connectionId: string): Promise -async listTables(connectionId: string): Promise -async getTable(ref: { connectionId: string } & KtxTableRef): Promise -async resolveDisplay(connectionId: string, display: string): Promise<{ resolved: KtxTableRef | null; candidates: KtxTableRef[]; dialect: string }> -async resolveDisplayTarget(connectionId: string, display: string): Promise -async searchByName(connectionId: string, query: string, limit: number): Promise -private loadCatalog(connectionId: string): Promise -private async readCatalog(connectionId: string): Promise -``` - -Within those methods, use `connectionId` for the cache key, raw artifact root, returned `TableDetail.connectionId`, and returned `RawSchemaHit.connectionId`. - -- [ ] **Step 4: Export the shared service** - -Add these exports to `packages/context/src/scan/index.ts` near the existing entity-details exports: - -```typescript -export type { - DisplayTargetResolution, - RawSchemaHit, - TableDetail, - WarehouseCatalogServiceDeps, -} from './warehouse-catalog.js'; -export { WarehouseCatalogService } from './warehouse-catalog.js'; -``` - -- [ ] **Step 5: Run the catalog test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/scan/warehouse-catalog.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the shared catalog move** - -Run: - -```bash -git add packages/context/src/scan/warehouse-catalog.ts packages/context/src/scan/warehouse-catalog.test.ts packages/context/src/scan/index.ts packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts -git commit -m "refactor(context): share warehouse catalog service" -``` - -## Task 3: Rename Ingest Warehouse-Verification Tool Inputs - -**Files:** -- Modify: `packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts` -- Modify: `packages/context/src/ingest/tools/warehouse-verification/index.ts` - -- [ ] **Step 1: Update imports from the shared scan service** - -In `entity-details.tool.ts`, use: - -```typescript -import { WarehouseCatalogService, type TableDetail } from '../../../scan/warehouse-catalog.js'; -``` - -In `discover-data.tool.ts`, use: - -```typescript -import { WarehouseCatalogService, type RawSchemaHit } from '../../../scan/warehouse-catalog.js'; -``` - -In `index.ts`, use: - -```typescript -import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -``` - -- [ ] **Step 2: Rename entity_details input and calls** - -In `entity-details.tool.ts`, update the schema: - -```typescript -const entityDetailsInputSchema = z.object({ - connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), - targets: z.array(targetSchema).min(1).max(50), -}); -``` - -Update `resolveTarget`: - -```typescript -async function resolveTarget( - catalog: WarehouseCatalogService, - connectionId: string, - target: EntityDetailsTarget, -): Promise<{ resolved: (KtxTableRef & { column?: string }) | null; candidates: KtxTableRef[] }> { - if ('display' in target) { - return catalog.resolveDisplayTarget(connectionId, target.display); - } - - const candidateResolution = await catalog.resolveDisplayTarget(connectionId, targetLabel(target)); - return { - resolved: { - catalog: target.catalog, - db: target.db, - name: target.name, - column: target.column, - }, - candidates: candidateResolution.candidates, - }; -} -``` - -Update the start of `call`: - -```typescript -async call(input: EntityDetailsInput, context: ToolContext): Promise> { - const allowed = allowedConnectionNames(context); - if (allowed && !allowed.has(input.connectionId)) { - return { - markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`, - structured: { resolved: [], missing: [], scanAvailable: false }, - }; - } - - const catalog = this.catalogFactory(context); - const scanAvailable = await catalog.hasScan(input.connectionId); - if (!scanAvailable) { - return { - markdown: `No live-database scan available for connection "${input.connectionId}"; run \`ktx scan\` first.`, - structured: { resolved: [], missing: [], scanAvailable: false }, - }; - } -``` - -Update the table lookup: - -```typescript -const resolution = await resolveTarget(catalog, input.connectionId, target); -const detail = await catalog.getTable({ connectionId: input.connectionId, ...resolution.resolved }); -``` - -- [ ] **Step 3: Rename sql_execution input and calls** - -In `sql-execution.tool.ts`, update the schema: - -```typescript -const sqlExecutionInputSchema = z.object({ - connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), - sql: z.string().min(1), - rowLimit: z.number().int().positive().max(1000).optional().default(100), -}); -``` - -Update the allowed-connection guard: - -```typescript -const allowed = context.session?.allowedConnectionNames; -if (allowed && !allowed.has(input.connectionId)) { - return { - markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`, - structured: { - headers: [], - rows: [], - rowCount: 0, - truncated: false, - sql: input.sql, - wrappedSql: '', - error: 'connection_not_allowed', - }, - }; -} -``` - -Update execution: - -```typescript -const result = await this.connections.executeQuery(input.connectionId, wrappedSql); -``` - -- [ ] **Step 4: Rename discover_data input, raw hits, and routing hints** - -In `discover-data.tool.ts`, update the schema: - -```typescript -const discoverDataInputSchema = z.object({ - query: z.string().optional(), - connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/).optional(), - limit: z.number().int().positive().max(50).optional().default(10), - sourceName: z.string().optional(), -}); -``` - -Update the out-of-scope check: - -```typescript -if (input.connectionId && allowed && !allowed.has(input.connectionId)) { - return { - markdown: `Connection "${input.connectionId}" is not available to this ingest stage.`, - structured: { wiki: null, sl: null, raw: null }, - }; -} -``` - -Update the source inspect mode: - -```typescript -const sl = await this.deps.slDiscoverTool.call( - { sourceName: input.sourceName, connectionId: input.connectionId }, - context, -); -``` - -Update the SL discover call: - -```typescript -const slResult = await this.deps.slDiscoverTool.call( - { query: query || undefined, connectionId: input.connectionId }, - context, -); -``` - -Update the raw search loop and hints: - -```typescript -const connections = input.connectionId ? [input.connectionId] : [...(allowed ?? [])].sort(); -const rawHits: RawSchemaHit[] = []; -for (const connectionId of connections) { - rawHits.push(...(await catalog.searchByName(connectionId, query, limit))); -} -if (rawHits.length > 0) { - parts.push( - '## Raw Warehouse Schema', - '> use `entity_details({connectionId, targets: [{display}]})` for full DDL + sample values', - ); - parts.push( - rawHits - .slice(0, limit) - .map( - (hit) => - `- ${hit.kind}: ${hit.display} [connectionId=${hit.connectionId}] (matched on ${hit.matchedOn}) - ` + - `follow up with \`entity_details({connectionId: "${hit.connectionId}", targets: [{display: "${hit.display}"}]})\``, - ) - .join('\n'), - ); - raw = { hits: rawHits.slice(0, limit) }; -} -``` - -- [ ] **Step 5: Run focused tool tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ - src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the ingest tool contract rename** - -Run: - -```bash -git add packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts packages/context/src/ingest/tools/warehouse-verification/index.ts packages/context/src/ingest/tools/warehouse-verification/*.test.ts -git commit -m "refactor(context): use connectionId in warehouse verification tools" -``` - -## Task 4: Update Prompt Assets And Runtime Tests - -**Files:** -- Modify: `packages/context/skills/_shared/identifier-verification.md` -- Modify: `packages/context/skills/dbt_ingest/SKILL.md` -- Modify: `packages/context/skills/historic_sql_patterns/SKILL.md` -- Modify: `packages/context/skills/historic_sql_table_digest/SKILL.md` -- Modify: `packages/context/skills/live_database_ingest/SKILL.md` -- Modify: `packages/context/skills/looker_ingest/SKILL.md` -- Modify: `packages/context/skills/lookml_ingest/SKILL.md` -- Modify: `packages/context/skills/metabase_ingest/SKILL.md` -- Modify: `packages/context/skills/metricflow_ingest/SKILL.md` -- Modify: `packages/context/skills/notion_synthesize/SKILL.md` -- Modify: `packages/context/skills/sl_capture/SKILL.md` -- Modify: `packages/context/skills/wiki_capture/SKILL.md` -- Modify: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Update the shared identifier verification protocol** - -Replace the tool-call examples in `packages/context/skills/_shared/identifier-verification.md` with: - -```markdown -2. `entity_details({connectionId, targets: [{display: ""}]})` - - confirm the identifier resolves; inspect native types, FK/PK, and - sampleValues. -3. For literal values from the source, such as status codes or plan tiers, - check whether they appear in `entity_details` sampleValues for the relevant - column. If sampleValues is short or the sample may have missed real values, - run a `sql_execution` probe with the same warehouse connection id: - `sql_execution({connectionId, sql: "SELECT DISTINCT FROM LIMIT 50"})`. -4. If the candidate identifier still does not resolve, do one of: - - Use `sql_execution({connectionId, sql: "SELECT 1 FROM LIMIT 0"})`. - If it errors, the identifier is fictional. -``` - -- [ ] **Step 2: Update copied skill assets** - -In the listed `packages/context/skills/*/SKILL.md` files, replace only KTX tool-call examples: - -```text -entity_details({connectionName, targets: -``` - -with: - -```text -entity_details({connectionId, targets: -``` - -Replace: - -```text -sql_execution({connectionName, sql: -``` - -with: - -```text -sql_execution({connectionId, sql: -``` - -Replace concrete KTX tool-call examples like: - -```text -sql_execution({connectionName: "warehouse", sql: -``` - -with: - -```text -sql_execution({connectionId: "warehouse", sql: -``` - -In `packages/context/skills/sl_capture/SKILL.md`, replace the JSON field inside the example object: - -```yaml -connectionName: "warehouse", -``` - -with: - -```yaml -connectionId: "warehouse", -``` - -Do not change `packages/context/skills/looker_ingest/SKILL.md` text that defines Looker runtime `connectionName`, and do not change LookML parser docs where `connectionName` names a LookML model property. - -- [ ] **Step 3: Update runtime asset tests** - -In `packages/context/src/ingest/ingest-runtime-assets.test.ts`, ensure the identifier test asserts the new examples: - -```typescript -expect(shared).toContain('sql_execution({connectionId, sql: "SELECT DISTINCT'); -expect(shared).toContain('sql_execution({connectionId, sql: "SELECT 1 FROM'); -expect(shared).not.toContain('entity_details({connectionName'); -expect(shared).not.toContain('sql_execution({connectionName'); -``` - -- [ ] **Step 4: Run prompt asset checks** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Verify stale tool-call examples are gone** - -Run: - -```bash -rg -n "entity_details\\(\\{connectionName|sql_execution\\(\\{connectionName|connectionName=" packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: no output. If this reports Looker/LookML prose that is not a KTX tool-call example, narrow the regex and keep the Looker/LookML prose unchanged. - -- [ ] **Step 6: Commit prompt asset updates** - -Run: - -```bash -git add packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts -git commit -m "docs(context): update ingest verification prompts for connectionId" -``` - -## Task 5: Final Verification - -**Files:** -- Verify all files changed in Tasks 1-4. - -- [ ] **Step 1: Run focused research-agent ingest tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/scan/warehouse-catalog.test.ts \ - src/ingest/tools/warehouse-verification/entity-details.tool.test.ts \ - src/ingest/tools/warehouse-verification/discover-data.tool.test.ts \ - src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts \ - src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run dead-code checks after TypeScript changes** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If Knip reports unrelated pre-existing findings, record the exact unrelated findings in the implementation handoff and do not add broad ignores. - -- [ ] **Step 4: Verify the v1-blocking old contract is gone** - -Run: - -```bash -rg -n "connectionName" packages/context/src/ingest/tools/warehouse-verification packages/context/src/scan/warehouse-catalog.ts packages/context/src/scan/warehouse-catalog.test.ts -``` - -Expected: no output. - -Run: - -```bash -rg -n "entity_details\\(\\{connectionName|sql_execution\\(\\{connectionName|connectionName=" packages/context/skills packages/context/src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: no output. - -- [ ] **Step 5: Inspect git status** - -Run: - -```bash -git status --short -``` - -Expected: only the intended scan catalog move, warehouse-verification tools/tests, prompt assets, and ingest runtime asset test changes are present. - -- [ ] **Step 6: Commit final fixes if verification required any** - -If Steps 1-5 required follow-up edits, commit those edits: - -```bash -git add packages/context/src packages/context/skills -git commit -m "test(context): verify warehouse verification connectionId contract" -``` - -If `git status --short` is empty after the earlier task commits, skip this commit. - -## Self-Review - -- Spec coverage: This plan covers the remaining v1 requirement that ingest-side warehouse verification uses `connectionId` and shares the raw-schema catalog service instead of preserving a divergent `connectionName` contract. -- Placeholder scan: The plan contains no deferred-work marker phrases. -- Type consistency: The plan uses `connectionId` consistently in public tool inputs, `TableDetail`, `RawSchemaHit`, `WarehouseCatalogService` method parameters, tests, and prompt assets. diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-setup-agents.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-setup-agents.md deleted file mode 100644 index 13f66764..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-setup-agents.md +++ /dev/null @@ -1,938 +0,0 @@ -# Research Agent MCP Setup Agents Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `ktx setup-agents` install the `ktx-research` skill and configure or print MCP client entries that point agents at the local `ktx mcp` HTTP endpoint. - -**Architecture:** Keep `packages/cli/src/setup-agents.ts` as the setup orchestration point. Add a small MCP-client config planner/writer in the same module, backed by `.ktx/mcp.json` when present, and install the research skill from a copied runtime asset so source checkouts and published CLI builds use the same `SKILL.md`. - -**Tech Stack:** TypeScript, Vitest, Node fs/path APIs, Commander setup options, KTX MCP daemon state, JSON config writers. - ---- - -## Current Audit - -Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md` - -Implemented v1 slices confirmed in current source: - -- MCP `sql_execution`, `entity_details`, `dictionary_search`, and `discover_data` are registered in `packages/context/src/mcp/context-tools.ts`. -- Local project MCP ports wire all four tools in `packages/context/src/mcp/local-project-ports.ts`. -- Parser-backed SQL validation exists in `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` and is exposed through `POST /sql/validate-read-only`. -- `ktx mcp start|stop|status|logs` exists in `packages/cli/src/commands/mcp-commands.ts`, with HTTP hosting in `packages/cli/src/mcp-http-server.ts` and daemon state in `packages/cli/src/managed-mcp-daemon.ts`. -- Targeted verification passed: - - `pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts src/search/discover.test.ts src/scan/entity-details.test.ts src/sl/dictionary-search.test.ts` - - `pnpm --filter @ktx/cli exec vitest run src/mcp-http-server.test.ts src/managed-mcp-daemon.test.ts src/commands/mcp-commands.test.ts src/setup-agents.test.ts` - -V1-blocking gaps remaining against the original spec: - -- `ktx setup-agents` still installs only the existing `ktx` agent files; it does not install `ktx-research`. -- `ktx setup-agents` does not write Claude Code or Cursor MCP JSON config entries. -- `ktx setup-agents` does not print Codex or opencode copy-paste snippets. -- `ktx setup-agents --remove` cannot remove written MCP JSON keys because none are written or tracked. -- The ingest-side warehouse-verification tools still use `connectionName`, `targets`, and `rowLimit`, and `WarehouseCatalogService` still exposes connection-name terminology. That is a separate v1-blocking subsystem and is not mixed into this setup-agent plan. - -Non-blocking or explicitly out-of-scope gaps: - -- Python code execution over MCP. -- Stdio MCP transport. -- OS-level auto-start. -- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing. -- Streaming SQL results. - -## File Structure - -Create: - -- `packages/cli/src/skills/research/SKILL.md` - - Canonical research skill body from the spec. - - Copied into `dist/skills/research/SKILL.md` during `@ktx/cli` build. -- `packages/cli/scripts/copy-runtime-assets.mjs` - - Copies `src/skills` into `dist/skills` after TypeScript compilation. - -Modify: - -- `packages/cli/package.json` - - Append the runtime asset copy step to the `build` script. -- `packages/cli/src/setup-agents.ts` - - Add `local` agent scope for Claude Code's per-project private config path. - - Add `research-skill` file entries in `plannedKtxAgentFiles()`. - - Read the research skill asset when writing research-skill entries. - - Add MCP endpoint resolution from `.ktx/mcp.json`, falling back to `http://localhost:7878/mcp`. - - Add JSON writers for Claude Code and Cursor MCP entries. - - Add printed snippets for Codex and opencode. - - Track written JSON keys in the install manifest. - - Print the daemon-start hint when the daemon is not currently running. -- `packages/cli/src/setup-agents.test.ts` - - Cover research skill install paths, MCP JSON writers, snippets, manifest removal, token handling, and no literal-token rendering. -- `packages/cli/src/commands/setup-commands.ts` - - Add `--local` for Claude Code local-scope setup. - - Reject `--local` with non-Claude targets and reject `--local --global`. -- `packages/cli/src/setup.ts` - - No behavior change beyond accepting `KtxAgentScope` with the new `local` value. -- `packages/cli/src/cli-program.ts` - - Keep the default bare setup `agentScope: 'project'`; no code change needed unless TypeScript requires the widened scope type in nearby annotations. - -## Task 1: Add The Research Skill Runtime Asset - -**Files:** -- Create: `packages/cli/src/skills/research/SKILL.md` -- Create: `packages/cli/scripts/copy-runtime-assets.mjs` -- Modify: `packages/cli/package.json` -- Modify: `packages/cli/src/setup-agents.test.ts` -- Modify: `packages/cli/src/setup-agents.ts` - -- [ ] **Step 1: Write the failing research-skill install tests** - -In `packages/cli/src/setup-agents.test.ts`, update the first test to expect `ktx-research` entries. Replace the project-scoped assertions with: - -```typescript - it('plans project-scoped CLI and research files for every target', () => { - expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-code', scope: 'project', mode: 'cli' })).toEqual([ - { kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md'), role: 'skill' }, - { kind: 'file', path: join(tempDir, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' }, - { kind: 'file', path: join(tempDir, '.claude/rules/ktx.md'), role: 'rule' }, - ]); - expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'codex', scope: 'project', mode: 'cli' })).toEqual([ - { kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md'), role: 'skill' }, - { kind: 'file', path: join(tempDir, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' }, - { kind: 'file', path: join(tempDir, '.codex/instructions/ktx.md'), role: 'rule' }, - ]); - expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'cursor', scope: 'project', mode: 'cli' })).toEqual([ - { kind: 'file', path: join(tempDir, '.cursor/rules/ktx.mdc') }, - { kind: 'file', path: join(tempDir, '.cursor/rules/ktx-research.mdc'), role: 'research-skill' }, - ]); - expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'opencode', scope: 'project', mode: 'cli' })).toEqual([ - { kind: 'file', path: join(tempDir, '.opencode/commands/ktx.md') }, - { kind: 'file', path: join(tempDir, '.opencode/commands/ktx-research.md'), role: 'research-skill' }, - ]); - expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'universal', scope: 'project', mode: 'cli' })).toEqual([ - { kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md') }, - { kind: 'file', path: join(tempDir, '.agents/skills/ktx-research/SKILL.md'), role: 'research-skill' }, - ]); - }); -``` - -Add this test after `installs target files, writes a manifest, and marks agents complete`: - -```typescript - it('installs the research skill from the runtime asset', async () => { - const io = makeIo(); - - await expect( - runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'universal', - scope: 'project', - mode: 'cli', - skipAgents: false, - }, - io.io, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - const researchSkill = await readFile(join(tempDir, '.agents/skills/ktx-research/SKILL.md'), 'utf-8'); - expect(researchSkill).toContain('name: ktx-research'); - expect(researchSkill).toContain('Always run `discover_data` before writing SQL.'); - expect(researchSkill).toContain('Treat a `dictionary_search` miss as non-authoritative.'); - }); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts -``` - -Expected: FAIL because `plannedKtxAgentFiles()` does not return `ktx-research` entries and the installed research skill file does not exist. - -- [ ] **Step 3: Add the research skill asset** - -Create `packages/cli/src/skills/research/SKILL.md`: - -```markdown ---- -name: ktx-research -description: Use when answering a question that needs data from a KTX-connected database - investigating, analyzing, "how many", "show me", "what's the breakdown of", finding records by value, exploring tables, comparing periods, or any data-investigation request. Triggers even when the user does not say "research"; if the answer requires querying a configured KTX connection, this skill applies. ---- - -# KTX Research Workflow - -You have access to KTX MCP tools for investigating data. Follow this workflow. - - -1. **Discover** - call `discover_data` first to see what exists across wiki, semantic-layer sources, and raw tables. Returns refs only. -2. **Inspect top hits in parallel** - for each promising ref: - - `kind: 'wiki'` -> `wiki_read` - - `kind: 'sl_source'`, `kind: 'sl_measure'`, or `kind: 'sl_dimension'` -> `sl_read_source` - - `kind: 'table'` or `kind: 'column'` -> `entity_details` -3. **Resolve literals** - if the user named a value such as "Acme Corp" or "status=shipped", call `dictionary_search` to find which column holds it. -4. **Query** - - - Prefer `sl_query` when the semantic layer covers the question. - - Use `sql_execution` only for questions the semantic layer does not cover. -5. **Capture learnings** - at the end of the turn, call `memory_capture` so future turns benefit. Skip when the answer carries no durable knowledge. - - - -- Always run `discover_data` before writing SQL. Do not guess table names. -- Prefer the semantic layer over raw SQL when both can answer the question; measures are the source of truth. -- Read entity details before writing SQL against an unfamiliar table. Do not assume column names. -- Treat `sql_execution` as read-only. Writes are rejected by the server. -- Validate value mentions with `dictionary_search` instead of guessing case or spelling. Treat a `dictionary_search` miss as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent. - - - -**Input:** "How many orders did Acme Corp place last month?" - -**Workflow:** -1. `dictionary_search({ values: ["Acme Corp"] })` finds `customers.name`. -2. `discover_data({ query: "orders customer monthly" })` finds an orders semantic-layer source. -3. `sl_read_source({ connectionId: "warehouse", sourceName: "orders_facts" })` confirms the source grain, measures, and dimensions. -4. `sl_query({ connectionId: "warehouse", measures: ["order_count"], filters: ["customer_name = 'Acme Corp'"] })` answers through the semantic layer. -5. `memory_capture({ userMessage, assistantMessage })` captures the durable finding. - ---- - -**Input:** "What columns does the events table have?" - -**Workflow:** -1. `discover_data({ query: "events table" })` returns a `table` ref. -2. `entity_details({ connectionId: "warehouse", entities: [{ table: "analytics.events" }] })` returns columns, types, and foreign keys. -3. Answer directly. No query is needed. - -``` - -- [ ] **Step 4: Copy skill assets during CLI build** - -Create `packages/cli/scripts/copy-runtime-assets.mjs`: - -```javascript -import { cp, mkdir, rm } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const packageRoot = fileURLToPath(new URL('..', import.meta.url)); -const skillsSource = join(packageRoot, 'src', 'skills'); -const skillsTarget = join(packageRoot, 'dist', 'skills'); - -await rm(skillsTarget, { recursive: true, force: true }); -await mkdir(dirname(skillsTarget), { recursive: true }); -await cp(skillsSource, skillsTarget, { recursive: true }); -``` - -Modify `packages/cli/package.json`: - -```json -"build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs" -``` - -- [ ] **Step 5: Add research-skill install entries and content loading** - -In `packages/cli/src/setup-agents.ts`, update the manifest entry role type: - -```typescript -| { kind: 'file'; path: string; role?: 'skill' | 'rule' | 'research-skill' } -``` - -Add this helper near `ktxCliLauncher()`: - -```typescript -async function readResearchSkillContent(): Promise { - const path = fileURLToPath(new URL('./skills/research/SKILL.md', import.meta.url)); - const content = await readFile(path, 'utf-8'); - return content.endsWith('\n') ? content : `${content}\n`; -} -``` - -Update `plannedKtxAgentFiles()` so every supported project target includes the `ktx-research` entry shown in Step 1. For global targets, return: - -```typescript -if (input.scope === 'global') { - if (input.target === 'claude-code') { - const home = process.env.HOME ?? ''; - return [ - { kind: 'file', path: join(home, '.claude/skills/ktx/SKILL.md'), role: 'skill' as const }, - { kind: 'file', path: join(home, '.claude/skills/ktx-research/SKILL.md'), role: 'research-skill' as const }, - { kind: 'file', path: join(home, '.claude/rules/ktx.md'), role: 'rule' as const }, - ]; - } - if (input.target === 'codex') { - const codexHome = process.env.CODEX_HOME ?? join(process.env.HOME ?? '', '.codex'); - return [ - { kind: 'file', path: join(codexHome, 'skills/ktx/SKILL.md'), role: 'skill' as const }, - { kind: 'file', path: join(codexHome, 'skills/ktx-research/SKILL.md'), role: 'research-skill' as const }, - { kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const }, - ]; - } - if (input.target === 'cursor' || input.target === 'opencode') { - return []; - } - throw new Error(`Global ${input.target} installation is not supported; omit --global.`); -} -``` - -In `installTarget()`, switch the file content selection to: - -```typescript -const content = - entry.role === 'rule' - ? ruleInstructionContent({ projectDir: input.projectDir }) - : entry.role === 'research-skill' - ? await readResearchSkillContent() - : cliInstructionContent({ projectDir: input.projectDir, launcher }); -``` - -- [ ] **Step 6: Run tests to verify the research skill passes** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts -``` - -Expected: PASS for the research skill install tests. MCP config tests are added in the next task and will fail until implemented. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/skills/research/SKILL.md packages/cli/scripts/copy-runtime-assets.mjs packages/cli/package.json packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts -git commit -m "feat(cli): install KTX research skill" -``` - -## Task 2: Add MCP Client Config Planning And Rendering - -**Files:** -- Modify: `packages/cli/src/setup-agents.test.ts` -- Modify: `packages/cli/src/setup-agents.ts` - -- [ ] **Step 1: Write failing MCP config planner tests** - -In `packages/cli/src/setup-agents.test.ts`, add these tests before `removes only manifest-listed files`: - -```typescript - it('writes Claude Code project MCP config and tracks the json key', async () => { - const io = makeIo(); - - await expect( - runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'claude-code', - scope: 'project', - mode: 'cli', - skipAgents: false, - }, - io.io, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - const mcpJson = JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8')) as { - mcpServers: { ktx: { type: string; url: string; headers?: Record } }; - }; - expect(mcpJson.mcpServers.ktx).toEqual({ type: 'http', url: 'http://localhost:7878/mcp' }); - expect(await readKtxAgentInstallManifest(tempDir)).toMatchObject({ - entries: expect.arrayContaining([{ kind: 'json-key', path: join(tempDir, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] }]), - }); - expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.'); - }); - - it('writes Cursor project MCP config', async () => { - const io = makeIo(); - - await runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'cursor', - scope: 'project', - mode: 'cli', - skipAgents: false, - }, - io.io, - ); - - const cursorJson = JSON.parse(await readFile(join(tempDir, '.cursor/mcp.json'), 'utf-8')) as { - mcpServers: { ktx: { url: string; headers?: Record } }; - }; - expect(cursorJson.mcpServers.ktx).toEqual({ url: 'http://localhost:7878/mcp' }); - }); - - it('prints Codex and opencode snippets without mutating printed-only config files', async () => { - const codexIo = makeIo(); - await runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'codex', - scope: 'project', - mode: 'cli', - skipAgents: false, - }, - codexIo.io, - ); - expect(codexIo.stdout()).toContain('[mcp_servers.ktx]'); - expect(codexIo.stdout()).toContain('url = "http://localhost:7878/mcp"'); - - const opencodeIo = makeIo(); - await runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'opencode', - scope: 'project', - mode: 'cli', - skipAgents: false, - }, - opencodeIo.io, - ); - expect(opencodeIo.stdout()).toContain('"mcp"'); - expect(opencodeIo.stdout()).toContain('"type": "remote"'); - await expect(readFile(join(tempDir, 'opencode.json'), 'utf-8')).rejects.toThrow(); - }); - - it('uses MCP daemon state for port and token metadata without rendering literal tokens', async () => { - await mkdir(join(tempDir, '.ktx'), { recursive: true }); - await writeFile( - join(tempDir, '.ktx/mcp.json'), - `${JSON.stringify( - { - schemaVersion: 1, - pid: 999999, - host: '127.0.0.1', - port: 8787, - tokenAuth: true, - projectDir: tempDir, - startedAt: '2026-05-14T00:00:00.000Z', - logPath: join(tempDir, '.ktx/logs/mcp.log'), - }, - null, - 2, - )}\n`, - 'utf-8', - ); - const io = makeIo(); - const previousToken = process.env.KTX_MCP_TOKEN; - process.env.KTX_MCP_TOKEN = 'secret-token'; - - try { - await runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'claude-code', - scope: 'project', - mode: 'cli', - skipAgents: false, - }, - io.io, - ); - - const rendered = JSON.stringify(JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8'))); - expect(rendered).toContain('http://127.0.0.1:8787/mcp'); - expect(rendered).toContain('Bearer ${KTX_MCP_TOKEN}'); - expect(rendered).not.toContain('secret-token'); - expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.'); - } finally { - process.env.KTX_MCP_TOKEN = previousToken; - } - }); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts -``` - -Expected: FAIL because no MCP config writer or snippet renderer exists. - -- [ ] **Step 3: Add JSON helpers and MCP endpoint resolution** - -In `packages/cli/src/setup-agents.ts`, add `existsSync` and `readKtxMcpDaemonStatus` imports: - -```typescript -import { existsSync } from 'node:fs'; -import { readKtxMcpDaemonStatus } from './managed-mcp-daemon.js'; -``` - -Add these types and helpers after `type InstallEntry`: - -```typescript -interface KtxMcpEndpointInfo { - url: string; - tokenAuth: boolean; - running: boolean; -} - -interface KtxMcpClientInstallResult { - entries: InstallEntry[]; - snippets: string[]; - notices: string[]; -} - -async function readJsonObject(path: string): Promise> { - if (!existsSync(path)) return {}; - const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error(`Expected JSON object in ${path}`); - } - return parsed as Record; -} - -function objectAtPath(root: Record, jsonPath: string[]): Record { - let cursor = root; - for (const segment of jsonPath) { - const current = cursor[segment]; - if (!current || typeof current !== 'object' || Array.isArray(current)) { - cursor[segment] = {}; - } - cursor = cursor[segment] as Record; - } - return cursor; -} - -async function writeJsonKey(path: string, jsonPath: string[], value: unknown): Promise { - const root = await readJsonObject(path); - const parent = objectAtPath(root, jsonPath.slice(0, -1)); - parent[jsonPath.at(-1) as string] = value; - await mkdir(dirname(path), { recursive: true }); - await writeFile(path, `${JSON.stringify(root, null, 2)}\n`, 'utf-8'); -} - -async function resolveMcpEndpoint(projectDir: string): Promise { - const status = await readKtxMcpDaemonStatus({ projectDir }).catch(() => null); - if (status?.kind === 'running') { - return { - url: status.url, - tokenAuth: status.state.tokenAuth, - running: true, - }; - } - if (status?.kind === 'stale' && status.state) { - return { - url: `http://${status.state.host}:${status.state.port}/mcp`, - tokenAuth: status.state.tokenAuth || Boolean(process.env.KTX_MCP_TOKEN), - running: false, - }; - } - return { - url: 'http://localhost:7878/mcp', - tokenAuth: Boolean(process.env.KTX_MCP_TOKEN), - running: false, - }; -} -``` - -- [ ] **Step 4: Add MCP entry renderers** - -Add these helpers after `resolveMcpEndpoint()`: - -```typescript -function tokenHeaders(endpoint: KtxMcpEndpointInfo): Record | undefined { - return endpoint.tokenAuth ? { Authorization: 'Bearer ${KTX_MCP_TOKEN}' } : undefined; -} - -function claudeMcpEntry(endpoint: KtxMcpEndpointInfo): Record { - return { - type: 'http', - url: endpoint.url, - ...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}), - }; -} - -function cursorMcpEntry(endpoint: KtxMcpEndpointInfo): Record { - return { - url: endpoint.url, - ...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}), - }; -} - -function codexSnippet(endpoint: KtxMcpEndpointInfo): string { - if (endpoint.tokenAuth) { - return [ - 'Codex MCP config does not currently document HTTP headers.', - 'Run KTX on loopback without token auth for Codex, or configure headers after Codex documents support.', - ].join('\n'); - } - return [`[mcp_servers.ktx]`, `url = "${endpoint.url}"`].join('\n'); -} - -function opencodeSnippet(endpoint: KtxMcpEndpointInfo): string { - return JSON.stringify( - { - mcp: { - ktx: { - type: 'remote', - url: endpoint.url, - enabled: true, - ...(tokenHeaders(endpoint) ? { headers: tokenHeaders(endpoint) } : {}), - }, - }, - }, - null, - 2, - ); -} - -function claudeConfigPath(projectDir: string, scope: KtxAgentScope): { path: string; jsonPath: string[] } { - const home = process.env.HOME ?? ''; - if (scope === 'global') { - return { path: join(home, '.claude.json'), jsonPath: ['mcpServers', 'ktx'] }; - } - if (scope === 'local') { - return { path: join(home, '.claude.json'), jsonPath: ['projects', resolve(projectDir), 'mcpServers', 'ktx'] }; - } - return { path: join(resolve(projectDir), '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] }; -} - -function cursorConfigPath(projectDir: string, scope: KtxAgentScope): { path: string; jsonPath: string[] } { - const home = process.env.HOME ?? ''; - return { - path: scope === 'global' ? join(home, '.cursor/mcp.json') : join(resolve(projectDir), '.cursor/mcp.json'), - jsonPath: ['mcpServers', 'ktx'], - }; -} -``` - -- [ ] **Step 5: Add the MCP client install planner** - -Add this function after the snippet helpers: - -```typescript -async function installMcpClientConfig(input: { - projectDir: string; - target: KtxAgentTarget; - scope: KtxAgentScope; -}): Promise { - const endpoint = await resolveMcpEndpoint(input.projectDir); - const entries: InstallEntry[] = []; - const snippets: string[] = []; - const notices: string[] = []; - - if (!endpoint.running) { - notices.push('Run `ktx mcp start` to enable the configured KTX MCP server.'); - } - - if (input.target === 'claude-code') { - const config = claudeConfigPath(input.projectDir, input.scope); - await writeJsonKey(config.path, config.jsonPath, claudeMcpEntry(endpoint)); - entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath }); - } else if (input.target === 'cursor') { - const config = cursorConfigPath(input.projectDir, input.scope); - await writeJsonKey(config.path, config.jsonPath, cursorMcpEntry(endpoint)); - entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath }); - } else if (input.target === 'codex') { - snippets.push(`Codex MCP snippet for ~/.codex/config.toml:\n${codexSnippet(endpoint)}`); - } else if (input.target === 'opencode') { - const path = - input.scope === 'global' ? '~/.config/opencode/opencode.json' : `${relative(input.projectDir, join(input.projectDir, 'opencode.json'))}`; - snippets.push(`opencode MCP snippet for ${path}:\n${opencodeSnippet(endpoint)}`); - } - - return { entries, snippets, notices }; -} -``` - -- [ ] **Step 6: Call the MCP planner during setup** - -Keep `installTarget()` responsible only for writing agent files and returning those file entries. - -In `runKtxSetupAgentsStep()`, replace the current install loop: - -```typescript - const entries: InstallEntry[] = []; - for (const install of installs) entries.push(...(await installTarget({ projectDir: args.projectDir, ...install }))); -``` - -with: - -```typescript - const entries: InstallEntry[] = []; - const snippets: string[] = []; - const notices = new Set(); - for (const install of installs) { - entries.push(...(await installTarget({ projectDir: args.projectDir, ...install }))); - const mcpResult = await installMcpClientConfig({ projectDir: args.projectDir, target: install.target, scope: install.scope }); - entries.push(...mcpResult.entries); - for (const snippet of mcpResult.snippets) snippets.push(snippet); - for (const notice of mcpResult.notices) notices.add(notice); - } -``` - -After the install summary write: - -```typescript - for (const snippet of snippets) { - io.stdout.write(`\n${snippet}\n`); - } - for (const notice of notices) { - io.stdout.write(`\n${notice}\n`); - } -``` - -- [ ] **Step 7: Run tests to verify MCP config passes** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts -``` - -Expected: PASS for research-skill and MCP config tests. - -- [ ] **Step 8: Commit** - -```bash -git add packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts -git commit -m "feat(cli): configure MCP clients in setup agents" -``` - -## Task 3: Add Claude Local Scope - -**Files:** -- Modify: `packages/cli/src/commands/setup-commands.ts` -- Modify: `packages/cli/src/setup-agents.ts` -- Modify: `packages/cli/src/setup-agents.test.ts` -- Modify: `packages/cli/src/setup.test.ts` -- Modify: `packages/cli/src/index.test.ts` - -- [ ] **Step 1: Write failing local-scope tests** - -Add this test to `packages/cli/src/setup-agents.test.ts`: - -```typescript - it('writes Claude Code local MCP config under the project key in ~/.claude.json', async () => { - const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); - const previousHome = process.env.HOME; - process.env.HOME = home; - try { - const io = makeIo(); - await runKtxSetupAgentsStep( - { - projectDir: tempDir, - inputMode: 'disabled', - yes: true, - agents: true, - target: 'claude-code', - scope: 'local', - mode: 'cli', - skipAgents: false, - }, - io.io, - ); - - const config = JSON.parse(await readFile(join(home, '.claude.json'), 'utf-8')) as { - projects: Record; - }; - expect(config.projects[tempDir].mcpServers.ktx).toEqual({ type: 'http', url: 'http://localhost:7878/mcp' }); - } finally { - process.env.HOME = previousHome; - await rm(home, { recursive: true, force: true }); - } - }); -``` - -Add these command-level tests after the existing `dispatches setup agent flags` test in `packages/cli/src/index.test.ts`: - -```typescript - it('rejects --local with non-Claude targets', async () => { - const setup = vi.fn(async () => 0); - const setupIo = makeIo(); - - await expect( - runKtxCli( - ['--project-dir', tempDir, 'setup', '--agents', '--target', 'cursor', '--local', '--no-input'], - setupIo.io, - { setup }, - ), - ).resolves.toBe(0); - - expect(setupIo.stderr()).toContain('--local is only supported with --target claude-code'); - expect(setup).not.toHaveBeenCalled(); - }); - - it('rejects --local and --global together', async () => { - const setup = vi.fn(async () => 0); - const setupIo = makeIo(); - - await expect( - runKtxCli( - ['--project-dir', tempDir, 'setup', '--agents', '--target', 'claude-code', '--local', '--global', '--no-input'], - setupIo.io, - { setup }, - ), - ).resolves.toBe(0); - - expect(setupIo.stderr()).toContain('Choose only one agent scope: --local or --global.'); - expect(setup).not.toHaveBeenCalled(); - }); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts src/index.test.ts -``` - -Expected: FAIL because `KtxAgentScope` does not include `local` and the setup command has no `--local` option. - -- [ ] **Step 3: Add the local scope type and command option** - -In `packages/cli/src/setup-agents.ts`, change: - -```typescript -export type KtxAgentScope = 'project' | 'global'; -``` - -to: - -```typescript -export type KtxAgentScope = 'project' | 'global' | 'local'; -``` - -In `packages/cli/src/commands/setup-commands.ts`, add `local` to `isOnlyAgentOptions()`: - -```typescript -'local', -``` - -Add the command option after `--global`: - -```typescript -.option('--local', 'Install Claude Code MCP config into the private per-project ~/.claude.json scope', false) -``` - -In the setup action before `const mode = ...`, add: - -```typescript - if (options.local && options.global) { - context.io.stderr.write('Choose only one agent scope: --local or --global.\n'); - context.setExitCode(1); - return; - } - if (options.local && options.target && options.target !== 'claude-code') { - context.io.stderr.write('--local is only supported with --target claude-code.\n'); - context.setExitCode(1); - return; - } -``` - -Replace: - -```typescript -const resolvedAgentScope = options.global ? 'global' : 'project'; -``` - -with: - -```typescript -const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project'; -``` - -- [ ] **Step 4: Run local-scope tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts src/index.test.ts -``` - -Expected: PASS for the new local-scope coverage. - -- [ ] **Step 5: Commit** - -```bash -git add packages/cli/src/commands/setup-commands.ts packages/cli/src/setup-agents.ts packages/cli/src/setup-agents.test.ts packages/cli/src/setup.test.ts packages/cli/src/index.test.ts -git commit -m "feat(cli): support Claude local MCP setup scope" -``` - -## Task 4: Final Verification - -**Files:** -- Verify all files changed in Tasks 1-3. - -- [ ] **Step 1: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts src/commands/mcp-commands.test.ts src/mcp-http-server.test.ts src/managed-mcp-daemon.test.ts -``` - -Expected: all selected test files pass. - -- [ ] **Step 2: Run CLI type-check** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -``` - -Expected: TypeScript completes with no errors. - -- [ ] **Step 3: Run CLI build** - -Run: - -```bash -pnpm --filter @ktx/cli run build -``` - -Expected: build succeeds and `packages/cli/dist/skills/research/SKILL.md` exists. - -- [ ] **Step 4: Run dead-code check for the changed TypeScript surface** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: Biome and Knip complete with no new findings from the setup-agent changes. - -- [ ] **Step 5: Inspect git status** - -Run: - -```bash -git status --short -``` - -Expected: only intended setup-agent, skill asset, package script, and test files are modified. - -## Self-Review - -Spec coverage: - -- Covers `ktx-research` skill installation paths for Claude Code, Codex, Cursor, opencode, and universal project targets. -- Covers Claude Code and Cursor JSON MCP writers. -- Covers Codex and opencode printed snippets. -- Covers token handling with `${KTX_MCP_TOKEN}` and no literal token rendering. -- Covers `.ktx/mcp.json` port selection and daemon-start hint. -- Covers manifest tracking for written JSON keys and removal through existing `json-key` cleanup. - -Known v1 gap not covered by this plan: - -- Ingest warehouse-verification contract convergence from `connectionName` to `connectionId`, shared service extraction, and caller/test updates remains v1-blocking and needs its own focused plan after this setup-agent slice lands. diff --git a/docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md b/docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md deleted file mode 100644 index 94611b90..00000000 --- a/docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md +++ /dev/null @@ -1,999 +0,0 @@ -# Research Agent MCP SQL Execution Foundation Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the parser-backed safety prerequisite and MCP `sql_execution` surface needed before the research-agent MCP tools can safely execute warehouse SQL. - -**Architecture:** Keep connector `executeReadOnly()` as the execution path, but make the MCP adapter require a sqlglot-backed validator before calling any connector. Extend the existing Python SQL-analysis daemon with a read-only validation endpoint, expose it through the TypeScript SQL-analysis port, then register an MCP `sql_execution` tool only when the host provides that validator and a local scan connector factory. - -**Tech Stack:** TypeScript, Vitest, Zod, Python, pytest, FastAPI, sqlglot, KTX MCP context ports, KTX scan connectors. - ---- - -## Audit Summary - -Original spec: `docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md` - -Implemented plans that overlap with the spec: - -- `docs/superpowers/plans/2026-05-11-managed-agent-mcp-semantic-runtime.md` is implemented for the existing in-process MCP semantic runtime. Current evidence: `packages/context/src/mcp/context-tools.ts` registers `connection_*`, `wiki_*`, `sl_*`, `ingest_*`, and `scan_*` tools, and `packages/context/src/mcp/local-project-ports.ts` provides local ports for those surfaces. -- `docs/superpowers/plans/2026-05-12-warehouse-verification-tools.md` plus its May 12 and May 13 closure plans are implemented for ingest-only warehouse verification. Current evidence: `packages/context/src/ingest/tools/warehouse-verification/{discover-data,entity-details,sql-execution,warehouse-catalog.service}.ts` exist and are wired for ingest agents. - -V1-blocking gaps remaining against the original spec: - -- The public MCP research tools are not registered. `KtxMcpContextPorts` has no `discover`, `entityDetails`, `dictionarySearch`, or `sqlExecution` ports. -- The existing ingest `discover_data`, `entity_details`, and `sql_execution` tools use `connectionName`, `targets`, and `rowLimit`, and return markdown plus structured output. The spec requires MCP-shaped `connectionId`, `entities` / `maxRows`, and pure structured outputs. -- `sql_execution` cannot be safely exposed yet: `packages/context/src/connections/read-only-sql.ts` still uses first-token regex checks. The spec requires a sqlglot/AST-backed guard or connector-side read-only session before MCP registration. -- `packages/context/src/scan/entity-details.ts`, `packages/context/src/sl/dictionary-search.ts`, and `packages/context/src/search/discover.ts` do not exist. -- `WarehouseCatalogService` caches by connection only and does not invalidate when latest scan artifact identity advances. -- `dictionary_search` has no MCP service, no coverage metadata, and no per-connection miss reasons. -- `discover_data` has no unified ranked MCP result shape with `summary`, `snippet`, `matchedOn`, `kind`, `tableRef`, and RRF fusion across wiki, SL, and raw schema. -- `ktx mcp start|stop|status|logs` does not exist, and no HTTP Streamable MCP daemon exists. -- `ktx setup-agents` installs only the existing `ktx` CLI skill/rules; it does not install `ktx-research` or MCP client config entries/snippets. - -Non-blocking or explicitly out-of-scope gaps: - -- Python code execution over MCP. -- Stdio MCP transport. -- OS-level auto-start. -- Native TLS, audit logging, rate limiting, per-tool authorization, and multi-project daemon routing. -- Streaming SQL results. -- Full DDL-style ingest `entity_details` markdown formatting and hard write-time validation in ingest writer tools. - -This plan covers the first prerequisite blocker: parser-backed SQL validation and MCP `sql_execution`. The remaining v1-blocking tool, daemon, and setup-agent work stays visible for subsequent plans. - -## File Structure - -Create no new files. - -Modify these files: - -- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`: add a sqlglot-backed read-only SQL validator. -- `python/ktx-daemon/src/ktx_daemon/app.py`: expose `POST /sql/validate-read-only`. -- `python/ktx-daemon/tests/test_sql_analysis.py`: cover accepted SELECT/WITH and rejected CTE-DML, multi-statement, command, pragma, and parse-error payloads. -- `python/ktx-daemon/tests/test_app.py`: cover the new HTTP endpoint. -- `packages/context/src/sql-analysis/ports.ts`: add `validateReadOnly()` to `SqlAnalysisPort`. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts`: call `/sql/validate-read-only` and map its response. -- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`: cover request and response mapping. -- `packages/context/src/mcp/types.ts`: add `KtxSqlExecutionMcpPort` and `sqlExecution` to `KtxMcpContextPorts`. -- `packages/context/src/mcp/context-tools.ts`: add the MCP `sql_execution` schema and registration. -- `packages/context/src/mcp/server.test.ts`: assert MCP registration and structured output for `sql_execution`. -- `packages/context/src/mcp/local-project-ports.ts`: expose local project SQL execution only when both `SqlAnalysisPort.validateReadOnly()` and a local scan connector factory are available. -- `packages/context/src/mcp/local-project-ports.test.ts`: cover validator success and validator rejection. - -### Task 1: Add sqlglot Read-Only Validation - -**Files:** -- Modify: `python/ktx-daemon/tests/test_sql_analysis.py` -- Modify: `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` -- Modify: `python/ktx-daemon/tests/test_app.py` -- Modify: `python/ktx-daemon/src/ktx_daemon/app.py` - -- [ ] **Step 1: Write failing sqlglot validator tests** - -In `python/ktx-daemon/tests/test_sql_analysis.py`, update the import block to include the new request model and function: - -```python -from ktx_daemon.sql_analysis import ( - AnalyzeSqlBatchItem, - AnalyzeSqlBatchRequest, - ValidateReadOnlySqlRequest, - _columns_from_nodes, - analyze_sql_batch_response, - validate_read_only_sql_response, -) -``` - -Add these tests after `test_columns_from_nodes_ignores_non_expression_clause_values`: - -```python -def test_validate_read_only_sql_accepts_select_and_with_queries() -> None: - select_response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest( - dialect="postgres", - sql="select id, status from public.orders where status = 'paid'", - ) - ) - with_response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest( - dialect="postgres", - sql=( - "with paid as (select * from public.orders where status = 'paid') " - "select count(*) from paid" - ), - ) - ) - - assert select_response.ok is True - assert select_response.error is None - assert with_response.ok is True - assert with_response.error is None - - -def test_validate_read_only_sql_rejects_cte_dml() -> None: - response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest( - dialect="postgres", - sql="with x as (insert into audit.events values (1) returning *) select * from x", - ) - ) - - assert response.ok is False - assert response.error == "SQL contains read/write operation: Insert" - - -def test_validate_read_only_sql_rejects_multi_statement_payloads() -> None: - response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest( - dialect="postgres", - sql="select * from public.orders; delete from public.orders", - ) - ) - - assert response.ok is False - assert response.error == "Only one SQL statement can be executed." - - -def test_validate_read_only_sql_rejects_commands_and_pragmas() -> None: - command_response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest(dialect="postgres", sql="call refresh_stats()") - ) - pragma_response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest(dialect="sqlite", sql="pragma table_info(users)") - ) - - assert command_response.ok is False - assert command_response.error == "SQL contains read/write operation: Command" - assert pragma_response.ok is False - assert pragma_response.error == "SQL contains read/write operation: Pragma" - - -def test_validate_read_only_sql_reports_parse_errors() -> None: - response = validate_read_only_sql_response( - ValidateReadOnlySqlRequest(dialect="postgres", sql="select * from where") - ) - - assert response.ok is False - assert response.error is not None - assert "Invalid expression" in response.error -``` - -- [ ] **Step 2: Run failing Python validator tests** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py -q -``` - -Expected: FAIL with an import error for `ValidateReadOnlySqlRequest` or `validate_read_only_sql_response`. - -- [ ] **Step 3: Implement the sqlglot validator** - -In `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`, add this model after `AnalyzeSqlBatchResponse`: - -```python -class ValidateReadOnlySqlRequest(BaseModel): - dialect: str - sql: str - - -class ValidateReadOnlySqlResponse(BaseModel): - ok: bool - error: str | None = None -``` - -Add this constant after the model definitions: - -```python -_READ_ONLY_ROOT_TYPES = (exp.Select, exp.Union) -_READ_WRITE_NODE_TYPES = ( - exp.Alter, - exp.Analyze, - exp.Cache, - exp.Command, - exp.Commit, - exp.Copy, - exp.Create, - exp.Delete, - exp.Describe, - exp.Drop, - exp.Execute, - exp.Grant, - exp.Insert, - exp.Merge, - exp.Pragma, - exp.Refresh, - exp.Revoke, - exp.Rollback, - exp.Set, - exp.Show, - exp.Transaction, - exp.TruncateTable, - exp.Uncache, - exp.Update, - exp.Use, -) -``` - -Add this function after `_analyze_payload`: - -```python -def validate_read_only_sql_response( - request: ValidateReadOnlySqlRequest, -) -> ValidateReadOnlySqlResponse: - try: - statements = sqlglot.parse(request.sql, read=request.dialect) - except sqlglot.errors.SqlglotError as exc: - return ValidateReadOnlySqlResponse(ok=False, error=str(exc)) - - if len(statements) != 1: - return ValidateReadOnlySqlResponse( - ok=False, - error="Only one SQL statement can be executed.", - ) - - tree = statements[0] - if tree is None: - return ValidateReadOnlySqlResponse(ok=False, error="SQL did not parse to a statement.") - if not isinstance(tree, _READ_ONLY_ROOT_TYPES): - return ValidateReadOnlySqlResponse( - ok=False, - error=f"SQL contains read/write operation: {type(tree).__name__}", - ) - - for node in tree.walk(): - if isinstance(node, _READ_WRITE_NODE_TYPES): - return ValidateReadOnlySqlResponse( - ok=False, - error=f"SQL contains read/write operation: {type(node).__name__}", - ) - - return ValidateReadOnlySqlResponse(ok=True, error=None) -``` - -- [ ] **Step 4: Run Python validator tests** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py -q -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing HTTP endpoint test** - -In `python/ktx-daemon/tests/test_app.py`, add this test after `test_sql_parse_table_identifier_endpoint`: - -```python -def test_sql_validate_read_only_endpoint() -> None: - client = TestClient(create_app()) - - ok_response = client.post( - "/sql/validate-read-only", - json={"dialect": "postgres", "sql": "select * from public.orders"}, - ) - bad_response = client.post( - "/sql/validate-read-only", - json={ - "dialect": "postgres", - "sql": "with x as (insert into audit.events values (1) returning *) select * from x", - }, - ) - - assert ok_response.status_code == 200 - assert ok_response.json() == {"ok": True, "error": None} - assert bad_response.status_code == 200 - assert bad_response.json() == { - "ok": False, - "error": "SQL contains read/write operation: Insert", - } -``` - -- [ ] **Step 6: Run failing HTTP endpoint test** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py -q -k validate_read_only -``` - -Expected: FAIL with HTTP 404 for `/sql/validate-read-only`. - -- [ ] **Step 7: Register the HTTP endpoint** - -In `python/ktx-daemon/src/ktx_daemon/app.py`, update the SQL-analysis import to include the new symbols: - -```python -from ktx_daemon.sql_analysis import ( - AnalyzeSqlBatchRequest, - AnalyzeSqlBatchResponse, - ValidateReadOnlySqlRequest, - ValidateReadOnlySqlResponse, - analyze_sql_batch_response, - validate_read_only_sql_response, -) -``` - -Add this endpoint immediately before the existing `@app.post("/sql/analyze-batch", ...)` route: - -```python - @app.post("/sql/validate-read-only", response_model=ValidateReadOnlySqlResponse) - async def sql_validate_read_only( - request: ValidateReadOnlySqlRequest, - ) -> ValidateReadOnlySqlResponse: - try: - return validate_read_only_sql_response(request) - except Exception as error: - logger.exception("SQL read-only validation failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"SQL read-only validation failed: {error}", - ) from error -``` - -- [ ] **Step 8: Run Python HTTP endpoint test** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_app.py -q -k validate_read_only -``` - -Expected: PASS. - -- [ ] **Step 9: Commit Python validator** - -Run: - -```bash -git add python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -git commit -m "feat(daemon): validate read-only SQL with sqlglot" -``` - -### Task 2: Expose Read-Only Validation Through the TypeScript SQL-Analysis Port - -**Files:** -- Modify: `packages/context/src/sql-analysis/ports.ts` -- Modify: `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` -- Modify: `packages/context/src/sql-analysis/http-sql-analysis-port.ts` - -- [ ] **Step 1: Add the port contract** - -In `packages/context/src/sql-analysis/ports.ts`, add this interface after `SqlAnalysisBatchResult`: - -```typescript -export interface SqlReadOnlyValidationResult { - ok: boolean; - error?: string | null; -} -``` - -Update `SqlAnalysisPort` to include the new method: - -```typescript -export interface SqlAnalysisPort { - analyzeForFingerprint(sql: string, dialect: SqlAnalysisDialect): Promise; - analyzeBatch( - items: SqlAnalysisBatchItem[], - dialect: SqlAnalysisDialect, - ): Promise>; - validateReadOnly(sql: string, dialect: SqlAnalysisDialect): Promise; -} -``` - -- [ ] **Step 2: Write failing HTTP port tests** - -In `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`, add this test inside the existing `describe('createHttpSqlAnalysisPort', ...)` block: - -```typescript - it('maps read-only SQL validation responses', async () => { - const requests: Array<{ path: string; payload: Record }> = []; - const port = createHttpSqlAnalysisPort({ - baseUrl: 'http://127.0.0.1:8765', - requestJson: async (path, payload) => { - requests.push({ path, payload }); - return { ok: false, error: 'SQL contains read/write operation: Insert' }; - }, - }); - - await expect(port.validateReadOnly('with x as (insert into t values (1)) select * from x', 'postgres')).resolves.toEqual({ - ok: false, - error: 'SQL contains read/write operation: Insert', - }); - expect(requests).toEqual([ - { - path: '/sql/validate-read-only', - payload: { - dialect: 'postgres', - sql: 'with x as (insert into t values (1)) select * from x', - }, - }, - ]); - }); -``` - -Add this test after it: - -```typescript - it('rejects malformed read-only validation responses', async () => { - const port = createHttpSqlAnalysisPort({ - baseUrl: 'http://127.0.0.1:8765', - requestJson: async () => ({ ok: 'yes' }), - }); - - await expect(port.validateReadOnly('select 1', 'postgres')).rejects.toThrow( - 'sql analysis response is missing boolean field ok', - ); - }); -``` - -- [ ] **Step 3: Run failing HTTP port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts -``` - -Expected: FAIL because `validateReadOnly` is not implemented. - -- [ ] **Step 4: Implement HTTP response mapping** - -In `packages/context/src/sql-analysis/http-sql-analysis-port.ts`, update the type import to include `SqlReadOnlyValidationResult`: - -```typescript - SqlReadOnlyValidationResult, -``` - -Add this helper after `requiredStringArray`: - -```typescript -function requiredBoolean(raw: Record, field: string): boolean { - const value = raw[field]; - if (typeof value !== 'boolean') { - throw new Error(`sql analysis response is missing boolean field ${field}`); - } - return value; -} -``` - -Add this mapper after `mapBatchResponse`: - -```typescript -function mapReadOnlyValidation(raw: Record): SqlReadOnlyValidationResult { - const error = optionalString(raw, 'error'); - return { - ok: requiredBoolean(raw, 'ok'), - ...(error !== undefined ? { error } : {}), - }; -} -``` - -Add this method to the object returned by `createHttpSqlAnalysisPort`: - -```typescript - async validateReadOnly(sql: string, dialect: SqlAnalysisDialect) { - const raw = await requestJson('/sql/validate-read-only', { - dialect, - sql, - }); - return mapReadOnlyValidation(raw); - }, -``` - -- [ ] **Step 5: Run HTTP port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit TypeScript SQL-analysis port** - -Run: - -```bash -git add packages/context/src/sql-analysis/ports.ts packages/context/src/sql-analysis/http-sql-analysis-port.ts packages/context/src/sql-analysis/http-sql-analysis-port.test.ts -git commit -m "feat(context): expose read-only SQL validation port" -``` - -### Task 3: Register the MCP `sql_execution` Tool Contract - -**Files:** -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/context-tools.ts` -- Modify: `packages/context/src/mcp/server.test.ts` - -- [ ] **Step 1: Add the MCP SQL execution port types** - -In `packages/context/src/mcp/types.ts`, add these interfaces immediately before `KtxMcpContextPorts`: - -```typescript -export interface KtxSqlExecutionResponse { - headers: string[]; - headerTypes?: string[]; - rows: unknown[][]; - rowCount: number; -} - -export interface KtxSqlExecutionMcpPort { - execute(input: { connectionId: string; sql: string; maxRows: number }): Promise; -} -``` - -Then add the new optional port to `KtxMcpContextPorts`: - -```typescript - sqlExecution?: KtxSqlExecutionMcpPort; -``` - -- [ ] **Step 2: Write failing MCP registration test** - -In `packages/context/src/mcp/server.test.ts`, update the type import from `./types.js` to include `KtxSqlExecutionMcpPort`. - -Add this test in `describe('createKtxMcpServer', ...)` after the existing connection-list registration test: - -```typescript - it('registers parser-gated sql_execution when the host provides a SQL execution port', async () => { - const fake = makeFakeServer(); - const sqlExecution: KtxSqlExecutionMcpPort = { - execute: vi.fn().mockResolvedValue({ - headers: ['status', 'count'], - headerTypes: ['text', 'bigint'], - rows: [['paid', 42]], - rowCount: 1, - }), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { - sqlExecution, - }, - }); - - expect(fake.tools.map((tool) => tool.name)).toEqual(['sql_execution']); - await expect( - getTool(fake.tools, 'sql_execution').handler({ - connectionId: 'warehouse', - sql: 'select status, count(*) from public.orders group by status', - maxRows: 50, - }), - ).resolves.toEqual({ - content: [ - { - type: 'text', - text: JSON.stringify( - { - headers: ['status', 'count'], - headerTypes: ['text', 'bigint'], - rows: [['paid', 42]], - rowCount: 1, - }, - null, - 2, - ), - }, - ], - structuredContent: { - headers: ['status', 'count'], - headerTypes: ['text', 'bigint'], - rows: [['paid', 42]], - rowCount: 1, - }, - }); - expect(sqlExecution.execute).toHaveBeenCalledWith({ - connectionId: 'warehouse', - sql: 'select status, count(*) from public.orders group by status', - maxRows: 50, - }); - }); -``` - -- [ ] **Step 3: Run failing MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t sql_execution -``` - -Expected: FAIL because `sql_execution` is not registered. - -- [ ] **Step 4: Add the MCP schema and registration** - -In `packages/context/src/mcp/context-tools.ts`, add this schema after `scanArtifactReadSchema`: - -```typescript -const sqlExecutionSchema = z.object({ - connectionId: connectionIdSchema, - sql: z.string().min(1), - maxRows: z.number().int().min(1).max(10_000).default(1000).optional(), -}); -``` - -Add this registration block in `registerKtxContextTools`, after the semantic-layer block and before the ingest block: - -```typescript - if (ports.sqlExecution) { - const sqlExecution = ports.sqlExecution; - registerParsedTool( - server, - 'sql_execution', - { - title: 'SQL Execution', - description: - 'Execute one parser-validated read-only SQL query against a configured KTX connection and return structured rows.', - inputSchema: sqlExecutionSchema.shape, - }, - sqlExecutionSchema, - async (input) => { - try { - return jsonToolResult( - await sqlExecution.execute({ - connectionId: input.connectionId, - sql: input.sql, - maxRows: input.maxRows ?? 1000, - }), - ); - } catch (error) { - return jsonErrorToolResult(error instanceof Error ? error.message : String(error)); - } - }, - ); - } -``` - -- [ ] **Step 5: Run MCP registration test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t sql_execution -``` - -Expected: PASS. - -- [ ] **Step 6: Commit MCP tool contract** - -Run: - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts -git commit -m "feat(context): register MCP sql execution tool" -``` - -### Task 4: Implement Local Project SQL Execution With Parser Validation - -**Files:** -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write failing local-port success test** - -In `packages/context/src/mcp/local-project-ports.test.ts`, update the imports from `../scan/index.js` to include `type KtxQueryResult`. - -Replace the existing `testConnector` helper with this version so tests can opt into read-only SQL: - -```typescript - function testConnector( - snapshot = testSnapshot(), - queryResult?: KtxQueryResult, - ): KtxScanConnector { - return { - id: `test:${snapshot.connectionId}`, - driver: snapshot.driver, - capabilities: createKtxConnectorCapabilities({ readOnlySql: queryResult !== undefined }), - introspect: vi.fn(async () => snapshot), - executeReadOnly: queryResult === undefined ? undefined : vi.fn(async () => queryResult), - cleanup: vi.fn(async () => {}), - }; - } -``` - -Add this test after `tests a local project connection through the native scan connector factory`: - -```typescript - it('executes MCP SQL only after parser-backed validation passes', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - const connector = testConnector(testSnapshot(), { - headers: ['id'], - headerTypes: ['integer'], - rows: [[1]], - totalRows: 1, - rowCount: 1, - }); - const createConnector = vi.fn(async () => connector); - const sqlAnalysis = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(), - validateReadOnly: vi.fn(async () => ({ ok: true, error: null })), - }; - const ports = createLocalProjectMcpContextPorts(project, { - sqlAnalysis, - localScan: { - createConnector, - }, - }); - - await expect( - ports.sqlExecution?.execute({ - connectionId: 'warehouse', - sql: 'select id from public.orders', - maxRows: 5, - }), - ).resolves.toEqual({ - headers: ['id'], - headerTypes: ['integer'], - rows: [[1]], - rowCount: 1, - }); - expect(sqlAnalysis.validateReadOnly).toHaveBeenCalledWith('select id from public.orders', 'postgres'); - expect(createConnector).toHaveBeenCalledWith('warehouse'); - expect(connector.executeReadOnly).toHaveBeenCalledWith( - { - connectionId: 'warehouse', - sql: 'select id from public.orders', - maxRows: 5, - }, - { runId: 'mcp-sql-execution' }, - ); - expect(connector.cleanup).toHaveBeenCalled(); - }); -``` - -- [ ] **Step 2: Write failing local-port rejection test** - -Add this test after the success test: - -```typescript - it('rejects MCP SQL before connector execution when parser validation fails', async () => { - const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - const connector = testConnector(testSnapshot(), { - headers: ['id'], - rows: [[1]], - totalRows: 1, - rowCount: 1, - }); - const sqlAnalysis = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(), - validateReadOnly: vi.fn(async () => ({ - ok: false, - error: 'SQL contains read/write operation: Insert', - })), - }; - const ports = createLocalProjectMcpContextPorts(project, { - sqlAnalysis, - localScan: { - createConnector: vi.fn(async () => connector), - }, - }); - - await expect( - ports.sqlExecution?.execute({ - connectionId: 'warehouse', - sql: 'with x as (insert into t values (1) returning *) select * from x', - maxRows: 1000, - }), - ).rejects.toThrow('SQL contains read/write operation: Insert'); - expect(connector.executeReadOnly).not.toHaveBeenCalled(); - }); -``` - -- [ ] **Step 3: Run failing local-port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "MCP SQL" -``` - -Expected: FAIL because `CreateLocalProjectMcpContextPortsOptions` has no `sqlAnalysis` option and no `sqlExecution` port. - -- [ ] **Step 4: Add SQL-analysis option and helper imports** - -In `packages/context/src/mcp/local-project-ports.ts`, add this import with the other context imports: - -```typescript -import type { SqlAnalysisDialect, SqlAnalysisPort } from '../sql-analysis/index.js'; -``` - -Add `sqlAnalysis` to `CreateLocalProjectMcpContextPortsOptions`: - -```typescript - sqlAnalysis?: SqlAnalysisPort; -``` - -Add this helper near `dialectForDriver`: - -```typescript -function sqlAnalysisDialectForDriver(driver: string | undefined): SqlAnalysisDialect { - return dialectForDriver(driver) as SqlAnalysisDialect; -} -``` - -- [ ] **Step 5: Implement the local SQL execution port** - -In `packages/context/src/mcp/local-project-ports.ts`, add this function before `createLocalProjectMcpContextPorts`: - -```typescript -async function executeValidatedReadOnlySql( - project: KtxLocalProject, - options: CreateLocalProjectMcpContextPortsOptions, - input: { connectionId: string; sql: string; maxRows: number }, -): Promise<{ headers: string[]; headerTypes?: string[]; rows: unknown[][]; rowCount: number }> { - const connectionId = assertSafeConnectionId(input.connectionId); - const connection = project.config.connections[connectionId]; - if (!connection) { - throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`); - } - if (!options.sqlAnalysis) { - throw new Error('sql_execution requires parser-backed SQL validation.'); - } - const validation = await options.sqlAnalysis.validateReadOnly( - input.sql, - sqlAnalysisDialectForDriver(connection.driver), - ); - if (!validation.ok) { - throw new Error(validation.error ?? 'SQL is not read-only.'); - } - const createConnector = options.localScan?.createConnector; - if (!createConnector) { - throw new Error('sql_execution requires a local scan connector factory.'); - } - - let connector: KtxScanConnector | null = null; - try { - connector = await createConnector(connectionId); - if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) { - throw new Error(`Connection "${connectionId}" does not support read-only SQL execution.`); - } - const result = await connector.executeReadOnly( - { - connectionId, - sql: input.sql, - maxRows: input.maxRows, - }, - { runId: 'mcp-sql-execution' }, - ); - return { - headers: result.headers, - ...(result.headerTypes ? { headerTypes: result.headerTypes } : {}), - rows: result.rows, - rowCount: result.rowCount ?? result.rows.length, - }; - } finally { - await cleanupConnector(connector); - } -} -``` - -In `createLocalProjectMcpContextPorts`, add this conditional block immediately after the initial `ports` object is created and before the existing `if (options.localIngest)` block: - -```typescript - if (options.sqlAnalysis && options.localScan?.createConnector) { - ports.sqlExecution = { - async execute(input) { - return executeValidatedReadOnlySql(project, options, input); - }, - }; - } -``` - -- [ ] **Step 6: Run local-port tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts -t "MCP SQL" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit local MCP SQL execution** - -Run: - -```bash -git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat(context): execute MCP SQL through validated connector path" -``` - -### Task 5: Verification - -**Files:** -- Verify: all modified files from Tasks 1-4 - -- [ ] **Step 1: Run Python SQL-analysis and app tests** - -Run: - -```bash -source .venv/bin/activate && uv run pytest python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -q -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused TypeScript tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sql-analysis/http-sql-analysis-port.test.ts src/mcp/server.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run Python pre-commit on changed Python files** - -Run: - -```bash -source .venv/bin/activate && uv run pre-commit run --files python/ktx-daemon/src/ktx_daemon/sql_analysis.py python/ktx-daemon/src/ktx_daemon/app.py python/ktx-daemon/tests/test_sql_analysis.py python/ktx-daemon/tests/test_app.py -``` - -Expected: PASS. If the repository has no usable pre-commit configuration in the active environment, record the exact error and keep the pytest results above as the closest Python verification. - -- [ ] **Step 5: Confirm the remaining v1 blockers are unchanged** - -Run: - -```bash -test -e packages/context/src/scan/entity-details.ts; printf 'entity-details:%s\n' "$?" -test -e packages/context/src/sl/dictionary-search.ts; printf 'dictionary-search:%s\n' "$?" -test -e packages/context/src/search/discover.ts; printf 'discover:%s\n' "$?" -test -e packages/cli/src/commands/mcp-commands.ts; printf 'mcp-commands:%s\n' "$?" -test -e packages/cli/src/skills/research/SKILL.md; printf 'research-skill:%s\n' "$?" -``` - -Expected: - -```text -entity-details:1 -dictionary-search:1 -discover:1 -mcp-commands:1 -research-skill:1 -``` - -These `1` exit-code markers confirm this plan landed only the SQL execution foundation and did not silently claim the remaining research-tool, daemon, or setup-agent v1 work. - -- [ ] **Step 6: Commit verification notes if any test docs changed** - -Run: - -```bash -git status --short -``` - -Expected: no uncommitted source changes after the task commits. If verification required a small documentation note, commit only that note with: - -```bash -git add docs/superpowers/plans/2026-05-14-research-agent-mcp-sql-execution-foundation.md -git commit -m "docs: record research MCP SQL execution plan" -``` diff --git a/docs/superpowers/plans/2026-05-15-claude-code-auth-probe-isolation-fix.md b/docs/superpowers/plans/2026-05-15-claude-code-auth-probe-isolation-fix.md deleted file mode 100644 index cee6774f..00000000 --- a/docs/superpowers/plans/2026-05-15-claude-code-auth-probe-isolation-fix.md +++ /dev/null @@ -1,678 +0,0 @@ -# Claude Code Auth Probe Isolation Fix Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the `claude-code` auth probe and runtime tolerate host-discovered -Claude Code init metadata while preserving KTX-owned tool, MCP, and plugin -restrictions. - -**Architecture:** Keep the existing Claude Code runtime and SDK option tuple. -Change the init-message assertion from "no host discovery appears" to "only the -KTX-controlled execution surface is active." Align the design spec and user docs -with the pinned SDK behavior: `settingSources: []` disables filesystem settings, -`skills: []` is a context filter, and deny-by-default `canUseTool` is the -runtime enforcement boundary. - -**Tech Stack:** TypeScript, pnpm, Vitest, Markdown, Fumadocs MDX, -`@anthropic-ai/claude-agent-sdk@0.3.142`. - ---- - -## Audit result - -The current strict isolation assertion is a v1-blocking bug. A real authenticated -Claude Code host can report non-empty `slash_commands`, `skills`, and `agents` -in the SDK init message even when KTX passes `settingSources: []`, `skills: []`, -`plugins: []`, `tools: []`, exact KTX MCP `allowedTools`, `disallowedTools`, and -deny-by-default `canUseTool`. - -Spec findings: - -- `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md:45-47` - requires host-discovered capabilities not to expand the KTX agent-loop tool - surface. That requirement is about invocation, not necessarily about zero - diagnostic metadata in the init message. -- `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md:254-265` - overreaches by asking the implementation to assert that unexpected - settings-derived commands, skills, agents, plugins, or MCP servers are - inactive from the SDK init message. In `@anthropic-ai/claude-agent-sdk@0.3.142`, - the available SDK controls cannot make `message.slash_commands`, - `message.skills`, or `message.agents` reliably empty on an authenticated host. -- `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md:266-267` - says skills are disabled with `skills: []`. The pinned SDK type definitions - document `skills` as a context filter, not a sandbox. -- `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md:543-545` - correctly requires the auth probe to pass the isolation option tuple and no - MCP servers. It does not require failing when host discovery metadata is - present. - -SDK evidence from -`node_modules/.pnpm/@anthropic-ai+claude-agent-sdk@0.3.142_zod@4.4.3/node_modules/@anthropic-ai/claude-agent-sdk/sdk.d.ts`: - -- Lines `1686-1695`: `settingSources: []` disables filesystem settings only. -- Lines `1697-1718`: `skills: []` is a context filter; unlisted skills are - hidden from listing and rejected by the Skill tool, but files remain on disk. -- Lines `1202-1213`: `allowedTools` is auto-approval, while `canUseTool` is the - permission handler for controlling tool execution. -- Lines `1224-1228`: `disallowedTools` removes listed tools from context and - prevents use. -- Lines `1255-1264`: `tools: []` disables built-in tools. -- Lines `1545-1558`: `plugins` loads plugins when supplied; KTX supplies `[]`. -- Lines `3465-3489`: the init message reports `agents`, `tools`, - `mcp_servers`, `slash_commands`, `skills`, and `plugins`. - -Implemented plan audit: - -- `2026-05-15-claude-code-backend-v1-runtime.md` is implemented for config, - runtime port, SDK dependency, model aliases, environment scrubbing, Claude Code - text/object/agent execution, setup/status/doctor support, docs, and LLM - call-site migration. -- `2026-05-15-claude-code-backend-v1-isolation-closure.md` is implemented, but - it converted the spec's ambiguous "assert inactive" line into an impossible - assertion against non-empty `slash_commands`, `skills`, and `agents`. -- `2026-05-15-claude-code-backend-v1-ingest-guidance-closure.md` is implemented - for the ingest missing-LLM guidance and associated CLI/context tests. - -Remaining v1-blocking gaps: - -- `packages/context/src/llm/claude-code-runtime.ts:94-101` throws on - host-discovered slash commands, skills, and agents. -- `packages/context/src/llm/claude-code-runtime.test.ts:158-178` encodes the - wrong behavior by requiring the runtime to reject any init message with - discovered agents. -- The auth probe has no regression coverage for an authenticated host whose init - message reports non-empty `slash_commands`, `skills`, and `agents`. -- User docs under `docs-site/content/docs/guides/` say KTX "disables" skills, - agents, hooks, and slash commands. That wording is stronger than the SDK - contract and must be changed to "not invokable by KTX agent loops." - -Non-blocking gaps: - -- Same-step AI SDK tool-call repair parity remains out of scope for v1. -- OTEL telemetry parity remains out of scope for v1. -- Embedding parity remains out of scope because embeddings are configured - separately. -- Full prompt-caching parity remains out of scope. V1 keeps warning on ignored - prompt-cache fields and avoids AI SDK cache markers on the Claude Code path. - -Decision: - -- Choose option (a): relax the assertion in code and align the spec text. Do not - rely on an invented SDK mechanism. The pinned type definitions expose - `settingSources`, `skills`, `plugins`, `tools`, `allowedTools`, - `disallowedTools`, and `canUseTool`, but they do not expose a query option that - disables all host-discovered slash commands or user-level subagent names in the - init message. - -## File structure - -Modify these files: - -- `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md` aligns the - design with the real SDK contract. -- `packages/context/src/llm/claude-code-runtime.test.ts` adds the failing - regression tests for auth probe and runtime init metadata. -- `packages/context/src/llm/claude-code-runtime.ts` relaxes init metadata checks - while tightening exact tool equality. -- `docs-site/content/docs/guides/llm-configuration.mdx` changes user docs from - "disabled" to "not invokable." -- `docs-site/content/docs/guides/building-context.mdx` applies the same - user-facing wording at the ingest guide boundary. - -### Task 1: Align the design spec with SDK reality - -**Files:** - -- Modify: `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md` - -- [ ] **Step 1: Update the tool-boundary goal** - -Replace the goal bullet at lines `45-47` with: - -```markdown -- Preserve KTX's curated tool boundaries. Claude Code built-ins, - filesystem-discovered MCP servers, hooks, skills, plugins, agents, and slash - commands must not become invokable in KTX agent loops. The Agent SDK init - message may still report host-discovered slash commands, skills, and agents; - KTX treats that metadata as diagnostic only and restricts execution through - `tools: []`, exact KTX MCP `allowedTools`, `disallowedTools`, and - deny-by-default `canUseTool`. -``` - -- [ ] **Step 2: Replace the over-broad init assertion requirement** - -Replace the bullet at lines `254-265` with: - -```markdown -- Filesystem settings are not loaded. The SDK's documented default for an - omitted `settingSources` is `["user", "project", "local"]` - (`@anthropic-ai/claude-agent-sdk@0.3.142` `sdk.d.ts:1686-1695`), - which would inherit the user's Claude Code filesystem settings. Every KTX - `query()` call site - agent loops, text generation, object generation, and - the auth probe - MUST pass `settingSources: []` explicitly, along with - `skills: []`, `plugins: []`, `tools: []`, `persistSession: false`, and no - `mcpServers` entries other than the KTX MCP server (omitted entirely when - the call site does not expose tools). The implementation MUST assert from - the SDK init message that the controlled execution surface matches KTX's - expectations: - - - `message.tools` equals the exact generated KTX MCP tool ids for the current - call. - - `message.mcp_servers` equals the expected KTX MCP server set: `[]` when the - call exposes no tools, or `["ktx"]` when it does. - - `message.plugins` is empty. - - The implementation MUST NOT reject a run solely because - `message.slash_commands`, `message.skills`, or `message.agents` contain - host-discovered names. In `@anthropic-ai/claude-agent-sdk@0.3.142`, those - fields can report host discovery even when KTX passes the isolation options. - They are not part of the KTX execution surface when `tools: []`, - `allowedTools`, `disallowedTools`, and deny-by-default `canUseTool` are set. -``` - -- [ ] **Step 3: Replace the skills/plugin wording** - -Replace the bullets at lines `266-289` with: - -```markdown -- `skills: []` is a context filter in the pinned SDK - (`sdk.d.ts:1697-1718`): unlisted skills are hidden from the model's skill - listing and rejected by the Skill tool, but discovered skill names may still - appear in init metadata. KTX must still pass `skills: []`. -- Plugins are disabled with `plugins: []`, and the runtime asserts that - `message.plugins` is empty in the init message. -- Built-in tools are disabled by setting `tools: []`. The pinned SDK type - (`@anthropic-ai/claude-agent-sdk@0.3.142`, `sdk.d.ts:1255-1264`) documents - `tools` as the base set of built-in tools, with `[]` meaning "disable all - built-ins"; `tools` does not accept MCP tool ids and cannot be used to - restrict MCP availability. -- MCP tool availability is granted by registering the KTX MCP server through - `mcpServers`. The SDK does not document a wildcard like `mcp__ktx__*` for - any tool field; KTX must enumerate exact generated MCP tool ids of the form - `mcp__ktx__` (derived from the tool map handed to - `createSdkMcpServer`) wherever a list of tool ids is required. -- Pre-approval under `permissionMode: "dontAsk"` is configured by listing those - same exact `mcp__ktx__` ids in `allowedTools` (documented as - auto-allow without prompting). Treat `allowedTools` as auto-approval, not - restriction. -- Defense-in-depth restriction uses `canUseTool`. The KTX runtime supplies a - `canUseTool` handler that allows only tool names in the current KTX MCP tool - map and denies everything else, so host-discovered slash commands, skills, - agents, future SDK defaults, or a misconfigured MCP server cannot expand the - execution surface. -- `disallowedTools` MUST additionally list the current built-in tool names - (`Agent`, `Task`, `AskUserQuestion`, `Bash`, `Read`, `Edit`, `Write`, `Glob`, - `Grep`, `WebFetch`, `WebSearch`, `TodoWrite`) as redundant insurance. -``` - -- [ ] **Step 4: Update auth probe acceptance text** - -After the auth probe option list at lines `543-545`, add: - -```markdown - The auth probe MUST tolerate init messages with non-empty - `slash_commands`, `skills`, and `agents` when `message.tools` is empty, - `message.mcp_servers` is empty, `message.plugins` is empty, and the query - options contain the KTX isolation tuple. Host discovery metadata is not an - auth failure. -``` - -- [ ] **Step 5: Update verified evidence and open items** - -Replace lines `621-623` with: - -```markdown -- The Agent SDK skills docs say the `skills` option is a context filter rather - than a sandbox. KTX must pass `skills: []`, but must not assert that - `message.skills` is empty in the SDK init message. -``` - -Replace open item `8` at lines `648-649` with: - -```markdown -8. Write tests proving a raw built-in Claude Code tool request is denied, - host-discovered Skill/Agent/SlashCommand requests are denied by `canUseTool`, - and only exact `mcp__ktx__*` tools are allowed during KTX agent loops. -``` - -Replace open item `9` at lines `650-654` with: - -```markdown -9. Write a test that asserts every KTX-originated `query()` invocation - (agent loop, text generation, object generation, auth probe) is called - with `settingSources: []`, `skills: []`, `plugins: []`, `tools: []`, and - `persistSession: false`, by spying on the SDK entry point. The test must - fail if any path falls back to SDK defaults for those fields. The test must - also prove that non-empty host-discovered `slash_commands`, `skills`, and - `agents` in the init message do not fail the auth probe or runtime when the - controlled tool, MCP server, and plugin surfaces match KTX expectations. -``` - -- [ ] **Step 6: Commit the spec alignment** - -Run: - -```bash -git add docs/superpowers/specs/2026-05-15-claude-code-backend-design.md -git commit -m "docs: align claude-code isolation spec with sdk metadata" -``` - -Expected: the design spec no longer requires zero host-discovery metadata in -the SDK init message. - -### Task 2: Add regression tests for host-discovered init metadata - -**Files:** - -- Modify: `packages/context/src/llm/claude-code-runtime.test.ts` - -- [ ] **Step 1: Replace the invalid agent rejection test** - -In `packages/context/src/llm/claude-code-runtime.test.ts`, replace the test named -`rejects settings-derived agents and non-KTX MCP servers from init messages` -with these tests: - -```ts - it('treats host-discovered commands skills and agents as non-fatal init metadata for text and auth probe', async () => { - const hostDiscoveredInit = initMessage({ - slash_commands: ['/help', '/compact', '/clear', '/user-command'], - skills: ['pdf', 'docx'], - agents: ['claude', 'Explore', 'general-purpose'], - }); - const textQuery = vi.fn((_input: any) => - stream([hostDiscoveredInit, resultMessage({ result: 'hello' })]), - ); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query: textQuery, - env: { ANTHROPIC_API_KEY: 'sk-ant-test', PATH: '/usr/bin' }, // pragma: allowlist secret - }); - - await expect(runtime.generateText({ role: 'default', prompt: 'say hello' })).resolves.toBe('hello'); - const textOptions = textQuery.mock.calls[0][0].options; - expect(textOptions).toMatchObject({ - settingSources: [], - skills: [], - plugins: [], - tools: [], - allowedTools: [], - permissionMode: 'dontAsk', - persistSession: false, - env: expect.not.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test' }), - }); - expect(textOptions.disallowedTools).toEqual(expect.arrayContaining(['Agent', 'Task', 'Bash'])); - expect(await textOptions.canUseTool('Agent', {}, { signal: new AbortController().signal, toolUseID: 'agent' })).toMatchObject({ - behavior: 'deny', - toolUseID: 'agent', - }); - expect(await textOptions.canUseTool('Skill', {}, { signal: new AbortController().signal, toolUseID: 'skill' })).toMatchObject({ - behavior: 'deny', - toolUseID: 'skill', - }); - expect( - await textOptions.canUseTool('SlashCommand', {}, { signal: new AbortController().signal, toolUseID: 'slash' }), - ).toMatchObject({ - behavior: 'deny', - toolUseID: 'slash', - }); - - const probeQuery = vi.fn((_input: any) => - stream([hostDiscoveredInit, resultMessage({ result: 'ok' })]), - ); - await expect( - runClaudeCodeAuthProbe({ - projectDir: '/tmp/project', - model: 'sonnet', - query: probeQuery, - env: { ANTHROPIC_AUTH_TOKEN: 'token', HOME: '/Users/test' }, - }), - ).resolves.toEqual({ ok: true }); - expect(probeQuery.mock.calls[0][0].options).toMatchObject({ - settingSources: [], - skills: [], - plugins: [], - tools: [], - allowedTools: [], - permissionMode: 'dontAsk', - persistSession: false, - env: expect.objectContaining({ HOME: '/Users/test' }), - }); - expect(probeQuery.mock.calls[0][0].options.env).not.toEqual( - expect.objectContaining({ ANTHROPIC_AUTH_TOKEN: 'token' }), - ); - }); - - it('allows host-discovered context during agent loops while requiring exact KTX MCP tools and servers', async () => { - const query = vi.fn((_input: any) => - stream([ - initMessage({ - tools: ['mcp__ktx__load_skill'], - mcp_servers: [{ name: 'ktx', status: 'connected' }], - slash_commands: ['/help', '/compact', '/clear'], - skills: ['memory-agent', 'doc-reader'], - agents: ['claude', 'Plan', 'Explore'], - }), - { - type: 'assistant', - message: { role: 'assistant', content: [] }, - parent_tool_use_id: null, - uuid: '00000000-0000-4000-8000-000000000006', - session_id: 'session-id', - } as unknown as SDKMessage, - resultMessage({ subtype: 'error_max_turns', is_error: true }), - ]), - ); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: {}, - }); - - await expect( - runtime.runAgentLoop({ - modelRole: 'default', - systemPrompt: 'system', - userPrompt: 'user', - toolSet: { - load_skill: { - name: 'load_skill', - description: 'Load skill.', - inputSchema: z.object({ name: z.string() }), - execute: async () => ({ markdown: 'loaded' }), - }, - }, - stepBudget: 1, - telemetryTags: { operationName: 'test' }, - }), - ).resolves.toEqual({ stopReason: 'budget' }); - - const options = query.mock.calls[0][0].options; - expect(options.allowedTools).toEqual(['mcp__ktx__load_skill']); - expect(await options.canUseTool('mcp__ktx__load_skill', {}, { signal: new AbortController().signal, toolUseID: '1' })).toEqual({ - behavior: 'allow', - toolUseID: '1', - }); - expect(await options.canUseTool('Task', {}, { signal: new AbortController().signal, toolUseID: '2' })).toMatchObject({ - behavior: 'deny', - toolUseID: '2', - }); - expect(await options.canUseTool('Skill', {}, { signal: new AbortController().signal, toolUseID: '3' })).toMatchObject({ - behavior: 'deny', - toolUseID: '3', - }); - }); - - it('still rejects unexpected tools, missing KTX tools, plugins, and non-KTX MCP servers from init messages', async () => { - const query = vi.fn((_input: any) => - stream([ - initMessage({ - tools: ['Bash'], - mcp_servers: [{ name: 'filesystem', status: 'connected' }], - plugins: [{ name: 'host-plugin', path: '/tmp/plugin' }], - }), - resultMessage({ result: 'hello' }), - ]), - ); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: {}, - }); - - await expect( - runtime.generateText({ - role: 'default', - prompt: 'say hello', - tools: { - load_skill: { - name: 'load_skill', - description: 'Load skill.', - inputSchema: z.object({ name: z.string() }), - execute: async () => ({ markdown: 'loaded' }), - }, - }, - }), - ).rejects.toThrow( - /Claude Code runtime isolation failed: .*tools=Bash.*missing_tools=mcp__ktx__load_skill.*mcp_servers=filesystem.*plugins=host-plugin/, - ); - }); -``` - -- [ ] **Step 2: Run the runtime test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts -``` - -Expected: FAIL. The first new test fails because `runClaudeCodeAuthProbe(...)` -returns `{ ok: false, ... }` and `generateText(...)` rejects when init metadata -contains non-empty `slash_commands`, `skills`, or `agents`. The second new test -fails because `runAgentLoop(...)` returns `{ stopReason: 'error', ... }` for the -same reason. - -- [ ] **Step 3: Commit the failing regression test** - -Run: - -```bash -git add packages/context/src/llm/claude-code-runtime.test.ts -git commit -m "test: cover claude-code host discovery metadata" -``` - -Expected: the commit contains tests that fail before the runtime assertion is -fixed. - -### Task 3: Relax init metadata assertions to the controlled execution surface - -**Files:** - -- Modify: `packages/context/src/llm/claude-code-runtime.ts` - -- [ ] **Step 1: Replace `assertInitIsolation`** - -In `packages/context/src/llm/claude-code-runtime.ts`, replace the full -`assertInitIsolation(...)` function with: - -```ts -function assertInitIsolation( - message: SDKMessage, - allowedToolIds: Set, - expectedMcpServerNames: Set, -): void { - if (message.type !== 'system' || message.subtype !== 'init') { - return; - } - const activeToolIds = new Set(message.tools); - const unexpectedTools = message.tools.filter((toolName) => !allowedToolIds.has(toolName)); - const missingTools = [...allowedToolIds].filter((toolName) => !activeToolIds.has(toolName)); - const activeMcpServerNames = message.mcp_servers.map((server) => server.name); - const unexpectedMcpServers = activeMcpServerNames.filter((name) => !expectedMcpServerNames.has(name)); - const missingMcpServers = [...expectedMcpServerNames].filter((name) => !activeMcpServerNames.includes(name)); - const unexpectedPlugins = message.plugins.map((plugin) => plugin.name); - if ( - unexpectedTools.length > 0 || - missingTools.length > 0 || - unexpectedMcpServers.length > 0 || - missingMcpServers.length > 0 || - unexpectedPlugins.length > 0 - ) { - throw new Error( - `Claude Code runtime isolation failed: tools=${unexpectedTools.join(',') || '(none)'} missing_tools=${ - missingTools.join(',') || '(none)' - } mcp_servers=${unexpectedMcpServers.join(',') || '(none)'} missing_mcp_servers=${ - missingMcpServers.join(',') || '(none)' - } plugins=${unexpectedPlugins.join(',') || '(none)'} host_slash_commands=${ - message.slash_commands.length - } host_skills=${message.skills.length} host_agents=${message.agents?.join(',') || '(none)'}`, - ); - } -} -``` - -This preserves strict checks for the KTX-controlled execution surface: - -- `message.tools` must exactly equal the generated KTX MCP tool ids for the - current call. -- `message.mcp_servers` must exactly equal the expected KTX MCP server names. -- `message.plugins` must be empty. - -It deliberately stops treating `message.slash_commands`, `message.skills`, and -`message.agents` as fatal because those fields can contain host-discovered -metadata that KTX cannot disable through the pinned SDK options. - -- [ ] **Step 2: Run the runtime test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Commit the runtime fix** - -Run: - -```bash -git add packages/context/src/llm/claude-code-runtime.ts packages/context/src/llm/claude-code-runtime.test.ts -git commit -m "fix: tolerate claude-code host discovery metadata" -``` - -Expected: the auth probe and runtime no longer fail solely because the SDK init -message reports host-discovered slash commands, skills, or agents. - -### Task 4: Correct user-facing docs wording - -**Files:** - -- Modify: `docs-site/content/docs/guides/llm-configuration.mdx` -- Modify: `docs-site/content/docs/guides/building-context.mdx` - -- [ ] **Step 1: Update the LLM configuration guide wording** - -In `docs-site/content/docs/guides/llm-configuration.mdx`, replace lines `39-41` -with: - -```mdx -`claude-code` keeps KTX tool boundaries intact. KTX exposes only the MCP tools -needed for the current KTX agent loop, disables Claude Code built-in tools, -keeps plugins empty, and denies every non-KTX tool request through -`canUseTool`. The Claude Agent SDK may still report host-discovered slash -commands, skills, and subagent names in init metadata; that metadata is not an -execution grant for KTX agent loops. -``` - -- [ ] **Step 2: Update the building context guide wording** - -In `docs-site/content/docs/guides/building-context.mdx`, replace lines `61-63` -with: - -```mdx -When you use `claude-code`, KTX still controls the tool surface for ingest and -memory capture. Claude Code built-in tools, discovered MCP servers, plugins, -skills, agents, and slash commands are not invokable by KTX agent loops unless -they are exact KTX MCP tools for the current run. -``` - -- [ ] **Step 3: Run docs tests** - -Run: - -```bash -pnpm --filter ktx-docs run test -``` - -Expected: PASS. - -- [ ] **Step 4: Commit docs wording** - -Run: - -```bash -git add docs-site/content/docs/guides/llm-configuration.mdx docs-site/content/docs/guides/building-context.mdx -git commit -m "docs: clarify claude-code host discovery metadata" -``` - -Expected: user docs describe invocation control rather than promising zero -host-discovery metadata. - -### Task 5: Final verification - -**Files:** - -- Verify: `docs/superpowers/specs/2026-05-15-claude-code-backend-design.md` -- Verify: `packages/context/src/llm/claude-code-runtime.ts` -- Verify: `packages/context/src/llm/claude-code-runtime.test.ts` -- Verify: `docs-site/content/docs/guides/llm-configuration.mdx` -- Verify: `docs-site/content/docs/guides/building-context.mdx` - -- [ ] **Step 1: Run targeted runtime tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts src/llm/runtime-tools.test.ts src/llm/claude-code-env.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run docs verification** - -Run: - -```bash -pnpm --filter ktx-docs run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS or only pre-existing unrelated findings. Investigate and fix any -finding caused by the runtime assertion or test changes. - -- [ ] **Step 5: Inspect git status** - -Run: - -```bash -git status --short -``` - -Expected: only files from this plan are modified, or the working tree is clean -if each task was committed. - -## Self-review - -- Spec coverage: This plan addresses the v1-blocking auth probe failure, - aligns the spec with the SDK contract, preserves the real KTX execution - boundary, and adds regression coverage for non-empty host-discovered - `slash_commands`, `skills`, and `agents` in both auth probe and runtime paths. -- Placeholder scan: No placeholder markers remain. Every code-changing step - includes exact file paths, code blocks, commands, and expected results. -- Type consistency: The plan uses existing names from the codebase: - `ClaudeCodeKtxLlmRuntime`, `runClaudeCodeAuthProbe`, `initMessage`, - `resultMessage`, `assertInitIsolation`, `mcpToolIds`, `KtxRuntimeToolSet`, and - `canUseTool`. diff --git a/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-ingest-guidance-closure.md b/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-ingest-guidance-closure.md deleted file mode 100644 index 5243ac31..00000000 --- a/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-ingest-guidance-closure.md +++ /dev/null @@ -1,160 +0,0 @@ -# Claude Code Backend V1 Ingest Guidance Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the `ktx ingest` missing-LLM guidance treat `claude-code` as a first-class setup path and restore the CLI ingest test suite. - -**Architecture:** Keep the existing Claude Code runtime implementation unchanged. Update the single local-ingest guard message so users see both the local Claude Code setup path and the Anthropic API setup path, then align the context and CLI tests with that user-facing copy. - -**Tech Stack:** TypeScript, pnpm, Vitest. - ---- - -## Audit summary - -The May 15 Claude Code backend runtime and isolation plans are implemented for -the core runtime path: config accepts `claude-code`, runtime calls use -`KtxLlmRuntimePort`, Claude SDK calls pass isolation options and scrubbed env, -setup/status/doctor validate Claude Code auth, and docs describe the backend. - -One v1-blocking issue remains: `packages/context/src/ingest/local-bundle-runtime.ts` -lists `claude-code` in the missing-LLM guard line but still tells users only to -"Configure an Anthropic provider." The full CLI ingest test suite currently -fails because `packages/cli/src/ingest.test.ts` still expects the old provider -list without `claude-code`. This is v1-blocking because CI is red and the -fallback guidance is not first-class for the new backend. - -Non-blocking gaps from the original spec remain unchanged: - -- Same-step AI SDK tool-call repair parity is out of scope for the Claude Code - runtime. -- OTEL telemetry parity is out of scope for the Claude Code runtime. -- Embedding parity is out of scope because embeddings stay independently - configured. -- Full prompt-caching parity for tools, history, and per-section TTLs is out of - scope; v1 only needs no AI SDK cache markers on `claude-code` and explicit - warnings for ignored fields. - -## File structure - -Modify these files: - -- `packages/context/src/ingest/local-bundle-runtime.ts` owns the missing-LLM - guard message used by local ingest and MCP-triggered ingest. -- `packages/context/src/ingest/local-bundle-runtime.test.ts` verifies the guard - message at the context boundary. -- `packages/cli/src/ingest.test.ts` verifies the user-facing CLI output. - -No `docs-site/` update is required because the existing public docs already -document `claude-code` setup and ingest behavior; this plan only fixes an -inline runtime error message. - -### Task 1: Update ingest LLM setup guidance - -**Files:** - -- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` -- Modify: `packages/cli/src/ingest.test.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` - -- [ ] **Step 1: Update the context guard-message test** - -In `packages/context/src/ingest/local-bundle-runtime.test.ts`, replace the -expected message in `requires an agent runner or configured local ingest LLM` -with this exact array: - -```ts -[ - 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', - 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:', - ` ktx setup --project-dir ${project.projectDir} --llm-backend claude-code --no-input`, - ` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`, -].join('\n') -``` - -- [ ] **Step 2: Update the CLI ingest test** - -In `packages/cli/src/ingest.test.ts`, replace the stale provider-list -assertion in `prints provider setup guidance when a skip-llm setup project runs -ingest` with: - -```ts -expect(runIo.stderr()).toContain( - 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', -); -expect(runIo.stderr()).toContain('Configure a local Claude Code session or API-backed LLM, then rerun ingest:'); -expect(runIo.stderr()).toContain(`ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`); -expect(runIo.stderr()).toContain( - `ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`, -); -``` - -- [ ] **Step 3: Run tests to verify the new expectations fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts -``` - -Expected: both suites fail because the source message still says -`Configure an Anthropic provider, then rerun ingest:` and does not include the -Claude Code setup command. - -- [ ] **Step 4: Update the ingest guard message** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, replace -`localIngestLlmProviderGuardMessage` with: - -```ts -function localIngestLlmProviderGuardMessage(projectDir: string): string { - return [ - 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', - 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:', - ` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`, - ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`, - ].join('\n'); -} -``` - -- [ ] **Step 5: Run the targeted tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts -``` - -Expected: both suites pass. - -- [ ] **Step 6: Run package type-checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: both commands pass. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add packages/context/src/ingest/local-bundle-runtime.ts packages/context/src/ingest/local-bundle-runtime.test.ts packages/cli/src/ingest.test.ts -git commit -m "fix: update claude-code ingest setup guidance" -``` - -## Self-review - -- Spec coverage: This plan closes the only remaining v1-blocking audit finding: - ingest setup guidance and CLI test expectations now include `claude-code` as - a first-class backend. -- Placeholder scan: No placeholders remain; every step includes exact paths, - code, commands, and expected output. -- Type consistency: The exact guard string is identical across the source and - both test updates. diff --git a/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-isolation-closure.md b/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-isolation-closure.md deleted file mode 100644 index 6295dd63..00000000 --- a/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-isolation-closure.md +++ /dev/null @@ -1,575 +0,0 @@ -# Claude Code Backend V1 Isolation Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking Claude Code backend gaps around SDK -init isolation assertions and setup-time prompt-caching warnings. - -**Architecture:** Keep the existing runtime port and Claude Code runtime. Add -the missing init-message checks inside the Claude runtime, then share the -prompt-caching warning formatter between status/doctor and setup so all -user-facing readiness flows report ignored Claude Code cache knobs consistently. - -**Tech Stack:** TypeScript, pnpm, Vitest, Zod, `@anthropic-ai/claude-agent-sdk@0.3.142`. - ---- - -## Audit Summary - -The May 15 Claude Code backend v1 plan is mostly implemented. Remaining -v1-blocking gaps from the original spec are: - -- `packages/context/src/llm/claude-code-runtime.ts` asserts init-message tools, - slash commands, skills, and plugins, but does not assert `agents` or - unexpected `mcp_servers`. The spec requires asserting that settings-derived - commands, skills, agents, plugins, and MCP servers are inactive. -- `packages/cli/src/setup-models.ts` validates Claude Code auth but does not - surface ignored `llm.promptCaching` fields during setup. The spec requires - setup, status, and doctor to surface ignored prompt-caching fields for the - `claude-code` backend. Status and doctor already warn. - -Non-blocking gaps: - -- Same-step tool-call repair parity remains out of scope for v1. -- OTEL telemetry parity remains out of scope for v1. -- Embedding parity remains out of scope because embeddings are configured - independently. -- Full prompt-caching parity for tools, history, and per-section TTLs remains - out of scope; v1 only needs explicit warnings and no AI SDK cache markers on - the Claude Code path. - -## File Structure - -Modify these files: - -- `packages/context/src/llm/claude-code-runtime.ts` adds complete init-message - isolation checks for agents and MCP servers. -- `packages/context/src/llm/claude-code-runtime.test.ts` adds regression tests - for rejected agents/MCP servers, object/agent env scrubbing, and callback - error handling. -- `packages/cli/src/claude-code-prompt-caching.ts` is created as the shared - formatter for ignored prompt-caching fields. -- `packages/cli/src/status-project.ts` imports the shared formatter instead of - keeping a local helper. -- `packages/cli/src/setup-models.ts` emits the shared warning when setup saves - `llm.provider.backend: claude-code` and existing prompt-caching fields are - present. -- `packages/cli/src/setup-models.test.ts` covers setup warning output. -- `packages/cli/src/doctor.test.ts` keeps coverage for doctor output using the - shared formatter. - -### Task 1: Complete Claude Code init isolation checks - -**Files:** - -- Modify: `packages/context/src/llm/claude-code-runtime.test.ts` -- Modify: `packages/context/src/llm/claude-code-runtime.ts` - -- [ ] **Step 1: Add failing isolation and runtime behavior tests** - -Add these tests inside `describe('ClaudeCodeKtxLlmRuntime', ...)` in -`packages/context/src/llm/claude-code-runtime.test.ts`: - -```ts - it('rejects settings-derived agents and non-KTX MCP servers from init messages', async () => { - const query = vi.fn((_input: any) => - stream([ - initMessage({ - agents: ['project-agent'], - mcp_servers: [{ name: 'filesystem', status: 'connected' }], - }), - resultMessage({ result: 'hello' }), - ]), - ); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: {}, - }); - - await expect(runtime.generateText({ role: 'default', prompt: 'say hello' })).rejects.toThrow( - /Claude Code runtime isolation failed: .*mcp_servers=filesystem.*agents=project-agent/, - ); - }); - - it('passes scrubbed env to object generation and agent loops', async () => { - const schema = z.object({ answer: z.string() }); - const objectQuery = vi.fn((_input: any) => - stream([initMessage(), resultMessage({ structured_output: { answer: 'yes' } })]), - ); - const objectRuntime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query: objectQuery, - env: { ANTHROPIC_API_KEY: 'sk-ant-test', AWS_PROFILE: 'prod', PATH: '/usr/bin' }, // pragma: allowlist secret - }); - - await expect(objectRuntime.generateObject({ role: 'default', prompt: 'json', schema })).resolves.toEqual({ - answer: 'yes', - }); - expect(objectQuery.mock.calls[0][0].options.env).toEqual( - expect.objectContaining({ PATH: '/usr/bin' }), - ); - expect(objectQuery.mock.calls[0][0].options.env).not.toEqual( - expect.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test', AWS_PROFILE: 'prod' }), // pragma: allowlist secret - ); - - const agentQuery = vi.fn((_input: any) => - stream([ - initMessage({ tools: ['mcp__ktx__load_skill'], mcp_servers: [{ name: 'ktx', status: 'connected' }] }), - { - type: 'assistant', - message: { role: 'assistant', content: [] }, - parent_tool_use_id: null, - uuid: '00000000-0000-4000-8000-000000000004', - session_id: 'session-id', - } as unknown as SDKMessage, - resultMessage({ subtype: 'error_max_turns', is_error: true }), - ]), - ); - const agentRuntime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query: agentQuery, - env: { ANTHROPIC_AUTH_TOKEN: 'token', CLAUDE_CODE_USE_VERTEX: '1', HOME: '/Users/test' }, - }); - - await agentRuntime.runAgentLoop({ - modelRole: 'default', - systemPrompt: 'system', - userPrompt: 'user', - toolSet: { - load_skill: { - name: 'load_skill', - description: 'Load skill.', - inputSchema: z.object({ name: z.string() }), - execute: async () => ({ markdown: 'loaded' }), - }, - }, - stepBudget: 1, - telemetryTags: { operationName: 'test' }, - }); - expect(agentQuery.mock.calls[0][0].options.env).toEqual(expect.objectContaining({ HOME: '/Users/test' })); - expect(agentQuery.mock.calls[0][0].options.env).not.toEqual( - expect.objectContaining({ ANTHROPIC_AUTH_TOKEN: 'token', CLAUDE_CODE_USE_VERTEX: '1' }), - ); - }); - - it('logs and ignores onStepFinish callback errors', async () => { - const query = vi.fn((_input: any) => - stream([ - initMessage(), - { - type: 'assistant', - message: { role: 'assistant', content: [] }, - parent_tool_use_id: null, - uuid: '00000000-0000-4000-8000-000000000005', - session_id: 'session-id', - } as unknown as SDKMessage, - resultMessage({ subtype: 'success', terminal_reason: 'completed' }), - ]), - ); - const logger = { - debug: vi.fn(), - log: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }; - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: {}, - logger, - }); - - await expect( - runtime.runAgentLoop({ - modelRole: 'default', - systemPrompt: 'system', - userPrompt: 'user', - toolSet: {}, - stepBudget: 1, - telemetryTags: { operationName: 'test' }, - onStepFinish: async () => { - throw new Error('callback exploded'); - }, - }), - ).resolves.toEqual({ stopReason: 'natural' }); - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('callback exploded')); - }); -``` - -- [ ] **Step 2: Run the Claude runtime test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts -``` - -Expected: FAIL because the new agents/MCP-server isolation test resolves -successfully instead of throwing. - -- [ ] **Step 3: Add expected MCP server metadata and complete init assertions** - -In `packages/context/src/llm/claude-code-runtime.ts`, replace -`assertInitIsolation` and add the helper below it: - -```ts -function assertInitIsolation( - message: SDKMessage, - allowedToolIds: Set, - expectedMcpServerNames: Set, -): void { - if (message.type !== 'system' || message.subtype !== 'init') { - return; - } - const unexpectedTools = message.tools.filter((toolName) => !allowedToolIds.has(toolName)); - const activeMcpServerNames = message.mcp_servers.map((server) => server.name); - const unexpectedMcpServers = activeMcpServerNames.filter((name) => !expectedMcpServerNames.has(name)); - const missingMcpServers = [...expectedMcpServerNames].filter((name) => !activeMcpServerNames.includes(name)); - const unexpectedAgents = message.agents ?? []; - if ( - unexpectedTools.length > 0 || - unexpectedMcpServers.length > 0 || - missingMcpServers.length > 0 || - message.slash_commands.length > 0 || - message.skills.length > 0 || - message.plugins.length > 0 || - unexpectedAgents.length > 0 - ) { - throw new Error( - `Claude Code runtime isolation failed: tools=${unexpectedTools.join(',') || '(none)'} mcp_servers=${ - unexpectedMcpServers.join(',') || '(none)' - } missing_mcp_servers=${missingMcpServers.join(',') || '(none)'} slash_commands=${ - message.slash_commands.length - } skills=${message.skills.length} plugins=${message.plugins.length} agents=${ - unexpectedAgents.join(',') || '(none)' - }`, - ); - } -} - -function expectedMcpServerNames(tools: KtxRuntimeToolSet | undefined): Set { - return tools && Object.keys(tools).length > 0 ? new Set(['ktx']) : new Set(); -} -``` - -Update `collectResult` parameters: - -```ts -async function collectResult(params: { - query: QueryFn; - prompt: string; - options: Options; - allowedToolIds: Set; - expectedMcpServerNames: Set; - onAssistantTurn?: () => Promise; -}): Promise { - let result: SDKResultMessage | undefined; - for await (const message of params.query({ prompt: params.prompt, options: params.options })) { - assertInitIsolation(message, params.allowedToolIds, params.expectedMcpServerNames); -``` - -Update the four `collectResult(...)` calls: - -```ts - const tools = input.tools ?? {}; - const result = await collectResult({ - query: this.runQuery, - prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'), - options, - allowedToolIds: new Set(mcpToolIds(tools)), - expectedMcpServerNames: expectedMcpServerNames(input.tools), - }); -``` - -For `runAgentLoop(...)`, use: - -```ts - const result = await collectResult({ - query: this.runQuery, - prompt: params.userPrompt, - options: { ...options, systemPrompt: params.systemPrompt }, - allowedToolIds: new Set(mcpToolIds(params.toolSet)), - expectedMcpServerNames: expectedMcpServerNames(params.toolSet), - onAssistantTurn: async () => { -``` - -For `runClaudeCodeAuthProbe(...)`, use: - -```ts - const result = await collectResult({ - query: input.query ?? defaultQuery, - prompt: 'Reply with exactly: ok', - options, - allowedToolIds: new Set(), - expectedMcpServerNames: new Set(), - }); -``` - -- [ ] **Step 4: Run the Claude runtime test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/llm/claude-code-runtime.ts packages/context/src/llm/claude-code-runtime.test.ts -git commit -m "fix: close claude-code runtime isolation checks" -``` - -### Task 2: Surface Claude Code prompt-caching warnings during setup - -**Files:** - -- Create: `packages/cli/src/claude-code-prompt-caching.ts` -- Modify: `packages/cli/src/status-project.ts` -- Modify: `packages/cli/src/setup-models.ts` -- Modify: `packages/cli/src/setup-models.test.ts` -- Modify: `packages/cli/src/doctor.test.ts` - -- [ ] **Step 1: Add failing setup warning test** - -Add this test to `packages/cli/src/setup-models.test.ts`: - -```ts - it('warns during Claude Code setup when existing prompt-caching fields will be ignored', async () => { - await writeFile( - join(tempDir, 'ktx.yaml'), - [ - 'llm:', - ' provider:', - ' backend: anthropic', - ' models:', - ' default: claude-sonnet-4-6', - ' promptCaching:', - ' enabled: true', - ' systemTtl: 1h', - ' toolsTtl: 1h', - ' historyTtl: 5m', - '', - ].join('\n'), - 'utf-8', - ); - const io = makeIo(); - - const result = await runKtxSetupAnthropicModelStep( - { - projectDir: tempDir, - inputMode: 'disabled', - llmBackend: 'claude-code', - skipLlm: false, - }, - io.io, - { - claudeCodeAuthProbe: async () => ({ ok: true as const }), - }, - ); - - expect(result.status).toBe('ready'); - expect(io.stderr()).toContain('claude-code ignores llm.promptCaching.systemTtl'); - expect(io.stderr()).toContain('Claude Agent SDK does not expose KTX prompt-cache TTL, tool, or history markers'); - }); -``` - -- [ ] **Step 2: Run setup tests to verify the new test fails** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-models.test.ts -``` - -Expected: FAIL because setup does not emit the ignored prompt-caching warning. - -- [ ] **Step 3: Create the shared prompt-caching warning helper** - -Create `packages/cli/src/claude-code-prompt-caching.ts`: - -```ts -import type { KtxProjectLlmConfig } from '@ktx/context/project'; - -const CLAUDE_CODE_IGNORED_PROMPT_CACHING_FIELDS = [ - 'systemTtl', - 'toolsTtl', - 'historyTtl', - 'vertexFallbackTo5m', -] as const; - -export function ignoredClaudeCodePromptCachingFields(config: KtxProjectLlmConfig): string[] { - if (config.provider.backend !== 'claude-code' || !config.promptCaching) { - return []; - } - return CLAUDE_CODE_IGNORED_PROMPT_CACHING_FIELDS.filter((key) => key in config.promptCaching).map( - (key) => `llm.promptCaching.${key}`, - ); -} - -export function formatClaudeCodePromptCachingWarning(fields: string[]): string | null { - if (fields.length === 0) { - return null; - } - return `claude-code ignores ${fields.join(', ')} because the Claude Agent SDK does not expose KTX prompt-cache TTL, tool, or history markers.`; -} - -export function formatClaudeCodePromptCachingFix(): string { - return 'Remove those promptCaching fields or use anthropic, vertex, or gateway when those cache knobs are required.'; -} -``` - -- [ ] **Step 4: Update status/doctor to use the shared helper** - -In `packages/cli/src/status-project.ts`, add: - -```ts -import { - formatClaudeCodePromptCachingFix, - formatClaudeCodePromptCachingWarning, - ignoredClaudeCodePromptCachingFields, -} from './claude-code-prompt-caching.js'; -``` - -Delete the local `ignoredClaudeCodePromptCachingFields(...)` function. - -Replace the warning block in `buildWarnings(...)` with: - -```ts - const warning = formatClaudeCodePromptCachingWarning(ignoredClaudeCodePromptCachingFields(config.llm)); - if (warning) { - warnings.push({ - message: warning, - fix: formatClaudeCodePromptCachingFix(), - }); - } -``` - -- [ ] **Step 5: Emit the setup warning before persisting Claude Code config** - -In `packages/cli/src/setup-models.ts`, add: - -```ts -import { - formatClaudeCodePromptCachingWarning, - ignoredClaudeCodePromptCachingFields, -} from './claude-code-prompt-caching.js'; -``` - -Inside the `backendChoice.backend === 'claude-code'` branch, immediately before -`await persistLlmConfig(...)`, add: - -```ts - const warning = formatClaudeCodePromptCachingWarning( - ignoredClaudeCodePromptCachingFields(buildProjectLlmConfig(project.config.llm, { backend: 'claude-code' }, model)), - ); - if (warning) { - io.stderr.write(`${warning}\n`); - } -``` - -- [ ] **Step 6: Run CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-models.test.ts src/doctor.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -Run: - -```bash -git add packages/cli/src/claude-code-prompt-caching.ts packages/cli/src/status-project.ts packages/cli/src/setup-models.ts packages/cli/src/setup-models.test.ts packages/cli/src/doctor.test.ts -git commit -m "fix: warn on claude-code prompt caching during setup" -``` - -### Task 3: Final verification - -**Files:** - -- Verify: `packages/context/src/llm/claude-code-runtime.ts` -- Verify: `packages/cli/src/setup-models.ts` -- Verify: `packages/cli/src/status-project.ts` - -- [ ] **Step 1: Run targeted tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts src/llm/runtime-tools.test.ts src/llm/claude-code-env.test.ts src/llm/claude-code-models.test.ts src/llm/runtime-local-config.test.ts -pnpm --filter @ktx/cli exec vitest run src/setup-models.test.ts src/doctor.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type-checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run the LLM boundary audit** - -Run: - -```bash -rg -n "generateKtxText\\(|generateKtxObject\\(|new AgentRunnerService\\(|AgentRunnerService\\b|llmProvider\\b|getModel\\(|getModelByName\\(" packages/context/src packages/cli/src packages/llm/src --glob '!**/*.test.ts' -``` - -Expected: remaining matches are limited to: - -- `packages/llm/src/**` -- `packages/context/src/llm/ai-sdk-runtime.ts` -- `packages/context/src/llm/local-config.ts` -- `packages/context/src/agent/agent-runner.service.ts` -- type/export declarations that intentionally preserve the AI SDK adapter - boundary. - -- [ ] **Step 4: Run dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS or only pre-existing unrelated findings. Investigate and fix -any finding caused by the new helper file. - -- [ ] **Step 5: Commit verification cleanup if needed** - -If verification required small cleanup, run: - -```bash -git add packages/context/src/llm/claude-code-runtime.ts packages/context/src/llm/claude-code-runtime.test.ts packages/cli/src/claude-code-prompt-caching.ts packages/cli/src/status-project.ts packages/cli/src/setup-models.ts packages/cli/src/setup-models.test.ts packages/cli/src/doctor.test.ts -git commit -m "chore: verify claude-code v1 closure" -``` - -If no files changed after verification, skip this commit. - -## Self-Review - -- Spec coverage: The plan closes the remaining v1-blocking isolation assertion - and setup-warning requirements from the original spec. -- Placeholder scan: No placeholders remain; every task includes file paths, - code, commands, and expected output. -- Type consistency: The helper names and runtime function signatures are used - consistently across tasks. diff --git a/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-runtime.md b/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-runtime.md deleted file mode 100644 index 9da58f86..00000000 --- a/docs/superpowers/plans/2026-05-15-claude-code-backend-v1-runtime.md +++ /dev/null @@ -1,2483 +0,0 @@ -# Claude Code Backend V1 Runtime Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `llm.provider.backend: claude-code` as a first-class KTX LLM backend for text generation, structured object generation, and agent-loop execution. - -**Architecture:** Keep `@ktx/llm` as the AI SDK provider package for `anthropic`, `vertex`, and `gateway`, and add a backend-neutral runtime port in `@ktx/context` for KTX operations. The AI SDK runtime wraps the existing provider behavior; the Claude Code runtime uses `@anthropic-ai/claude-agent-sdk@0.3.142` with explicit isolation options, a scrubbed environment, exact MCP tool ids, and KTX-owned tool descriptors. - -**Tech Stack:** TypeScript, pnpm, Vitest, AI SDK v6, Zod v4, `@anthropic-ai/claude-agent-sdk@0.3.142`, Commander, Fumadocs MDX. - ---- - -## Audit Result - -No implemented plan exists for the May 15 Claude Code backend spec. The latest -plans in `docs/superpowers/plans/` stop at May 14 research-agent MCP work, -which configures external agent clients but does not make `claude-code` a KTX -LLM backend. - -Current v1-blocking gaps: - -- `packages/context/src/project/config.ts` accepts only `none`, `anthropic`, - `vertex`, and `gateway`. -- `packages/llm/src/types.ts` defines `KtxLlmBackend` without `claude-code`. -- `@anthropic-ai/claude-agent-sdk` is not a workspace dependency. -- `packages/llm/src/model-provider.ts` falls through to gateway for unknown - non-`anthropic` and non-`vertex` backends instead of throwing. -- No `KtxLlmRuntimePort` exists, and LLM call sites still depend directly on - `KtxLlmProvider`, `AgentRunnerService`, `generateKtxText`, and - `generateKtxObject`. -- Agent-loop tools are still AI SDK `Tool` objects. Several inline tools return - bare strings or plain objects to the model path. -- `ktx setup`, `ktx status`, and doctor output do not understand - `claude-code` as an LLM provider or validate local Claude Code authentication. -- Docs do not describe `claude-code` as a local Claude Code session backend or - document prompt-caching divergence. - -Non-blocking gaps from the spec: - -- Same-step AI SDK tool-call repair parity can remain absent on the Claude Code - runtime. Schema/tool errors can surface as normal tool failures and - next-turn self-correction. -- OTEL telemetry parity can remain absent for the Claude Code runtime. -- Embedding parity is out of scope because embeddings stay configured under - `ingest.embeddings` and scan enrichment embedding settings. -- Session persistence for Claude Code debugging is out of scope for v1 because - the required runtime behavior sets `persistSession: false`. -- Full prompt-caching parity for tools, history, and per-section TTLs is out of - scope. V1 must only avoid AI-SDK cache markers on `claude-code` and warn when - users configure ignored prompt-caching fields. - -## File Structure - -Create these files: - -- `packages/context/src/llm/runtime-port.ts` defines `KtxLlmRuntimePort`, - text/object inputs, runtime tool descriptors, runtime tool outputs, and - `AgentRunnerPort`. -- `packages/context/src/llm/runtime-tools.ts` converts runtime descriptors to - AI SDK tools and Claude SDK MCP tools, normalizes markdown/structured output, - and rejects non-object tool schemas. -- `packages/context/src/llm/ai-sdk-runtime.ts` implements - `KtxLlmRuntimePort` for existing AI SDK backends. -- `packages/context/src/llm/claude-code-env.ts` owns the Claude Code - environment denylist and scrubber. -- `packages/context/src/llm/claude-code-models.ts` maps `sonnet`, `opus`, and - `haiku` aliases and validates full model ids. -- `packages/context/src/llm/claude-code-runtime.ts` implements text, object, - auth probe, and agent loops through the Claude Agent SDK. -- `packages/context/src/llm/runtime-local-config.test.ts`, - `packages/context/src/llm/runtime-tools.test.ts`, - `packages/context/src/llm/claude-code-env.test.ts`, - `packages/context/src/llm/claude-code-models.test.ts`, and - `packages/context/src/llm/claude-code-runtime.test.ts` cover the new runtime - boundary. - -Modify these files: - -- `packages/context/package.json` adds the pinned Claude Agent SDK dependency. -- `packages/llm/src/types.ts`, `packages/llm/src/model-provider.ts`, - `packages/llm/src/model-provider.test.ts`, and - `packages/llm/src/model-health.test.ts` add backend typing and explicit - unsupported-provider behavior. -- `packages/context/src/project/config.ts` and - `packages/context/src/project/config.test.ts` parse and serialize - `claude-code`. -- `packages/context/src/llm/local-config.ts` and - `packages/context/src/llm/index.ts` create and export the runtime factory. -- `packages/context/src/llm/generation.ts` makes `generateKtxText` and - `generateKtxObject` runtime-backed helpers. -- `packages/context/src/agent/agent-runner.service.ts` uses runtime tool - descriptors on the AI SDK path and exposes `AgentRunnerPort`. -- `packages/context/src/tools/base-tool.ts` adds `toRuntimeTool`. -- `packages/context/src/ingest/local-bundle-runtime.ts`, - `packages/context/src/ingest/local-ingest.ts`, - `packages/context/src/ingest/ports.ts`, - `packages/context/src/ingest/page-triage/page-triage.service.ts`, - `packages/context/src/ingest/stages/stage-3-work-units.ts`, - `packages/context/src/ingest/stages/stage-4-reconciliation.ts`, - `packages/context/src/ingest/context-candidates/curator-pagination.service.ts`, - `packages/context/src/ingest/ingest-bundle.runner.ts`, - `packages/context/src/ingest/stages/build-wu-context.ts`, and - `packages/context/src/ingest/stages/build-reconcile-context.ts` move local - ingest paths to the runtime boundary. -- `packages/context/src/memory/types.ts`, - `packages/context/src/memory/local-memory.ts`, and - `packages/context/src/memory/memory-agent.service.ts` move memory capture to - runtime-backed agent loops. -- `packages/context/src/scan/local-scan.ts`, - `packages/context/src/scan/local-enrichment.ts`, - `packages/context/src/scan/description-generation.ts`, and - `packages/context/src/scan/relationship-llm-proposal.ts` move scan - enrichment and relationship proposals to runtime text/object operations. -- `packages/context/src/mcp/local-project-ports.ts` passes runtime-backed local - ingest options into MCP-triggered ingest. -- `packages/cli/src/setup-commands.ts`, `packages/cli/src/setup-models.ts`, - `packages/cli/src/setup-models.test.ts`, `packages/cli/src/status-project.ts`, - and `packages/cli/src/doctor.test.ts` expose setup/status/doctor support. -- `docs-site/content/docs/getting-started/quickstart.mdx`, - `docs-site/content/docs/cli-reference/ktx-setup.mdx`, - `docs-site/content/docs/cli-reference/ktx-status.mdx`, - `docs-site/content/docs/guides/building-context.mdx`, - `docs-site/content/docs/guides/llm-configuration.mdx`, and - `docs-site/content/docs/guides/meta.json` describe the backend. - -### Task 1: Config, Dependency, and No-Fallback Guard - -**Files:** - -- Modify: `packages/context/package.json` -- Modify: `packages/context/src/project/config.ts` -- Modify: `packages/context/src/project/config.test.ts` -- Modify: `packages/llm/src/types.ts` -- Modify: `packages/llm/src/model-provider.ts` -- Modify: `packages/llm/src/model-provider.test.ts` -- Modify: `packages/llm/src/model-health.test.ts` - -- [ ] **Step 1: Write failing config and provider tests** - -Add this test to `packages/context/src/project/config.test.ts`: - -```ts -it('parses Claude Code as a first-class LLM backend', () => { - const config = parseKtxProjectConfig(` -llm: - provider: - backend: claude-code - models: - default: sonnet - triage: haiku - candidateExtraction: sonnet - curator: sonnet - reconcile: sonnet - repair: opus -`); - - expect(config.llm.provider.backend).toBe('claude-code'); - expect(config.llm.models).toEqual({ - default: 'sonnet', - triage: 'haiku', - candidateExtraction: 'sonnet', - curator: 'sonnet', - reconcile: 'sonnet', - repair: 'opus', - }); -}); -``` - -Add this test to `packages/llm/src/model-provider.test.ts`: - -```ts -it('throws instead of falling through when an unsupported LLM backend is passed to the AI SDK provider factory', () => { - expect(() => - createKtxLlmProvider({ - backend: 'claude-code', - modelSlots: { default: 'sonnet' }, - promptCaching: { enabled: false }, - }), - ).toThrow('claude-code is not an AI SDK LanguageModel backend'); -}); -``` - -Add this test to `packages/llm/src/model-health.test.ts`: - -```ts -it('reports claude-code as unsupported by the AI SDK health check', async () => { - const result = await runKtxLlmHealthCheck({ - backend: 'claude-code', - modelSlots: { default: 'sonnet' }, - promptCaching: { enabled: false }, - }); - - expect(result).toEqual({ - ok: false, - message: expect.stringContaining('claude-code is not an AI SDK LanguageModel backend'), - }); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts -pnpm --filter @ktx/llm exec vitest run src/model-provider.test.ts src/model-health.test.ts -``` - -Expected: the config test rejects `claude-code`, and the `@ktx/llm` tests fail -because `KtxLlmBackend` does not include `claude-code`. - -- [ ] **Step 3: Add the pinned SDK dependency** - -In `packages/context/package.json`, add this dependency inside -`dependencies`: - -```json -"@anthropic-ai/claude-agent-sdk": "0.3.142" -``` - -Run: - -```bash -pnpm install -``` - -Expected: `pnpm-lock.yaml` records `@anthropic-ai/claude-agent-sdk@0.3.142`. - -- [ ] **Step 4: Extend backend config and types** - -In `packages/context/src/project/config.ts`, update the backend list and -description: - -```ts -const KTX_LLM_BACKENDS = ['none', 'anthropic', 'vertex', 'gateway', 'claude-code'] as const; -``` - -```ts -.describe( - 'LLM provider backend. "none" disables LLM features; "anthropic" / "vertex" / "gateway" require the matching nested credentials block; "claude-code" uses the local Claude Code session.', -), -``` - -In `packages/llm/src/types.ts`, update the backend type: - -```ts -export type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code'; -``` - -- [ ] **Step 5: Make unsupported AI SDK provider backends explicit** - -In `packages/llm/src/model-provider.ts`, replace the gateway fallthrough in -`createModelFactory` with an explicit gateway branch and a final throw: - -```ts - if (config.backend === 'gateway') { - const gateway = (deps.createGateway ?? createGateway)({ - ...(config.gateway?.apiKey ? { apiKey: config.gateway.apiKey } : {}), - ...(config.gateway?.baseURL ? { baseURL: config.gateway.baseURL } : {}), - headers: { - 'anthropic-beta': ANTHROPIC_BETA_HEADER, - }, - }); - return (modelId) => gateway(modelId); - } - - throw new Error(`${config.backend} is not an AI SDK LanguageModel backend; use KtxLlmRuntimePort`); -``` - -- [ ] **Step 6: Run tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/project/config.test.ts -pnpm --filter @ktx/llm exec vitest run src/model-provider.test.ts src/model-health.test.ts -``` - -Expected: all selected tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/package.json pnpm-lock.yaml packages/context/src/project/config.ts packages/context/src/project/config.test.ts packages/llm/src/types.ts packages/llm/src/model-provider.ts packages/llm/src/model-provider.test.ts packages/llm/src/model-health.test.ts -git commit -m "feat: recognize claude-code llm backend" -``` - -### Task 2: Runtime Port, Tool Descriptors, and AI SDK Adapter - -**Files:** - -- Create: `packages/context/src/llm/runtime-port.ts` -- Create: `packages/context/src/llm/runtime-tools.ts` -- Create: `packages/context/src/llm/ai-sdk-runtime.ts` -- Create: `packages/context/src/llm/runtime-tools.test.ts` -- Modify: `packages/context/src/tools/base-tool.ts` -- Modify: `packages/context/src/agent/agent-runner.service.ts` -- Modify: `packages/context/src/llm/generation.ts` -- Modify: `packages/context/src/llm/index.ts` - -- [ ] **Step 1: Write failing runtime tool tests** - -Create `packages/context/src/llm/runtime-tools.test.ts`: - -```ts -import { describe, expect, it, vi } from 'vitest'; -import { z } from 'zod'; -import { createAiSdkToolSet, createClaudeSdkTools, normalizeKtxRuntimeToolOutput } from './runtime-tools.js'; -import type { KtxRuntimeToolDescriptor } from './runtime-port.js'; - -describe('runtime tool descriptors', () => { - const descriptor: KtxRuntimeToolDescriptor<{ id: string }, { ok: boolean }> = { - name: 'read_thing', - description: 'Read one thing.', - inputSchema: z.object({ id: z.string() }), - execute: vi.fn(async (input) => ({ - markdown: `Read ${input.id}`, - structured: { ok: true }, - })), - }; - - it('normalizes string and object tool outputs into markdown plus optional structured payload', () => { - expect(normalizeKtxRuntimeToolOutput('plain text')).toEqual({ markdown: 'plain text' }); - expect(normalizeKtxRuntimeToolOutput({ markdown: 'shown', structured: { id: 1 } })).toEqual({ - markdown: 'shown', - structured: { id: 1 }, - }); - expect(normalizeKtxRuntimeToolOutput({ name: 'skill', content: 'body' })).toEqual({ - markdown: '```json\n{\n "name": "skill",\n "content": "body"\n}\n```', - structured: { name: 'skill', content: 'body' }, - }); - }); - - it('builds AI SDK tools that expose markdown to the model', async () => { - const tools = createAiSdkToolSet({ read_thing: descriptor }); - const output = await tools.read_thing.execute?.({ id: 'a' }, { toolCallId: 'call-1', messages: [] } as never); - const modelOutput = tools.read_thing.toModelOutput?.({ output } as never); - - expect(modelOutput).toEqual({ type: 'content', value: [{ type: 'text', text: 'Read a' }] }); - }); - - it('builds Claude SDK tools that return text content only', async () => { - const tools = createClaudeSdkTools({ read_thing: descriptor }); - const result = await tools[0].handler({ id: 'b' } as never, {}); - - expect(result).toEqual({ content: [{ type: 'text', text: 'Read b' }] }); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/runtime-tools.test.ts -``` - -Expected: FAIL because `runtime-tools.ts` and `runtime-port.ts` do not exist. - -- [ ] **Step 3: Define the runtime port** - -Create `packages/context/src/llm/runtime-port.ts`: - -```ts -import type { KtxModelRole } from '@ktx/llm'; -import type { z } from 'zod'; - -export interface KtxRuntimeToolOutput { - markdown: string; - structured?: TOutput; -} - -export interface KtxRuntimeToolDescriptor { - name: string; - description: string; - inputSchema: z.ZodObject; - execute(input: TInput): Promise>; -} - -export type KtxRuntimeToolSet = Record; - -export type RunLoopStopReason = 'budget' | 'natural' | 'error'; - -export interface RunLoopStepInfo { - stepIndex: number; - stepBudget: number; -} - -export interface RunLoopParams { - modelRole: KtxModelRole; - systemPrompt: string; - userPrompt: string; - toolSet: KtxRuntimeToolSet; - stepBudget: number; - telemetryTags: Record; - onStepFinish?: (info: RunLoopStepInfo) => void | Promise; -} - -export interface RunLoopResult { - stopReason: RunLoopStopReason; - error?: Error; -} - -export interface KtxGenerateTextInput { - role: KtxModelRole; - prompt: string; - system?: string; - tools?: KtxRuntimeToolSet; - temperature?: number; -} - -export interface KtxGenerateObjectInput> { - role: KtxModelRole; - prompt: string; - system?: string; - tools?: KtxRuntimeToolSet; - temperature?: number; - schema: TSchema; -} - -export interface KtxLlmRuntimePort { - generateText(input: KtxGenerateTextInput): Promise; - generateObject>( - input: KtxGenerateObjectInput, - ): Promise; - runAgentLoop(params: RunLoopParams): Promise; -} - -export interface AgentRunnerPort { - runLoop(params: RunLoopParams): Promise; -} - -export class RuntimeAgentRunner implements AgentRunnerPort { - constructor(private readonly runtime: KtxLlmRuntimePort) {} - - runLoop(params: RunLoopParams): Promise { - return this.runtime.runAgentLoop(params); - } -} -``` - -- [ ] **Step 4: Implement runtime tool conversion** - -Create `packages/context/src/llm/runtime-tools.ts`: - -```ts -import { tool as aiTool, type ToolSet } from 'ai'; -import { - tool as claudeTool, - type SdkMcpToolDefinition, - type CallToolResult, -} from '@anthropic-ai/claude-agent-sdk'; -import type { z } from 'zod'; -import type { KtxRuntimeToolDescriptor, KtxRuntimeToolOutput, KtxRuntimeToolSet } from './runtime-port.js'; - -function isRuntimeOutput(value: unknown): value is KtxRuntimeToolOutput { - return Boolean(value && typeof value === 'object' && 'markdown' in value && typeof (value as { markdown?: unknown }).markdown === 'string'); -} - -export function normalizeKtxRuntimeToolOutput(value: unknown): KtxRuntimeToolOutput { - if (isRuntimeOutput(value)) { - return 'structured' in value - ? { markdown: value.markdown, structured: value.structured } - : { markdown: value.markdown }; - } - if (typeof value === 'string') { - return { markdown: value }; - } - return { - markdown: `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\``, - structured: value, - }; -} - -function assertObjectSchema(name: string, schema: z.ZodType): asserts schema is z.ZodObject { - if (schema.def.type !== 'object') { - throw new Error(`KTX runtime tool "${name}" must use z.object input schema for claude-code`); - } -} - -export function createAiSdkToolSet(tools: KtxRuntimeToolSet = {}): ToolSet { - return Object.fromEntries( - Object.entries(tools).map(([name, descriptor]) => [ - name, - aiTool({ - description: descriptor.description, - inputSchema: descriptor.inputSchema, - execute: async (input) => descriptor.execute(input), - toModelOutput: ({ output }) => { - const normalized = normalizeKtxRuntimeToolOutput(output); - return { type: 'content', value: [{ type: 'text', text: normalized.markdown }] }; - }, - }), - ]), - ); -} - -export function createClaudeSdkTools(tools: KtxRuntimeToolSet = {}): Array> { - return Object.values(tools).map((descriptor) => { - assertObjectSchema(descriptor.name, descriptor.inputSchema); - const sdkTool = claudeTool( - descriptor.name, - descriptor.description, - descriptor.inputSchema.shape, - async (input): Promise => { - const normalized = normalizeKtxRuntimeToolOutput(await descriptor.execute(input)); - return { content: [{ type: 'text', text: normalized.markdown }] }; - }, - ); - return Object.assign(sdkTool, { handler: sdkTool.handler }); - }); -} - -export function mcpToolIds(tools: KtxRuntimeToolSet = {}): string[] { - return Object.keys(tools).map((name) => `mcp__ktx__${name}`); -} -``` - -- [ ] **Step 5: Add `BaseTool.toRuntimeTool`** - -In `packages/context/src/tools/base-tool.ts`, add this import: - -```ts -import type { KtxRuntimeToolDescriptor } from '../llm/runtime-port.js'; -import { normalizeKtxRuntimeToolOutput } from '../llm/runtime-tools.js'; -``` - -Add this method beside `toAiSdkTool`: - -```ts - toRuntimeTool(context: ToolContext): KtxRuntimeToolDescriptor { - const toolName = this.name; - return { - name: toolName, - description: this.description, - inputSchema: this.inputSchema as KtxRuntimeToolDescriptor['inputSchema'], - execute: async (params) => { - const callContext = { ...context }; - if (!callContext.userId) { - throw new Error('Authentication required: userId must be provided in ToolContext'); - } - const parsedInput = this.parseInput(params as Record); - return normalizeKtxRuntimeToolOutput(await this.call(parsedInput, callContext)); - }, - }; - } -``` - -- [ ] **Step 6: Implement the AI SDK runtime adapter** - -Create `packages/context/src/llm/ai-sdk-runtime.ts`: - -```ts -import { KtxMessageBuilder, splitKtxSystemMessages, type KtxLlmProvider } from '@ktx/llm'; -import { generateText, Output, stepCountIs, type FlexibleSchema } from 'ai'; -import type { z } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; -import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from './debug-request-recorder.js'; -import { createAiSdkToolSet } from './runtime-tools.js'; -import type { - KtxGenerateObjectInput, - KtxGenerateTextInput, - KtxLlmRuntimePort, - RunLoopParams, - RunLoopResult, -} from './runtime-port.js'; - -export interface AiSdkKtxLlmRuntimeDeps { - llmProvider: KtxLlmProvider; - logger?: KtxLogger; - debugRequestRecorder?: KtxLlmDebugRequestRecorder; -} - -export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { - private readonly logger: KtxLogger; - - constructor(private readonly deps: AiSdkKtxLlmRuntimeDeps) { - this.logger = deps.logger ?? noopLogger; - } - - async generateText(input: KtxGenerateTextInput): Promise { - const model = this.deps.llmProvider.getModel(input.role); - if ((model as { provider?: string }).provider === 'deterministic') { - return `Deterministic description for ${input.prompt.slice(0, 64).trim() || 'data source'}`; - } - const tools = createAiSdkToolSet(input.tools ?? {}); - const built = new KtxMessageBuilder(this.deps.llmProvider).wrapSimple({ - system: input.system, - messages: [{ role: 'user', content: input.prompt }], - tools, - model, - }); - const split = splitKtxSystemMessages(built.messages); - const result = await generateText({ - model, - temperature: input.temperature ?? 0, - ...(split.system ? { system: split.system } : {}), - messages: split.messages, - tools: built.tools, - ...(Object.keys(tools).length > 0 - ? { - experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({ - source: `ktx-${input.role}`, - }), - } - : {}), - }); - if (typeof result.text !== 'string') { - throw new Error('KTX LLM text generation returned no text'); - } - return result.text; - } - - async generateObject>( - input: KtxGenerateObjectInput, - ): Promise { - const model = this.deps.llmProvider.getModel(input.role); - const tools = createAiSdkToolSet(input.tools ?? {}); - const built = new KtxMessageBuilder(this.deps.llmProvider).wrapSimple({ - system: input.system, - messages: [{ role: 'user', content: input.prompt }], - tools, - model, - }); - const split = splitKtxSystemMessages(built.messages); - const result = await generateText({ - model, - temperature: input.temperature ?? 0, - ...(split.system ? { system: split.system } : {}), - messages: split.messages, - tools: built.tools, - ...(Object.keys(tools).length > 0 - ? { - experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({ - source: `ktx-${input.role}`, - }), - } - : {}), - output: Output.object({ schema: input.schema as unknown as FlexibleSchema }), - }); - if (result.output == null) { - throw new Error('KTX LLM object generation returned no output'); - } - return result.output as TOutput; - } - - async runAgentLoop(params: RunLoopParams): Promise { - let stepIndex = 0; - try { - const model = this.deps.llmProvider.getModel(params.modelRole); - const tools = createAiSdkToolSet(params.toolSet); - const builder = new KtxMessageBuilder(this.deps.llmProvider); - const built = builder.wrapSimple({ - system: params.systemPrompt, - messages: [{ role: 'user', content: params.userPrompt }], - tools, - model, - }); - const promptMessages = splitKtxSystemMessages(built.messages); - await this.deps.debugRequestRecorder?.record( - summarizeKtxLlmDebugRequest({ - operationName: params.telemetryTags.operationName ?? 'ktx-agent-runner', - source: params.telemetryTags.source, - jobId: params.telemetryTags.jobId, - unitKey: params.telemetryTags.unitKey, - modelRole: params.modelRole, - modelId: (model as { modelId?: string }).modelId ?? params.modelRole, - messages: built.messages, - tools: built.tools as Record, - }), - ); - await generateText({ - model, - temperature: 0, - stopWhen: stepCountIs(params.stepBudget), - experimental_telemetry: this.deps.llmProvider.telemetryConfig(), - experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({ - source: params.telemetryTags.operationName ?? 'ktx-agent-runner', - }), - ...(promptMessages.system ? { system: promptMessages.system } : {}), - messages: promptMessages.messages, - tools: built.tools, - onStepFinish: async () => { - stepIndex += 1; - if (!params.onStepFinish) return; - try { - await params.onStepFinish({ stepIndex, stepBudget: params.stepBudget }); - } catch (err) { - this.logger.warn(`[agent-runner] onStepFinish callback threw; ignoring: ${err instanceof Error ? err.message : String(err)}`); - } - }, - }); - return { stopReason: 'natural' }; - } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); - this.logger.warn(`[agent-runner] loop failed: ${err.message}`); - return { stopReason: 'error', error: err }; - } - } -} -``` - -- [ ] **Step 7: Keep `AgentRunnerService` as the AI SDK class** - -In `packages/context/src/agent/agent-runner.service.ts`, import and re-export -the runtime loop types: - -```ts -import { AiSdkKtxLlmRuntime } from '../llm/ai-sdk-runtime.js'; -import type { AgentRunnerPort, RunLoopParams, RunLoopResult } from '../llm/runtime-port.js'; -export type { - AgentRunnerPort, - RunLoopParams, - RunLoopResult, - RunLoopStepInfo, - RunLoopStopReason, -} from '../llm/runtime-port.js'; -``` - -Then replace the existing implementation with delegation to -`AiSdkKtxLlmRuntime`: - -```ts -export class AgentRunnerService implements AgentRunnerPort { - private readonly runtime: AiSdkKtxLlmRuntime; - - constructor(deps: AgentRunnerServiceDeps) { - this.runtime = new AiSdkKtxLlmRuntime(deps); - } - - runLoop(params: RunLoopParams): Promise { - return this.runtime.runAgentLoop(params); - } -} -``` - -- [ ] **Step 8: Re-export the runtime API and adapt generation helpers** - -In `packages/context/src/llm/index.ts`, export the new modules: - -```ts -export { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; -export type { - AgentRunnerPort, - KtxGenerateObjectInput, - KtxGenerateTextInput, - KtxLlmRuntimePort, - KtxRuntimeToolDescriptor, - KtxRuntimeToolOutput, - KtxRuntimeToolSet, -} from './runtime-port.js'; -export { RuntimeAgentRunner } from './runtime-port.js'; -export { createAiSdkToolSet, createClaudeSdkTools, normalizeKtxRuntimeToolOutput } from './runtime-tools.js'; -``` - -In `packages/context/src/llm/generation.ts`, replace direct provider use with -runtime-backed helpers: - -```ts -import type { z } from 'zod'; -import type { KtxGenerateObjectInput, KtxGenerateTextInput, KtxLlmRuntimePort } from './runtime-port.js'; - -export async function generateKtxText(input: KtxGenerateTextInput & { runtime: KtxLlmRuntimePort }): Promise { - return input.runtime.generateText(input); -} - -export async function generateKtxObject>( - input: KtxGenerateObjectInput & { runtime: KtxLlmRuntimePort }, -): Promise { - return input.runtime.generateObject(input); -} -``` - -- [ ] **Step 9: Run runtime tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/runtime-tools.test.ts src/agent/agent-runner.service.test.ts -``` - -Expected: selected tests pass after call sites in tests use runtime tool -descriptors. - -- [ ] **Step 10: Commit** - -```bash -git add packages/context/src/llm/runtime-port.ts packages/context/src/llm/runtime-tools.ts packages/context/src/llm/ai-sdk-runtime.ts packages/context/src/llm/runtime-tools.test.ts packages/context/src/tools/base-tool.ts packages/context/src/agent/agent-runner.service.ts packages/context/src/llm/generation.ts packages/context/src/llm/index.ts packages/context/src/agent/agent-runner.service.test.ts -git commit -m "feat: add ktx llm runtime port" -``` - -### Task 3: Claude Code Runtime, Auth Boundary, and Stop Reasons - -**Files:** - -- Create: `packages/context/src/llm/claude-code-env.ts` -- Create: `packages/context/src/llm/claude-code-env.test.ts` -- Create: `packages/context/src/llm/claude-code-models.ts` -- Create: `packages/context/src/llm/claude-code-models.test.ts` -- Create: `packages/context/src/llm/claude-code-runtime.ts` -- Create: `packages/context/src/llm/claude-code-runtime.test.ts` -- Modify: `packages/context/src/llm/index.ts` - -- [ ] **Step 1: Write failing environment and model tests** - -Create `packages/context/src/llm/claude-code-env.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { CLAUDE_CODE_PROVIDER_ENV_DENYLIST, createKtxClaudeCodeEnv } from './claude-code-env.js'; - -describe('createKtxClaudeCodeEnv', () => { - it('strips provider-routing credentials from the Claude Code child environment', () => { - const seeded = Object.fromEntries(CLAUDE_CODE_PROVIDER_ENV_DENYLIST.map((key) => [key, `${key}-value`])); - const env = createKtxClaudeCodeEnv({ - ...seeded, - PATH: '/usr/bin', - HOME: '/Users/test', - }); - - for (const key of CLAUDE_CODE_PROVIDER_ENV_DENYLIST) { - expect(env).not.toHaveProperty(key); - } - expect(env.PATH).toBe('/usr/bin'); - expect(env.HOME).toBe('/Users/test'); - }); -}); -``` - -Create `packages/context/src/llm/claude-code-models.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { resolveClaudeCodeModel } from './claude-code-models.js'; - -describe('resolveClaudeCodeModel', () => { - it.each([ - ['sonnet', 'claude-sonnet-4-6'], - ['opus', 'claude-opus-4-7'], - ['haiku', 'claude-haiku-4-5'], - ['claude-sonnet-4-6', 'claude-sonnet-4-6'], - ])('maps %s to %s', (input, expected) => { - expect(resolveClaudeCodeModel(input)).toBe(expected); - }); - - it('rejects unsupported aliases', () => { - expect(() => resolveClaudeCodeModel('gpt-5')).toThrow('Unsupported Claude Code model'); - }); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-env.test.ts src/llm/claude-code-models.test.ts -``` - -Expected: FAIL because the files do not exist. - -- [ ] **Step 3: Implement environment scrubbing** - -Create `packages/context/src/llm/claude-code-env.ts`: - -```ts -export const CLAUDE_CODE_PROVIDER_ENV_DENYLIST = [ - 'ANTHROPIC_API_KEY', - 'ANTHROPIC_AUTH_TOKEN', - 'ANTHROPIC_BASE_URL', - 'ANTHROPIC_MODEL', - 'ANTHROPIC_VERTEX_PROJECT_ID', - 'CLOUD_ML_REGION', - 'GOOGLE_APPLICATION_CREDENTIALS', - 'GOOGLE_CLOUD_PROJECT', - 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY', - 'AWS_SESSION_TOKEN', - 'AWS_REGION', - 'AWS_PROFILE', - 'CLAUDE_CODE_USE_BEDROCK', - 'CLAUDE_CODE_USE_VERTEX', -] as const; - -const DENYLIST = new Set(CLAUDE_CODE_PROVIDER_ENV_DENYLIST); - -export function createKtxClaudeCodeEnv(env: NodeJS.ProcessEnv = process.env): Record { - return Object.fromEntries(Object.entries(env).filter(([key]) => !DENYLIST.has(key))); -} -``` - -- [ ] **Step 4: Implement model alias resolution** - -Create `packages/context/src/llm/claude-code-models.ts`: - -```ts -const CLAUDE_CODE_MODEL_ALIASES: Record = { - sonnet: 'claude-sonnet-4-6', - opus: 'claude-opus-4-7', - haiku: 'claude-haiku-4-5', -}; - -const FULL_MODEL_ID = /^claude-(sonnet|opus|haiku)-[0-9]+-[0-9]+$/; - -export function resolveClaudeCodeModel(model: string): string { - const normalized = model.trim(); - const alias = CLAUDE_CODE_MODEL_ALIASES[normalized]; - if (alias) { - return alias; - } - if (FULL_MODEL_ID.test(normalized)) { - return normalized; - } - throw new Error(`Unsupported Claude Code model "${model}". Use sonnet, opus, haiku, or a claude-* model id.`); -} -``` - -- [ ] **Step 5: Run environment and model tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-env.test.ts src/llm/claude-code-models.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Write failing runtime tests for isolation, text, objects, tools, and progress** - -Create `packages/context/src/llm/claude-code-runtime.test.ts`: - -```ts -import { describe, expect, it, vi } from 'vitest'; -import { z } from 'zod'; -import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk'; -import { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from './claude-code-runtime.js'; - -async function* stream(messages: SDKMessage[]): AsyncGenerator { - for (const message of messages) { - yield message; - } -} - -function initMessage(overrides: Partial> = {}): Extract { - return { - type: 'system', - subtype: 'init', - apiKeySource: 'none', // pragma: allowlist secret - claude_code_version: '0.3.142', - cwd: '/tmp/project', - tools: [], - mcp_servers: [], - model: 'claude-sonnet-4-6', - permissionMode: 'dontAsk', - slash_commands: [], - output_style: 'default', - skills: [], - plugins: [], - ...overrides, - }; -} - -function resultMessage(overrides: Partial> = {}): Extract { - return { - type: 'result', - subtype: 'success', - duration_ms: 1, - duration_api_ms: 1, - is_error: false, - num_turns: 1, - result: 'ok', - stop_reason: null, - total_cost_usd: 0, - usage: {} as never, - modelUsage: {}, - permission_denials: [], - uuid: 'result-id', - session_id: 'session-id', - ...overrides, - }; -} - -describe('ClaudeCodeKtxLlmRuntime', () => { - it('passes isolation options and scrubbed env to text generation', async () => { - const query = vi.fn(() => stream([initMessage(), resultMessage({ result: 'hello' })])); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: { ANTHROPIC_API_KEY: 'sk-ant-test', PATH: '/usr/bin' }, // pragma: allowlist secret - }); - - await expect(runtime.generateText({ role: 'default', prompt: 'say hello' })).resolves.toBe('hello'); - expect(query).toHaveBeenCalledWith({ - prompt: 'say hello', - options: expect.objectContaining({ - cwd: '/tmp/project', - model: 'claude-sonnet-4-6', - maxTurns: 1, - settingSources: [], - skills: [], - plugins: [], - tools: [], - allowedTools: [], - permissionMode: 'dontAsk', - persistSession: false, - env: expect.not.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test' }), - }), - }); - }); - - it('validates structured output with the caller schema', async () => { - const schema = z.object({ answer: z.string() }); - const query = vi.fn(() => stream([initMessage(), resultMessage({ structured_output: { answer: 'yes' } })])); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: {}, - }); - - await expect(runtime.generateObject({ role: 'default', prompt: 'json', schema })).resolves.toEqual({ answer: 'yes' }); - expect(query.mock.calls[0][0].options.outputFormat).toMatchObject({ - type: 'json_schema', - schema: expect.objectContaining({ type: 'object' }), - }); - }); - - it('registers only exact KTX MCP tool ids and denies non-KTX tools', async () => { - const query = vi.fn(() => - stream([ - initMessage({ tools: ['mcp__ktx__load_skill'], mcp_servers: [{ name: 'ktx', status: 'connected' }] }), - { type: 'assistant', message: { role: 'assistant', content: [] }, parent_tool_use_id: null, uuid: 'assistant-1', session_id: 'session-id' } as SDKMessage, - resultMessage({ subtype: 'error_max_turns', is_error: true }), - ]), - ); - const runtime = new ClaudeCodeKtxLlmRuntime({ - projectDir: '/tmp/project', - modelSlots: { default: 'sonnet' }, - query, - env: {}, - }); - const onStepFinish = vi.fn(); - - await runtime.runAgentLoop({ - modelRole: 'default', - systemPrompt: 'system', - userPrompt: 'user', - toolSet: { - load_skill: { - name: 'load_skill', - description: 'Load skill.', - inputSchema: z.object({ name: z.string() }), - execute: async () => ({ markdown: 'loaded' }), - }, - }, - stepBudget: 1, - telemetryTags: { operationName: 'test' }, - onStepFinish, - }); - - const options = query.mock.calls[0][0].options; - expect(options.allowedTools).toEqual(['mcp__ktx__load_skill']); - expect(await options.canUseTool('mcp__ktx__load_skill', {}, { signal: new AbortController().signal, toolUseID: '1' })).toEqual({ behavior: 'allow' }); - expect(await options.canUseTool('Bash', {}, { signal: new AbortController().signal, toolUseID: '2' })).toMatchObject({ behavior: 'deny' }); - expect(onStepFinish).toHaveBeenCalledWith({ stepIndex: 1, stepBudget: 1 }); - }); - - it('maps max-turn terminal reasons to budget', () => { - expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'error_max_turns' }))).toBe('budget'); - expect(mapClaudeCodeStopReason(resultMessage({ terminal_reason: 'max_turns' }))).toBe('budget'); - expect(mapClaudeCodeStopReason(resultMessage({ stop_reason: 'max_turns' }))).toBe('budget'); - expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'success', terminal_reason: 'completed' }))).toBe('natural'); - expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'error_during_execution' }))).toBe('error'); - }); - - it('auth probe uses isolation options and a scrubbed env', async () => { - const query = vi.fn(() => stream([initMessage(), resultMessage({ result: 'ok' })])); - - await expect(runClaudeCodeAuthProbe({ projectDir: '/tmp/project', model: 'sonnet', query, env: { ANTHROPIC_API_KEY: 'sk-ant-test' } })).resolves.toEqual({ ok: true }); // pragma: allowlist secret - expect(query.mock.calls[0][0].options).toMatchObject({ - settingSources: [], - skills: [], - plugins: [], - tools: [], - allowedTools: [], - persistSession: false, - env: expect.not.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test' }), - }); - }); -}); -``` - -- [ ] **Step 7: Run runtime tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-runtime.test.ts -``` - -Expected: FAIL because `claude-code-runtime.ts` does not exist. - -- [ ] **Step 8: Implement Claude Code runtime** - -Create `packages/context/src/llm/claude-code-runtime.ts` with these exported -types and functions: - -```ts -import { - createSdkMcpServer, - query as defaultQuery, - type Options, - type SDKMessage, - type SDKResultMessage, -} from '@anthropic-ai/claude-agent-sdk'; -import { z } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; -import type { RunLoopParams, RunLoopResult, RunLoopStopReason } from './runtime-port.js'; -import { createKtxClaudeCodeEnv } from './claude-code-env.js'; -import { resolveClaudeCodeModel } from './claude-code-models.js'; -import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js'; -import type { KtxGenerateObjectInput, KtxGenerateTextInput, KtxLlmRuntimePort, KtxRuntimeToolSet } from './runtime-port.js'; - -type QueryFn = typeof defaultQuery; - -export interface ClaudeCodeKtxLlmRuntimeDeps { - projectDir: string; - modelSlots: { default: string } & Partial>; - query?: QueryFn; - env?: NodeJS.ProcessEnv; - logger?: KtxLogger; -} - -const BUILTIN_TOOLS = [ - 'Agent', - 'Task', - 'AskUserQuestion', - 'Bash', - 'Read', - 'Edit', - 'Write', - 'Glob', - 'Grep', - 'WebFetch', - 'WebSearch', - 'TodoWrite', -]; - -function isResult(message: SDKMessage): message is SDKResultMessage { - return message.type === 'result'; -} - -function resultError(result: SDKResultMessage): Error | undefined { - if (result.subtype === 'success') return undefined; - const details = result.errors.length > 0 ? `: ${result.errors.join('; ')}` : ''; - return new Error(`Claude Code query failed (${result.subtype})${details}`); -} - -export function mapClaudeCodeStopReason(result: SDKResultMessage): RunLoopStopReason { - if (result.subtype === 'error_max_turns') return 'budget'; - if (result.subtype === 'success') return result.terminal_reason && result.terminal_reason !== 'completed' ? result.terminal_reason === 'max_turns' ? 'budget' : 'error' : 'natural'; - if (result.terminal_reason === 'max_turns') return 'budget'; - if (result.stop_reason === 'max_turns') return 'budget'; - return 'error'; -} - -function jsonSchema(schema: z.ZodType): Record { - return z.toJSONSchema(schema, { target: 'draft-7' }) as Record; -} - -function modelForRole(modelSlots: ClaudeCodeKtxLlmRuntimeDeps['modelSlots'], role: string): string { - return resolveClaudeCodeModel(modelSlots[role] ?? modelSlots.default); -} - -function assertInitIsolation(message: SDKMessage, allowedToolIds: Set): void { - if (message.type !== 'system' || message.subtype !== 'init') return; - const unexpectedTools = message.tools.filter((tool) => !allowedToolIds.has(tool)); - if (unexpectedTools.length > 0 || message.slash_commands.length > 0 || message.skills.length > 0 || message.plugins.length > 0) { - throw new Error( - `Claude Code runtime isolation failed: tools=${unexpectedTools.join(',') || '(none)'} slash_commands=${message.slash_commands.length} skills=${message.skills.length} plugins=${message.plugins.length}`, - ); - } -} - -function baseOptions(input: { - projectDir: string; - model: string; - env: NodeJS.ProcessEnv | undefined; - maxTurns: number; - tools?: KtxRuntimeToolSet; -}): Options { - const toolIds = mcpToolIds(input.tools ?? {}); - const allowedToolIds = new Set(toolIds); - return { - cwd: input.projectDir, - model: input.model, - maxTurns: input.maxTurns, - settingSources: [], - skills: [], - plugins: [], - tools: [], - allowedTools: toolIds, - disallowedTools: BUILTIN_TOOLS, - canUseTool: async (toolName, _toolInput, options) => - allowedToolIds.has(toolName) - ? { behavior: 'allow', toolUseID: options.toolUseID } - : { - behavior: 'deny', - message: `KTX claude-code runtime only permits current KTX MCP tools; denied ${toolName}.`, - toolUseID: options.toolUseID, - }, - permissionMode: 'dontAsk', - persistSession: false, - env: createKtxClaudeCodeEnv(input.env), - ...(input.tools && Object.keys(input.tools).length > 0 - ? { mcpServers: { ktx: createSdkMcpServer({ name: 'ktx', tools: createClaudeSdkTools(input.tools) }) } } - : {}), - }; -} - -async function collectResult(params: { - query: QueryFn; - prompt: string; - options: Options; - allowedToolIds: Set; - onAssistantTurn?: () => Promise; -}): Promise { - let result: SDKResultMessage | undefined; - for await (const message of params.query({ prompt: params.prompt, options: params.options })) { - assertInitIsolation(message, params.allowedToolIds); - if (message.type === 'assistant' && message.parent_tool_use_id === null) { - await params.onAssistantTurn?.(); - } - if (isResult(message)) { - result = message; - } - } - if (!result) { - throw new Error('Claude Code query returned no result message'); - } - return result; -} - -export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { - private readonly runQuery: QueryFn; - private readonly logger: KtxLogger; - - constructor(private readonly deps: ClaudeCodeKtxLlmRuntimeDeps) { - this.runQuery = deps.query ?? defaultQuery; - this.logger = deps.logger ?? noopLogger; - } - - async generateText(input: KtxGenerateTextInput): Promise { - const options = baseOptions({ - projectDir: this.deps.projectDir, - model: modelForRole(this.deps.modelSlots, input.role), - env: this.deps.env, - maxTurns: 1, - tools: input.tools, - }); - const result = await collectResult({ - query: this.runQuery, - prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'), - options, - allowedToolIds: new Set(mcpToolIds(input.tools ?? {})), - }); - const error = resultError(result); - if (error) throw error; - return result.result; - } - - async generateObject>( - input: KtxGenerateObjectInput, - ): Promise { - const options = { - ...baseOptions({ - projectDir: this.deps.projectDir, - model: modelForRole(this.deps.modelSlots, input.role), - env: this.deps.env, - maxTurns: 1, - tools: input.tools, - }), - outputFormat: { type: 'json_schema' as const, schema: jsonSchema(input.schema as z.ZodType) }, - }; - const result = await collectResult({ - query: this.runQuery, - prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'), - options, - allowedToolIds: new Set(mcpToolIds(input.tools ?? {})), - }); - const error = resultError(result); - if (error) throw error; - return (input.schema as z.ZodType).parse(result.structured_output); - } - - async runAgentLoop(params: RunLoopParams): Promise { - let stepIndex = 0; - try { - const options = baseOptions({ - projectDir: this.deps.projectDir, - model: modelForRole(this.deps.modelSlots, params.modelRole), - env: this.deps.env, - maxTurns: params.stepBudget, - tools: params.toolSet, - }); - const result = await collectResult({ - query: this.runQuery, - prompt: params.userPrompt, - options: { ...options, systemPrompt: params.systemPrompt }, - allowedToolIds: new Set(mcpToolIds(params.toolSet)), - onAssistantTurn: async () => { - stepIndex += 1; - if (!params.onStepFinish) return; - try { - await params.onStepFinish({ stepIndex, stepBudget: params.stepBudget }); - } catch (error) { - this.logger.warn(`[claude-code-runner] onStepFinish callback threw; ignoring: ${error instanceof Error ? error.message : String(error)}`); - } - }, - }); - const stopReason = mapClaudeCodeStopReason(result); - return { stopReason, ...(stopReason === 'error' && resultError(result) ? { error: resultError(result) } : {}) }; - } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); - return { stopReason: 'error', error: err }; - } - } -} - -export async function runClaudeCodeAuthProbe(input: { - projectDir: string; - model: string; - query?: QueryFn; - env?: NodeJS.ProcessEnv; -}): Promise<{ ok: true } | { ok: false; message: string }> { - try { - const options = baseOptions({ - projectDir: input.projectDir, - model: resolveClaudeCodeModel(input.model), - env: input.env, - maxTurns: 1, - }); - const result = await collectResult({ - query: input.query ?? defaultQuery, - prompt: 'Reply with exactly: ok', - options, - allowedToolIds: new Set(), - }); - const error = resultError(result); - if (error) throw error; - return { ok: true }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - ok: false, - message: `Claude Code authentication is not usable. Authenticate Claude Code locally with the Claude Code CLI, then rerun setup or the command. ${message}`, - }; - } -} -``` - -- [ ] **Step 9: Export Claude Code runtime modules** - -In `packages/context/src/llm/index.ts`, add: - -```ts -export { createKtxClaudeCodeEnv, CLAUDE_CODE_PROVIDER_ENV_DENYLIST } from './claude-code-env.js'; -export { resolveClaudeCodeModel } from './claude-code-models.js'; -export { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from './claude-code-runtime.js'; -``` - -- [ ] **Step 10: Run runtime tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/claude-code-env.test.ts src/llm/claude-code-models.test.ts src/llm/claude-code-runtime.test.ts -``` - -Expected: all selected tests pass. - -- [ ] **Step 11: Commit** - -```bash -git add packages/context/src/llm/claude-code-env.ts packages/context/src/llm/claude-code-env.test.ts packages/context/src/llm/claude-code-models.ts packages/context/src/llm/claude-code-models.test.ts packages/context/src/llm/claude-code-runtime.ts packages/context/src/llm/claude-code-runtime.test.ts packages/context/src/llm/index.ts -git commit -m "feat: add claude-code llm runtime" -``` - -### Task 4: Local Runtime Factory and Non-Agent LLM Call Sites - -**Files:** - -- Create: `packages/context/src/llm/runtime-local-config.test.ts` -- Modify: `packages/context/src/llm/local-config.ts` -- Modify: `packages/context/src/llm/index.ts` -- Modify: `packages/context/src/ingest/page-triage/page-triage.service.ts` -- Modify: `packages/context/src/ingest/page-triage/page-triage.service.test.ts` -- Modify: `packages/context/src/scan/description-generation.ts` -- Modify: `packages/context/src/scan/description-generation.test.ts` -- Modify: `packages/context/src/scan/relationship-llm-proposal.ts` -- Modify: `packages/context/src/scan/relationship-llm-proposal.test.ts` -- Modify: `packages/context/src/scan/local-enrichment.ts` -- Modify: `packages/context/src/scan/local-scan.ts` -- Modify: `packages/context/src/scan/local-scan.test.ts` - -- [ ] **Step 1: Write failing local runtime factory tests** - -Create `packages/context/src/llm/runtime-local-config.test.ts`: - -```ts -import { describe, expect, it, vi } from 'vitest'; -import { createLocalKtxLlmProviderFromConfig, createLocalKtxLlmRuntimeFromConfig } from './local-config.js'; - -describe('local KTX LLM runtime config', () => { - it('creates a Claude Code runtime for claude-code backend without creating an AI SDK provider', () => { - const runtime = createLocalKtxLlmRuntimeFromConfig( - { - provider: { backend: 'claude-code' }, - models: { default: 'sonnet', triage: 'haiku' }, - }, - { env: {}, projectDir: '/tmp/project', createClaudeCodeRuntime: vi.fn((deps) => ({ deps })) }, - ); - - expect(runtime).toMatchObject({ deps: expect.objectContaining({ projectDir: '/tmp/project' }) }); - }); - - it('returns null from the AI SDK provider factory for claude-code backend', () => { - expect( - createLocalKtxLlmProviderFromConfig({ - provider: { backend: 'claude-code' }, - models: { default: 'sonnet' }, - }), - ).toBeNull(); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/runtime-local-config.test.ts -``` - -Expected: FAIL because `createLocalKtxLlmRuntimeFromConfig` does not exist. - -- [ ] **Step 3: Implement the local runtime factory** - -In `packages/context/src/llm/local-config.ts`, extend `LocalConfigDeps`: - -```ts - projectDir?: string; - createClaudeCodeRuntime?: (deps: ConstructorParameters[0]) => KtxLlmRuntimePort; - createAiSdkRuntime?: (deps: { llmProvider: KtxLlmProvider }) => KtxLlmRuntimePort; -``` - -Add imports: - -```ts -import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; -import { ClaudeCodeKtxLlmRuntime } from './claude-code-runtime.js'; -import type { KtxLlmRuntimePort } from './runtime-port.js'; -``` - -Update `createLocalKtxLlmProviderFromConfig`: - -```ts - const resolved = resolveLocalKtxLlmConfig(config, deps.env ?? process.env); - if (!resolved || resolved.backend === 'claude-code') { - return null; - } - return (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved); -``` - -Add `createLocalKtxLlmRuntimeFromConfig`: - -```ts -export function createLocalKtxLlmRuntimeFromConfig( - config: KtxProjectLlmConfig, - deps: LocalConfigDeps = {}, -): KtxLlmRuntimePort | null { - const resolved = resolveLocalKtxLlmConfig(config, deps.env ?? process.env); - if (!resolved) { - return null; - } - if (resolved.backend === 'claude-code') { - const projectDir = deps.projectDir; - if (!projectDir) { - throw new Error('projectDir is required when creating the claude-code LLM runtime'); - } - return (deps.createClaudeCodeRuntime ?? ((runtimeDeps) => new ClaudeCodeKtxLlmRuntime(runtimeDeps)))({ - projectDir, - modelSlots: resolved.modelSlots, - env: deps.env, - }); - } - const llmProvider = (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved); - return (deps.createAiSdkRuntime ?? ((runtimeDeps) => new AiSdkKtxLlmRuntime(runtimeDeps)))({ llmProvider }); -} -``` - -Export it from `packages/context/src/llm/index.ts`: - -```ts -export { createLocalKtxLlmRuntimeFromConfig } from './local-config.js'; -``` - -- [ ] **Step 4: Migrate page triage to runtime text generation** - -In `packages/context/src/ingest/page-triage/page-triage.service.ts`, replace -the dependency: - -```ts -import type { KtxLlmRuntimePort } from '../../llm/index.js'; -``` - -```ts - llmRuntime: KtxLlmRuntimePort; -``` - -Replace `callModel` with: - -```ts - private async callModel(params: { - operationName: 'page-triage' | 'light-extraction'; - system: string; - prompt: string; - sourceKey: string; - jobId: string; - unitKey: string; - }): Promise { - return this.deps.llmRuntime.generateText({ - role: 'triage', - system: params.system, - prompt: params.prompt, - temperature: 0, - }); - } -``` - -In `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, replace -provider fakes with: - -```ts -const llmRuntime = { - generateText: vi.fn(async () => JSON.stringify({ action: 'keep', confidence: 0.9, reason: 'relevant' })), - generateObject: vi.fn(), - runAgentLoop: vi.fn(), -}; -``` - -Assert the runtime call: - -```ts -expect(llmRuntime.generateText).toHaveBeenCalledWith(expect.objectContaining({ role: 'triage' })); -``` - -- [ ] **Step 5: Migrate scan text and object generation** - -In `packages/context/src/scan/description-generation.ts`, replace the provider -field with: - -```ts -import type { KtxLlmRuntimePort } from '../llm/index.js'; -``` - -```ts - llmRuntime: KtxLlmRuntimePort; -``` - -Update `generateAiDescription`: - -```ts - const text = await generateKtxText({ - runtime: this.llmRuntime, - role: 'candidateExtraction', - system: prompt.system, - prompt: prompt.user, - temperature: this.settings.temperature, - }); -``` - -In `packages/context/src/scan/relationship-llm-proposal.ts`, change the input: - -```ts - llmRuntime: KtxLlmRuntimePort | null; -``` - -Remove `modelIsDeterministic` and skip only when `!input.llmRuntime`. Replace -the generation call: - -```ts - const generated = await generateKtxObject< - KtxRelationshipLlmProposalOutput, - typeof relationshipLlmProposalSchema - >({ - runtime: input.llmRuntime, - role: 'candidateExtraction', - system, - prompt, - schema: relationshipLlmProposalSchema, - }); -``` - -In `packages/context/src/scan/local-scan.ts`, create runtime providers for LLM -mode: - -```ts -const llmRuntime = createLocalKtxLlmRuntimeFromConfig(llmConfig, { - ...deps, - projectDir: deps.projectDir, -}); -``` - -Thread `llmRuntime` through `KtxLocalScanEnrichmentProviders`. - -- [ ] **Step 6: Update non-agent tests** - -In each changed test file, use this runtime fake when the test needs LLM output: - -```ts -const runtime = { - generateText: vi.fn(async () => 'Generated description'), - generateObject: vi.fn(async () => ({ relationships: [] })), - runAgentLoop: vi.fn(), -}; -``` - -Update assertions from `llmProvider.getModel` to the operation used: - -```ts -expect(runtime.generateText).toHaveBeenCalledWith(expect.objectContaining({ role: 'candidateExtraction' })); -expect(runtime.generateObject).toHaveBeenCalledWith(expect.objectContaining({ role: 'candidateExtraction' })); -``` - -- [ ] **Step 7: Run non-agent runtime tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/llm/runtime-local-config.test.ts src/ingest/page-triage/page-triage.service.test.ts src/scan/description-generation.test.ts src/scan/relationship-llm-proposal.test.ts src/scan/local-scan.test.ts -``` - -Expected: all selected tests pass. - -- [ ] **Step 8: Commit** - -```bash -git add packages/context/src/llm/local-config.ts packages/context/src/llm/index.ts packages/context/src/llm/runtime-local-config.test.ts packages/context/src/ingest/page-triage/page-triage.service.ts packages/context/src/ingest/page-triage/page-triage.service.test.ts packages/context/src/scan/description-generation.ts packages/context/src/scan/description-generation.test.ts packages/context/src/scan/relationship-llm-proposal.ts packages/context/src/scan/relationship-llm-proposal.test.ts packages/context/src/scan/local-enrichment.ts packages/context/src/scan/local-scan.ts packages/context/src/scan/local-scan.test.ts -git commit -m "feat: route non-agent llm calls through runtime" -``` - -### Task 5: Agent Loops, Local Ingest, Memory, and MCP Ingest - -**Files:** - -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` -- Modify: `packages/context/src/ingest/local-ingest.ts` -- Modify: `packages/context/src/ingest/ports.ts` -- Modify: `packages/context/src/ingest/stages/stage-3-work-units.ts` -- Modify: `packages/context/src/ingest/stages/stage-3-work-units.test.ts` -- Modify: `packages/context/src/ingest/stages/stage-4-reconciliation.ts` -- Modify: `packages/context/src/ingest/stages/stage-4-reconciliation.test.ts` -- Modify: `packages/context/src/ingest/context-candidates/curator-pagination.service.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/stages/build-wu-context.ts` -- Modify: `packages/context/src/ingest/stages/build-reconcile-context.ts` -- Modify: `packages/context/src/memory/types.ts` -- Modify: `packages/context/src/memory/local-memory.ts` -- Modify: `packages/context/src/memory/memory-agent.service.ts` -- Modify: `packages/context/src/memory/memory-agent.service.ingest.test.ts` -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Write failing runtime injection tests** - -Add this test to `packages/context/src/ingest/local-bundle-runtime.test.ts`: - -```ts -it('uses a runtime-backed agent runner when claude-code is configured', () => { - const runtime = { - generateText: vi.fn(), - generateObject: vi.fn(), - runAgentLoop: vi.fn(async () => ({ stopReason: 'natural' as const })), - }; - project.config.llm = { - provider: { backend: 'claude-code' }, - models: { default: 'sonnet' }, - }; - const createLlmRuntime = vi.fn(() => runtime); - - const created = createLocalBundleIngestRuntime({ - project, - adapters: [], - createLlmRuntime, - }); - - expect(created).toBeDefined(); - expect(createLlmRuntime).toHaveBeenCalledWith( - project.config.llm, - expect.objectContaining({ projectDir: project.projectDir }), - ); -}); -``` - -Add this test to `packages/context/src/memory/memory-agent.service.ingest.test.ts`: - -```ts -it('normalizes load_skill output to markdown while preserving structured payload', async () => { - const agentRunner = { - runLoop: vi.fn(async (params) => { - const result = await params.toolSet.load_skill.execute({ name: 'memory_agent' }); - expect(result.markdown).toContain('memory_agent'); - expect(result.structured).toMatchObject({ name: 'memory_agent' }); - return { stopReason: 'natural' as const }; - }), - }; - const mocks = buildMocks({ - agentRunner, - skillsRegistry: { - listSkills: vi.fn().mockResolvedValue([{ name: 'memory_agent', path: '/tmp/skills/memory_agent' }]), - buildSkillsPrompt: vi.fn().mockReturnValue(''), - getSkill: vi.fn().mockResolvedValue({ name: 'memory_agent', path: '/tmp/skills/memory_agent' }), - stripFrontmatter: vi.fn().mockReturnValue('Skill body'), - }, - }); - const svc = buildService(mocks); - - await svc.ingest(baseInput); - expect(agentRunner.runLoop).toHaveBeenCalled(); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts src/memory/memory-agent.service.ingest.test.ts -``` - -Expected: FAIL because runtime injection and runtime tool descriptors are not -wired through local ingest and memory. - -- [ ] **Step 3: Change agent-runner dependency types to the port** - -Replace imports of concrete `AgentRunnerService` where services only call -`runLoop`: - -```ts -import type { AgentRunnerPort } from '../llm/index.js'; -``` - -or, from nested ingest stage files: - -```ts -import type { AgentRunnerPort } from '../../llm/index.js'; -``` - -Change fields such as: - -```ts - agentRunner: AgentRunnerService; -``` - -to: - -```ts - agentRunner: AgentRunnerPort; -``` - -Apply this in `ports.ts`, `stage-3-work-units.ts`, -`stage-4-reconciliation.ts`, `curator-pagination.service.ts`, and -`memory/types.ts`. - -- [ ] **Step 4: Create runtime-backed local ingest runners** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, add runtime factory -support: - -```ts -import { - createLocalKtxLlmRuntimeFromConfig, - RuntimeAgentRunner, - type KtxLlmRuntimePort, -} from '../llm/index.js'; -``` - -Extend `CreateLocalBundleIngestRuntimeOptions`: - -```ts - llmRuntime?: KtxLlmRuntimePort; - createLlmRuntime?: typeof createLocalKtxLlmRuntimeFromConfig; -``` - -Replace `resolveAgentRunner` with: - -```ts -function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): { - agentRunner: AgentRunnerPort; - llmRuntime?: KtxLlmRuntimePort; -} { - const llmRuntime = - options.llmRuntime ?? - (options.createLlmRuntime ?? createLocalKtxLlmRuntimeFromConfig)(options.project.config.llm, { - projectDir: options.project.projectDir, - env: process.env, - }) ?? - undefined; - - if (options.agentRunner) { - return { agentRunner: options.agentRunner, ...(llmRuntime ? { llmRuntime } : {}) }; - } - - if (!llmRuntime) { - throw new Error(localIngestLlmProviderGuardMessage(options.project.projectDir)); - } - - return { - agentRunner: new RuntimeAgentRunner(llmRuntime), - llmRuntime, - }; -} -``` - -Update the guard message: - -```ts -'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.' -``` - -Pass `llmRuntime` to `PageTriageService`: - -```ts - pageTriage: llmRuntime - ? new PageTriageService({ - store: contextStore, - llmRuntime, - settings: { - enabled: true, - maxConcurrency: 2, - lightExtractionEnabled: true, - classifierModel: null, - lightExtractionMaxCandidates: 5, - }, - promptService, - logger, - }) - : undefined, -``` - -- [ ] **Step 5: Normalize BaseTool factory outputs** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, replace -`toAiSdkTool(context)` with `toRuntimeTool(context)`: - -```ts -return { - ...Object.fromEntries(this.tools.map((tool) => [tool.name, tool.toRuntimeTool(context)])), - ...this.sourceTools, -}; -``` - -In `packages/context/src/memory/local-memory.ts`, make the same replacement: - -```ts -return Object.fromEntries(this.tools.map((tool) => [tool.name, tool.toRuntimeTool(context)])); -``` - -Update `MemoryToolSetLike` in `packages/context/src/memory/types.ts`: - -```ts -toRuntimeTools(context: ToolContext): KtxRuntimeToolSet; -``` - -- [ ] **Step 6: Convert inline ingest and memory tools to runtime descriptors** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, replace each inline -`tool(...)` wrapper with a `KtxRuntimeToolDescriptor`. For `load_skill`, use: - -```ts -const loadSkillTool: KtxRuntimeToolSet = { - load_skill: { - name: 'load_skill', - description: - 'Load a skill to get specialized instructions. Call this when a skill listed in the system prompt matches the current task.', - inputSchema: z.object({ name: z.string() }), - execute: async ({ name }) => { - const skill = await this.deps.skillsRegistry.getSkill(name, 'memory_agent'); - if (!skill) { - const available = - (await this.deps.skillsRegistry.listSkills('memory_agent')).map((s) => s.name).join(', ') || '(none)'; - return { markdown: `Skill "${name}" not available. Available: ${available}` }; - } - const body = await readFile(join(skill.path, 'SKILL.md'), 'utf-8'); - if (!skillsLoadedPerWu.includes(skill.name)) { - skillsLoadedPerWu.push(skill.name); - } - const structured = { - name: skill.name, - skillDirectory: skill.path, - content: this.deps.skillsRegistry.stripFrontmatter(body), - }; - return { - markdown: `# ${structured.name}\n\n${structured.content}`, - structured, - }; - }, - }, -}; -``` - -Use the same shape for reconciliation `rcLoadSkill`, including -`skillDirectory` in the structured payload. - -In `packages/context/src/memory/memory-agent.service.ts`, use: - -```ts -const loadSkillTool: KtxRuntimeToolSet = { - load_skill: { - name: 'load_skill', - description: - 'Load a skill to get specialized instructions. Call this when a skill listed in the system prompt matches the current task.', - inputSchema: z.object({ - name: z.string().describe('The skill name as listed in the system prompt.'), - }), - execute: async ({ name }) => { - const skill = await this.deps.skillsRegistry.getSkill(name, 'memory_agent'); - if (!skill) { - const available = - (await this.deps.skillsRegistry.listSkills('memory_agent')).map((s) => s.name).join(', ') || '(none)'; - return { markdown: `Skill "${name}" not available to the memory agent. Available: ${available}` }; - } - try { - const body = await readFile(join(skill.path, 'SKILL.md'), 'utf-8'); - if (!skillsLoaded.includes(skill.name)) { - skillsLoaded.push(skill.name); - } - const structured = { - name: skill.name, - skillDirectory: skill.path, - content: this.deps.skillsRegistry.stripFrontmatter(body), - }; - return { - markdown: `# ${structured.name}\n\n${structured.content}`, - structured, - }; - } catch (error) { - return { markdown: `Error loading skill "${name}": ${error instanceof Error ? error.message : String(error)}` }; - } - }, - }, -}; -``` - -- [ ] **Step 7: Convert ingest tool helper factories** - -For every helper under `packages/context/src/ingest/tools/*.tool.ts` that -currently returns `tool({ ... })`, change the return type to -`KtxRuntimeToolDescriptor` and return: - -```ts -return { - name: '', - description: '', - inputSchema, - execute: async (input) => normalizeKtxRuntimeToolOutput(await existingExecution(input)), -}; -``` - -Apply this to: - -- `stage-diff.tool.ts` -- `stage-list.tool.ts` -- `eviction-list.tool.ts` -- `read-raw-file.tool.ts` -- `read-raw-span.tool.ts` -- `emit-conflict-resolution.tool.ts` -- `emit-eviction-decision.tool.ts` -- `emit-artifact-resolution.tool.ts` -- `emit-unmapped-fallback.tool.ts` -- `verification-ledger.tool.ts` -- `adapters/historic-sql/evidence-tool.ts` -- `adapters/looker/tools/looker-query-to-sl.tool.ts` - -- [ ] **Step 8: Pass runtime through local ingest and MCP trigger options** - -In `packages/context/src/ingest/local-ingest.ts`, add `llmRuntime?: KtxLlmRuntimePort` -to local ingest options and pass it into `createLocalBundleIngestRuntime`. - -In `packages/context/src/mcp/local-project-ports.ts`, pass -`options.localIngest?.llmRuntime` into `runLocalMetabaseIngest` and -`runLocalIngest`: - -```ts -llmRuntime: options.localIngest?.llmRuntime, -``` - -Update `packages/context/src/ingest/ports.ts` to expose `llmRuntime` beside -`agentRunner` only for local wiring. - -- [ ] **Step 9: Run agent-loop tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/stages/stage-3-work-units.test.ts src/ingest/stages/stage-4-reconciliation.test.ts src/ingest/local-bundle-runtime.test.ts src/memory/memory-agent.service.ingest.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: all selected tests pass, and test fakes call `runLoop` through -`AgentRunnerPort`. - -- [ ] **Step 10: Commit** - -```bash -git add packages/context/src/ingest/local-bundle-runtime.ts packages/context/src/ingest/local-bundle-runtime.test.ts packages/context/src/ingest/local-ingest.ts packages/context/src/ingest/ports.ts packages/context/src/ingest/stages/stage-3-work-units.ts packages/context/src/ingest/stages/stage-3-work-units.test.ts packages/context/src/ingest/stages/stage-4-reconciliation.ts packages/context/src/ingest/stages/stage-4-reconciliation.test.ts packages/context/src/ingest/context-candidates/curator-pagination.service.ts packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/stages/build-wu-context.ts packages/context/src/ingest/stages/build-reconcile-context.ts packages/context/src/ingest/tools packages/context/src/memory/types.ts packages/context/src/memory/local-memory.ts packages/context/src/memory/memory-agent.service.ts packages/context/src/memory/memory-agent.service.ingest.test.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat: route agent loops through llm runtime" -``` - -### Task 6: Setup, Status, Doctor, Prompt-Caching Warnings, and Docs - -**Files:** - -- Modify: `packages/cli/src/setup-commands.ts` -- Modify: `packages/cli/src/setup-models.ts` -- Modify: `packages/cli/src/setup-models.test.ts` -- Modify: `packages/cli/src/status-project.ts` -- Modify: `packages/cli/src/doctor.test.ts` -- Modify: `docs-site/content/docs/getting-started/quickstart.mdx` -- Modify: `docs-site/content/docs/cli-reference/ktx-setup.mdx` -- Modify: `docs-site/content/docs/cli-reference/ktx-status.mdx` -- Modify: `docs-site/content/docs/guides/building-context.mdx` -- Create: `docs-site/content/docs/guides/llm-configuration.mdx` -- Modify: `docs-site/content/docs/guides/meta.json` - -- [ ] **Step 1: Write failing CLI tests** - -Add this test to `packages/cli/src/setup-models.test.ts`: - -```ts -it('configures Claude Code backend and validates local auth', async () => { - const io = makeIo(); - const authProbe = vi.fn(async () => ({ ok: true as const })); - - const result = await runKtxSetupAnthropicModelStep( - { - projectDir: tempDir, - inputMode: 'disabled', - llmBackend: 'claude-code', - skipLlm: false, - }, - io.io, - { claudeCodeAuthProbe: authProbe }, - ); - - expect(result.status).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.llm).toMatchObject({ - provider: { backend: 'claude-code' }, - models: { default: 'sonnet' }, - }); - expect(authProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'sonnet' })); -}); -``` - -Add this test to `packages/cli/src/doctor.test.ts`: - -```ts -it('reports Claude Code auth failures and ignored prompt-caching fields in project doctor output', async () => { - await writeFile( - join(tempDir, 'ktx.yaml'), - [ - 'llm:', - ' provider:', - ' backend: claude-code', - ' models:', - ' default: sonnet', - ' promptCaching:', - ' enabled: true', - ' systemTtl: 1h', - ' toolsTtl: 1h', - ' historyTtl: 5m', - '', - ].join('\n'), - 'utf-8', - ); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, - testIo.io, - { - claudeCodeAuthProbe: async () => ({ - ok: false as const, - message: 'Authenticate Claude Code locally.', - }), - }, - ), - ).resolves.toBe(1); - - expect(testIo.stdout()).toContain('claude-code'); - expect(testIo.stdout()).toContain('Authenticate Claude Code locally'); - expect(testIo.stdout()).toContain('claude-code ignores llm.promptCaching'); -}); -``` - -- [ ] **Step 2: Run CLI tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-models.test.ts src/doctor.test.ts -``` - -Expected: FAIL because CLI backend parsing and status probing do not support -`claude-code`. - -- [ ] **Step 3: Add setup backend parsing and auth probe dependency** - -In `packages/cli/src/setup-models.ts`, update the backend type: - -```ts -export type KtxSetupLlmBackend = 'anthropic' | 'vertex' | 'claude-code'; -``` - -Add the dependency: - -```ts - claudeCodeAuthProbe?: (input: { projectDir: string; model: string; env?: NodeJS.ProcessEnv }) => Promise<{ ok: true } | { ok: false; message: string }>; -``` - -Update `buildProjectLlmConfig` to accept Claude Code: - -```ts - | { backend: 'claude-code' }, -``` - -```ts - if (provider.backend === 'claude-code') { - return { - provider: { backend: 'claude-code' }, - models: { ...existing.models, default: model }, - promptCaching: existing.promptCaching, - }; - } -``` - -When `args.llmBackend === 'claude-code'`, set: - -```ts -const model = args.anthropicModel ?? 'sonnet'; -const probe = deps.claudeCodeAuthProbe ?? runClaudeCodeAuthProbe; -const health = await probe({ projectDir: args.projectDir, model, env: deps.env ?? process.env }); -if (!health.ok) { - io.stderr.write(`${health.message}\n`); - return { status: 'failed', projectDir: args.projectDir }; -} -project.config.llm = buildProjectLlmConfig(project.config.llm, { backend: 'claude-code' }, model); -``` - -In `packages/cli/src/setup-commands.ts`, update the hidden parser to accept -`claude-code`: - -```ts -if (value === 'anthropic' || value === 'vertex' || value === 'claude-code') { - return value; -} -``` - -- [ ] **Step 4: Add status and doctor auth validation** - -In `packages/cli/src/status-project.ts`, extend `BuildProjectStatusOptions`: - -```ts - claudeCodeAuthProbe?: (input: { projectDir: string; model: string; env?: NodeJS.ProcessEnv }) => Promise<{ ok: true } | { ok: false; message: string }>; -``` - -Make `buildLlmStatus` async and add: - -```ts - if (backend === 'claude-code') { - const modelName = model ?? 'sonnet'; - const probe = options.claudeCodeAuthProbe ?? runClaudeCodeAuthProbe; - const auth = await probe({ projectDir, model: modelName, env }); - if (auth.ok) { - return { backend, model: modelName, status: 'ok', detail: 'local Claude Code session authenticated' }; - } - return { - backend, - model: modelName, - status: 'fail', - detail: auth.message, - fix: 'Authenticate Claude Code locally with the Claude Code CLI, then rerun `ktx status`.', - }; - } -``` - -Add prompt-caching warnings: - -```ts -function ignoredClaudeCodePromptCachingFields(config: KtxProjectLlmConfig): string[] { - if (config.provider.backend !== 'claude-code' || !config.promptCaching) return []; - return Object.keys(config.promptCaching).map((key) => `llm.promptCaching.${key}`); -} -``` - -Append this warning in `buildWarnings`: - -```ts -const ignored = ignoredClaudeCodePromptCachingFields(config.llm); -if (ignored.length > 0) { - warnings.push({ - level: 'warn', - message: `claude-code ignores ${ignored.join(', ')} because the Claude Agent SDK does not expose KTX prompt-cache TTL, tool, or history markers.`, - fix: 'Remove those promptCaching fields or use anthropic, vertex, or gateway when those cache knobs are required.', - }); -} -``` - -- [ ] **Step 5: Update docs** - -In `docs-site/content/docs/getting-started/quickstart.mdx`, add this provider -example in the LLM setup section: - -````mdx -To use your local Claude Code session instead of an API key, set: - -```yaml -llm: - provider: - backend: claude-code - models: - default: sonnet - triage: haiku - candidateExtraction: sonnet - curator: sonnet - reconcile: sonnet - repair: sonnet -``` - -`claude-code` uses the Claude Code authentication already configured on your -machine. It doesn't use `ANTHROPIC_API_KEY`, Vertex credentials, AI Gateway -tokens, or Bedrock credentials. -```` - -In `docs-site/content/docs/cli-reference/ktx-setup.mdx`, document: - -```mdx -| `--llm-backend claude-code` | Use the local Claude Code session for KTX LLM calls | - | -``` - -In `docs-site/content/docs/cli-reference/ktx-status.mdx`, add: - -```mdx -For `llm.provider.backend: claude-code`, `ktx status` checks that the local -Claude Code session is usable. If auth fails, run the Claude Code CLI login -flow, then rerun `ktx status`. -``` - -In `docs-site/content/docs/guides/building-context.mdx`, add: - -```mdx -When you use `claude-code`, KTX still controls the tool surface for ingest and -memory capture. Claude Code built-in tools, discovered MCP servers, hooks, -skills, plugins, agents, and slash commands are not exposed to KTX agent loops. -``` - -Create `docs-site/content/docs/guides/llm-configuration.mdx`: - -````mdx ---- -title: LLM configuration -description: Configure KTX LLM providers, model roles, and prompt caching. ---- - -KTX uses the top-level `llm` block in `ktx.yaml` for text generation, -structured extraction, and ingest or memory agent loops. - -## Backends - -Set `llm.provider.backend` to one of these values: - -- `anthropic`: Use the Anthropic API through `ANTHROPIC_API_KEY` or the - configured `api_key` reference. -- `vertex`: Use Vertex AI Anthropic models through Google Cloud credentials. -- `gateway`: Use AI Gateway-compatible Anthropic model ids. -- `claude-code`: Use your local Claude Code session through the Claude Agent - SDK. KTX removes provider-routing environment variables from Claude Code - child processes, so this backend doesn't silently fall back to - `ANTHROPIC_API_KEY`, Vertex, Gateway, or Bedrock credentials. - -## Claude Code - -Use aliases or full Claude model ids in `llm.models`: - -```yaml -llm: - provider: - backend: claude-code - models: - default: sonnet - triage: haiku - candidateExtraction: sonnet - curator: sonnet - reconcile: sonnet - repair: sonnet -``` - -`claude-code` keeps KTX tool boundaries intact. KTX exposes only the MCP tools -needed for the current KTX agent loop and disables Claude Code built-in tools, -filesystem settings, skills, plugins, agents, hooks, and slash commands. - -## Prompt caching - -`llm.promptCaching` has partial parity on `claude-code`. KTX doesn't pass -Anthropic cache-control markers to the Claude Agent SDK. Status and doctor warn -when you configure prompt-cache TTL, tool, or history fields that the Claude -Agent SDK backend ignores. -```` - -In `docs-site/content/docs/guides/meta.json`, add the page: - -```json -{ - "title": "Guides", - "defaultOpen": true, - "pages": ["building-context", "llm-configuration", "writing-context", "serving-agents"] -} -``` - -- [ ] **Step 6: Run CLI and docs tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-models.test.ts src/doctor.test.ts -pnpm --filter ktx-docs run test -``` - -Expected: selected CLI tests and docs tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add packages/cli/src/setup-commands.ts packages/cli/src/setup-models.ts packages/cli/src/setup-models.test.ts packages/cli/src/status-project.ts packages/cli/src/doctor.test.ts docs-site/content/docs/getting-started/quickstart.mdx docs-site/content/docs/cli-reference/ktx-setup.mdx docs-site/content/docs/cli-reference/ktx-status.mdx docs-site/content/docs/guides/building-context.mdx docs-site/content/docs/guides/llm-configuration.mdx docs-site/content/docs/guides/meta.json -git commit -m "feat: support claude-code setup and status" -``` - -### Task 7: Repo-Wide Audit and Final Verification - -**Files:** - -- Modify: any files still found by the required grep audit. -- Modify: `knip.json` only if the new Agent SDK entrypoints are intentionally - dynamic and Knip cannot infer them. - -- [ ] **Step 1: Run the required LLM call-site audit** - -Run: - -```bash -rg -n "getModel\\(|generateKtxText\\(|generateKtxObject\\(|AgentRunnerService|llmProvider" packages/context packages/cli packages/llm -g '!**/dist/**' -``` - -Expected remaining allowed matches: - -- `packages/llm/src/model-provider.ts` and its tests for AI SDK provider - construction. -- `packages/llm/src/model-health.ts` and its tests for AI SDK health checks. -- `packages/context/src/llm/ai-sdk-runtime.ts` for the AI SDK runtime adapter. -- `packages/context/src/llm/local-config.ts` for provider construction on - non-`claude-code` backends. -- Test fakes where the test name explicitly covers AI SDK provider behavior. - -Every runtime call site under ingest, memory, MCP-triggered ingest, page -triage, scan description generation, and relationship LLM proposals must use -`KtxLlmRuntimePort` or `AgentRunnerPort`. - -- [ ] **Step 2: Fix disallowed audit matches** - -For every disallowed match from Step 1, replace the dependency with one of -these patterns: - -```ts -import type { KtxLlmRuntimePort } from '../llm/index.js'; -``` - -```ts -import type { AgentRunnerPort } from '../llm/index.js'; -``` - -Then call: - -```ts -await runtime.generateText({ role, system, prompt, temperature }); -await runtime.generateObject({ role, system, prompt, schema, temperature }); -await agentRunner.runLoop(params); -``` - -- [ ] **Step 3: Run targeted package tests** - -Run: - -```bash -pnpm --filter @ktx/llm run test -pnpm --filter @ktx/context run test -pnpm --filter @ktx/cli run test -pnpm --filter ktx-docs run test -``` - -Expected: all selected package tests pass. - -- [ ] **Step 4: Run type-checks** - -Run: - -```bash -pnpm run type-check -``` - -Expected: TypeScript compilation passes across packages. - -- [ ] **Step 5: Run build/export verification** - -Run: - -```bash -pnpm run build -``` - -Expected: all package builds pass and exports resolve. - -- [ ] **Step 6: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: Biome and Knip pass. If Knip reports the Agent SDK dependency as -unused despite static imports in `claude-code-runtime.ts`, inspect the report -and fix the import path or package dependency location before adding an ignore. - -- [ ] **Step 7: Run full workspace test** - -Run: - -```bash -pnpm run test -``` - -Expected: workspace tests pass. - -- [ ] **Step 8: Commit verification cleanup** - -```bash -git status --short -git add packages docs-site pnpm-lock.yaml knip.json -git commit -m "test: verify claude-code backend runtime" -``` - -Use `git add packages docs-site pnpm-lock.yaml knip.json` only after inspecting -`git status --short` and confirming every staged file belongs to this backend -implementation. - -## Self-Review - -- Spec coverage: This plan covers first-class config, runtime port, text - generation, structured object generation, agent loops, tool boundaries, exact - MCP ids, `canUseTool`, isolation options, scrubbed environment, auth probe, - stop reason mapping, `onStepFinish`, prompt-caching warnings, setup/status, - docs, and the required grep audit. -- V1-blocking coverage: All blocking gaps from the audit are assigned to Tasks - 1 through 7. -- Non-blocking exclusions: Same-step repair parity, OTEL parity, embedding - parity, persisted Claude sessions, and full prompt-caching parity are - intentionally left outside v1. diff --git a/docs/superpowers/plans/2026-05-15-semantic-layer-docs.md b/docs/superpowers/plans/2026-05-15-semantic-layer-docs.md deleted file mode 100644 index 59e5d5bf..00000000 --- a/docs/superpowers/plans/2026-05-15-semantic-layer-docs.md +++ /dev/null @@ -1,328 +0,0 @@ -# Semantic Layer Docs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. - -**Goal:** Add a standalone, scannable Concepts page that explains the semantic-layer internals while positioning KTX as a broader context layer. - -**Architecture:** Implement this as docs-only MDX content inside the existing Fumadocs tree. The new page uses inline MDX diagrams and Fumadocs color tokens, matching the custom diagram pattern already used in `the-context-layer.mdx`. - -**Tech Stack:** MDX, Fumadocs content, Next.js docs site, pnpm workspace commands. - ---- - -### Task 1: Add Concepts Navigation Entry - -**Files:** -- Modify: `docs-site/content/docs/concepts/meta.json` - -- [x] **Step 1: Update the Concepts page order** - -Replace the `pages` array with: - -```json -{ - "title": "Concepts", - "defaultOpen": true, - "pages": ["the-context-layer", "semantic-layer-internals", "context-as-code"] -} -``` - -- [x] **Step 2: Verify JSON parses** - -Run: - -```bash -node -e "JSON.parse(require('node:fs').readFileSync('docs-site/content/docs/concepts/meta.json', 'utf8')); console.log('concepts meta ok')" -``` - -Expected output: - -```text -concepts meta ok -``` - -### Task 2: Create the Semantic Layer Internals Page - -**Files:** -- Create: `docs-site/content/docs/concepts/semantic-layer-internals.mdx` - -- [x] **Step 1: Add frontmatter and opening positioning** - -Create the page with this frontmatter and opening section: - -```mdx ---- -title: Semantic Layer Internals -description: How KTX uses join graphs, grain, and relationship metadata to turn context into safe SQL. ---- - -KTX is a context layer for agents. Its semantic layer is the query-planning core -that turns reviewed context into safe SQL. - -Use this page to understand the mechanics behind KTX's semantic execution: -the join graph, how KTX builds and maintains it, and how that graph prevents -classic analytics errors like fan-out and ambiguous join paths. - -| KTX is | KTX is not just | -|---|---| -| A context layer for agents | A metric definition store | -| A system for ingesting, reviewing, and serving analytics context | A markdown saver | -| A semantic execution layer plus wiki pages, scans, provenance, and agent workflows | A replacement for every BI semantic layer | -``` - -- [x] **Step 2: Add the system-fit diagram** - -Add a `Where the semantic layer fits` section with a custom `not-prose` diagram. -The diagram must show: - -```text -Context inputs -> Semantic layer engine -> Agent workflows -``` - -The semantic-layer box must be visually prominent and list: - -```text -join graph -grain -measures -relationships -safe query planning -``` - -- [x] **Step 3: Add the join graph section** - -Add `## The join graph` with: - -- one short paragraph defining nodes and edges; -- bullets for why the graph matters; -- an inline diagram using `orders`, `customers`, `order_items`, and `refunds`. - -The section must include this claim in plain language: - -```text -The graph lets KTX choose valid paths, reject unsafe paths, and reason about -whether a join preserves or multiplies rows before SQL is generated. -``` - -- [x] **Step 4: Add build and maintenance sections** - -Add `## How KTX builds it` and `## How KTX maintains it`. - -`How KTX builds it` must cover these inputs: - -```text -declared primary keys -declared foreign keys -inferred relationships -dbt, MetricFlow, and LookML imports -query history -analyst review -``` - -`How KTX maintains it` must show this loop: - -```text -ingest evidence -> YAML diff -> validation -> analyst review -> agent use -> corrections -``` - -- [x] **Step 5: Add the fan-out and safe execution sections** - -Add `## Why grain and relationships matter` with a fan-out example comparing -orders joined to order items. Include a compact table with columns: - -```text -Problem -What happens -How KTX avoids it -``` - -Add `## How the execution engine uses the graph` with a before/after table: - -```text -Naive SQL shape -Semantic-layer SQL shape -``` - -The safe path must mention: - -```text -pre-aggregates fact measures at their own grain before joining dimensions -``` - -- [x] **Step 6: Add agent outcome links** - -Add a closing `## What this means for agents` section with bullets explaining -that agents can: - -```text -search semantic sources -compile SQL through ktx sl query -validate changes before review -patch YAML and Markdown files in git -explain provenance and metric meaning -``` - -End with links to: - -```mdx -[Writing Context](/docs/guides/writing-context) -[ktx sl](/docs/cli-reference/ktx-sl) -``` - -### Task 3: Add the Cross-Link from The Context Layer - -**Files:** -- Modify: `docs-site/content/docs/concepts/the-context-layer.mdx` - -- [x] **Step 1: Replace the semantic sources paragraph with a scannable block** - -Find the `**Semantic sources**` paragraph under `KTX organizes context into four pillars`. -Replace the long paragraph with: - -```mdx -**Semantic sources** are YAML definitions that describe your data in terms -agents can reason about: - -- source tables or SQL queries; -- row grain; -- typed columns; -- valid joins; -- named measures, filters, and segments. - -This is where "revenue means `sum(amount)` excluding refunds" lives. For the -join graph, fan-out protections, and execution mechanics, read -[Semantic Layer Internals](/docs/concepts/semantic-layer-internals). -``` - -- [x] **Step 2: Confirm the page still owns the product positioning** - -Search the edited file: - -```bash -rg -n "context layer|Semantic Layer Internals|semantic layer - that's a critical component" docs-site/content/docs/concepts/the-context-layer.mdx -``` - -Expected: output includes the existing context-layer framing and the new internals link. - -### Task 4: Fix Mobile Docs Header Overflow - -**Files:** -- Modify: `docs-site/app/docs/[[...slug]]/page.tsx` - -- [x] **Step 1: Stack title actions on narrow screens** - -Replace the non-hero page header wrapper: - -```tsx -
    -``` - -with: - -```tsx -
    -``` - -This keeps desktop layout unchanged while preventing the action buttons from -forcing horizontal overflow on mobile. - -- [x] **Step 2: Allow the docs article to shrink in the layout grid** - -Update the `DocsPage` and `DocsBody` wrappers: - -```tsx - -``` - -```tsx - -``` - -This prevents tables, code blocks, and custom diagrams from forcing the -Fumadocs main article column wider than the mobile viewport, overrides the -library's built-in max-width rule on mobile, aligns the article to the left on -mobile, and preserves the normal centered desktop max width. - -If long words still clip under mobile viewport capture, add the same wrapping -behavior used by the Fumadocs sidebar: - -```tsx - - {page.data.description} - -``` - -```tsx - -``` - -- [x] **Step 3: Recheck mobile render** - -Capture or inspect a 390px-wide render of: - -```text -http://127.0.0.1:3000/docs/concepts/semantic-layer-internals -``` - -Expected: the title, description, action buttons, and positioning block stay -within the viewport. - -### Task 5: Verify Docs Content and Build - -**Files:** -- Check: `docs-site/content/docs/concepts/semantic-layer-internals.mdx` -- Check: `docs-site/content/docs/concepts/the-context-layer.mdx` -- Check: `docs-site/content/docs/concepts/meta.json` -- Check: `docs-site/app/docs/[[...slug]]/page.tsx` - -- [x] **Step 1: Run content checks** - -Run: - -```bash -rg -n "KTX is a context layer|markdown saver|fan-out|join graph|pre-aggregates|Semantic Layer Internals" docs-site/content/docs/concepts -``` - -Expected: matches appear in the new page and the cross-link appears in -`the-context-layer.mdx`. - -- [x] **Step 2: Build the docs site** - -Run: - -```bash -pnpm --filter ktx-docs build -``` - -Expected: build exits 0. - -- [x] **Step 3: Preview locally** - -Run: - -```bash -pnpm --filter ktx-docs dev -``` - -Open: - -```text -http://localhost:3000/docs/concepts/semantic-layer-internals -``` - -Inspect desktop and mobile widths. The opening should clearly position KTX as a -context layer, the Concepts navigation should list the new page, and diagrams -should not overlap or produce unreadable text. - -- [x] **Step 4: Commit implementation** - -Run: - -```bash -git status --short -git add docs-site/content/docs/concepts/meta.json docs-site/content/docs/concepts/semantic-layer-internals.mdx docs-site/content/docs/concepts/the-context-layer.mdx docs-site/app/docs/[[...slug]]/page.tsx docs/superpowers/plans/2026-05-15-semantic-layer-docs.md -git commit -m "docs: add semantic layer internals concept" -``` diff --git a/docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-metadata-progress.md b/docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-metadata-progress.md deleted file mode 100644 index 28221089..00000000 --- a/docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-metadata-progress.md +++ /dev/null @@ -1,1459 +0,0 @@ -# MCP Tool Polish V1 Metadata and Progress Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish the remaining v1-blocking MCP polish work after the surface -change: tool metadata, schemas, in-band errors, normalization, resolved-source -invariants, and progress notifications. - -**Architecture:** Keep the 11-tool research surface already implemented. Add -metadata and output schemas through the shared `registerParsedTool` path, keep -runtime handlers small, and plumb progress as optional callbacks through the -MCP ports that execute work. - -**Tech Stack:** TypeScript, Zod v4, MCP SDK 1.29, Vitest, pnpm workspace -commands. - ---- - -## Audit summary - -The original spec is -`docs/superpowers/specs/2026-05-16-mcp-tool-polish-design.md`. - -Already implemented by -`docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-surface-change.md`: - -- The MCP surface is reduced to 11 registered tools in - `packages/context/src/mcp/context-tools.ts`. -- `memory_capture` and `memory_capture_status` are replaced by - `memory_ingest` and `memory_ingest_status`. -- Memory ingest runs through `registerKtxContextTools`, so it shares the same - registration path as the other retained tools. -- `packages/cli/src/skills/analytics/SKILL.md` uses `memory_ingest` and - documents the multi-connection rule. -- `docs-site/content/docs/integrations/agent-clients.mdx` says memory ingest. - -Remaining v1-blocking gaps covered by this plan: - -- Add MCP tool annotations and `outputSchema` for all 11 retained tools. -- Add `.describe()` to every input field and rewrite tool descriptions with - concrete argument examples. -- Move in-band runtime error wrapping into `registerParsedTool` and remove the - local `sql_execution` catch. -- Normalize `sl_query.dimensions` Cube-style `{ dimension, granularity }`. -- Normalize `entity_details.entities[].table` SQL-style - `{ schema, table }` into `{ catalog: null, db: schema, name: table }`. -- Type-narrow `jsonToolResult` so bare arrays do not type-check. -- Add the `toResolvedWire` invariant comment and narrow compute-port source - types to resolved sources. -- Emit progress notifications for `sql_execution` and `sl_query` when the MCP - request includes `_meta.progressToken`. - -Non-blocking gaps left outside this plan: - -- Delete admin tool implementation code after a future `ktx-admin` skill lands. -- MCP resources, MCP prompts, elicitation, sampling, tool icons, code execution, - multi-tenancy, telemetry, and rate limiting. -- More exhaustive multi-client manual smoke beyond the automated in-memory MCP - SDK coverage in this plan. - -## File structure - -- `packages/context/src/mcp/types.ts`: expand the local MCP server facade with - output schemas, annotations, handler context, and progress callback types. -- `packages/context/src/mcp/context-tools.ts`: add output schemas, annotations, - input descriptions, tool descriptions, centralized error wrapping, - normalization, type-narrowed `jsonToolResult`, and progress callback wiring. -- `packages/context/src/mcp/server.test.ts`: add schema, annotation, - normalization, in-band error, progress, and type-narrowing coverage. -- `packages/context/src/daemon/semantic-layer-compute.ts`: document and type - the resolved-source invariant for daemon-backed semantic-layer calls. -- `packages/context/src/sl/local-query.ts`: accept an optional progress - callback and emit semantic-layer query stages. -- `packages/context/src/mcp/local-project-ports.ts`: pass progress callbacks - into `compileLocalSlQuery` and emit SQL execution stages. -- `packages/context/src/mcp/local-project-ports.test.ts`: verify local port - progress stages. -- `packages/context/src/sl/local-query.test.ts`: verify compile and execution - progress stages. - -### Task 1: Add failing MCP metadata, schema, normalization, error, and progress tests - -**Files:** - -- Modify: `packages/context/src/mcp/server.test.ts` - -- [ ] **Step 1: Update imports and fake server types** - -In `packages/context/src/mcp/server.test.ts`, replace the import from -`./server.js` and the MCP type import with: - -```typescript -import { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; -import { z } from 'zod'; -import { createDefaultKtxMcpServer, createKtxMcpServer } from './server.js'; -import { jsonToolResult } from './context-tools.js'; -import type { - KtxDiscoverDataMcpPort, - KtxDictionarySearchMcpPort, - KtxEntityDetailsMcpPort, - KtxKnowledgeMcpPort, - KtxMcpContextPorts, - KtxMcpToolHandlerContext, - KtxSemanticLayerMcpPort, - KtxSqlExecutionMcpPort, - KtxSqlExecutionResponse, - MemoryIngestPort, -} from './types.js'; -``` - -Replace the `RegisteredTool` type with: - -```typescript -type RegisteredTool = { - name: string; - config: { - title?: string; - description?: string; - inputSchema: unknown; - outputSchema?: unknown; - annotations?: Record; - }; - handler: (input: Record, context?: KtxMcpToolHandlerContext) => Promise; -}; -``` - -- [ ] **Step 2: Add shared test helpers** - -After `getTool`, add: - -```typescript -const retainedToolNames = [ - 'connection_list', - 'dictionary_search', - 'discover_data', - 'entity_details', - 'memory_ingest', - 'memory_ingest_status', - 'sl_query', - 'sl_read_source', - 'sql_execution', - 'wiki_read', - 'wiki_search', -] as const; - -function makeAllContextTools(): KtxMcpContextPorts { - return { - connections: { - list: vi.fn().mockResolvedValue([{ id: 'warehouse', name: 'Warehouse', connectionType: 'POSTGRES' }]), - }, - knowledge: { - search: vi.fn().mockResolvedValue({ results: [], totalFound: 0 }), - read: vi.fn().mockResolvedValue({ - key: 'revenue', - summary: 'Paid order value', - content: '# Revenue', - scope: 'GLOBAL', - tags: ['finance'], - refs: [], - slRefs: ['orders'], - }), - }, - semanticLayer: { - readSource: vi.fn().mockResolvedValue({ - sourceName: 'orders', - yaml: 'name: orders\n', - }), - query: vi.fn().mockResolvedValue({ - sql: 'select 1', - headers: ['count'], - rows: [[1]], - totalRows: 1, - plan: { sources: ['orders'] }, - }), - }, - entityDetails: { - read: vi.fn().mockResolvedValue({ results: [] }), - }, - dictionarySearch: { - search: vi.fn().mockResolvedValue({ searched: [], results: [] }), - }, - discover: { - search: vi.fn().mockResolvedValue([]), - }, - sqlExecution: { - execute: vi.fn().mockResolvedValue({ - headers: ['count'], - headerTypes: ['integer'], - rows: [[1]], - rowCount: 1, - }), - }, - memoryIngest: { - ingest: vi.fn().mockResolvedValue({ runId: 'run-1' }), - status: vi.fn().mockResolvedValue({ - runId: 'run-1', - status: 'done', - stage: 'done', - done: true, - captured: { wiki: [], sl: [], xrefs: [] }, - error: null, - commitHash: null, - skillsLoaded: [], - signalDetected: false, - }), - }, - }; -} - -async function listToolsThroughSdk(contextTools: KtxMcpContextPorts) { - const server = createDefaultKtxMcpServer({ - name: 'ktx-test', - version: '0.0.0-test', - userContext: { userId: 'mcp-user' }, - contextTools, - }); - const client = new Client({ name: 'ktx-test-client', version: '0.0.0-test' }); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - - await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); - try { - return await client.listTools(); - } finally { - await client.close(); - await server.close(); - } -} -``` - -- [ ] **Step 3: Add annotations and output schema assertions** - -Inside `describe('createKtxMcpServer', () => {`, add: - -```typescript - it('registers annotations and output schemas for every retained tool', async () => { - const fake = makeFakeServer(); - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'mcp-user' }, - contextTools: makeAllContextTools(), - }); - - expect(fake.tools.map((tool) => tool.name).sort()).toEqual([...retainedToolNames].sort()); - - const expectedAnnotations: Record> = { - connection_list: { title: 'Connection List', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - discover_data: { title: 'Discover Data', readOnlyHint: true, openWorldHint: false }, - wiki_search: { title: 'Wiki Search', readOnlyHint: true, openWorldHint: false }, - wiki_read: { title: 'Wiki Read', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - entity_details: { title: 'Entity Details', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - dictionary_search: { title: 'Dictionary Search', readOnlyHint: true, openWorldHint: false }, - sl_read_source: { - title: 'Semantic Layer Read Source', - readOnlyHint: true, - idempotentHint: true, - openWorldHint: false, - }, - sl_query: { title: 'Semantic Layer Query', readOnlyHint: true, openWorldHint: false }, - sql_execution: { title: 'SQL Execution', readOnlyHint: true, openWorldHint: false }, - memory_ingest: { title: 'Memory Ingest', destructiveHint: true, openWorldHint: false }, - memory_ingest_status: { title: 'Memory Ingest Status', readOnlyHint: true, openWorldHint: false }, - }; - - for (const toolName of retainedToolNames) { - const tool = getTool(fake.tools, toolName); - expect(tool.config.title).toBe(expectedAnnotations[toolName]?.title); - expect(tool.config.annotations).toEqual(expectedAnnotations[toolName]); - expect(tool.config.outputSchema).toBeDefined(); - const inputShape = tool.config.inputSchema as Record; - for (const inputSchema of Object.values(inputShape)) { - expect(inputSchema.description).toEqual(expect.any(String)); - } - } - }); -``` - -- [ ] **Step 4: Add the SDK tools/list schema snapshot test** - -Add: - -```typescript - it('exposes annotations and output schemas through the SDK tools/list response', async () => { - const result = await listToolsThroughSdk(makeAllContextTools()); - const toolNames = result.tools.map((tool) => tool.name).sort(); - expect(toolNames).toEqual([...retainedToolNames].sort()); - - await expect(result.tools).toMatchFileSnapshot('__snapshots__/mcp-tools-list.json'); - }); -``` - -- [ ] **Step 5: Add normalization tests for the two remaining drift shapes** - -Add: - -```typescript - it('sl_query normalizes cube-style dimensions to field dimensions', async () => { - const fake = makeFakeServer(); - const semanticLayer = makeAllContextTools().semanticLayer!; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { semanticLayer }, - }); - - await getTool(fake.tools, 'sl_query').handler({ - connectionId: 'warehouse', - measures: ['orders.count'], - dimensions: [{ dimension: 'orders.created_at', granularity: 'month' }, 'orders.status'], - }); - - expect(semanticLayer.query).toHaveBeenCalledWith( - { - connectionId: 'warehouse', - query: expect.objectContaining({ - dimensions: [{ field: 'orders.created_at', granularity: 'month' }, { field: 'orders.status' }], - }), - }, - undefined, - ); - }); - - it('entity_details normalizes sql-style schema table refs', async () => { - const fake = makeFakeServer(); - const entityDetails = makeAllContextTools().entityDetails!; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { entityDetails }, - }); - - await getTool(fake.tools, 'entity_details').handler({ - connectionId: 'warehouse', - entities: [{ table: { schema: 'public', table: 'orders' }, columns: ['id'] }], - }); - - expect(entityDetails.read).toHaveBeenCalledWith({ - connectionId: 'warehouse', - entities: [{ table: { catalog: null, db: 'public', name: 'orders' }, columns: ['id'] }], - }); - }); -``` - -- [ ] **Step 6: Add centralized runtime error wrapping tests** - -Add: - -```typescript - it('wraps handler exceptions in-band for non-sql tools', async () => { - const fake = makeFakeServer(); - const knowledge: KtxKnowledgeMcpPort = { - search: vi.fn().mockRejectedValue(new Error('wiki index unavailable')), - read: vi.fn(), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { knowledge }, - }); - - await expect(getTool(fake.tools, 'wiki_search').handler({ query: 'revenue' })).resolves.toEqual({ - content: [{ type: 'text', text: 'wiki index unavailable' }], - isError: true, - }); - }); -``` - -- [ ] **Step 7: Add MCP progress notification tests** - -Add: - -```typescript - it('wires sql_execution progress to MCP notifications when a progress token is present', async () => { - const fake = makeFakeServer(); - const notifications: unknown[] = []; - const sqlExecution: KtxSqlExecutionMcpPort = { - execute: vi.fn().mockImplementation(async (_input, options) => { - await options?.onProgress?.({ progress: 0, message: 'Validating SQL' }); - await options?.onProgress?.({ progress: 0.3, message: 'Executing' }); - await options?.onProgress?.({ progress: 1, message: 'Fetched 1 rows' }); - return { headers: ['count'], rows: [[1]], rowCount: 1 }; - }), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local-user' }, - contextTools: { sqlExecution }, - }); - - await getTool(fake.tools, 'sql_execution').handler( - { connectionId: 'warehouse', sql: 'select 1' }, - { - _meta: { progressToken: 'progress-1' }, - sendNotification: async (notification) => { - notifications.push(notification); - }, - }, - ); - - expect(notifications).toEqual([ - { - method: 'notifications/progress', - params: { progressToken: 'progress-1', progress: 0, message: 'Validating SQL' }, - }, - { - method: 'notifications/progress', - params: { progressToken: 'progress-1', progress: 0.3, message: 'Executing' }, - }, - { - method: 'notifications/progress', - params: { progressToken: 'progress-1', progress: 1, message: 'Fetched 1 rows' }, - }, - ]); - }); -``` - -- [ ] **Step 8: Add the compile-time array rejection assertion** - -Add this test near the bottom of the describe block: - -```typescript - it('keeps jsonToolResult typed to non-array objects', () => { - expect(jsonToolResult({ ok: true }).structuredContent).toEqual({ ok: true }); - - if (false) { - // @ts-expect-error bare arrays are not valid MCP structuredContent objects in KTX - jsonToolResult([]); - } - }); -``` - -- [ ] **Step 9: Run MCP tests and confirm they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -``` - -Expected: FAIL with missing annotations, missing output schemas, missing -normalization, missing centralized error wrapping, missing progress callback -wiring, and a missing snapshot. - -### Task 2: Implement MCP annotations, output schemas, descriptions, normalization, and in-band error wrapping - -**Files:** - -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/context-tools.ts` -- Modify: `packages/context/src/mcp/server.test.ts` - -- [ ] **Step 1: Extend MCP facade types** - -In `packages/context/src/mcp/types.ts`, replace `KtxMcpToolResult`, -`KtxMcpServerLike`, `KtxSemanticLayerMcpPort`, and -`KtxSqlExecutionMcpPort` with: - -```typescript -export type NonArrayObject = object & { length?: never }; - -export interface KtxMcpTextContent { - type: 'text'; - text: string; -} - -export interface KtxMcpToolResult { - content: KtxMcpTextContent[]; - structuredContent?: T; - isError?: true; -} - -export interface KtxMcpProgressEvent { - progress: number; - total?: number; - message: string; -} - -export type KtxMcpProgressCallback = (event: KtxMcpProgressEvent) => void | Promise; - -export interface KtxMcpToolHandlerContext { - _meta?: { progressToken?: string | number; [key: string]: unknown }; - sendNotification?: (notification: { - method: 'notifications/progress'; - params: { - progressToken: string | number; - progress: number; - total?: number; - message?: string; - }; - }) => Promise; -} - -export interface KtxMcpServerLike { - registerTool( - name: string, - config: { - title?: string; - description?: string; - inputSchema: unknown; - outputSchema?: unknown; - annotations?: Record; - }, - handler: (input: Record, context?: KtxMcpToolHandlerContext) => Promise, - ): void; -} - -export interface KtxSemanticLayerMcpPort { - readSource(input: { connectionId: string; sourceName: string }): Promise; - query( - input: { connectionId?: string; query: SemanticLayerQueryInput }, - options?: { onProgress?: KtxMcpProgressCallback }, - ): Promise; -} - -export interface KtxSqlExecutionMcpPort { - execute( - input: { connectionId: string; sql: string; maxRows: number }, - options?: { onProgress?: KtxMcpProgressCallback }, - ): Promise; -} -``` - -- [ ] **Step 2: Add output schemas and annotations** - -In `packages/context/src/mcp/context-tools.ts`, add this import: - -```typescript -import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; -``` - -Replace the MCP type import with: - -```typescript -import type { - KtxMcpContextPorts, - KtxMcpProgressCallback, - KtxMcpServerLike, - KtxMcpToolHandlerContext, - KtxMcpToolResult, - KtxMcpUserContext, - NonArrayObject, -} from './types.js'; -``` - -After `const connectionIdSchema = z.string().min(1);`, add: - -```typescript -const unknownRecordSchema = z.record(z.string(), z.unknown()); -const tableRefSchema = z.object({ - catalog: z.string().nullable(), - db: z.string().nullable(), - name: z.string(), -}); - -const toolAnnotations = { - connection_list: { title: 'Connection List', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - discover_data: { title: 'Discover Data', readOnlyHint: true, openWorldHint: false }, - wiki_search: { title: 'Wiki Search', readOnlyHint: true, openWorldHint: false }, - wiki_read: { title: 'Wiki Read', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - entity_details: { title: 'Entity Details', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - dictionary_search: { title: 'Dictionary Search', readOnlyHint: true, openWorldHint: false }, - sl_read_source: { title: 'Semantic Layer Read Source', readOnlyHint: true, idempotentHint: true, openWorldHint: false }, - sl_query: { title: 'Semantic Layer Query', readOnlyHint: true, openWorldHint: false }, - sql_execution: { title: 'SQL Execution', readOnlyHint: true, openWorldHint: false }, - memory_ingest: { title: 'Memory Ingest', destructiveHint: true, openWorldHint: false }, - memory_ingest_status: { title: 'Memory Ingest Status', readOnlyHint: true, openWorldHint: false }, -} satisfies Record; - -const toolDescriptions = { - connection_list: - 'List configured read-only data connections available to this KTX project. Use this before connection-scoped tools when the project may have multiple warehouses.', - discover_data: - 'Search across KTX wiki pages, semantic-layer sources, measures, dimensions, raw tables, and columns. Example: discover_data({ query: "monthly orders by customer", connectionId: "warehouse", kinds: ["sl_source", "table"] }).', - wiki_search: - 'Search KTX wiki pages for reusable business context. Example: wiki_search({ query: "revenue recognition", limit: 5 }).', - wiki_read: - 'Read a KTX wiki page by key returned from wiki_search. Example: wiki_read({ key: "global/revenue" }).', - entity_details: - 'Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: "warehouse", entities: [{ table: { schema: "public", table: "orders" }, columns: ["id"] }] }).', - dictionary_search: - 'Search profile-sampled warehouse values to locate likely source columns for business values. Example: dictionary_search({ values: ["Acme Corp"], connectionId: "warehouse" }).', - sl_read_source: - 'Read a semantic-layer YAML source by connection id and source name. Example: sl_read_source({ connectionId: "warehouse", sourceName: "orders" }).', - sl_query: - 'Execute a semantic-layer query and return rows, headers, generated SQL, and plan details. Example: sl_query({ connectionId: "warehouse", measures: ["orders.order_count"], dimensions: [{ dimension: "orders.created_at", granularity: "month" }] }).', - sql_execution: - 'Execute one parser-validated read-only SQL query against a configured KTX connection. Example: sql_execution({ connectionId: "warehouse", sql: "select count(*) from public.orders", maxRows: 100 }).', - memory_ingest: - 'Ingest free-form markdown knowledge into durable KTX memory. Use this for business rules, metric definitions, schema gotchas, recurring findings, or explicit user requests to remember something. Example: memory_ingest({ connectionId: "warehouse", content: "ARR is reported in cents in this warehouse." }).', - memory_ingest_status: - 'Read the current or final status for a memory ingest run. Example: memory_ingest_status({ runId: "memory-run-1" }).', -} satisfies Record; -``` - -After `memoryIngestStatusSchema`, add: - -```typescript -const connectionListOutputSchema = z.object({ - connections: z.array( - z.object({ - id: z.string(), - name: z.string(), - connectionType: z.string(), - }), - ), -}); - -const wikiSearchOutputSchema = z.object({ - results: z.array( - z.object({ - key: z.string(), - path: z.string(), - scope: z.enum(['GLOBAL', 'USER']), - summary: z.string(), - score: z.number(), - matchReasons: z.array(z.string()).optional(), - lanes: z - .array( - z.object({ - lane: z.string(), - status: z.string(), - requestedCandidatePoolLimit: z.number(), - effectiveCandidatePoolLimit: z.number(), - returnedCandidateCount: z.number(), - weight: z.number(), - reason: z.string().optional(), - }), - ) - .optional(), - }), - ), - totalFound: z.number(), -}); - -const wikiReadOutputSchema = z.object({ - key: z.string(), - summary: z.string(), - content: z.string(), - scope: z.enum(['GLOBAL', 'USER']), - tags: z.array(z.string()).optional(), - refs: z.array(z.string()).optional(), - slRefs: z.array(z.string()).optional(), -}); - -const slReadSourceOutputSchema = z.object({ - sourceName: z.string(), - yaml: z.string(), -}); - -const slQueryOutputSchema = z.object({ - connectionId: z.string().optional(), - dialect: z.string().optional(), - sql: z.string(), - headers: z.array(z.string()), - rows: z.array(z.array(z.unknown())), - totalRows: z.number(), - plan: unknownRecordSchema.optional(), -}); - -const entityDetailsSnapshotOutputSchema = z.object({ - syncId: z.string(), - extractedAt: z.string(), - scanRunId: z.string().nullable(), -}); - -const entityDetailsColumnOutputSchema = z.object({ - name: z.string(), - nativeType: z.string(), - normalizedType: z.string(), - dimensionType: z.enum(['time', 'string', 'number', 'boolean']), - nullable: z.boolean(), - primaryKey: z.boolean(), - comment: z.string().nullable(), -}); - -const entityDetailsForeignKeyOutputSchema = z.object({ - fromColumn: z.string(), - toCatalog: z.string().nullable(), - toDb: z.string().nullable(), - toTable: z.string(), - toColumn: z.string(), - constraintName: z.string().nullable(), -}); - -const entityDetailsOutputSchema = z.object({ - results: z.array( - z.union([ - z.object({ - ok: z.literal(true), - connectionId: z.string(), - tableRef: tableRefSchema, - display: z.string(), - kind: z.enum(['table', 'view', 'external', 'event_stream']), - comment: z.string().nullable(), - estimatedRows: z.number().nullable(), - columns: z.array(entityDetailsColumnOutputSchema), - foreignKeys: z.array(entityDetailsForeignKeyOutputSchema), - snapshot: entityDetailsSnapshotOutputSchema, - }), - z.object({ - ok: z.literal(false), - connectionId: z.string(), - table: z.union([z.string(), tableRefSchema]), - snapshot: entityDetailsSnapshotOutputSchema.optional(), - error: z.object({ - code: z.enum(['scan_missing', 'table_not_found', 'ambiguous_table', 'column_not_found']), - message: z.string(), - candidates: z.union([z.array(z.object({ tableRef: tableRefSchema, display: z.string() })), z.array(z.string())]).optional(), - }), - }), - ]), - ), -}); - -const dictionarySearchOutputSchema = z.object({ - searched: z.array( - z.object({ - connectionId: z.string(), - coverage: z.object({ - sampledRows: z.number().nullable(), - valuesPerColumn: z.number().nullable(), - profiledColumns: z.number(), - syncId: z.string().nullable(), - profiledAt: z.string().nullable(), - }), - status: z.enum(['ready', 'no_profile_artifact', 'no_candidate_columns']), - }), - ), - results: z.array( - z.object({ - value: z.string(), - matches: z.array( - z.object({ - connectionId: z.string(), - sourceName: z.string(), - columnName: z.string(), - matchedValue: z.string(), - cardinality: z.number().nullable(), - }), - ), - misses: z.array( - z.object({ - connectionId: z.string(), - reason: z.enum(['no_profile_artifact', 'no_candidate_columns', 'value_not_in_sample']), - }), - ), - }), - ), -}); - -const discoverDataOutputSchema = z.object({ - refs: z.array( - z.object({ - kind: discoverDataKindSchema, - id: z.string(), - score: z.number(), - summary: z.string().nullable(), - snippet: z.string().nullable(), - matchedOn: z.enum(['name', 'display', 'description', 'comment', 'expr', 'sample_value', 'body']), - connectionId: z.string().optional(), - tableRef: tableRefSchema.optional(), - columnName: z.string().optional(), - }), - ), -}); - -const sqlExecutionOutputSchema = z.object({ - headers: z.array(z.string()), - headerTypes: z.array(z.string()).optional(), - rows: z.array(z.array(z.unknown())), - rowCount: z.number(), -}); - -const memoryIngestOutputSchema = z.object({ - runId: z.string(), -}); - -const memoryIngestStatusOutputSchema = z.object({ - runId: z.string(), - status: z.enum(['running', 'done', 'error']), - stage: z.string(), - done: z.boolean(), - captured: z.object({ - wiki: z.array(z.string()), - sl: z.array(z.string()), - xrefs: z.array(z.string()), - }), - error: z.string().nullable(), - commitHash: z.string().nullable(), - skillsLoaded: z.array(z.string()), - signalDetected: z.boolean(), -}); -``` - -- [ ] **Step 3: Replace input schemas with described and normalized versions** - -In `context-tools.ts`, replace the input schema section from -`connectionListSchema` through `entityDetailsSchema` with: - -```typescript -const connectionListSchema = z.object({}); - -const knowledgeSearchSchema = z.object({ - query: z.string().min(1).describe('Natural-language wiki search query, e.g. "revenue recognition policy".'), - limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return. Defaults to 10.'), -}); - -const knowledgeReadSchema = z.object({ - key: z.string().min(1).describe('Wiki page key returned by wiki_search, e.g. "global/revenue".'), -}); - -const slReadSourceSchema = z.object({ - connectionId: connectionIdSchema.describe('Connection id that owns the semantic-layer source.'), - sourceName: z.string().min(1).describe('Semantic-layer source name without ".yaml", e.g. "orders".'), -}); - -const slQueryMeasureSchema = z.union([ - z.string().describe('Semantic-layer measure key, e.g. "orders.order_count".'), - z.object({ - expr: z.string().min(1).describe('Ad hoc aggregate expression, e.g. "sum(orders.amount)".'), - name: z.string().min(1).describe('Alias for the ad hoc measure, e.g. "gross_revenue".'), - }), -]); - -const slQueryDimensionSchema = z.preprocess( - (value) => { - if (typeof value === 'string') return { field: value }; - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('field' in obj) && typeof obj.dimension === 'string') obj.field = obj.dimension; - return obj; - } - return value; - }, - z.object({ - field: z.string().min(1).describe('Dimension to group by, e.g. "orders.created_at" or "orders.status".'), - granularity: z.string().min(1).optional().describe('Time grain for time dimensions: day, week, month, quarter, or year.'), - }), -); -``` - -Keep the existing `slQueryOrderBySchema` preprocess and replace -`slQuerySchema` plus `entityDetailsTableRefSchema` with: - -```typescript -const slQuerySchema = z.object({ - connectionId: connectionIdSchema - .optional() - .describe('Connection id to query. Omit only when the project has exactly one configured connection.'), - measures: z.array(slQueryMeasureSchema).min(1).describe('Measures to select. Use semantic-layer keys when available.'), - dimensions: z.array(slQueryDimensionSchema).default([]).describe('Dimensions to group by. Strings and {dimension, granularity} are accepted.'), - filters: z.array(z.string().describe('Semantic-layer filter expression, e.g. "orders.status = paid".')).default([]), - segments: z.array(z.string().describe('Semantic-layer segment key to apply.')).default([]), - order_by: z.array(slQueryOrderBySchema).default([]).describe('Sort clauses. Strings and Cube-style {id, desc} are accepted.'), - limit: z.number().int().min(0).default(1000).describe('Maximum rows to return. Defaults to 1000.'), - include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups. Defaults to true.'), -}); - -const entityDetailsTableRefSchema = z.preprocess( - (value) => { - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('db' in obj) && typeof obj.schema === 'string') obj.db = obj.schema; - if (!('name' in obj) && typeof obj.table === 'string') obj.name = obj.table; - if (!('catalog' in obj)) obj.catalog = null; - return obj; - } - return value; - }, - z.object({ - catalog: z.string().nullable().describe('Catalog/project/database. Use null when not applicable.'), - db: z.string().nullable().describe('Schema/database/dataset. Use null when not applicable.'), - name: z.string().min(1).describe('Table name.'), - }), -); - -const entityDetailsSchema = z.object({ - connectionId: connectionIdSchema.describe('Connection id whose latest scan snapshot should be read.'), - entities: z - .array( - z.object({ - table: z - .union([z.string().min(1), entityDetailsTableRefSchema]) - .describe('Table display string or object ref. {schema, table} is accepted as an alias for {db, name}.'), - columns: z.array(z.string().min(1).describe('Column name to inspect.')).optional().describe('Optional column filter.'), - }), - ) - .min(1) - .max(20) - .describe('Tables or columns to inspect. Maximum 20 entities.'), -}); -``` - -Replace `dictionarySearchSchema`, `discoverDataSchema`, and -`sqlExecutionSchema` with: - -```typescript -const dictionarySearchSchema = z.object({ - values: z - .array(z.string().min(1).describe('Business value to locate, e.g. "Acme Corp" or "enterprise".')) - .min(1) - .max(20) - .describe('Values to search for in sampled warehouse dictionaries.'), - connectionId: connectionIdSchema - .optional() - .describe('Optional connection id. Pass it when user intent pins a specific warehouse.'), -}); - -const discoverDataKindSchema = z.enum(['wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column']); - -const discoverDataSchema = z.object({ - query: z.string().min(1).describe('Natural-language discovery query, e.g. "monthly orders by customer".'), - connectionId: connectionIdSchema - .optional() - .describe('Optional connection id. Pass it when user intent pins a specific warehouse.'), - kinds: z.array(discoverDataKindSchema.describe('Reference kind to include.')).optional().describe('Optional kind filter.'), - limit: z.number().int().min(1).max(50).default(15).optional().describe('Maximum refs to return. Defaults to 15.'), -}); - -const sqlExecutionSchema = z.object({ - connectionId: connectionIdSchema.describe('Connection id to execute against. Required for raw SQL.'), - sql: z.string().min(1).describe('Parser-validated read-only SQL, e.g. "select count(*) from public.orders".'), - maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return. Defaults to 1000.'), -}); -``` - -- [ ] **Step 4: Replace `jsonToolResult`, `formatToolError`, and `registerParsedTool`** - -Replace `jsonToolResult`, `jsonErrorToolResult`, and `registerParsedTool` -with: - -```typescript -export function jsonToolResult(structuredContent: T): KtxMcpToolResult { - return { - content: [{ type: 'text', text: JSON.stringify(structuredContent, null, 2) }], - structuredContent, - }; -} - -export function jsonErrorToolResult(text: string): KtxMcpToolResult> { - return { - content: [{ type: 'text', text }], - isError: true, - }; -} - -function formatToolError(error: unknown): string { - if (error instanceof z.ZodError) { - return error.issues - .map((issue) => `${issue.path.length > 0 ? issue.path.join('.') : ''}: ${issue.message}`) - .join('\n'); - } - return error instanceof Error ? error.message : String(error); -} - -function mcpProgressCallback(context?: KtxMcpToolHandlerContext): KtxMcpProgressCallback | undefined { - const progressToken = context?._meta?.progressToken; - if (progressToken === undefined || !context?.sendNotification) { - return undefined; - } - return async (event) => { - await context.sendNotification?.({ - method: 'notifications/progress', - params: { - progressToken, - progress: event.progress, - ...(event.total !== undefined ? { total: event.total } : {}), - message: event.message, - }, - }); - }; -} - -function registerParsedTool( - server: KtxMcpServerLike, - name: string, - config: { - title: string; - description: string; - inputSchema: unknown; - outputSchema: unknown; - annotations: ToolAnnotations; - }, - schema: TSchema, - handler: (input: z.infer, context?: KtxMcpToolHandlerContext) => Promise, -): void { - server.registerTool(name, config, async (input, context) => { - try { - return await handler(schema.parse(input), context); - } catch (error) { - return jsonErrorToolResult(formatToolError(error)); - } - }); -} -``` - -- [ ] **Step 5: Update every registration config** - -For each `registerParsedTool` call, add `annotations` and `outputSchema`. -For example, replace the `connection_list` config with: - -```typescript - { - title: toolAnnotations.connection_list.title!, - description: toolDescriptions.connection_list, - inputSchema: connectionListSchema.shape, - outputSchema: connectionListOutputSchema, - annotations: toolAnnotations.connection_list, - }, -``` - -Use these exact output schemas: - -```typescript -connection_list -> connectionListOutputSchema -wiki_search -> wikiSearchOutputSchema -wiki_read -> wikiReadOutputSchema -sl_read_source -> slReadSourceOutputSchema -sl_query -> slQueryOutputSchema -entity_details -> entityDetailsOutputSchema -dictionary_search -> dictionarySearchOutputSchema -discover_data -> discoverDataOutputSchema -sql_execution -> sqlExecutionOutputSchema -memory_ingest -> memoryIngestOutputSchema -memory_ingest_status -> memoryIngestStatusOutputSchema -``` - -Use `toolAnnotations.` and `toolDescriptions.` for the -matching tool. - -- [ ] **Step 6: Remove the local sql_execution catch and wire progress callbacks** - -Replace the `sql_execution` handler with: - -```typescript - async (input, context) => { - const onProgress = mcpProgressCallback(context); - return jsonToolResult( - await sqlExecution.execute( - { - connectionId: input.connectionId, - sql: input.sql, - maxRows: input.maxRows ?? 1000, - }, - onProgress ? { onProgress } : undefined, - ), - ); - }, -``` - -Replace the `sl_query` handler with: - -```typescript - async (input, context) => { - const onProgress = mcpProgressCallback(context); - return jsonToolResult( - await semanticLayer.query( - { - connectionId: input.connectionId, - query: { - measures: input.measures, - dimensions: input.dimensions, - filters: input.filters, - segments: input.segments, - order_by: input.order_by, - limit: input.limit, - include_empty: input.include_empty, - }, - }, - onProgress ? { onProgress } : undefined, - ), - ); - }, -``` - -- [ ] **Step 7: Run MCP tests and update the snapshot** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -u -``` - -Expected: PASS. The new snapshot file is created at -`packages/context/src/mcp/__snapshots__/mcp-tools-list.json`. - -- [ ] **Step 8: Commit** - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/__snapshots__/mcp-tools-list.json -git commit -m "feat(context): polish mcp tool metadata" -``` - -### Task 3: Enforce resolved semantic-layer compute sources - -**Files:** - -- Modify: `packages/context/src/daemon/semantic-layer-compute.ts` -- Modify: `packages/context/src/sl/local-query.ts` - -- [ ] **Step 1: Narrow compute port source types and add invariant comments** - -In `packages/context/src/daemon/semantic-layer-compute.ts`, replace the import -from `../sl/index.js` with: - -```typescript -import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js'; -``` - -Replace the `query` and `validateSources` signatures in -`KtxSemanticLayerComputePort` with: - -```typescript - /** - * Callers must pass sources sanitized through toResolvedWire. The Python - * daemon rejects authoring-only fields such as usage and inherits_columns_from. - */ - query(input: { - sources: ResolvedSemanticLayerSource[]; - query: SemanticLayerQueryInput; - dialect: string; - }): Promise; - - /** - * Callers must pass sources sanitized through toResolvedWire. The Python - * daemon rejects authoring-only fields such as usage and inherits_columns_from. - */ - validateSources(input: { - sources: ResolvedSemanticLayerSource[]; - dialect: string; - recentlyTouched?: string[]; - }): Promise; -``` - -- [ ] **Step 2: Remove the unnecessary cast in local query loading** - -In `packages/context/src/sl/local-query.ts`, replace `loadComputableSources` -with: - -```typescript -async function loadComputableSources( - project: KtxLocalProject, - connectionId: string, -): Promise[]> { - return (await loadLocalSlSourceRecords(project, { connectionId: assertSafeConnectionId(connectionId) })) - .filter((record) => record.source.table || record.source.sql) - .map((record) => toResolvedWire(record.source)); -} -``` - -- [ ] **Step 3: Run type-check and relevant semantic-layer tests** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/context exec vitest run src/sl/local-query.test.ts -``` - -Expected: PASS. - -- [ ] **Step 4: Commit** - -```bash -git add packages/context/src/daemon/semantic-layer-compute.ts packages/context/src/sl/local-query.ts -git commit -m "fix(context): enforce resolved semantic layer compute sources" -``` - -### Task 4: Add local progress stages for sl_query and sql_execution - -**Files:** - -- Modify: `packages/context/src/sl/local-query.ts` -- Modify: `packages/context/src/sl/local-query.test.ts` -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` - -- [ ] **Step 1: Add failing local-query progress tests** - -In `packages/context/src/sl/local-query.test.ts`, add a test that calls -`compileLocalSlQuery` with execution enabled and captures events: - -```typescript - it('emits progress while compiling and executing a local semantic-layer query', async () => { - const progress: Array<{ progress: number; message: string }> = []; - const queryExecutor = { - execute: vi.fn(async () => ({ - headers: ['status', 'order_count'], - rows: [['paid', 2]], - totalRows: 1, - command: 'SELECT', - rowCount: 1, - })), - }; - - const result = await compileLocalSlQuery(project, { - connectionId: 'warehouse', - query: { - measures: ['orders.order_count'], - dimensions: ['orders.status'], - limit: 25, - }, - compute, - execute: true, - maxRows: 10, - queryExecutor, - onProgress: (event) => progress.push({ progress: event.progress, message: event.message }), - }); - - expect(result.totalRows).toBe(1); - expect(progress).toEqual([ - { progress: 0, message: 'Compiling query' }, - { progress: 0.3, message: 'Generating SQL' }, - { progress: 0.6, message: 'Executing' }, - { progress: 1, message: 'Fetched 1 rows' }, - ]); - }); -``` - -- [ ] **Step 2: Implement local-query progress** - -In `packages/context/src/sl/local-query.ts`, import the progress type: - -```typescript -import type { KtxMcpProgressCallback } from '../mcp/types.js'; -``` - -Add the option: - -```typescript - onProgress?: KtxMcpProgressCallback; -``` - -In `compileLocalSlQuery`, emit stages in this order: - -```typescript - await options.onProgress?.({ progress: 0, message: 'Compiling query' }); - const connectionId = resolveLocalConnectionId(project, options.connectionId); - const dialect = dialectForDriver(project.config.connections[connectionId]?.driver); - const sources = await loadComputableSources(project, connectionId); - - await options.onProgress?.({ progress: 0.3, message: 'Generating SQL' }); - const response = await options.compute.query({ - sources, - dialect, - query: options.query, - }); -``` - -Before the query-executor call, add: - -```typescript - await options.onProgress?.({ progress: 0.6, message: 'Executing' }); -``` - -After the query-executor call, add: - -```typescript - await options.onProgress?.({ progress: 1, message: `Fetched ${execution.totalRows} rows` }); -``` - -In the compile-only branch, before returning, add: - -```typescript - await options.onProgress?.({ progress: 1, message: 'Fetched 0 rows' }); -``` - -- [ ] **Step 3: Add failing local SQL execution progress test** - -In `packages/context/src/mcp/local-project-ports.test.ts`, add: - -```typescript - it('emits sql_execution progress stages from local MCP ports', async () => { - const project = await initKtxProject({ projectDir: tempDir }); - project.config.connections.warehouse = { - driver: 'postgres', - url: 'env:DATABASE_URL', - }; - const connector = testConnector(testSnapshot(), { - headers: ['id'], - headerTypes: ['integer'], - rows: [[1]], - totalRows: 1, - rowCount: 1, - }); - const createConnector = vi.fn(async () => connector); - const sqlAnalysis = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(), - validateReadOnly: vi.fn(async () => ({ ok: true, error: null })), - }; - const progress: Array<{ progress: number; message: string }> = []; - const ports = createLocalProjectMcpContextPorts(project, { - sqlAnalysis, - localScan: { - createConnector, - }, - }); - - const result = await ports.sqlExecution?.execute( - { connectionId: 'warehouse', sql: 'select id from public.orders', maxRows: 5 }, - { onProgress: (event) => progress.push({ progress: event.progress, message: event.message }) }, - ); - - expect(result?.rowCount).toBe(1); - expect(progress).toEqual([ - { progress: 0, message: 'Validating SQL' }, - { progress: 0.3, message: 'Executing' }, - { progress: 1, message: 'Fetched 1 rows' }, - ]); - }); -``` - -- [ ] **Step 4: Implement local SQL execution progress** - -In `packages/context/src/mcp/local-project-ports.ts`, import the progress type: - -```typescript -import type { KtxMcpContextPorts, KtxMcpProgressCallback, KtxSqlExecutionResponse } from './types.js'; -``` - -Change `executeValidatedReadOnlySql` to accept progress: - -```typescript -async function executeValidatedReadOnlySql( - project: KtxLocalProject, - options: CreateLocalProjectMcpContextPortsOptions, - input: { connectionId: string; sql: string; maxRows: number }, - onProgress?: KtxMcpProgressCallback, -): Promise { -``` - -At the start of the function, add: - -```typescript - await onProgress?.({ progress: 0, message: 'Validating SQL' }); -``` - -Immediately before `connector.executeReadOnly`, add: - -```typescript - await onProgress?.({ progress: 0.3, message: 'Executing' }); -``` - -Replace the direct return with: - -```typescript - const response = { - headers: result.headers, - ...(result.headerTypes ? { headerTypes: result.headerTypes } : {}), - rows: result.rows, - rowCount: result.rowCount ?? result.rows.length, - }; - await onProgress?.({ progress: 1, message: `Fetched ${response.rowCount} rows` }); - return response; -``` - -Pass progress through the port: - -```typescript - async execute(input, executionOptions) { - return executeValidatedReadOnlySql(project, options, input, executionOptions?.onProgress); - }, -``` - -Pass semantic-layer progress through: - -```typescript - return compileLocalSlQuery(project, { - connectionId: input.connectionId, - query: input.query, - compute: options.semanticLayerCompute, - execute: Boolean(options.queryExecutor), - maxRows: input.query.limit, - queryExecutor: options.queryExecutor, - onProgress: executionOptions?.onProgress, - }); -``` - -- [ ] **Step 5: Run local progress tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/sl/local-query.test.ts src/mcp/local-project-ports.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/sl/local-query.ts packages/context/src/sl/local-query.test.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -git commit -m "feat(context): emit mcp query progress stages" -``` - -### Task 5: Final verification - -**Files:** - -- Verify: TypeScript workspace checks. - -- [ ] **Step 1: Run context tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -pnpm --filter @ktx/context run test:slow -``` - -Expected: PASS. - -- [ ] **Step 2: Run type-checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. - -- [ ] **Step 5: Inspect final diff** - -Run: - -```bash -git status --short -git diff --stat -git diff -- packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.test.ts packages/context/src/daemon/semantic-layer-compute.ts packages/context/src/sl/local-query.ts packages/context/src/sl/local-query.test.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts -``` - -Expected: only intended MCP polish and progress files are changed. diff --git a/docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-surface-change.md b/docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-surface-change.md deleted file mode 100644 index 2e259463..00000000 --- a/docs/superpowers/plans/2026-05-16-mcp-tool-polish-v1-surface-change.md +++ /dev/null @@ -1,1305 +0,0 @@ -# MCP Tool Polish V1 Surface Change Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Land the atomic MCP surface change from the MCP tool polish spec: -retain only the research-loop tools, replace `memory_capture` with -`memory_ingest`, and update the installed analytics skill in the same change. - -**Architecture:** Keep the existing context, memory, and CLI services, but make -the MCP server register only the v1 research surface. Move memory ingest into -`registerKtxContextTools` so the next polish plan can apply annotations, -`outputSchema`, descriptions, and in-band error handling through one path. - -**Tech Stack:** TypeScript, Zod, MCP SDK, Vitest, pnpm workspace commands. - ---- - -## Audit summary - -The original spec is -`docs/superpowers/specs/2026-05-16-mcp-tool-polish-design.md`. - -Implemented before this plan: - -- `discover_data` already returns an object shape: - `jsonToolResult({ refs: await discover.search(input) })`. -- `sl_query.order_by` already accepts bare strings and Cube-style - `{ id, desc }` objects through `z.preprocess`. -- The local `sl_query` path already sanitizes sources with `toResolvedWire`. - -Remaining v1 blockers: - -- The MCP server still registers the broad admin surface: - `connection_test`, `wiki_write`, `sl_list_sources`, `sl_write_source`, - `sl_validate`, `ingest_*`, and `scan_*`. -- The MCP memory tools are still `memory_capture` and - `memory_capture_status`, with `userMessage` and `assistantMessage` input. -- Memory tools are still registered directly in `server.ts`, bypassing - `registerParsedTool`. -- The analytics skill and agent client docs still say "memory capture." - -Remaining v1 blockers after this plan: - -- Per-tool polish kit: annotations, `outputSchema`, input field - descriptions, long tool descriptions, in-band error wrapping, union-drift - normalization, `jsonToolResult` type narrowing, and `toResolvedWire` - invariant enforcement for `validateSources`. -- Progress notifications for `sql_execution` and `sl_query`. - -Non-blocking items from the spec: - -- Deleting admin tool implementation code after a future `ktx-admin` skill - lands. -- MCP resources, MCP prompts, elicitation, sampling, tool icons, code - execution, multi-tenancy, telemetry, and rate limiting. -- Error-message redaction for `formatToolError`, which belongs to the polish - kit plan. - -## File structure - -- `packages/context/src/memory/memory-runs.ts`: rename the memory run service - API from capture to ingest with no compatibility wrapper. -- `packages/context/src/memory/local-memory.ts`: rename the local factory to - `createLocalProjectMemoryIngest`. -- `packages/context/src/memory/index.ts`: re-export the new memory ingest - names only. -- `packages/context/src/mcp/types.ts`: rename `MemoryCapturePort` to - `MemoryIngestPort`, add `memoryIngest` to `KtxMcpContextPorts`, and remove - MCP context ports for removed admin tool families. -- `packages/context/src/mcp/context-tools.ts`: remove removed tool - registrations and register `memory_ingest` plus `memory_ingest_status`. -- `packages/context/src/mcp/server.ts`: delete direct memory tool - registration and route all tools through `registerKtxContextTools`. -- `packages/context/src/mcp/local-project-ports.ts`: stop assembling MCP - ports for removed admin tools. -- `packages/cli/src/mcp-server-factory.ts`: create the local memory ingest - port and include it in `contextTools.memoryIngest`. -- `packages/cli/src/text-ingest.ts`: rename CLI text ingest dependency names - from capture to ingest while preserving behavior. -- `packages/cli/src/skills/analytics/SKILL.md`: replace memory capture - guidance with memory ingest guidance and add multi-connection routing. -- `docs-site/content/docs/integrations/agent-clients.mdx`: replace the - existing memory capture wording. -- Tests: - `packages/context/src/mcp/server.test.ts`, - `packages/context/src/memory/memory-runs.test.ts`, - `packages/context/src/memory/local-memory.test.ts`, - `packages/cli/src/text-ingest.test.ts`, - `packages/cli/src/setup-agents.test.ts`. - -### Task 1: Lock the new MCP surface with failing tests - -**Files:** - -- Modify: `packages/context/src/mcp/server.test.ts` - -- [ ] **Step 1: Update the imports for new memory names** - -In `packages/context/src/mcp/server.test.ts`, replace the memory imports at -the top with: - -```typescript -import { - createLocalProjectMemoryIngest, - detectCaptureSignals, - type MemoryAgentInput, -} from '../memory/index.js'; -``` - -In the MCP type import from `./types.js`, replace `MemoryCapturePort` with -`MemoryIngestPort`: - -```typescript -import type { - KtxDiscoverDataMcpPort, - KtxDictionarySearchMcpPort, - KtxEntityDetailsMcpPort, - KtxKnowledgeMcpPort, - KtxMcpContextPorts, - KtxSemanticLayerMcpPort, - KtxSqlExecutionMcpPort, - KtxSqlExecutionResponse, - MemoryIngestPort, -} from './types.js'; -``` - -- [ ] **Step 2: Replace the standalone memory capture test** - -Replace the test named -`registers memory capture tools without host app dependencies` with this test: - -```typescript - it('registers memory ingest tools through the context tool surface', async () => { - const fake = makeFakeServer(); - let receivedInput: MemoryAgentInput | undefined; - const ingest: MemoryIngestPort = { - ingest: vi.fn().mockImplementation(async (input) => { - receivedInput = input; - return { runId: 'run-1' }; - }), - status: vi.fn().mockResolvedValue({ - runId: 'run-1', - status: 'done', - stage: 'done', - done: true, - captured: { wiki: ['revenue'], sl: [], xrefs: [] }, - error: null, - commitHash: 'abc123', - skillsLoaded: ['wiki_capture'], - signalDetected: true, - }), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'mcp-user' }, - contextTools: { memoryIngest: ingest }, - }); - - expect(fake.tools.map((tool) => tool.name).sort()).toEqual([ - 'memory_ingest', - 'memory_ingest_status', - ]); - - const content = [ - 'view: orders {', - ' sql_table_name: public.orders ;;', - ' measure: gross_revenue {', - ' type: sum', - ' sql: ${TABLE}.gross_revenue_cents ;;', - ' }', - '}', - ].join('\n'); - const memoryIngest = getTool(fake.tools, 'memory_ingest'); - await expect( - memoryIngest.handler({ - content, - connectionId: '00000000-0000-4000-8000-000000000001', - }), - ).resolves.toEqual({ - content: [{ type: 'text', text: JSON.stringify({ runId: 'run-1' }, null, 2) }], - structuredContent: { runId: 'run-1' }, - }); - expect(ingest.ingest).toHaveBeenCalledWith({ - userId: 'mcp-user', - chatId: expect.stringMatching(/^mcp-/), - userMessage: 'Ingest external knowledge into KTX memory.', - assistantMessage: content, - connectionId: '00000000-0000-4000-8000-000000000001', - sourceType: 'external_ingest', - }); - - const cliEquivalentInput: MemoryAgentInput = { - userId: 'mcp-user', - chatId: 'cli-text-ingest-test-1', - userMessage: 'Ingest external text artifact "orders lookml" into KTX memory.', - assistantMessage: content, - connectionId: '00000000-0000-4000-8000-000000000001', - sourceType: 'external_ingest', - }; - expect(detectCaptureSignals(receivedInput!)).toEqual(detectCaptureSignals(cliEquivalentInput)); - - const memoryStatus = getTool(fake.tools, 'memory_ingest_status'); - await expect(memoryStatus.handler({ runId: 'run-1' })).resolves.toEqual({ - content: [ - { - type: 'text', - text: JSON.stringify( - { - runId: 'run-1', - status: 'done', - stage: 'done', - done: true, - captured: { wiki: ['revenue'], sl: [], xrefs: [] }, - error: null, - commitHash: 'abc123', - skillsLoaded: ['wiki_capture'], - signalDetected: true, - }, - null, - 2, - ), - }, - ], - structuredContent: { - runId: 'run-1', - status: 'done', - stage: 'done', - done: true, - captured: { wiki: ['revenue'], sl: [], xrefs: [] }, - error: null, - commitHash: 'abc123', - skillsLoaded: ['wiki_capture'], - signalDetected: true, - }, - }); - }); -``` - -- [ ] **Step 3: Replace the missing memory run test** - -Replace the test that looks up `memory_capture_status` for a missing run with: - -```typescript - it('returns an in-band error when a memory ingest run is missing', async () => { - const fake = makeFakeServer(); - const ingest: MemoryIngestPort = { - ingest: vi.fn(), - status: vi.fn().mockResolvedValue(null), - }; - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'mcp-user' }, - contextTools: { memoryIngest: ingest }, - }); - - const memoryStatus = getTool(fake.tools, 'memory_ingest_status'); - await expect(memoryStatus.handler({ runId: 'missing-run' })).resolves.toEqual({ - content: [{ type: 'text', text: 'Memory ingest run "missing-run" was not found.' }], - isError: true, - }); - }); -``` - -- [ ] **Step 4: Update the local project MCP memory test** - -Rename the test `runs MCP memory_capture against a local project memory port` -to `runs MCP memory_ingest against a local project memory port`. - -Inside that test, rename the factory call and handler calls: - -```typescript - const memoryIngest = createLocalProjectMemoryIngest(project, { - agentRunner, - llmProvider, - runIdFactory: () => 'memory-run-mcp', - }); - - createKtxMcpServer({ - server: fake.server, - userContext: { userId: 'local' }, - contextTools: { memoryIngest }, - }); - - const capture = await getTool(fake.tools, 'memory_ingest').handler({ - content: 'Revenue means paid order value.', - connectionId: 'warehouse', - }); - - await memoryIngest.waitForRun('memory-run-mcp'); - const status = await getTool(fake.tools, 'memory_ingest_status').handler({ - runId: 'memory-run-mcp', - }); -``` - -Keep the existing wiki assertion in the test. Update its expected memory-agent -input to use: - -```typescript -{ - userId: 'local', - chatId: expect.stringMatching(/^mcp-/), - userMessage: 'Ingest external knowledge into KTX memory.', - assistantMessage: 'Revenue means paid order value.', - connectionId: 'warehouse', - sourceType: 'external_ingest', -} -``` - -- [ ] **Step 5: Update the full-surface registration assertion** - -In the large registration test, replace the expected tool-name list with the -retained v1 list: - -```typescript - expect(fake.tools.map((tool) => tool.name).sort()).toEqual([ - 'connection_list', - 'dictionary_search', - 'discover_data', - 'entity_details', - 'memory_ingest', - 'memory_ingest_status', - 'sl_query', - 'sl_read_source', - 'sql_execution', - 'wiki_read', - 'wiki_search', - ]); -``` - -Delete assertions that call removed tools: -`connection_test`, `wiki_write`, `sl_list_sources`, `sl_write_source`, -`sl_validate`, `ingest_trigger`, `ingest_status`, `ingest_report`, -`ingest_replay`, `scan_trigger`, `scan_status`, `scan_report`, -`scan_list_artifacts`, and `scan_read_artifact`. - -- [ ] **Step 6: Run the MCP tests and confirm they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "memory ingest|registers all available" -``` - -Expected: FAIL. The current implementation still registers `memory_capture`, -accepts `userMessage` and `assistantMessage`, and exposes removed admin tools. - -### Task 2: Rename memory capture internals to memory ingest - -**Files:** - -- Modify: `packages/context/src/memory/memory-runs.ts` -- Modify: `packages/context/src/memory/memory-runs.test.ts` -- Modify: `packages/context/src/memory/local-memory.ts` -- Modify: `packages/context/src/memory/local-memory.test.ts` -- Modify: `packages/context/src/memory/index.ts` - -- [ ] **Step 1: Update memory run tests to the new API** - -In `packages/context/src/memory/memory-runs.test.ts`, replace the import with: - -```typescript -import { MemoryIngestService, type MemoryRunStorePort } from './memory-runs.js'; -``` - -Replace `MemoryCaptureService` with `MemoryIngestService`, rename local -variables from `capture` to `ingest`, and replace `.capture(` calls with -`.ingest(` calls. The shared test setup type becomes: - -```typescript -let ingest: MemoryIngestService; -``` - -The service construction becomes: - -```typescript -ingest = new MemoryIngestService({ memoryAgent, runs: store }); -``` - -- [ ] **Step 2: Update local memory tests to the new factory** - -In `packages/context/src/memory/local-memory.test.ts`, replace the import with: - -```typescript -import { createLocalProjectMemoryIngest } from './local-memory.js'; -``` - -Rename the describe block to: - -```typescript -describe('createLocalProjectMemoryIngest', () => { -``` - -Replace `createLocalProjectMemoryCapture(` with -`createLocalProjectMemoryIngest(` and replace local variables named `capture` -with `ingest`. - -- [ ] **Step 3: Run the renamed memory tests and confirm they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runs.test.ts src/memory/local-memory.test.ts -``` - -Expected: FAIL with missing exports and missing `.ingest()` method. - -- [ ] **Step 4: Rename the memory run service** - -In `packages/context/src/memory/memory-runs.ts`, replace the capture-specific -type and class declarations with: - -```typescript -export interface MemoryIngestServiceDeps { - memoryAgent: Pick; - runs: MemoryRunStorePort; -} - -export interface MemoryIngestStartResult { - runId: string; -} - -export interface MemoryIngestStatus { - runId: string; - status: MemoryRunStatus; - stage: string; - done: boolean; - captured: { - wiki: string[]; - sl: string[]; - xrefs: string[]; - }; - error: string | null; - commitHash: string | null; - skillsLoaded: string[]; - signalDetected: boolean; -} -``` - -Update `capturedKeys` to return the renamed status type: - -```typescript -function capturedKeys(actions: MemoryAction[]): MemoryIngestStatus['captured'] { -``` - -Replace the class with: - -```typescript -export class MemoryIngestService { - private readonly inFlight = new Map>(); - - constructor(private readonly deps: MemoryIngestServiceDeps) {} - - async ingest(input: MemoryAgentInput): Promise { - const row = await this.deps.runs.createRunning({ - inputHash: inputHash(input), - chatId: input.chatId, - }); - - await this.deps.runs.markRunning(row.id, 'ingesting'); - - const run = this.runIngest(row.id, input); - this.inFlight.set(row.id, run); - run.finally(() => this.inFlight.delete(row.id)).catch(() => undefined); - - return { runId: row.id }; - } - - async waitForRun(runId: string): Promise { - await this.inFlight.get(runId); - } - - private async runIngest(runId: string, input: MemoryAgentInput): Promise { - try { - const outputSummary = await this.deps.memoryAgent.ingest(input); - await this.deps.runs.markDone(runId, outputSummary); - } catch (error) { - await this.deps.runs.markError(runId, error instanceof Error ? error.message : String(error)); - } - } - - async status(runId: string): Promise { - const row = await this.deps.runs.findById(runId); - if (!row) { - return null; - } - - const output = row.outputSummary; - return { - runId: row.id, - status: row.status, - stage: row.stage, - done: row.status !== 'running', - captured: output ? capturedKeys(output.actions) : { wiki: [], sl: [], xrefs: [] }, - error: row.error, - commitHash: output?.commitHash ?? null, - skillsLoaded: output?.skillsLoaded ?? [], - signalDetected: output?.signalDetected ?? false, - }; - } -} -``` - -- [ ] **Step 5: Rename the local memory factory** - -In `packages/context/src/memory/local-memory.ts`, replace the service import: - -```typescript -import { MemoryIngestService } from './memory-runs.js'; -``` - -Rename the options interface and factory: - -```typescript -export interface CreateLocalProjectMemoryIngestOptions { - llmProvider?: KtxLlmProvider; - agentRunner?: AgentRunnerService; - memoryModel?: string; - semanticLayerCompute?: KtxSemanticLayerComputePort; - queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise }; - runIdFactory?: () => string; - logger?: KtxLogger; -} - -export function createLocalProjectMemoryIngest( - project: KtxLocalProject, - options: CreateLocalProjectMemoryIngestOptions = {}, -): MemoryIngestService { -``` - -Update the error string: - -```typescript -throw new Error('createLocalProjectMemoryIngest requires llm.provider.backend or an injected agentRunner'); -``` - -Return the renamed service: - -```typescript - return new MemoryIngestService({ - memoryAgent, - runs: new LocalMemoryRunStore({ projectDir: project.projectDir, idFactory: options.runIdFactory }), - }); -``` - -- [ ] **Step 6: Update memory exports** - -In `packages/context/src/memory/index.ts`, replace the memory run exports with: - -```typescript -export { createLocalProjectMemoryIngest, type CreateLocalProjectMemoryIngestOptions } from './local-memory.js'; -export { LocalMemoryRunStore, type LocalMemoryRunStoreOptions } from './local-memory-runs.js'; -export { - MemoryIngestService, - type MemoryIngestServiceDeps, - type MemoryIngestStartResult, - type MemoryIngestStatus, - type MemoryRunRecord, - type MemoryRunStatus, - type MemoryRunStorePort, -} from './memory-runs.js'; -``` - -- [ ] **Step 7: Run memory tests and commit** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/memory/memory-runs.test.ts src/memory/local-memory.test.ts -``` - -Expected: PASS. - -Commit: - -```bash -git add packages/context/src/memory/memory-runs.ts packages/context/src/memory/memory-runs.test.ts packages/context/src/memory/local-memory.ts packages/context/src/memory/local-memory.test.ts packages/context/src/memory/index.ts -git commit -m "refactor(context): rename memory capture service to ingest" -``` - -### Task 3: Move memory ingest into the shared MCP context tool path - -**Files:** - -- Modify: `packages/context/src/mcp/types.ts` -- Modify: `packages/context/src/mcp/context-tools.ts` -- Modify: `packages/context/src/mcp/server.ts` -- Modify: `packages/context/src/mcp/server.test.ts` - -- [ ] **Step 1: Update MCP types** - -In `packages/context/src/mcp/types.ts`, replace the memory import with: - -```typescript -import type { MemoryIngestService } from '../memory/index.js'; -``` - -Replace `MemoryCapturePort` with: - -```typescript -export interface MemoryIngestPort { - ingest: MemoryIngestService['ingest']; - status: MemoryIngestService['status']; -} -``` - -Reduce the retained MCP port interfaces to the v1 surface: - -```typescript -export interface KtxConnectionsMcpPort { - list(): Promise; -} - -export interface KtxKnowledgeMcpPort { - search(input: { userId: string; query: string; limit: number }): Promise; - read(input: { userId: string; key: string }): Promise; -} - -export interface KtxSemanticLayerMcpPort { - readSource(input: { connectionId: string; sourceName: string }): Promise; - query(input: { connectionId?: string; query: SemanticLayerQueryInput }): Promise; -} - -export interface KtxMcpContextPorts { - connections?: KtxConnectionsMcpPort; - knowledge?: KtxKnowledgeMcpPort; - semanticLayer?: KtxSemanticLayerMcpPort; - entityDetails?: KtxEntityDetailsMcpPort; - dictionarySearch?: KtxDictionarySearchMcpPort; - discover?: KtxDiscoverDataMcpPort; - sqlExecution?: KtxSqlExecutionMcpPort; - memoryIngest?: MemoryIngestPort; -} - -export interface KtxMcpServerDeps { - server: KtxMcpServerLike; - userContext: KtxMcpUserContext; - contextTools?: KtxMcpContextPorts; -} -``` - -- [ ] **Step 2: Add memory ingest schemas to `context-tools.ts`** - -At the top of `packages/context/src/mcp/context-tools.ts`, add: - -```typescript -import { randomUUID } from 'node:crypto'; -import type { MemoryAgentInput } from '../memory/index.js'; -``` - -After `sqlExecutionSchema`, add: - -```typescript -const memoryIngestSchema = z.object({ - content: z - .string() - .min(1) - .describe( - 'Free-form markdown to ingest. Include the knowledge itself plus any context (source, the user question, why this came up) that the memory agent should consider when triaging into wiki/SL.', - ), - connectionId: connectionIdSchema - .optional() - .describe( - 'Scope this memory to a specific connection. Required when the knowledge is warehouse-specific, including measure definitions, schema gotchas, or anything tied to a particular warehouse. Omit only for global wiki knowledge.', - ), -}); - -const memoryIngestStatusSchema = z.object({ - runId: z.string().min(1).describe('The memory ingest run id returned by memory_ingest.'), -}); -``` - -- [ ] **Step 3: Delete removed registration blocks** - -In `registerKtxContextTools`, delete the registration blocks for these tool -names: - -```text -connection_test -wiki_write -sl_list_sources -sl_write_source -sl_validate -ingest_trigger -ingest_status -ingest_report -ingest_replay -scan_trigger -scan_status -scan_report -scan_list_artifacts -scan_read_artifact -``` - -Also delete their now-unused input schemas from `context-tools.ts`: -`connectionTestSchema`, `historicSqlUsageFrontmatterSchema`, -`knowledgeWriteSchema`, `slListSourcesSchema`, `slWriteSourceSchema`, -`slValidateSchema`, `ingestTriggerSchema`, `ingestStatusSchema`, -`ingestReportSchema`, `ingestReplaySchema`, `scanTriggerSchema`, -`scanStatusSchema`, and `scanArtifactReadSchema`. - -- [ ] **Step 4: Register memory ingest through `registerParsedTool`** - -Add this block near the end of `registerKtxContextTools`, after -`sql_execution`: - -```typescript - if (ports.memoryIngest) { - const memoryIngest = ports.memoryIngest; - registerParsedTool( - server, - 'memory_ingest', - { - title: 'Memory Ingest', - description: - 'Ingest free-form markdown knowledge into KTX durable memory. Use this for business rules, metric definitions, schema gotchas, recurring findings, or explicit user requests to remember something.', - inputSchema: memoryIngestSchema.shape, - }, - memoryIngestSchema, - async (input) => { - const ingestInput: MemoryAgentInput = { - userId: userContext.userId, - chatId: `mcp-${randomUUID()}`, - userMessage: 'Ingest external knowledge into KTX memory.', - assistantMessage: input.content, - connectionId: input.connectionId, - sourceType: 'external_ingest', - }; - return jsonToolResult(await memoryIngest.ingest(ingestInput)); - }, - ); - - registerParsedTool( - server, - 'memory_ingest_status', - { - title: 'Memory Ingest Status', - description: 'Read the current or final status for a memory ingest run.', - inputSchema: memoryIngestStatusSchema.shape, - }, - memoryIngestStatusSchema, - async (input) => { - const status = await memoryIngest.status(input.runId); - return status ? jsonToolResult(status) : jsonErrorToolResult(`Memory ingest run "${input.runId}" was not found.`); - }, - ); - } -``` - -- [ ] **Step 5: Simplify `server.ts`** - -Replace `packages/context/src/mcp/server.ts` with: - -```typescript -import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { registerKtxContextTools } from './context-tools.js'; -import type { KtxMcpServerDeps, KtxMcpServerLike } from './types.js'; - -export function createKtxMcpServer(deps: KtxMcpServerDeps): KtxMcpServerDeps['server'] { - if (deps.contextTools) { - registerKtxContextTools({ - server: deps.server, - ports: deps.contextTools, - userContext: deps.userContext, - }); - } - - return deps.server; -} - -export function createDefaultKtxMcpServer( - deps: Omit & { name?: string; version?: string }, -): McpServer { - const server = new McpServer({ - name: deps.name ?? 'ktx', - version: deps.version ?? '0.0.0-private', - }); - createKtxMcpServer({ - server: server as KtxMcpServerLike, - userContext: deps.userContext, - contextTools: deps.contextTools, - }); - return server; -} -``` - -- [ ] **Step 6: Run MCP tests and commit** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/server.test.ts -t "memory ingest|registers all available" -``` - -Expected: PASS for the new memory ingest and retained surface tests. - -Commit: - -```bash -git add packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.ts packages/context/src/mcp/server.test.ts -git commit -m "feat(mcp): slim research tool surface" -``` - -### Task 4: Slim local MCP port assembly and CLI server factory - -**Files:** - -- Modify: `packages/context/src/mcp/local-project-ports.ts` -- Modify: `packages/context/src/mcp/local-project-ports.test.ts` -- Modify: `packages/cli/src/mcp-server-factory.ts` - -- [ ] **Step 1: Remove local admin MCP port assembly** - -In `packages/context/src/mcp/local-project-ports.ts`, remove the -`localIngest` option: - -```typescript -interface CreateLocalProjectMcpContextPortsOptions { - semanticLayerCompute?: KtxSemanticLayerComputePort; - queryExecutor?: KtxSqlQueryExecutorPort; - sqlAnalysis?: SqlAnalysisPort; - localScan?: LocalScanMcpOptions; - embeddingService?: KtxEmbeddingPort | null; -} -``` - -Inside `createLocalProjectMcpContextPorts`, remove these object members: - -```typescript - async test(input) { - return testLocalConnection(project, options, input.connectionId); - }, -``` - -```typescript - async write(input) { - const existing = await readLocalKnowledgePage(project, { - key: input.key, - userId: input.userId, - }); - await writeLocalKnowledgePage(project, { - key: input.key, - scope: 'GLOBAL', - userId: input.userId, - summary: input.summary, - content: input.content, - tags: input.tags, - refs: input.refs, - slRefs: input.slRefs, - source: input.source, - intent: input.intent, - tables: input.tables, - representativeSql: input.representativeSql, - usage: input.usage, - fingerprints: input.fingerprints, - }); - return { success: true, key: input.key, action: existing ? 'updated' : 'created' }; - }, -``` - -Remove `semanticLayer.listSources`, `semanticLayer.writeSource`, and -`semanticLayer.validate` from the returned semantic-layer port. Keep only -`readSource` and `query`. - -Delete the `if (options.localIngest) { ... }` block and the -`if (options.localScan) { ... }` block at the bottom of the function. Keep -the `options.localScan` value available to `sql_execution`, because -`executeValidatedReadOnlySql` still uses it. - -- [ ] **Step 2: Remove local-project helper code that became unused** - -In `packages/context/src/mcp/local-project-ports.ts`, delete these helper -functions when no references remain: - -```text -testLocalConnection -scanArtifactType -listArtifactsForReport -readScanArtifact -loadComputableSources -validateSourceRecord -localIngestSourceDir -rawFileCountFromIngestReport -statusFromIngestReport -``` - -Remove now-unused imports from `../ingest/index.js`, `../wiki/local-knowledge.js`, -`yaml`, and `./types.js`. Keep imports used by `connection_list`, -`wiki_search`, `wiki_read`, `sl_read_source`, `sl_query`, `entity_details`, -`dictionary_search`, `discover_data`, and `sql_execution`. - -- [ ] **Step 3: Update local-project port tests** - -In `packages/context/src/mcp/local-project-ports.test.ts`, remove assertions -that depend on `ports.connections.test`, `ports.knowledge.write`, -`ports.semanticLayer.listSources`, `ports.semanticLayer.writeSource`, -`ports.semanticLayer.validate`, `ports.ingest`, or `ports.scan`. - -Add this retained-surface assertion to the test that constructs local ports: - -```typescript -expect(Object.keys(ports).sort()).toEqual([ - 'connections', - 'dictionarySearch', - 'discover', - 'entityDetails', - 'knowledge', - 'semanticLayer', - 'sqlExecution', -]); -expect(Object.keys(ports.connections ?? {}).sort()).toEqual(['list']); -expect(Object.keys(ports.knowledge ?? {}).sort()).toEqual(['read', 'search']); -expect(Object.keys(ports.semanticLayer ?? {}).sort()).toEqual(['query', 'readSource']); -``` - -- [ ] **Step 4: Update the CLI MCP server factory** - -In `packages/cli/src/mcp-server-factory.ts`, replace the memory import: - -```typescript -import { createLocalProjectMemoryIngest } from '@ktx/context/memory'; -``` - -Remove the `localIngest` block from the call to -`createLocalProjectMcpContextPorts`. Keep `semanticLayerCompute`, -`queryExecutor`, `sqlAnalysis`, and `localScan`. - -Replace the memory creation block with: - -```typescript - let memoryIngest: ReturnType | undefined; - try { - memoryIngest = createLocalProjectMemoryIngest(input.project, { semanticLayerCompute, queryExecutor }); - } catch (error) { - input.io?.stderr.write(`KTX MCP memory_ingest disabled: ${error instanceof Error ? error.message : String(error)}\n`); - } -``` - -Pass memory ingest through the context tools object: - -```typescript - return () => - createDefaultKtxMcpServer({ - name: 'ktx', - version: input.cliVersion, - userContext: { userId: 'local' }, - contextTools: { - ...contextTools, - ...(memoryIngest ? { memoryIngest } : {}), - }, - }); -``` - -- [ ] **Step 5: Run local MCP and CLI factory tests and commit** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/mcp/local-project-ports.test.ts src/mcp/server.test.ts -pnpm --filter @ktx/cli exec vitest run src/commands/mcp-commands.test.ts src/mcp-http-server.test.ts src/managed-mcp-daemon.test.ts -``` - -Expected: PASS. - -Commit: - -```bash -git add packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts packages/cli/src/mcp-server-factory.ts -git commit -m "refactor(mcp): remove admin ports from server factory" -``` - -### Task 5: Rename CLI text ingest dependencies - -**Files:** - -- Modify: `packages/cli/src/text-ingest.ts` -- Modify: `packages/cli/src/text-ingest.test.ts` - -- [ ] **Step 1: Update text-ingest tests** - -In `packages/cli/src/text-ingest.test.ts`, replace -`MemoryCaptureStatus` with `MemoryIngestStatus` and -`TextMemoryCapturePort` with `TextMemoryIngestPort`. - -Rename helper functions and dependency keys: - -```typescript -function createMemoryIngestStub( - status: MemoryIngestStatus | null, -): TextMemoryIngestPort { -``` - -Replace `createMemoryCapture` dependency uses with `createMemoryIngest`. - -- [ ] **Step 2: Run text ingest tests and confirm they fail** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/text-ingest.test.ts -``` - -Expected: FAIL with missing `MemoryIngestStatus`, -`TextMemoryIngestPort`, and `createMemoryIngest`. - -- [ ] **Step 3: Update `text-ingest.ts` imports and types** - -In `packages/cli/src/text-ingest.ts`, replace the memory import with: - -```typescript -import { createLocalProjectMemoryIngest, type MemoryAgentInput, type MemoryIngestStatus } from '@ktx/context/memory'; -``` - -Replace the text port and dependency types with: - -```typescript -export interface TextMemoryIngestPort { - ingest(input: MemoryAgentInput): Promise<{ runId: string }>; - waitForRun(runId: string): Promise; - status(runId: string): Promise; -} -``` - -```typescript -export interface KtxTextIngestDeps { - loadProject?: (options: { projectDir: string }) => Promise; - createMemoryIngest?: (project: KtxLocalProject) => TextMemoryIngestPort; - readFile?: (path: string) => Promise; - readStdin?: () => Promise; - now?: () => number; -} -``` - -Update the default factory: - -```typescript -function defaultCreateMemoryIngest(project: KtxLocalProject): TextMemoryIngestPort { - return createLocalProjectMemoryIngest(project); -} -``` - -Replace `MemoryCaptureStatus` type references with `MemoryIngestStatus`. - -- [ ] **Step 4: Update the text ingest runtime calls** - -In `runKtxTextIngest`, replace: - -```typescript - const memoryCapture = (deps.createMemoryCapture ?? defaultCreateMemoryCapture)(project); -``` - -with: - -```typescript - const memoryIngest = (deps.createMemoryIngest ?? defaultCreateMemoryIngest)(project); -``` - -Replace the run block with: - -```typescript - const ingestInput: MemoryAgentInput = { - userId: args.userId, - chatId: `cli-text-ingest-${batchId}-${index + 1}`, - userMessage: `Ingest external text artifact ${artifactReference(item.label)} into KTX memory.`, - assistantMessage: item.content.trim(), - ...(args.connectionId ? { connectionId: args.connectionId } : {}), - sourceType: 'external_ingest', - }; - const ingest = await memoryIngest.ingest(ingestInput); - runId = ingest.runId; - await memoryIngest.waitForRun(runId); - const status = await memoryIngest.status(runId); - if (!status) { - throw new Error(`Memory ingest run "${runId}" was not found.`); - } -``` - -- [ ] **Step 5: Run text ingest tests and commit** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/text-ingest.test.ts -``` - -Expected: PASS. - -Commit: - -```bash -git add packages/cli/src/text-ingest.ts packages/cli/src/text-ingest.test.ts -git commit -m "refactor(cli): rename text ingest memory port" -``` - -### Task 6: Update analytics skill and docs - -**Files:** - -- Modify: `packages/cli/src/skills/analytics/SKILL.md` -- Modify: `packages/cli/src/setup-agents.test.ts` -- Modify: `docs-site/content/docs/integrations/agent-clients.mdx` - -- [ ] **Step 1: Update the analytics skill text** - -In `packages/cli/src/skills/analytics/SKILL.md`, replace line 8 with: - -```markdown -You have access to KTX MCP tools for data discovery, semantic-layer analysis, raw read-only SQL, wiki context, and memory ingest. Follow this workflow. -``` - -Replace workflow step 7 with: - -```markdown -7. **Capture durable learnings** - call `memory_ingest` whenever a turn produces something worth remembering (business rules, metric definitions, schema gotchas, recurring findings) **or** whenever the user asks you to remember something. Pass markdown in `content` including any source context the memory agent should weigh. Each call is a feedback loop; better notes today mean smarter `discover_data` and `wiki_search` results tomorrow. -``` - -Add this rule under `` after the `dictionary_search` rule: - -```markdown -- When `connection_list` shows multiple connections, pass an explicit `connectionId` to every tool that takes one and where user intent pins a specific warehouse. Required: `entity_details`, `sl_read_source`, and `sql_execution`. Required when user intent is warehouse-specific, including wording like "in our warehouse" or "this warehouse": `memory_ingest`; without `connectionId`, the memory agent cannot update the semantic layer and the knowledge lands as wiki-only. Pass `connectionId` when intent pins a warehouse, otherwise omit for unscoped discovery: `sl_query`, `discover_data`, and `dictionary_search`. Never pass `connectionId` to `connection_list`, `wiki_search`, `wiki_read`, or `memory_ingest_status`. If intent is ambiguous for a required-or-scoped tool, ask the user which warehouse before calling. -``` - -In the first example, replace step 5 with: - -```markdown -5. `memory_ingest({ connectionId: "warehouse", content: "Acme Corp order analysis used orders_facts.order_count filtered by customers.name = 'Acme Corp'. Source: current analysis turn." })` captures the durable finding. -``` - -Add this example before ``: - -```markdown ---- - -**Input:** "Heads up: ARR is always reported in cents in our warehouse." - -**Workflow:** -1. If multiple connections exist, call `connection_list` and identify the warehouse the user means. Ask if ambiguous. -2. `memory_ingest({ connectionId: "warehouse", content: "ARR is reported in cents (not dollars) in this warehouse. Multiply by 0.01 for dollar amounts. Source: user clarification." })` remembers the warehouse-specific rule without running an analysis turn. -``` - -- [ ] **Step 2: Add setup-agent skill assertions** - -In `packages/cli/src/setup-agents.test.ts`, find the test that reads -`.agents/skills/ktx-analytics/SKILL.md` and currently asserts -`name: ktx-analytics`. Extend it with: - -```typescript -expect(analyticsSkill).toContain('memory_ingest'); -expect(analyticsSkill).toContain('ARR is reported in cents'); -expect(analyticsSkill).not.toContain('memory_capture'); -``` - -- [ ] **Step 3: Update docs-site memory wording** - -In `docs-site/content/docs/integrations/agent-clients.mdx`, replace: - -```markdown -semantic-layer queries, wiki search, SQL execution, and memory capture. The -``` - -with: - -```markdown -semantic-layer queries, wiki search, SQL execution, and memory ingest. The -``` - -- [ ] **Step 4: Run skill and docs tests and commit** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/setup-agents.test.ts -pnpm --filter ktx-docs run build -pnpm --filter ktx-docs run test -``` - -Expected: PASS. - -Commit: - -```bash -git add packages/cli/src/skills/analytics/SKILL.md packages/cli/src/setup-agents.test.ts docs-site/content/docs/integrations/agent-clients.mdx -git commit -m "docs: update analytics skill for memory ingest" -``` - -### Task 7: Full verification and cleanup - -**Files:** - -- Verify: all files changed in Tasks 1-6 - -- [ ] **Step 1: Check for stale capture names** - -Run: - -```bash -rg -n "memory_capture|memory_capture_status|MemoryCapture|createLocalProjectMemoryCapture|TextMemoryCapturePort|memoryCapture" packages/context/src packages/cli/src docs-site/content/docs/integrations/agent-clients.mdx -``` - -Expected: no matches in MCP, memory service, CLI setup, analytics skill, text -ingest, or docs-site files. Matches in historical `docs/superpowers/` files -are allowed and are intentionally excluded from the command. - -- [ ] **Step 2: Check retained MCP tool registration names** - -Run: - -```bash -rg -n "'(connection_test|wiki_write|sl_list_sources|sl_write_source|sl_validate|ingest_trigger|ingest_status|ingest_report|ingest_replay|scan_trigger|scan_status|scan_report|scan_list_artifacts|scan_read_artifact)'" packages/context/src/mcp packages/cli/src -``` - -Expected: no matches. - -- [ ] **Step 3: Run required context checks** - -Run: - -```bash -pnpm --filter @ktx/context run test -pnpm --filter @ktx/context run test:slow -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run required CLI checks** - -Run: - -```bash -pnpm --filter @ktx/cli run type-check -pnpm --filter @ktx/cli run test -``` - -Expected: PASS. - -- [ ] **Step 5: Run docs-site checks** - -Run: - -```bash -pnpm --filter ktx-docs run build -pnpm --filter ktx-docs run test -``` - -Expected: PASS. - -- [ ] **Step 6: Run dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. If Knip reports only exports intentionally kept for future -admin CLI work, add narrow `knip.json` entries for the exact symbols. Delete -private unused MCP-only helpers instead of ignoring them. - -- [ ] **Step 7: Run pre-commit on changed files** - -Run this command with the actual changed files from `git diff --name-only`: - -```bash -uv run pre-commit run --files packages/context/src/memory/memory-runs.ts packages/context/src/memory/memory-runs.test.ts packages/context/src/memory/local-memory.ts packages/context/src/memory/local-memory.test.ts packages/context/src/memory/index.ts packages/context/src/mcp/types.ts packages/context/src/mcp/context-tools.ts packages/context/src/mcp/server.ts packages/context/src/mcp/server.test.ts packages/context/src/mcp/local-project-ports.ts packages/context/src/mcp/local-project-ports.test.ts packages/cli/src/mcp-server-factory.ts packages/cli/src/text-ingest.ts packages/cli/src/text-ingest.test.ts packages/cli/src/skills/analytics/SKILL.md packages/cli/src/setup-agents.test.ts docs-site/content/docs/integrations/agent-clients.mdx -``` - -Expected: PASS. If pre-commit reports missing local tool versions without -changing files, record the exact error in the final handoff and rely on the -passing package checks above. - -- [ ] **Step 8: Commit final verification cleanup** - -Run: - -```bash -git status --short -``` - -Expected: only intentional files from this plan are modified. - -If verification cleanup changed files, commit them: - -```bash -git add packages/context/src packages/cli/src docs-site/content/docs/integrations/agent-clients.mdx knip.json -git commit -m "chore: verify mcp surface rename" -``` - -If no files changed after the previous commits, do not create an empty commit. - -## Self-review - -- Spec coverage: This plan covers PR 1 from the spec: tool surface reduction, - `memory_capture` to `memory_ingest` rename, memory input contract, memory - registration through the shared context tool path, analytics skill updates, - docs-site wording, CLI factory wiring, text ingest naming, and tests. -- Deferred v1 coverage: PR 2 polish kit and PR 3 progress notifications remain - v1-blocking follow-up plans after this lands. -- Red-flag scan: The plan avoids deferred-work markers, migration shims, - compatibility wrappers, and incomplete implementation instructions. -- Type consistency: All new names use `MemoryIngestService`, - `MemoryIngestPort`, `MemoryIngestStatus`, - `createLocalProjectMemoryIngest`, `TextMemoryIngestPort`, - `memory_ingest`, and `memory_ingest_status`. diff --git a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-core.md b/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-core.md deleted file mode 100644 index 89e8ed6c..00000000 --- a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-core.md +++ /dev/null @@ -1,2938 +0,0 @@ -# Isolated Diff Ingestion V1 Core Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the first production isolated-diff ingestion path, with persistent -postmortem traces, final artifact gates, and Metabase regression coverage. - -**Architecture:** Keep the existing shared-worktree runner as the fallback path -while a private runner-owned source allowlist enables isolated diffs for -Metabase and tests. The isolated path creates one integration worktree, runs -optional deterministic projection there, executes each work unit in a child -worktree from the same ingestion base commit, collects binary Git patches, and -applies accepted patches back to the integration worktree in deterministic -order before reconciliation, final gates, and squash. Every ingestion step emits -structured JSONL trace events under `.ktx/ingest-traces//trace.jsonl` -and references that path in reports and CLI status output. - -**Tech Stack:** TypeScript ESM/NodeNext, simple-git, Node `fs/promises`, Vitest, -existing `GitService`, `SessionWorktreeService`, `IngestBundleRunner`, -`SemanticLayerService`, `KnowledgeWikiService`, and ingest report schemas. - ---- - -## Audit summary - -This audit read -`docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md`, searched -`docs/superpowers/plans/`, and inspected the current ingest runner under -`packages/context/src/ingest/`. - -No existing plan or implementation covers isolated-diff ingestion. Searches for -the exact implementation terms from the spec, including `git apply --3way`, -`--binary --no-renames`, `integration worktree`, `global semantic gate`, and -`wiki body reference`, returned no plan or code matches. Existing May 13 unified -ingest plans are implemented public CLI and UX work; May 15 Claude Code plans -cover LLM backend isolation, not ingestion diff isolation. - -Implemented foundations that this plan reuses: - -- `SessionWorktreeService` can create Git worktrees from a base SHA. -- `GitService` already supports `addWorktree`, `removeWorktree`, - `resetHardTo`, `diffNameStatus`, `assertWorktreeClean`, and - `squashMergeIntoMain`. -- `IngestBundleRunner` already stages raw snapshots in a session worktree, - chunks adapters into `WorkUnit[]`, runs WorkUnit and reconciliation agent - loops, records tool transcripts, writes reports, inserts provenance, and - squashes into main. -- `buildWuToolSet()` already withholds `sl_write_source` and `sl_edit_source` - for `slDisallowed` WorkUnits. - -Current gaps that block v1: - -- WorkUnits still run against one mutable session worktree. In - `ingest-bundle.runner.ts`, the runner creates one `sessionWorktree` and each - WorkUnit uses `sessionWorktree.workdir`, `sessionWorktree.config`, and - `sessionWorktree.git`. -- WorkUnits do not produce durable Git patch proposal artifacts. -- There is no artifact-aware patch integration layer using - `git apply --3way --index`. -- There is no integration rollback and structured conflict classification for - failed patch application or semantic gate failures. -- Deterministic imports run as post-processors after WorkUnits and - reconciliation, while the spec requires projection before child worktree - creation. -- Final gates do not validate wiki body inline-code references to semantic - layer entities or raw tables. -- Provenance insertion accepts unknown raw hashes instead of failing before - insertion. -- `slDisallowed` is enforced at tool construction only; there is no integration - patch rejection for `semantic-layer/**`. -- Existing progress events and tool transcripts are useful but not sufficient - persistent traces. They do not capture the input snapshot, every routing - decision, patch collection, patch application timing, gate timing, rollback - context, and final outcome in one inspectable trace file. - -Non-blocking gaps for this plan: - -- Migrating Notion, LookML, Looker, dbt, MetricFlow, and historic-SQL direct - durable writes to the isolated path. This plan enables the path privately for - Metabase and test fixtures. -- Promoting isolated diffs as the default for every connector. -- Removing the old shared-worktree WorkUnit path. -- Interactive, CLI, or agent-driven conflict resolution. -- Auto-merging semantic conflicts that cannot be proven correct. -- Transitive SQL-projection closure for semantic-layer dependency expansion. -- Moving provenance to worktree files. -- Public connector knobs such as `executionMode`, `planningStrategy`, or - `conflictPolicy`. - -## File structure - -- Create `packages/context/src/ingest/ingest-trace.ts`. - Owns persistent JSONL trace writing, trace timing helpers, error - serialization, and trace path construction. -- Create `packages/context/src/ingest/ingest-trace.test.ts`. - Covers JSONL trace persistence, timing events, error context, and path layout. -- Modify `packages/context/src/ingest/ports.ts`. - Adds trace storage and private isolated-diff settings. -- Modify `packages/context/src/ingest/local-bundle-runtime.ts`. - Stores traces under `.ktx/ingest-traces//trace.jsonl` and enables the - isolated path for Metabase. -- Modify `packages/context/src/ingest/reports.ts` and - `packages/context/src/ingest/report-snapshot.ts`. - Adds `tracePath` and isolated-diff outcome fields to reports. -- Modify `packages/cli/src/ingest.ts`. - Prints `Trace: ` in stored ingest status. -- Modify `packages/context/src/core/git.service.ts` and tests. - Adds binary patch collection, patch application, staged commit, and path - inspection helpers needed by patch integration. -- Create `packages/context/src/ingest/isolated-diff/git-patch.ts`. - Owns patch metadata parsing, path restrictions, mode-change checks, and - binary/text artifact rejection. -- Create `packages/context/src/ingest/isolated-diff/git-patch.test.ts`. - Covers path parsing, `slDisallowed`, text-artifact binary rejection, and - executable-mode rejection. -- Create `packages/context/src/ingest/wiki-body-refs.ts`. - Parses and validates explicit wiki body references. -- Create `packages/context/src/ingest/wiki-body-refs.test.ts`. - Covers the `source.entity`, `connectionId/source.entity`, - `source:source_name`, and `table:qualified_table_name` grammar. -- Create `packages/context/src/ingest/artifact-gates.ts`. - Runs WorkUnit-local and final global artifact gates for SL, wiki refs, - wiki `sl_refs`, wiki body refs, and provenance rows. -- Create `packages/context/src/ingest/artifact-gates.test.ts`. - Covers the stale `total_contract_arr_cents` incident and provenance raw-path - failure. -- Create `packages/context/src/ingest/isolated-diff/work-unit-executor.ts`. - Executes a WorkUnit inside a child worktree, records traces, persists - transcripts, runs local gates, collects its patch, and cleans up the child. -- Create `packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts`. - Covers child-worktree base SHA usage, patch collection, child cleanup, and - trace emission on success and failure. -- Create `packages/context/src/ingest/isolated-diff/patch-integrator.ts`. - Applies accepted WorkUnit patches into the integration worktree, commits each - accepted patch, rolls back on textual or semantic conflict, and records - trace events. -- Create `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`. - Covers clean patch integration, textual conflict rollback, semantic conflict - rollback, and `slDisallowed` rejection. -- Modify `packages/context/src/ingest/types.ts`. - Adds the optional `SourceAdapter.project()` hook for deterministic projection. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Adds the isolated-diff execution branch, final gates, trace lifecycle, and - report integration. -- Create `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Adds the known Metabase-style stale wiki reference regression, clean - different-page integration, textual conflict, hybrid projection, Notion-style - invalid `sl_refs`, and LookML-style `slDisallowed` rejection tests. -- Modify `packages/context/src/ingest/index.ts`. - Exports new trace, artifact gate, and isolated-diff testing types. - ---- - -### Task 1: Persistent ingestion trace sink - -**Files:** -- Create: `packages/context/src/ingest/ingest-trace.ts` -- Create: `packages/context/src/ingest/ingest-trace.test.ts` -- Modify: `packages/context/src/ingest/ports.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Modify: `packages/context/src/ingest/reports.ts` -- Modify: `packages/context/src/ingest/report-snapshot.ts` -- Modify: `packages/cli/src/ingest.ts` - -- [ ] **Step 1: Write failing trace sink tests** - -Create `packages/context/src/ingest/ingest-trace.test.ts`: - -```ts -import { mkdtemp, readFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import { FileIngestTraceWriter, ingestTracePathForJob, traceTimed } from './ingest-trace.js'; - -describe('FileIngestTraceWriter', () => { - it('persists structured trace events as JSONL', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-trace-')); - const tracePath = ingestTracePathForJob(root, 'job-1'); - const trace = new FileIngestTraceWriter({ - tracePath, - jobId: 'job-1', - connectionId: 'metabase-main', - sourceKey: 'metabase', - level: 'debug', - }); - - await trace.event('debug', 'snapshot', 'input_snapshot', { - baseSha: 'abc123', - rawFileCount: 2, - diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 }, - }); - - const lines = (await readFile(tracePath, 'utf-8')).trim().split('\n').map((line) => JSON.parse(line)); - expect(lines).toHaveLength(1); - expect(lines[0]).toMatchObject({ - schemaVersion: 1, - jobId: 'job-1', - connectionId: 'metabase-main', - sourceKey: 'metabase', - level: 'debug', - phase: 'snapshot', - event: 'input_snapshot', - data: { - baseSha: 'abc123', - rawFileCount: 2, - diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 }, - }, - }); - expect(typeof lines[0].at).toBe('string'); - }); - - it('records timing and error context for postmortem inspection', async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date('2026-05-17T12:00:00.000Z')); - const root = await mkdtemp(join(tmpdir(), 'ktx-trace-')); - const tracePath = ingestTracePathForJob(root, 'job-2'); - const trace = new FileIngestTraceWriter({ - tracePath, - jobId: 'job-2', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - await expect( - traceTimed(trace, 'integration', 'apply_patch', { unitKey: 'wu-1' }, async () => { - vi.advanceTimersByTime(17); - throw new Error('patch conflict'); - }), - ).rejects.toThrow('patch conflict'); - - const lines = (await readFile(tracePath, 'utf-8')).trim().split('\n').map((line) => JSON.parse(line)); - expect(lines.map((line) => line.event)).toEqual(['apply_patch_started', 'apply_patch_failed']); - expect(lines[1]).toMatchObject({ - level: 'error', - phase: 'integration', - data: { unitKey: 'wu-1' }, - error: { name: 'Error', message: 'patch conflict' }, - }); - expect(lines[1].durationMs).toBe(17); - vi.useRealTimers(); - }); - - it('uses the documented trace path layout', () => { - expect(ingestTracePathForJob('/project/.ktx', 'job-3')).toBe('/project/.ktx/ingest-traces/job-3/trace.jsonl'); - }); -}); -``` - -- [ ] **Step 2: Run the failing trace sink tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-trace.test.ts -``` - -Expected: FAIL because `packages/context/src/ingest/ingest-trace.ts` does not -exist. - -- [ ] **Step 3: Add the trace sink implementation** - -Create `packages/context/src/ingest/ingest-trace.ts`: - -```ts -import { appendFile, mkdir } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { performance } from 'node:perf_hooks'; - -export type IngestTraceLevel = 'info' | 'debug' | 'trace' | 'error'; - -const TRACE_LEVEL_RANK: Record = { - error: 0, - info: 1, - debug: 2, - trace: 3, -}; - -export interface IngestTraceContext { - tracePath: string; - jobId: string; - connectionId: string; - sourceKey: string; - runId?: string; - syncId?: string; - level?: IngestTraceLevel; -} - -export interface IngestTraceEvent { - schemaVersion: 1; - at: string; - level: IngestTraceLevel; - jobId: string; - connectionId: string; - sourceKey: string; - runId?: string; - syncId?: string; - phase: string; - event: string; - durationMs?: number; - data?: Record; - error?: { - name: string; - message: string; - stack?: string; - }; -} - -export interface IngestTraceWriter { - readonly tracePath: string; - readonly context: IngestTraceContext; - withContext(context: Partial>): IngestTraceWriter; - event( - level: IngestTraceLevel, - phase: string, - event: string, - data?: Record, - error?: unknown, - durationMs?: number, - ): Promise; -} - -export function ingestTracePathForJob(homeDir: string, jobId: string): string { - return join(homeDir, 'ingest-traces', jobId, 'trace.jsonl'); -} - -function serializeError(error: unknown): IngestTraceEvent['error'] | undefined { - if (error === undefined || error === null) { - return undefined; - } - if (error instanceof Error) { - return { - name: error.name, - message: error.message, - ...(error.stack ? { stack: error.stack } : {}), - }; - } - return { name: 'Error', message: String(error) }; -} - -function shouldWrite(configured: IngestTraceLevel, incoming: IngestTraceLevel): boolean { - return TRACE_LEVEL_RANK[incoming] <= TRACE_LEVEL_RANK[configured]; -} - -export class FileIngestTraceWriter implements IngestTraceWriter { - readonly tracePath: string; - readonly context: IngestTraceContext; - - constructor(context: IngestTraceContext) { - this.context = { ...context, level: context.level ?? 'debug' }; - this.tracePath = context.tracePath; - } - - withContext(context: Partial>): IngestTraceWriter { - return new FileIngestTraceWriter({ ...this.context, ...context, tracePath: this.tracePath }); - } - - async event( - level: IngestTraceLevel, - phase: string, - event: string, - data?: Record, - error?: unknown, - durationMs?: number, - ): Promise { - if (!shouldWrite(this.context.level ?? 'debug', level)) { - return; - } - const payload: IngestTraceEvent = { - schemaVersion: 1, - at: new Date().toISOString(), - level, - jobId: this.context.jobId, - connectionId: this.context.connectionId, - sourceKey: this.context.sourceKey, - ...(this.context.runId ? { runId: this.context.runId } : {}), - ...(this.context.syncId ? { syncId: this.context.syncId } : {}), - phase, - event, - ...(durationMs !== undefined ? { durationMs } : {}), - ...(data ? { data } : {}), - ...(serializeError(error) ? { error: serializeError(error) } : {}), - }; - await mkdir(dirname(this.tracePath), { recursive: true }); - await appendFile(this.tracePath, `${JSON.stringify(payload)}\n`, 'utf-8'); - } -} - -export class NoopIngestTraceWriter implements IngestTraceWriter { - readonly tracePath = ''; - readonly context: IngestTraceContext = { - tracePath: '', - jobId: '', - connectionId: '', - sourceKey: '', - level: 'error', - }; - - withContext(): IngestTraceWriter { - return this; - } - - async event(): Promise {} -} - -export async function traceTimed( - trace: IngestTraceWriter, - phase: string, - event: string, - data: Record, - fn: () => Promise, -): Promise { - await trace.event('debug', phase, `${event}_started`, data); - const started = performance.now(); - try { - const result = await fn(); - await trace.event('debug', phase, `${event}_finished`, data, undefined, performance.now() - started); - return result; - } catch (error) { - await trace.event('error', phase, `${event}_failed`, data, error, performance.now() - started); - throw error; - } -} -``` - -- [ ] **Step 4: Add trace storage and report fields** - -In `packages/context/src/ingest/ports.ts`, import `IngestTraceLevel`: - -```ts -import type { IngestTraceLevel } from './ingest-trace.js'; -``` - -Then extend `IngestSettingsPort` and `IngestStoragePort`: - -```ts -export interface IngestSettingsPort { - memoryIngestionModel: string; - probeRowCount: number; - workUnitMaxConcurrency?: number; - workUnitStepBudget?: number; - workUnitFailureMode?: 'abort' | 'continue'; - isolatedDiffSourceKeys?: string[]; - ingestTraceLevel?: IngestTraceLevel; -} - -export interface IngestStoragePort { - homeDir: string; - systemGitAuthor: IngestGitAuthor; - resolveUploadDir(uploadId: string): string; - resolvePullDir(jobId: string): string; - resolveTranscriptDir(jobId: string): string; - resolveTracePath(jobId: string): string; -} -``` - -In `packages/context/src/ingest/local-bundle-runtime.ts`, import -`ingestTracePathForJob`: - -```ts -import { ingestTracePathForJob } from './ingest-trace.js'; -``` - -Then add the storage method: - -```ts - resolveTracePath(jobId: string): string { - return ingestTracePathForJob(this.homeDir, jobId); - } -``` - -When creating the runner settings in `createLocalBundleIngestRuntime()`, set: - -```ts - settings: { - memoryIngestionModel: options.memoryModel ?? project.config.llm.memoryIngestionModel, - probeRowCount: project.config.ai.slValidation.probeRowCount, - workUnitMaxConcurrency: project.config.ingest.workUnitMaxConcurrency, - workUnitStepBudget: project.config.ingest.workUnitStepBudget, - workUnitFailureMode: project.config.ingest.workUnitFailureMode, - isolatedDiffSourceKeys: ['metabase'], - ingestTraceLevel: 'debug', - }, -``` - -In `packages/context/src/ingest/reports.ts`, add report fields: - -```ts -export interface IngestReportBody { - syncId: string; - diffSummary: IngestDiffSummary; - fetch?: SourceFetchReport; - commitSha: string | null; - tracePath?: string; - isolatedDiff?: { - enabled: boolean; - integrationWorktreePath?: string; - ingestionBaseSha?: string; - projectionSha?: string | null; - acceptedPatches: number; - textualConflicts: number; - semanticConflicts: number; - }; - workUnits: IngestReportWorkUnit[]; - failedWorkUnits: string[]; - reconciliationSkipped: boolean; - reconciliationActions?: MemoryAction[]; - conflictsResolved: ConflictResolvedRecord[]; - evictionsApplied: EvictionAppliedRecord[]; - unmappedFallbacks: UnmappedFallbackRecord[]; - artifactResolutions?: ArtifactResolutionRecord[]; - evictionInputs: string[]; - unresolvedCards: UnresolvedCardInfo[]; - supersededBy: string | null; - overrideOf: string | null; - provenanceRows: IngestReportProvenanceDetail[]; - toolTranscripts: IngestReportToolTranscriptSummary[]; - postProcessor?: IngestReportPostProcessorOutcome; - wikiSlRefRepairs?: WikiSlRefRepair[]; - wikiSlRefRepairWarnings?: string[]; - memoryFlow?: MemoryFlowReplayInput; -} -``` - -In `packages/context/src/ingest/report-snapshot.ts`, add this schema inside -`body`: - -```ts - tracePath: z.string().optional(), - isolatedDiff: z - .object({ - enabled: z.boolean(), - integrationWorktreePath: z.string().optional(), - ingestionBaseSha: z.string().optional(), - projectionSha: z.string().nullable().optional(), - acceptedPatches: z.number().int().min(0), - textualConflicts: z.number().int().min(0), - semanticConflicts: z.number().int().min(0), - }) - .optional(), -``` - -In `packages/cli/src/ingest.ts`, update `writeReportStatus()`: - -```ts - if (report.body.tracePath) { - io.stdout.write(`Trace: ${report.body.tracePath}\n`); - } -``` - -Place it after the `Job:` line so a failed run's trace path is visible near the -run identifiers. - -- [ ] **Step 5: Run trace sink tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-trace.test.ts src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/ingest-trace.ts \ - packages/context/src/ingest/ingest-trace.test.ts \ - packages/context/src/ingest/ports.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/reports.ts \ - packages/context/src/ingest/report-snapshot.ts \ - packages/cli/src/ingest.ts -git commit -m "feat: persist ingest trace events" -``` - ---- - -### Task 2: Git patch contract helpers - -**Files:** -- Modify: `packages/context/src/core/git.service.ts` -- Create: `packages/context/src/core/git.service.patch.test.ts` -- Create: `packages/context/src/ingest/isolated-diff/git-patch.ts` -- Create: `packages/context/src/ingest/isolated-diff/git-patch.test.ts` - -- [ ] **Step 1: Write failing GitService patch tests** - -Create `packages/context/src/core/git.service.patch.test.ts`: - -```ts -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; -import { GitService } from './git.service.js'; - -async function makeGit() { - const homeDir = await mkdtemp(join(tmpdir(), 'ktx-git-patch-')); - const configDir = join(homeDir, 'config'); - const git = new GitService({ - storage: { configDir, homeDir }, - git: { - userName: 'System User', - userEmail: 'system@example.com', - bootstrapMessage: 'init', - bootstrapAuthor: 'system', - bootstrapAuthorEmail: 'system@example.com', - }, - }); - await git.onModuleInit(); - return { homeDir, configDir, git }; -} - -describe('GitService patch helpers', () => { - it('collects binary-safe no-rename patches and applies them with --3way --index', async () => { - const { homeDir, configDir, git } = await makeGit(); - await mkdir(join(configDir, 'wiki/global'), { recursive: true }); - await writeFile(join(configDir, 'wiki/global/page.md'), 'old\n'); - await git.commitFiles(['wiki/global/page.md'], 'add page', 'System User', 'system@example.com'); - const base = await git.revParseHead(); - - await writeFile(join(configDir, 'wiki/global/page.md'), 'new\n'); - await git.commitFiles(['wiki/global/page.md'], 'edit page', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'proposal.patch'); - await git.writeBinaryNoRenamePatch(base, 'HEAD', patchPath); - - const targetDir = join(homeDir, 'target'); - await git.addWorktree(targetDir, 'target', base); - const targetGit = git.forWorktree(targetDir); - await targetGit.applyPatchFile3WayIndex(patchPath); - await targetGit.commitStaged('apply proposal', 'System User', 'system@example.com'); - - await expect(readFile(join(targetDir, 'wiki/global/page.md'), 'utf-8')).resolves.toBe('new\n'); - }); -}); -``` - -- [ ] **Step 2: Run failing GitService patch tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/core/git.service.patch.test.ts -``` - -Expected: FAIL because `writeBinaryNoRenamePatch`, `applyPatchFile3WayIndex`, -and `commitStaged` are missing. - -- [ ] **Step 3: Add GitService patch helpers** - -At the top of `packages/context/src/core/git.service.ts`, change: - -```ts -import { join } from 'node:path'; -``` - -to: - -```ts -import { dirname, join } from 'node:path'; -``` - -Then add these methods to the `GitService` class: - -```ts - async writeBinaryNoRenamePatch(from: string, to: string, patchPath: string): Promise { - await this.withMutationQueue(async () => { - const patch = await this.git.raw(['diff', '--binary', '--no-renames', `${from}..${to}`]); - await fs.mkdir(dirname(patchPath), { recursive: true }); - await fs.writeFile(patchPath, patch, 'utf-8'); - }); - } - - async applyPatchFile3WayIndex(patchPath: string): Promise { - await this.withMutationQueue(async () => { - await this.git.raw(['apply', '--3way', '--index', patchPath]); - }); - } - - async commitStaged(commitMessage: string, author: string, authorEmail: string): Promise { - return this.withMutationQueue(async () => { - const stagedChanges = await this.git.diff(['--cached', '--name-only']); - if (!stagedChanges.trim()) { - const head = (await this.git.revparse(['HEAD'])).trim(); - const log = await this.git.log({ maxCount: 1 }); - const latest = log.latest; - return { - commitHash: head, - shortHash: head.substring(0, 7), - message: latest?.message ?? '', - author: latest?.author_name ?? '', - authorEmail: latest?.author_email ?? '', - timestamp: latest?.date ?? new Date(0).toISOString(), - committedDate: latest?.date ? new Date(latest.date).toISOString() : new Date(0).toISOString(), - created: false, - }; - } - await this.git.commit(commitMessage, { '--author': `${author} <${authorEmail}>` }); - const head = (await this.git.revparse(['HEAD'])).trim(); - const log = await this.git.log({ maxCount: 1 }); - const latest = log.latest; - return { - commitHash: head, - shortHash: head.substring(0, 7), - message: latest?.message ?? commitMessage, - author: latest?.author_name ?? author, - authorEmail: latest?.author_email ?? authorEmail, - timestamp: latest?.date ?? new Date().toISOString(), - committedDate: latest?.date ? new Date(latest.date).toISOString() : new Date().toISOString(), - created: true, - }; - }); - } -``` - -- [ ] **Step 4: Write failing patch contract tests** - -Create `packages/context/src/ingest/isolated-diff/git-patch.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { - assertPatchAllowedForWorkUnit, - parsePatchTouchedPaths, - textArtifactRoots, -} from './git-patch.js'; - -describe('isolated diff patch contract', () => { - it('parses touched paths from no-rename git patches', () => { - const patch = [ - 'diff --git a/wiki/global/a.md b/wiki/global/a.md', - 'index 1111111..2222222 100644', - '--- a/wiki/global/a.md', - '+++ b/wiki/global/a.md', - '@@ -1 +1 @@', - '-old', - '+new', - 'diff --git a/semantic-layer/c1/orders.yaml b/semantic-layer/c1/orders.yaml', - 'new file mode 100644', - '--- /dev/null', - '+++ b/semantic-layer/c1/orders.yaml', - '@@ -0,0 +1 @@', - '+name: orders', - '', - ].join('\n'); - - expect(parsePatchTouchedPaths(patch)).toEqual([ - { path: 'wiki/global/a.md', oldPath: 'wiki/global/a.md', newPath: 'wiki/global/a.md', mode: '100644', binary: false }, - { - path: 'semantic-layer/c1/orders.yaml', - oldPath: 'semantic-layer/c1/orders.yaml', - newPath: 'semantic-layer/c1/orders.yaml', - mode: '100644', - binary: false, - }, - ]); - }); - - it('rejects semantic-layer paths for slDisallowed work units', () => { - const patch = 'diff --git a/semantic-layer/c1/orders.yaml b/semantic-layer/c1/orders.yaml\nindex 1..2 100644\n'; - - expect(() => - assertPatchAllowedForWorkUnit({ - unitKey: 'lookml-mismatch', - patch, - slDisallowed: true, - }), - ).toThrow(/slDisallowed WorkUnit lookml-mismatch touched semantic-layer\/c1\/orders.yaml/); - }); - - it('rejects executable and binary changes under known text artifact roots', () => { - expect(textArtifactRoots).toEqual(['wiki/', 'semantic-layer/']); - - const executablePatch = - 'diff --git a/wiki/global/a.md b/wiki/global/a.md\nold mode 100644\nnew mode 100755\nindex 1..2\n'; - expect(() => - assertPatchAllowedForWorkUnit({ - unitKey: 'wu-1', - patch: executablePatch, - slDisallowed: false, - }), - ).toThrow(/unexpected executable mode under wiki\/global\/a.md/); - - const binaryPatch = [ - 'diff --git a/semantic-layer/c1/orders.yaml b/semantic-layer/c1/orders.yaml', - 'index 1111111..2222222 100644', - 'GIT binary patch', - 'literal 0', - '', - ].join('\n'); - expect(() => - assertPatchAllowedForWorkUnit({ - unitKey: 'wu-2', - patch: binaryPatch, - slDisallowed: false, - }), - ).toThrow(/unexpected binary patch under semantic-layer\/c1\/orders.yaml/); - }); -}); -``` - -- [ ] **Step 5: Add patch contract helpers** - -Create `packages/context/src/ingest/isolated-diff/git-patch.ts`: - -```ts -export const textArtifactRoots = ['wiki/', 'semantic-layer/'] as const; - -export interface PatchTouchedPath { - path: string; - oldPath: string; - newPath: string; - mode: string | null; - binary: boolean; -} - -export interface PatchPolicyInput { - unitKey: string; - patch: string; - slDisallowed: boolean; -} - -function stripPrefix(path: string): string { - return path.replace(/^[ab]\//, ''); -} - -function isTextArtifactPath(path: string): boolean { - return textArtifactRoots.some((root) => path.startsWith(root)); -} - -export function parsePatchTouchedPaths(patch: string): PatchTouchedPath[] { - const lines = patch.split('\n'); - const entries: PatchTouchedPath[] = []; - let current: PatchTouchedPath | null = null; - - const pushCurrent = () => { - if (current) { - entries.push(current); - } - }; - - for (const line of lines) { - const diffMatch = /^diff --git (.+) (.+)$/.exec(line); - if (diffMatch) { - pushCurrent(); - const oldPath = stripPrefix(diffMatch[1] ?? ''); - const newPath = stripPrefix(diffMatch[2] ?? ''); - current = { - path: newPath === '/dev/null' ? oldPath : newPath, - oldPath, - newPath, - mode: null, - binary: false, - }; - continue; - } - if (!current) { - continue; - } - const indexMode = /^index [0-9a-f]+\.\.[0-9a-f]+(?: [0-7]{6})?$/.exec(line); - if (indexMode && line.includes(' ')) { - current.mode = line.split(' ').at(-1) ?? current.mode; - } - const newMode = /^new mode ([0-7]{6})$/.exec(line); - if (newMode) { - current.mode = newMode[1] ?? current.mode; - } - if (line === 'GIT binary patch' || line.startsWith('Binary files ')) { - current.binary = true; - } - } - - pushCurrent(); - return entries; -} - -export function assertPatchAllowedForWorkUnit(input: PatchPolicyInput): PatchTouchedPath[] { - const touched = parsePatchTouchedPaths(input.patch); - for (const entry of touched) { - if (input.slDisallowed && entry.path.startsWith('semantic-layer/')) { - throw new Error(`slDisallowed WorkUnit ${input.unitKey} touched ${entry.path}`); - } - if (!isTextArtifactPath(entry.path)) { - continue; - } - if (entry.binary) { - throw new Error(`unexpected binary patch under ${entry.path}`); - } - if (entry.mode && entry.mode !== '100644') { - throw new Error(`unexpected executable mode under ${entry.path}: ${entry.mode}`); - } - } - return touched; -} -``` - -- [ ] **Step 6: Run patch helper tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/core/git.service.patch.test.ts src/ingest/isolated-diff/git-patch.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/src/core/git.service.ts \ - packages/context/src/core/git.service.patch.test.ts \ - packages/context/src/ingest/isolated-diff/git-patch.ts \ - packages/context/src/ingest/isolated-diff/git-patch.test.ts -git commit -m "feat: add isolated ingest patch helpers" -``` - ---- - -### Task 3: Wiki body reference parser and validator - -**Files:** -- Create: `packages/context/src/ingest/wiki-body-refs.ts` -- Create: `packages/context/src/ingest/wiki-body-refs.test.ts` - -- [ ] **Step 1: Write failing wiki body reference tests** - -Create `packages/context/src/ingest/wiki-body-refs.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { findInvalidWikiBodyRefs, parseWikiBodyRefs } from './wiki-body-refs.js'; - -const sources = [ - { - name: 'mart_account_segments', - grain: ['account_id'], - columns: [{ name: 'account_id', type: 'string' }, { name: 'segment', type: 'string' }], - joins: [], - measures: [{ name: 'total_contract_arr', expr: 'sum(contract_arr)' }], - segments: [{ name: 'enterprise', expr: "segment = 'enterprise'" }], - table: 'analytics.mart_account_segments', - }, -]; - -describe('wiki body refs', () => { - it('parses only explicit inline-code body references outside fenced blocks', () => { - const body = [ - 'Valid `mart_account_segments.total_contract_arr` and `source:mart_account_segments`.', - 'Also `warehouse/mart_account_segments.segment` and `table:analytics.mart_account_segments`.', - 'Ignore prose mart_account_segments.total_contract_arr_cents.', - 'Ignore `single_token`.', - '```sql', - 'select `mart_account_segments.total_contract_arr_cents`', - '```', - ].join('\n'); - - expect(parseWikiBodyRefs(body)).toEqual([ - { kind: 'sl_entity', connectionId: null, sourceName: 'mart_account_segments', entityName: 'total_contract_arr' }, - { kind: 'sl_source', connectionId: null, sourceName: 'mart_account_segments' }, - { kind: 'sl_entity', connectionId: 'warehouse', sourceName: 'mart_account_segments', entityName: 'segment' }, - { kind: 'table', connectionId: null, tableRef: 'analytics.mart_account_segments' }, - ]); - }); - - it('rejects stale inline-code semantic-layer references', async () => { - const invalid = await findInvalidWikiBodyRefs({ - pageKey: 'account-segments', - body: 'ARR is documented as `mart_account_segments.total_contract_arr_cents`.', - visibleConnectionIds: ['warehouse'], - loadSources: async () => sources, - tableExists: async () => true, - }); - - expect(invalid).toEqual([ - 'account-segments: unknown semantic-layer entity mart_account_segments.total_contract_arr_cents', - ]); - }); - - it('validates source, dimension, segment, measure, and table references', async () => { - const invalid = await findInvalidWikiBodyRefs({ - pageKey: 'account-segments', - body: [ - '`mart_account_segments.total_contract_arr`', - '`mart_account_segments.segment`', - '`mart_account_segments.enterprise`', - '`source:mart_account_segments`', - '`table:analytics.mart_account_segments`', - ].join('\n'), - visibleConnectionIds: ['warehouse'], - loadSources: async () => sources, - tableExists: async (_connectionId, tableRef) => tableRef === 'analytics.mart_account_segments', - }); - - expect(invalid).toEqual([]); - }); -}); -``` - -- [ ] **Step 2: Run failing wiki body reference tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/wiki-body-refs.test.ts -``` - -Expected: FAIL because `wiki-body-refs.ts` does not exist. - -- [ ] **Step 3: Add parser and validator** - -Create `packages/context/src/ingest/wiki-body-refs.ts`: - -```ts -import type { SemanticLayerSource } from '../sl/index.js'; - -export type WikiBodyRef = - | { kind: 'sl_entity'; connectionId: string | null; sourceName: string; entityName: string } - | { kind: 'sl_source'; connectionId: string | null; sourceName: string } - | { kind: 'table'; connectionId: string | null; tableRef: string }; - -export interface WikiBodyRefValidationInput { - pageKey: string; - body: string; - visibleConnectionIds: string[]; - loadSources(connectionId: string): Promise; - tableExists(connectionId: string, tableRef: string): Promise; -} - -const inlineCodePattern = /`([^`\n]+)`/g; - -function visibleLinesOutsideFences(body: string): string[] { - const lines: string[] = []; - let fenced = false; - for (const line of body.split('\n')) { - if (/^\s*```/.test(line)) { - fenced = !fenced; - continue; - } - if (!fenced) { - lines.push(line); - } - } - return lines; -} - -function parseConnectionScoped(value: string): { connectionId: string | null; body: string } { - const slash = value.indexOf('/'); - if (slash <= 0) { - return { connectionId: null, body: value }; - } - return { connectionId: value.slice(0, slash), body: value.slice(slash + 1) }; -} - -export function parseWikiBodyRefs(body: string): WikiBodyRef[] { - const refs: WikiBodyRef[] = []; - for (const line of visibleLinesOutsideFences(body)) { - for (const match of line.matchAll(inlineCodePattern)) { - const token = (match[1] ?? '').trim(); - if (!token) { - continue; - } - const scoped = parseConnectionScoped(token); - if (scoped.body.startsWith('source:')) { - const sourceName = scoped.body.slice('source:'.length).trim(); - if (sourceName) { - refs.push({ kind: 'sl_source', connectionId: scoped.connectionId, sourceName }); - } - continue; - } - if (scoped.body.startsWith('table:')) { - const tableRef = scoped.body.slice('table:'.length).trim(); - if (tableRef) { - refs.push({ kind: 'table', connectionId: scoped.connectionId, tableRef }); - } - continue; - } - const parts = scoped.body.split('.'); - if (parts.length === 2 && parts[0] && parts[1]) { - refs.push({ - kind: 'sl_entity', - connectionId: scoped.connectionId, - sourceName: parts[0], - entityName: parts[1], - }); - } - } - } - return refs; -} - -function entityNames(source: SemanticLayerSource): Set { - return new Set([ - ...(source.measures ?? []).map((measure) => measure.name), - ...(source.columns ?? []).map((column) => column.name), - ...(source.segments ?? []).map((segment) => segment.name), - ]); -} - -export async function findInvalidWikiBodyRefs(input: WikiBodyRefValidationInput): Promise { - const errors: string[] = []; - const sourceCache = new Map(); - const loadSources = async (connectionId: string): Promise => { - const cached = sourceCache.get(connectionId); - if (cached) { - return cached; - } - const sources = await input.loadSources(connectionId); - sourceCache.set(connectionId, sources); - return sources; - }; - - for (const ref of parseWikiBodyRefs(input.body)) { - const connectionIds = ref.connectionId ? [ref.connectionId] : input.visibleConnectionIds; - if (ref.kind === 'table') { - const found = await Promise.all(connectionIds.map((connectionId) => input.tableExists(connectionId, ref.tableRef))); - if (!found.some(Boolean)) { - errors.push(`${input.pageKey}: unknown raw table ${ref.connectionId ? `${ref.connectionId}/` : ''}${ref.tableRef}`); - } - continue; - } - - let source: SemanticLayerSource | undefined; - for (const connectionId of connectionIds) { - source = (await loadSources(connectionId)).find((candidate) => candidate.name === ref.sourceName); - if (source) { - break; - } - } - if (!source) { - errors.push(`${input.pageKey}: unknown semantic-layer source ${ref.sourceName}`); - continue; - } - if (ref.kind === 'sl_entity' && !entityNames(source).has(ref.entityName)) { - errors.push(`${input.pageKey}: unknown semantic-layer entity ${ref.sourceName}.${ref.entityName}`); - } - } - - return errors; -} -``` - -- [ ] **Step 4: Run wiki body reference tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/wiki-body-refs.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/context/src/ingest/wiki-body-refs.ts \ - packages/context/src/ingest/wiki-body-refs.test.ts -git commit -m "feat: validate wiki body semantic references" -``` - ---- - -### Task 4: Artifact gates and provenance validation - -**Files:** -- Create: `packages/context/src/ingest/artifact-gates.ts` -- Create: `packages/context/src/ingest/artifact-gates.test.ts` - -- [ ] **Step 1: Write failing artifact gate tests** - -Create `packages/context/src/ingest/artifact-gates.test.ts`: - -```ts -import { describe, expect, it, vi } from 'vitest'; -import { validateFinalIngestArtifacts, validateProvenanceRawPaths } from './artifact-gates.js'; - -describe('artifact gates', () => { - it('fails the final tree when wiki body references a stale semantic-layer measure', async () => { - const wikiService = { - readPage: vi.fn().mockResolvedValue({ - pageKey: 'account-segments', - frontmatter: { - summary: 'Account segments', - usage_mode: 'auto', - sl_refs: ['mart_account_segments'], - }, - content: 'ARR is `mart_account_segments.total_contract_arr_cents`.', - }), - }; - const semanticLayerService = { - loadAllSources: vi.fn().mockResolvedValue({ - sources: [ - { - name: 'mart_account_segments', - grain: ['account_id'], - columns: [{ name: 'account_id', type: 'string' }], - joins: [], - measures: [{ name: 'total_contract_arr', expr: 'sum(contract_arr)' }], - table: 'analytics.mart_account_segments', - }, - ], - loadErrors: [], - }), - }; - - await expect( - validateFinalIngestArtifacts({ - connectionIds: ['warehouse'], - changedWikiPageKeys: ['account-segments'], - touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'mart_account_segments' }], - wikiService: wikiService as never, - semanticLayerService: semanticLayerService as never, - validateTouchedSources: async () => ({ invalidSources: [], validSources: ['mart_account_segments'] }), - tableExists: async () => true, - }), - ).rejects.toThrow(/unknown semantic-layer entity mart_account_segments\.total_contract_arr_cents/); - }); - - it('fails before provenance insertion when a raw path cannot be tied to the current snapshot or eviction set', () => { - expect(() => - validateProvenanceRawPaths({ - rows: [{ rawPath: 'cards/missing.json' }], - currentRawPaths: new Set(['cards/present.json']), - deletedRawPaths: new Set(['cards/deleted.json']), - }), - ).toThrow(/provenance row references raw path outside this snapshot: cards\/missing\.json/); - }); -}); -``` - -- [ ] **Step 2: Run failing artifact gate tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -``` - -Expected: FAIL because `artifact-gates.ts` does not exist. - -- [ ] **Step 3: Add artifact gate implementation** - -Create `packages/context/src/ingest/artifact-gates.ts`: - -```ts -import type { SemanticLayerService } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; -import { findInvalidWikiBodyRefs } from './wiki-body-refs.js'; - -export interface TouchedValidationResult { - invalidSources: string[]; - validSources: string[]; -} - -export interface FinalArtifactGateInput { - connectionIds: string[]; - changedWikiPageKeys: string[]; - touchedSlSources: TouchedSlSource[]; - wikiService: KnowledgeWikiService; - semanticLayerService: SemanticLayerService; - validateTouchedSources(touched: TouchedSlSource[]): Promise; - tableExists(connectionId: string, tableRef: string): Promise; -} - -export interface ProvenanceRawPathValidationInput { - rows: Array<{ rawPath: string }>; - currentRawPaths: Set; - deletedRawPaths: Set; -} - -function bareSlRef(ref: string): string { - const withoutConnection = ref.includes('/') ? ref.slice(ref.indexOf('/') + 1) : ref; - return withoutConnection.split('.')[0] ?? withoutConnection; -} - -async function validateWikiSlRefs(input: FinalArtifactGateInput): Promise { - const errors: string[] = []; - const sourcesByConnection = new Map>(); - for (const connectionId of input.connectionIds) { - const { sources } = await input.semanticLayerService.loadAllSources(connectionId); - sourcesByConnection.set(connectionId, new Set(sources.map((source) => source.name))); - } - - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - for (const ref of page.frontmatter.sl_refs ?? []) { - const sourceName = bareSlRef(ref); - const connectionId = ref.includes('/') ? ref.slice(0, ref.indexOf('/')) : null; - const sourceSets = connectionId ? [sourcesByConnection.get(connectionId)] : [...sourcesByConnection.values()]; - if (!sourceSets.some((set) => set?.has(sourceName))) { - errors.push(`${pageKey}: unknown sl_refs entry ${ref}`); - } - } - } - return errors; -} - -export async function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise { - const validation = await input.validateTouchedSources(input.touchedSlSources); - const errors: string[] = validation.invalidSources.map((source) => `semantic-layer validation failed for ${source}`); - errors.push(...(await validateWikiSlRefs(input))); - - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - errors.push( - ...(await findInvalidWikiBodyRefs({ - pageKey, - body: page.content, - visibleConnectionIds: input.connectionIds, - loadSources: async (connectionId) => { - const { sources } = await input.semanticLayerService.loadAllSources(connectionId); - return sources; - }, - tableExists: input.tableExists, - })), - ); - } - - if (errors.length > 0) { - throw new Error(`final artifact gates failed:\n${errors.join('\n')}`); - } -} - -export function validateProvenanceRawPaths(input: ProvenanceRawPathValidationInput): void { - for (const row of input.rows) { - if (!input.currentRawPaths.has(row.rawPath) && !input.deletedRawPaths.has(row.rawPath)) { - throw new Error(`provenance row references raw path outside this snapshot: ${row.rawPath}`); - } - } -} -``` - -- [ ] **Step 4: Run artifact gate tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts -git commit -m "feat: add final ingest artifact gates" -``` - ---- - -### Task 5: Isolated WorkUnit executor - -**Files:** -- Create: `packages/context/src/ingest/isolated-diff/work-unit-executor.ts` -- Create: `packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts` -- Modify: `packages/context/src/ingest/stages/stage-3-work-units.ts` - -- [ ] **Step 1: Write failing isolated WorkUnit executor tests** - -Create `packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts`: - -```ts -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../core/index.js'; -import { FileIngestTraceWriter } from '../ingest-trace.js'; -import { runIsolatedWorkUnit } from './work-unit-executor.js'; - -async function makeGit() { - const homeDir = await mkdtemp(join(tmpdir(), 'ktx-isolated-wu-')); - const configDir = join(homeDir, 'config'); - const git = new GitService({ - storage: { configDir, homeDir }, - git: { - userName: 'System User', - userEmail: 'system@example.com', - bootstrapMessage: 'init', - bootstrapAuthor: 'system', - bootstrapAuthorEmail: 'system@example.com', - }, - }); - await git.onModuleInit(); - await mkdir(join(configDir, 'raw-sources/c1/fake/s'), { recursive: true }); - await writeFile(join(configDir, 'raw-sources/c1/fake/s/a.json'), '{}\n'); - await git.commitFiles(['raw-sources/c1/fake/s/a.json'], 'raw snapshot', 'System User', 'system@example.com'); - return { homeDir, configDir, git, baseSha: await git.revParseHead() }; -} - -describe('runIsolatedWorkUnit', () => { - it('creates a child worktree at the ingestion base and persists a patch proposal', async () => { - const { homeDir, git, baseSha } = await makeGit(); - const childDir = join(homeDir, '.worktrees/session-job-1-wu-1'); - const childGit = git.forWorktree(childDir); - const sessionWorktreeService = { - create: vi.fn(async (_key: string, startSha: string) => { - await mkdir(join(homeDir, '.worktrees'), { recursive: true }); - await git.addWorktree(childDir, 'session/job-1-wu-1', startSha); - return { chatId: 'job-1-wu-1', workdir: childDir, branch: 'session/job-1-wu-1', baseSha: startSha, createdAt: new Date(), git: childGit, config: {} }; - }), - cleanup: vi.fn(async () => undefined), - }; - const tracePath = join(homeDir, '.ktx/ingest-traces/job-1/trace.jsonl'); - const trace = new FileIngestTraceWriter({ - tracePath, - jobId: 'job-1', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await runIsolatedWorkUnit({ - unitIndex: 0, - ingestionBaseSha: baseSha, - sessionWorktreeService: sessionWorktreeService as never, - patchDir: join(homeDir, '.ktx/ingest-patches/job-1'), - trace, - run: async (child) => { - await mkdir(join(child.workdir, 'wiki/global'), { recursive: true }); - await writeFile(join(child.workdir, 'wiki/global/a.md'), '---\nsummary: A\nusage_mode: auto\n---\n\nBody\n'); - await child.git.commitFiles(['wiki/global/a.md'], 'test: write wiki', 'KTX Test', 'system@ktx.local'); - return { - unitKey: 'wu-1', - status: 'success', - preSha: baseSha, - postSha: await child.git.revParseHead(), - actions: [{ target: 'wiki', type: 'created', key: 'a', detail: 'A' }], - touchedSlSources: [], - }; - }, - workUnit: { unitKey: 'wu-1', rawFiles: ['a.json'], peerFileIndex: [], dependencyPaths: [] }, - }); - - expect(sessionWorktreeService.create).toHaveBeenCalledWith('job-1-wu-1', baseSha); - expect(sessionWorktreeService.cleanup).toHaveBeenCalledWith(expect.any(Object), 'success'); - expect(result.status).toBe('success'); - expect(result.patchPath).toContain('0000-wu-1.patch'); - await expect(readFile(result.patchPath, 'utf-8')).resolves.toContain('wiki/global/a.md'); - await expect(readFile(tracePath, 'utf-8')).resolves.toContain('work_unit_child_created'); - }); -}); -``` - -- [ ] **Step 2: Run failing isolated WorkUnit executor tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/work-unit-executor.test.ts -``` - -Expected: FAIL because `work-unit-executor.ts` does not exist. - -- [ ] **Step 3: Add patch metadata to WorkUnitOutcome** - -In `packages/context/src/ingest/stages/stage-3-work-units.ts`, extend -`WorkUnitOutcome`: - -```ts -export interface WorkUnitOutcome { - unitKey: string; - status: 'success' | 'failed'; - reason?: string; - preSha: string; - postSha: string; - actions: MemoryAction[]; - touchedSlSources: TouchedSlSource[]; - slDisallowed?: boolean; - slDisallowedReason?: 'lookml_connection_mismatch'; - patchPath?: string; - patchTouchedPaths?: string[]; - childWorktreePath?: string; -} -``` - -- [ ] **Step 4: Add isolated WorkUnit executor** - -Create `packages/context/src/ingest/isolated-diff/work-unit-executor.ts`: - -```ts -import { mkdir, readFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import type { SessionOutcome } from '../../core/index.js'; -import type { IngestSessionWorktree, IngestSessionWorktreePort } from '../ports.js'; -import type { WorkUnit } from '../types.js'; -import type { IngestTraceWriter } from '../ingest-trace.js'; -import type { WorkUnitOutcome } from '../stages/stage-3-work-units.js'; -import { assertPatchAllowedForWorkUnit } from './git-patch.js'; - -export interface RunIsolatedWorkUnitInput { - unitIndex: number; - ingestionBaseSha: string; - sessionWorktreeService: IngestSessionWorktreePort; - patchDir: string; - trace: IngestTraceWriter; - workUnit: WorkUnit; - run(child: IngestSessionWorktree): Promise; -} - -function patchFileName(unitIndex: number, unitKey: string): string { - const safeKey = unitKey.replace(/[^a-zA-Z0-9_.-]+/g, '-'); - return `${String(unitIndex).padStart(4, '0')}-${safeKey}.patch`; -} - -export async function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Promise { - const sessionKey = `${input.trace.context.jobId}-${input.workUnit.unitKey}`; - let cleanupOutcome: SessionOutcome = 'crash'; - const child = await input.sessionWorktreeService.create(sessionKey, input.ingestionBaseSha); - await input.trace.event('debug', 'work_unit', 'work_unit_child_created', { - unitKey: input.workUnit.unitKey, - unitIndex: input.unitIndex, - worktreePath: child.workdir, - baseSha: input.ingestionBaseSha, - }); - - try { - const outcome = await input.run(child); - if (outcome.status !== 'success') { - cleanupOutcome = 'crash'; - await input.trace.event('error', 'work_unit', 'work_unit_failed_before_patch', { - unitKey: input.workUnit.unitKey, - reason: outcome.reason ?? 'unknown failure', - }); - return { ...outcome, childWorktreePath: child.workdir }; - } - - await mkdir(input.patchDir, { recursive: true }); - const patchPath = join(input.patchDir, patchFileName(input.unitIndex, input.workUnit.unitKey)); - await child.git.writeBinaryNoRenamePatch(input.ingestionBaseSha, 'HEAD', patchPath); - const patch = await readFile(patchPath, 'utf-8'); - const touched = assertPatchAllowedForWorkUnit({ - unitKey: input.workUnit.unitKey, - patch, - slDisallowed: input.workUnit.slDisallowed === true, - }); - cleanupOutcome = 'success'; - await input.trace.event('debug', 'work_unit', 'work_unit_patch_collected', { - unitKey: input.workUnit.unitKey, - patchPath, - touchedPaths: touched.map((entry) => entry.path), - patchBytes: Buffer.byteLength(patch), - }); - return { - ...outcome, - patchPath, - patchTouchedPaths: touched.map((entry) => entry.path), - childWorktreePath: child.workdir, - }; - } catch (error) { - cleanupOutcome = 'crash'; - await input.trace.event( - 'error', - 'work_unit', - 'work_unit_child_failed', - { unitKey: input.workUnit.unitKey, worktreePath: child.workdir }, - error, - ); - throw error; - } finally { - await input.sessionWorktreeService.cleanup(child, cleanupOutcome); - await input.trace.event('trace', 'work_unit', 'work_unit_child_cleanup', { - unitKey: input.workUnit.unitKey, - outcome: cleanupOutcome, - worktreePath: child.workdir, - }); - } -} -``` - -- [ ] **Step 5: Run isolated WorkUnit executor tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/work-unit-executor.test.ts src/ingest/stages/stage-3-work-units.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add packages/context/src/ingest/isolated-diff/work-unit-executor.ts \ - packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts \ - packages/context/src/ingest/stages/stage-3-work-units.ts -git commit -m "feat: execute ingest work units in child worktrees" -``` - ---- - -### Task 6: Patch integration and rollback - -**Files:** -- Create: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- Create: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` - -- [ ] **Step 1: Write failing patch integrator tests** - -Create `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`: - -```ts -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../core/index.js'; -import { FileIngestTraceWriter } from '../ingest-trace.js'; -import { integrateWorkUnitPatch } from './patch-integrator.js'; - -async function makeRepo() { - const homeDir = await mkdtemp(join(tmpdir(), 'ktx-integrate-')); - const configDir = join(homeDir, 'config'); - const git = new GitService({ - storage: { configDir, homeDir }, - git: { - userName: 'System User', - userEmail: 'system@example.com', - bootstrapMessage: 'init', - bootstrapAuthor: 'system', - bootstrapAuthorEmail: 'system@example.com', - }, - }); - await git.onModuleInit(); - await mkdir(join(configDir, 'wiki/global'), { recursive: true }); - await writeFile(join(configDir, 'wiki/global/a.md'), 'old\n'); - await git.commitFiles(['wiki/global/a.md'], 'base', 'System User', 'system@example.com'); - return { homeDir, configDir, git, baseSha: await git.revParseHead() }; -} - -describe('integrateWorkUnitPatch', () => { - it('applies a clean patch, runs semantic gates, and commits accepted changes', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - const childDir = join(homeDir, 'child'); - await git.addWorktree(childDir, 'child', baseSha); - const childGit = git.forWorktree(childDir); - await writeFile(join(childDir, 'wiki/global/a.md'), 'new\n'); - await childGit.commitFiles(['wiki/global/a.md'], 'edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/wu.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-1/trace.jsonl'), - jobId: 'job-1', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-1', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree: vi.fn().mockResolvedValue(undefined), - slDisallowed: false, - }); - - expect(result.status).toBe('accepted'); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('new\n'); - await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('patch_apply_finished'); - }); - - it('rolls back and classifies semantic conflicts', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - const childDir = join(homeDir, 'child-semantic'); - await git.addWorktree(childDir, 'child-semantic', baseSha); - const childGit = git.forWorktree(childDir); - await writeFile(join(childDir, 'wiki/global/a.md'), 'bad\n'); - await childGit.commitFiles(['wiki/global/a.md'], 'bad edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/bad.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-2/trace.jsonl'), - jobId: 'job-2', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-bad', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree: vi.fn().mockRejectedValue(new Error('final artifact gates failed')), - slDisallowed: false, - }); - - expect(result.status).toBe('semantic_conflict'); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('old\n'); - }); -}); -``` - -- [ ] **Step 2: Run failing patch integrator tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: FAIL because `patch-integrator.ts` does not exist. - -- [ ] **Step 3: Add patch integrator** - -Create `packages/context/src/ingest/isolated-diff/patch-integrator.ts`: - -```ts -import { readFile } from 'node:fs/promises'; -import type { GitService } from '../../core/index.js'; -import type { IngestTraceWriter } from '../ingest-trace.js'; -import { traceTimed } from '../ingest-trace.js'; -import { assertPatchAllowedForWorkUnit } from './git-patch.js'; - -export type PatchIntegrationResult = - | { status: 'accepted'; commitSha: string; touchedPaths: string[] } - | { status: 'textual_conflict'; reason: string; touchedPaths: string[] } - | { status: 'semantic_conflict'; reason: string; touchedPaths: string[] }; - -export interface IntegrateWorkUnitPatchInput { - unitKey: string; - patchPath: string; - integrationGit: GitService; - trace: IngestTraceWriter; - author: { name: string; email: string }; - slDisallowed: boolean; - validateAppliedTree(touchedPaths: string[]): Promise; -} - -export async function integrateWorkUnitPatch(input: IntegrateWorkUnitPatchInput): Promise { - const preApplyHead = await input.integrationGit.revParseHead(); - const patch = await readFile(input.patchPath, 'utf-8'); - const touched = assertPatchAllowedForWorkUnit({ - unitKey: input.unitKey, - patch, - slDisallowed: input.slDisallowed, - }); - const touchedPaths = touched.map((entry) => entry.path); - - try { - await traceTimed(input.trace, 'integration', 'patch_apply', { unitKey: input.unitKey, patchPath: input.patchPath, touchedPaths }, async () => { - await input.integrationGit.applyPatchFile3WayIndex(input.patchPath); - await input.integrationGit.assertWorktreeClean(); - }); - } catch (error) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - await input.trace.event('error', 'integration', 'patch_textual_conflict', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason: error instanceof Error ? error.message : String(error), - }); - return { - status: 'textual_conflict', - reason: error instanceof Error ? error.message : String(error), - touchedPaths, - }; - } - - try { - await traceTimed(input.trace, 'integration', 'semantic_gate', { unitKey: input.unitKey, touchedPaths }, async () => { - await input.validateAppliedTree(touchedPaths); - }); - } catch (error) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - await input.trace.event('error', 'integration', 'patch_semantic_conflict', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason: error instanceof Error ? error.message : String(error), - }); - return { - status: 'semantic_conflict', - reason: error instanceof Error ? error.message : String(error), - touchedPaths, - }; - } - - const commit = await input.integrationGit.commitStaged( - `ingest: accept WorkUnit ${input.unitKey}`, - input.author.name, - input.author.email, - ); - await input.trace.event('debug', 'integration', 'patch_accepted', { - unitKey: input.unitKey, - commitSha: commit.commitHash, - touchedPaths, - }); - return { status: 'accepted', commitSha: commit.commitHash, touchedPaths }; -} -``` - -- [ ] **Step 4: Run patch integrator tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts -git commit -m "feat: integrate isolated work unit patches" -``` - ---- - -### Task 7: Runner-owned isolated-diff execution path - -**Files:** -- Modify: `packages/context/src/ingest/types.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Add deterministic projection hook to SourceAdapter** - -In `packages/context/src/ingest/types.ts`, add these interfaces before -`SourceAdapter`: - -```ts -export interface DeterministicProjectionContext { - connectionId: string; - sourceKey: string; - syncId: string; - jobId: string; - runId: string; - stagedDir: string; - workdir: string; - parseArtifacts?: unknown; -} - -export interface ProjectionResult { - warnings: string[]; - errors: string[]; - touchedSources: Array<{ connectionId: string; sourceName: string }>; - changedWikiPageKeys: string[]; - result?: unknown; -} -``` - -Then add the optional adapter method: - -```ts - project?(ctx: DeterministicProjectionContext): Promise; -``` - -Keep existing adapter fields unchanged. - -- [ ] **Step 2: Add isolated-diff exports** - -In `packages/context/src/ingest/index.ts`, export the new modules: - -```ts -export * from './ingest-trace.js'; -export * from './artifact-gates.js'; -export * from './wiki-body-refs.js'; -export * from './isolated-diff/git-patch.js'; -export * from './isolated-diff/work-unit-executor.js'; -export * from './isolated-diff/patch-integrator.js'; -``` - -- [ ] **Step 3: Refactor shared runner helpers** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add imports: - -```ts -import { validateFinalIngestArtifacts, validateProvenanceRawPaths } from './artifact-gates.js'; -import { FileIngestTraceWriter, type IngestTraceWriter, traceTimed } from './ingest-trace.js'; -import { integrateWorkUnitPatch } from './isolated-diff/patch-integrator.js'; -import { runIsolatedWorkUnit } from './isolated-diff/work-unit-executor.js'; -``` - -Add these private helpers inside `IngestBundleRunner`: - -```ts - private isIsolatedDiffEnabled(sourceKey: string): boolean { - return (this.deps.settings.isolatedDiffSourceKeys ?? []).includes(sourceKey); - } - - private createTrace(job: IngestBundleJob): IngestTraceWriter { - return new FileIngestTraceWriter({ - tracePath: this.deps.storage.resolveTracePath(job.jobId), - jobId: job.jobId, - connectionId: job.connectionId, - sourceKey: job.sourceKey, - level: this.deps.settings.ingestTraceLevel ?? 'debug', - }); - } - - private wikiPageKeysFromPaths(paths: string[]): string[] { - return [ - ...new Set( - paths - .filter((path) => path.startsWith('wiki/global/') && path.endsWith('.md')) - .map((path) => path.slice('wiki/global/'.length, -'.md'.length)), - ), - ].sort(); - } - - private touchedSlSourcesFromPaths(paths: string[]): TouchedSlSource[] { - return paths - .filter((path) => path.startsWith('semantic-layer/') && path.endsWith('.yaml') && !path.includes('/_schema/')) - .map((path) => { - const [, connectionId, fileName] = path.split('/'); - return { connectionId: connectionId ?? '', sourceName: (fileName ?? '').replace(/\.yaml$/, '') }; - }) - .filter((source) => source.connectionId.length > 0 && source.sourceName.length > 0); - } -``` - -- [ ] **Step 4: Add isolated branch after planning** - -In `runInner()`, create the trace immediately after `syncId`: - -```ts - const trace = this.createTrace(job); - await trace.event('info', 'run', 'ingest_started', { - trigger: job.trigger, - bundleRefKind: job.bundleRef.kind, - }); -``` - -After `runs.create()`, bind run and sync context: - -```ts - const runTrace = trace.withContext({ runId: runRow.id, syncId }); - await runTrace.event('debug', 'snapshot', 'input_snapshot', { - baseSha, - stagedDir, - rawFileCount: currentHashes.size, - rawDirInWorktree, - diffSummary, - scopeFingerprint: scopeDescriptor?.fingerprint ?? null, - }); -``` - -After `workUnits` are planned and `stageIndex` is initialized, branch: - -```ts - const isolatedDiffEnabled = !overrideReport && this.isIsolatedDiffEnabled(job.sourceKey); - const isolatedDiffSummary = { - enabled: isolatedDiffEnabled, - integrationWorktreePath: isolatedDiffEnabled ? sessionWorktree.workdir : undefined, - ingestionBaseSha: undefined as string | undefined, - projectionSha: null as string | null, - acceptedPatches: 0, - textualConflicts: 0, - semanticConflicts: 0, - }; -``` - -Replace only the current `if (!overrideReport) { ...run work units... }` block -with a two-path branch: - -```ts - if (!overrideReport && isolatedDiffEnabled) { - await runTrace.event('info', 'routing', 'isolated_diff_enabled', { - sourceKey: job.sourceKey, - workUnitCount: workUnits.length, - integrationWorktreePath: sessionWorktree.workdir, - }); - - let projectionTouchedSources: TouchedSlSource[] = []; - let projectionChangedWikiPageKeys: string[] = []; - if (adapter.project) { - const projection = await traceTimed( - runTrace, - 'projection', - 'deterministic_projection', - { sourceKey: job.sourceKey }, - () => - adapter.project!({ - connectionId: job.connectionId, - sourceKey: job.sourceKey, - syncId, - jobId: job.jobId, - runId: runRow.id, - stagedDir, - workdir: sessionWorktree.workdir, - parseArtifacts, - }), - ); - if (projection.errors.length > 0) { - await this.deps.runs.markFailed(runRow.id); - throw new Error(`deterministic projection failed: ${projection.errors.join('; ')}`); - } - projectionTouchedSources = projection.touchedSources; - projectionChangedWikiPageKeys = projection.changedWikiPageKeys; - const projectionCommit = await sessionWorktree.git.commitStaged( - `ingest(${job.sourceKey}): deterministic projection syncId=${syncId}`, - this.deps.storage.systemGitAuthor.name, - this.deps.storage.systemGitAuthor.email, - ); - isolatedDiffSummary.projectionSha = projectionCommit.created ? projectionCommit.commitHash : null; - } - - const ingestionBaseSha = await sessionWorktree.git.revParseHead(); - isolatedDiffSummary.ingestionBaseSha = ingestionBaseSha; - const patchDir = join(this.deps.storage.homeDir, 'ingest-patches', job.jobId); - const workUnitSettings = { - maxConcurrency: this.deps.settings.workUnitMaxConcurrency ?? 1, - stepBudget: this.deps.settings.workUnitStepBudget ?? 40, - failureMode: this.deps.settings.workUnitFailureMode ?? 'continue', - }; - const limitWorkUnit = pLimit(workUnitSettings.maxConcurrency); - const workUnitOutcomesByIndex: WorkUnitOutcome[] = []; - let completedWorkUnits = 0; - - await Promise.all( - workUnits.map((wu, index) => - limitWorkUnit(async () => { - const outcome = await runIsolatedWorkUnit({ - unitIndex: index, - ingestionBaseSha, - sessionWorktreeService: this.deps.sessionWorktreeService, - patchDir, - trace: runTrace, - workUnit: wu, - run: async (child) => { - const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir); - const scopedSemanticLayerService = this.deps.semanticLayerService.forWorktree(child.workdir); - return this.runWorkUnitInWorktree({ - job, - wu, - worktree: child, - stagedDir, - contextReport, - ingestToolMetadata, - slConnectionIds, - wikiIndex, - slIndex, - priorProvenance: await this.deps.provenance.findLatestArtifactsForRawPaths( - job.connectionId, - job.sourceKey, - wu.rawFiles, - ), - scopedWikiService, - scopedSemanticLayerService, - baseFraming, - skillsPrompt, - canonicalPins, - workUnitSettings, - transcriptDir, - transcriptSummaries, - recordTranscriptEntry, - stageIndex, - currentTableExists: (tableRef) => - this.tableRefExistsInSemanticLayer(scopedSemanticLayerService, slConnectionIds, tableRef), - onStepFinish: ({ stepIndex, stepBudget }) => { - memoryFlow?.emit({ type: 'work_unit_step', unitKey: wu.unitKey, stepIndex, stepBudget }); - }, - }); - }, - }); - workUnitOutcomesByIndex[index] = outcome; - memoryFlow?.emit({ - type: 'work_unit_finished', - unitKey: outcome.unitKey, - status: outcome.status, - ...(outcome.reason ? { reason: outcome.reason } : {}), - }); - completedWorkUnits += 1; - await stage3?.updateProgress( - completedWorkUnits / workUnits.length, - `${completedWorkUnits} of ${workUnits.length} work units complete`, - ); - }), - ), - ); - - workUnitOutcomes.push(...workUnitOutcomesByIndex.filter((outcome): outcome is WorkUnitOutcome => Boolean(outcome))); - failedWorkUnits.push(...workUnitOutcomes.filter((outcome) => outcome.status === 'failed').map((outcome) => outcome.unitKey)); - stageIndex.workUnits = workUnitOutcomes.map((o) => ({ - unitKey: o.unitKey, - rawFiles: workUnits.find((w) => w.unitKey === o.unitKey)?.rawFiles ?? [], - status: o.status, - reason: o.reason, - actions: o.actions, - touchedSlSources: o.touchedSlSources, - slDisallowed: o.slDisallowed, - slDisallowedReason: o.slDisallowedReason, - })); - - for (const [index, outcome] of workUnitOutcomes.entries()) { - if (outcome.status !== 'success' || !outcome.patchPath) { - continue; - } - const wu = workUnits[index]!; - const integration = await integrateWorkUnitPatch({ - unitKey: outcome.unitKey, - patchPath: outcome.patchPath, - integrationGit: sessionWorktree.git, - trace: runTrace, - author: this.deps.storage.systemGitAuthor, - slDisallowed: wu.slDisallowed === true, - validateAppliedTree: async (touchedPaths) => { - await validateFinalIngestArtifacts({ - connectionIds: slConnectionIds, - changedWikiPageKeys: this.wikiPageKeysFromPaths(touchedPaths), - touchedSlSources: this.touchedSlSourcesFromPaths(touchedPaths), - wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - validateTouchedSources: (touched) => - validateWuTouchedSources({ - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - connections: this.deps.connections, - configService: sessionWorktree.config, - gitService: sessionWorktree.git, - slSourcesRepository: this.deps.slSourcesRepository, - probeRowCount: this.deps.settings.probeRowCount, - slValidator: this.deps.slValidator, - }, touched), - tableExists: (connectionId, tableRef) => - this.tableRefExistsInSemanticLayer( - this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - [connectionId], - tableRef, - ), - }); - }, - }); - if (integration.status === 'textual_conflict') { - isolatedDiffSummary.textualConflicts += 1; - await this.deps.runs.markFailed(runRow.id); - cleanupOutcome = 'conflict'; - throw new Error(`isolated diff textual conflict in ${outcome.unitKey}: ${integration.reason}`); - } - if (integration.status === 'semantic_conflict') { - isolatedDiffSummary.semanticConflicts += 1; - await this.deps.runs.markFailed(runRow.id); - cleanupOutcome = 'conflict'; - throw new Error(`isolated diff semantic conflict in ${outcome.unitKey}: ${integration.reason}`); - } - isolatedDiffSummary.acceptedPatches += 1; - } - - await validateFinalIngestArtifacts({ - connectionIds: slConnectionIds, - changedWikiPageKeys: [ - ...new Set([ - ...projectionChangedWikiPageKeys, - ...workUnitOutcomes.flatMap((outcome) => outcome.patchTouchedPaths ?? []).flatMap((path) => this.wikiPageKeysFromPaths([path])), - ]), - ], - touchedSlSources: [ - ...projectionTouchedSources, - ...workUnitOutcomes.flatMap((outcome) => outcome.touchedSlSources), - ], - wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - validateTouchedSources: (touched) => - validateWuTouchedSources({ - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - connections: this.deps.connections, - configService: sessionWorktree.config, - gitService: sessionWorktree.git, - slSourcesRepository: this.deps.slSourcesRepository, - probeRowCount: this.deps.settings.probeRowCount, - slValidator: this.deps.slValidator, - }, touched), - tableExists: (connectionId, tableRef) => - this.tableRefExistsInSemanticLayer( - this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - [connectionId], - tableRef, - ), - }); - } else if (!overrideReport) { - await runTrace.event('info', 'routing', 'shared_worktree_path_enabled', { sourceKey: job.sourceKey }); - // Keep the existing shared-worktree WorkUnit block here unchanged. - } -``` - -Extract the existing inner `runSingleWorkUnit()` implementation into a private -method named `runWorkUnitInWorktree()` before this replacement. Its code is the -current body of `runSingleWorkUnit()` with these explicit parameters: - -```ts - private async runWorkUnitInWorktree(input: { - job: IngestBundleJob; - wu: WorkUnit; - worktree: IngestSessionWorktree; - stagedDir: string; - contextReport: ContextEvidenceIndexSummary | null; - ingestToolMetadata: { runId: string; jobId: string; syncId: string; sourceKey: string }; - slConnectionIds: string[]; - wikiIndex: string; - slIndex: string; - priorProvenance: Map; - scopedWikiService: ReturnType; - scopedSemanticLayerService: ReturnType; - baseFraming: string; - skillsPrompt: string; - canonicalPins: CanonicalPin[]; - workUnitSettings: { maxConcurrency: number; stepBudget: number; failureMode: 'abort' | 'continue' }; - transcriptDir: string; - transcriptSummaries: Map; - recordTranscriptEntry(path: string): (entry: ToolCallLogEntry) => void; - stageIndex: StageIndex; - currentTableExists(tableRef: string): Promise; - onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void; - }): Promise -``` - -The method must preserve the current tool sessions, transcript wrapping, skill -loading behavior, unmapped fallback behavior, `validateWikiRefs`, and -`validateTouchedSources`. The only value changes are: - -- Use `input.worktree.workdir`, `input.worktree.git`, and - `input.worktree.config`. -- Use `input.scopedWikiService` and `input.scopedSemanticLayerService`. -- Use `input.priorProvenance` instead of loading it inside the method. -- Use `input.onStepFinish`. - -- [ ] **Step 5: Add report trace and isolated summary** - -In the final `reportBody`, add: - -```ts - tracePath: runTrace.tracePath, - isolatedDiff: isolatedDiffEnabled ? isolatedDiffSummary : undefined, -``` - -Before provenance insertion, replace unknown-hash fallback with validation: - -```ts - validateProvenanceRawPaths({ - rows: provenanceRows, - currentRawPaths: new Set(currentHashes.keys()), - deletedRawPaths: new Set(eviction?.deletedRawPaths ?? []), - }); -``` - -Then change: - -```ts -const hash = currentHashes.get(rawPath) ?? 'unknown'; -``` - -to: - -```ts -const hash = currentHashes.get(rawPath) ?? ''; -``` - -for action and artifact-resolution provenance. The validation above guarantees -that non-eviction rows from current actions have a current hash. - -At the end of a successful run, before `return`, add: - -```ts - await runTrace.event('info', 'run', 'ingest_finished', { - status: 'completed', - commitSha, - failedWorkUnits, - tracePath: runTrace.tracePath, - }); -``` - -In the outer `catch` path in `run()`, add a trace event if `runInner()` throws -after trace creation by wrapping `runInner()` errors inside `runInner()`: - -```ts - } catch (error) { - await trace.event('error', 'run', 'ingest_failed', { - tracePath: trace.tracePath, - }, error); - throw error; - } -``` - -Place that catch around the body of `runInner()` after `const trace = -this.createTrace(job);`. - -- [ ] **Step 6: Run focused runner tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-bundle.runner.test.ts \ - src/ingest/ingest-trace.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add packages/context/src/ingest/types.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/index.ts -git commit -m "feat: route selected ingest sources through isolated diffs" -``` - ---- - -### Task 8: V1 regression coverage and Metabase rollout - -**Files:** -- Create: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` - -- [ ] **Step 1: Write isolated-diff regression tests** - -Create `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -with these six tests: - -```ts -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import { GitService, SessionWorktreeService } from '../core/index.js'; -import { LocalGitFileStore } from '../project/local-git-file-store.js'; -import { addTouchedSlSource } from '../tools/index.js'; -import { IngestBundleRunner } from './ingest-bundle.runner.js'; -import type { IngestBundleRunnerDeps } from './ports.js'; - -async function makeRealGitRuntime() { - const homeDir = await mkdtemp(join(tmpdir(), 'ktx-isolated-runner-')); - const configDir = join(homeDir, 'config'); - const git = new GitService({ - storage: { configDir, homeDir }, - git: { - userName: 'System User', - userEmail: 'system@example.com', - bootstrapMessage: 'init', - bootstrapAuthor: 'system', - bootstrapAuthorEmail: 'system@example.com', - }, - }); - await git.onModuleInit(); - const configService = new LocalGitFileStore(configDir); - const sessionWorktreeService = new SessionWorktreeService({ - coreConfig: { - storage: { configDir, homeDir }, - git: { - userName: 'System User', - userEmail: 'system@example.com', - bootstrapMessage: 'init', - bootstrapAuthor: 'system', - bootstrapAuthorEmail: 'system@example.com', - }, - }, - gitService: git, - configService, - }); - return { homeDir, configDir, git, configService, sessionWorktreeService }; -} - -function makeDeps(runtime: Awaited>) { - const adapter = { - source: 'metabase', - skillNames: [], - detect: vi.fn().mockResolvedValue(true), - chunk: vi.fn().mockResolvedValue({ - workUnits: [ - { unitKey: 'card-wiki', rawFiles: ['cards/wiki.json'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }, - ], - }), - }; - const scopedWikiService = { - readPage: vi.fn(async (_scope: string, _scopeId: string | null, key: string) => { - const path = join(runtime.configDir, 'wiki/global', `${key}.md`); - const raw = await readFile(path, 'utf-8').catch(() => null); - if (!raw) return null; - const [, yaml = '', content = ''] = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(raw) ?? []; - const slRefs = /sl_refs:\n((?: - .+\n?)*)/.exec(yaml)?.[1]?.split('\n').map((line) => line.trim().replace(/^- /, '')).filter(Boolean) ?? []; - return { pageKey: key, frontmatter: { summary: key, usage_mode: 'auto', sl_refs: slRefs }, content: content.trim() }; - }), - listPageKeys: vi.fn().mockResolvedValue(['account-segments']), - }; - const semanticLayerService = { - forWorktree: vi.fn(() => semanticLayerService), - loadAllSources: vi.fn(async () => { - const raw = await readFile(join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), 'utf-8').catch(() => ''); - const hasCents = raw.includes('total_contract_arr_cents'); - return { - sources: [ - { - name: 'mart_account_segments', - grain: ['account_id'], - columns: [{ name: 'account_id', type: 'string' }], - joins: [], - measures: [{ name: hasCents ? 'total_contract_arr_cents' : 'total_contract_arr', expr: 'sum(contract_arr)' }], - table: 'analytics.mart_account_segments', - }, - ], - loadErrors: [], - }; - }), - listFilesForConnection: vi.fn().mockResolvedValue(['mart_account_segments.yaml']), - }; - const deps: IngestBundleRunnerDeps = { - runs: { create: vi.fn().mockResolvedValue({ id: 'run-1' }), markCompleted: vi.fn(), markFailed: vi.fn() }, - provenance: { insertMany: vi.fn(), findLatestHashesForCompletedSyncs: vi.fn().mockResolvedValue(new Map()), findLatestArtifactsForRawPaths: vi.fn().mockResolvedValue(new Map()) }, - reports: { create: vi.fn().mockResolvedValue({ id: 'report-1' }), findByJobId: vi.fn().mockResolvedValue(null), markSuperseded: vi.fn() }, - canonicalPins: { listPins: vi.fn().mockResolvedValue([]) }, - registry: { get: vi.fn().mockReturnValue(adapter), register: vi.fn(), has: vi.fn(), list: vi.fn() }, - diffSetService: { compute: vi.fn().mockResolvedValue({ added: ['cards/wiki.json', 'cards/source.json'], modified: [], deleted: [], unchanged: [] }) }, - sessionWorktreeService: runtime.sessionWorktreeService, - agentRunner: { runLoop: vi.fn() }, - gitService: runtime.git, - lockingService: { withLock: vi.fn(async (_key, fn) => fn()) }, - storage: { - homeDir: join(runtime.configDir, '.ktx'), - systemGitAuthor: { name: 'KTX Test', email: 'system@ktx.local' }, - resolveUploadDir: (id) => join(runtime.homeDir, 'upload', id), - resolvePullDir: (id) => join(runtime.homeDir, 'pull', id), - resolveTranscriptDir: (id) => join(runtime.configDir, '.ktx/ingest-transcripts', id), - resolveTracePath: (id) => join(runtime.configDir, '.ktx/ingest-traces', id, 'trace.jsonl'), - }, - settings: { memoryIngestionModel: 'test', probeRowCount: 1, isolatedDiffSourceKeys: ['metabase'], ingestTraceLevel: 'trace' }, - skillsRegistry: { listSkills: vi.fn().mockResolvedValue([]), getSkill: vi.fn().mockResolvedValue(null), buildSkillsPrompt: vi.fn().mockReturnValue(''), stripFrontmatter: vi.fn((body) => body) }, - promptService: { loadPrompt: vi.fn().mockResolvedValue('base') }, - wikiService: { forWorktree: vi.fn(() => scopedWikiService), readPage: scopedWikiService.readPage, syncFromCommit: vi.fn() }, - knowledgeIndex: { listPagesForUser: vi.fn().mockResolvedValue([]) }, - knowledgeSlRefs: { syncFromWiki: vi.fn() }, - semanticLayerService: semanticLayerService as never, - slSearchService: { indexSources: vi.fn() }, - slSourcesRepository: {}, - slValidator: { validateSingleSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }) }, - connections: { listEnabledConnections: vi.fn().mockResolvedValue([]), getConnectionById: vi.fn() } as never, - toolsetFactory: { createIngestWuToolset: vi.fn(() => ({ toRuntimeTools: vi.fn(() => ({})) })) }, - commitMessages: { enqueueForExternalCommit: vi.fn() }, - }; - return { deps, adapter }; -} - -describe('IngestBundleRunner isolated diff path', () => { - it('rejects the Metabase stale-measure wiki body regression before squash', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps } = makeDeps(runtime); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.unitKey === 'card-wiki') { - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n', - ); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nARR is `mart_account_segments.total_contract_arr_cents`.\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ target: 'wiki', type: 'created', key: 'account-segments', detail: 'Account segments' }); - currentSession.actions.push({ target: 'sl', type: 'created', key: 'mart_account_segments', detail: 'Cents measure', targetConnectionId: 'warehouse' }); - await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], 'wu wiki', 'KTX Test', 'system@ktx.local'); - } - if (params.telemetryTags.unitKey === 'card-source') { - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ target: 'sl', type: 'updated', key: 'mart_account_segments', detail: 'Dollar measure', targetConnectionId: 'warehouse' }); - await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source', 'KTX Test', 'system@ktx.local'); - } - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async ({ worktreeRoot }: any) => { - const rawDir = join(worktreeRoot, 'raw-sources/warehouse/metabase/s'); - await mkdir(rawDir, { recursive: true }); - await writeFile(join(rawDir, 'wiki.json'), '{}'); - await writeFile(join(rawDir, 'source.json'), '{}'); - return { currentHashes: new Map([['cards/wiki.json', 'h1'], ['cards/source.json', 'h2']]), rawDirInWorktree: 'raw-sources/warehouse/metabase/s' }; - }); - - await expect( - runner.run({ jobId: 'job-1', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/total_contract_arr_cents/); - await expect(readFile(join(runtime.configDir, '.ktx/ingest-traces/job-1/trace.jsonl'), 'utf-8')).resolves.toContain('patch_semantic_conflict'); - expect(deps.gitService.squashMergeIntoMain).toBeDefined(); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); - -}); -``` - -Add these five additional `it()` blocks inside the same `describe()` block. -They use the same `makeRealGitRuntime()` and `makeDeps()` helpers from the -first test: - -```ts - it('accepts two isolated work units that edit different wiki pages', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { unitKey: 'page-a', rawFiles: ['pages/a.json'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'page-b', rawFiles: ['pages/b.json'], peerFileIndex: [], dependencyPaths: [] }, - ], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const unitKey = params.telemetryTags.unitKey; - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, `wiki/global/${unitKey}.md`), - `---\nsummary: ${unitKey}\nusage_mode: auto\n---\n\n${unitKey}\n`, - ); - currentSession.actions.push({ target: 'wiki', type: 'created', key: unitKey, detail: unitKey }); - await currentSession.gitService.commitFiles([`wiki/global/${unitKey}.md`], `wu ${unitKey}`, 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' }; - }) as never; - const runner = new IngestBundleRunner(deps); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async ({ worktreeRoot }: any) => { - const rawDir = join(worktreeRoot, 'raw-sources/warehouse/metabase/s'); - await mkdir(rawDir, { recursive: true }); - await writeFile(join(rawDir, 'a.json'), '{}'); - await writeFile(join(rawDir, 'b.json'), '{}'); - return { currentHashes: new Map([['pages/a.json', 'h1'], ['pages/b.json', 'h2']]), rawDirInWorktree: 'raw-sources/warehouse/metabase/s' }; - }); - - const result = await runner.run({ jobId: 'job-clean', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }); - expect(result.failedWorkUnits).toEqual([]); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-clean/trace.jsonl'), 'utf-8'); - expect(trace.match(/patch_accepted/g)).toHaveLength(2); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); - - it('classifies same-source patch application failure as a textual conflict', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { unitKey: 'orders-a', rawFiles: ['orders/a.json'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'orders-b', rawFiles: ['orders/b.json'], peerFileIndex: [], dependencyPaths: [] }, - ], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const suffix = params.telemetryTags.unitKey === 'orders-a' ? 'a' : 'b'; - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse/orders.yaml'), - `name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures:\n - name: order_count_${suffix}\n expr: count(*)\n`, - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'orders'); - currentSession.actions.push({ target: 'sl', type: 'updated', key: 'orders', detail: suffix, targetConnectionId: 'warehouse' }); - await currentSession.gitService.commitFiles(['semantic-layer/warehouse/orders.yaml'], `wu ${suffix}`, 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' }; - }) as never; - const runner = new IngestBundleRunner(deps); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async ({ worktreeRoot }: any) => { - const rawDir = join(worktreeRoot, 'raw-sources/warehouse/metabase/s'); - await mkdir(rawDir, { recursive: true }); - return { currentHashes: new Map([['orders/a.json', 'h1'], ['orders/b.json', 'h2']]), rawDirInWorktree: 'raw-sources/warehouse/metabase/s' }; - }); - - await expect( - runner.run({ jobId: 'job-text-conflict', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/isolated diff textual conflict/); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); - - it('makes deterministic projection visible to child worktrees before WorkUnit synthesis', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'wiki-projected', rawFiles: ['projected/wiki.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - adapter.project = vi.fn(async ({ workdir }) => { - await mkdir(join(workdir, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(workdir, 'semantic-layer/warehouse/projected_orders.yaml'), - 'name: projected_orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures:\n - name: order_count\n expr: count(*)\n', - ); - return { warnings: [], errors: [], touchedSources: [{ connectionId: 'warehouse', sourceName: 'projected_orders' }], changedWikiPageKeys: [], result: { sourcesCreated: 1 } }; - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - await expect( - readFile(join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse/projected_orders.yaml'), 'utf-8'), - ).resolves.toContain('order_count'); - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global/projected-orders.md'), - '---\nsummary: Projected orders\nusage_mode: auto\nsl_refs:\n - projected_orders\n---\n\nBad ref `projected_orders.missing_measure`.\n', - ); - currentSession.actions.push({ target: 'wiki', type: 'created', key: 'projected-orders', detail: 'Projected orders' }); - await currentSession.gitService.commitFiles(['wiki/global/projected-orders.md'], 'wu projected wiki', 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' }; - }) as never; - const runner = new IngestBundleRunner(deps); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async ({ worktreeRoot }: any) => { - const rawDir = join(worktreeRoot, 'raw-sources/warehouse/metabase/s'); - await mkdir(rawDir, { recursive: true }); - return { currentHashes: new Map([['projected/wiki.json', 'h1']]), rawDirInWorktree: 'raw-sources/warehouse/metabase/s' }; - }); - - await expect( - runner.run({ jobId: 'job-projection', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/projected_orders\.missing_measure/); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); - - it('rejects Notion-style changed wiki pages with invalid sl_refs', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'notion-page', rawFiles: ['pages/notion.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'wiki/global/notion-page.md'), - '---\nsummary: Notion page\nusage_mode: auto\nsl_refs:\n - missing_source\n---\n\nBody\n', - ); - currentSession.actions.push({ target: 'wiki', type: 'created', key: 'notion-page', detail: 'Notion page' }); - await currentSession.gitService.commitFiles(['wiki/global/notion-page.md'], 'wu notion', 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' }; - }) as never; - const runner = new IngestBundleRunner(deps); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async () => ({ currentHashes: new Map([['pages/notion.json', 'h1']]), rawDirInWorktree: 'raw-sources/warehouse/metabase/s' })); - - await expect( - runner.run({ jobId: 'job-invalid-slrefs', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/unknown sl_refs entry missing_source/); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); - - it('rejects slDisallowed patches that touch semantic-layer files', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'lookml-mismatch', rawFiles: ['views/orders.lkml'], peerFileIndex: [], dependencyPaths: [], slDisallowed: true, slDisallowedReason: 'lookml_connection_mismatch' }], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - await mkdir(join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(currentSession.configService.rootDir ?? runtime.configDir, 'semantic-layer/warehouse/orders.yaml'), - 'name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures: []\n', - ); - currentSession.actions.push({ target: 'sl', type: 'created', key: 'orders', detail: 'forbidden', targetConnectionId: 'warehouse' }); - await currentSession.gitService.commitFiles(['semantic-layer/warehouse/orders.yaml'], 'forbidden sl', 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' }; - }) as never; - const runner = new IngestBundleRunner(deps); - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async () => ({ currentHashes: new Map([['views/orders.lkml', 'h1']]), rawDirInWorktree: 'raw-sources/warehouse/metabase/s' })); - - await expect( - runner.run({ jobId: 'job-sl-disallowed', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/slDisallowed WorkUnit lookml-mismatch touched semantic-layer\/warehouse\/orders.yaml/); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Run failing isolated regression tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: FAIL until the runner branch from Task 7 is complete. - -- [ ] **Step 3: Confirm Metabase remains privately allowlisted** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, verify settings still -include: - -```ts - isolatedDiffSourceKeys: ['metabase'], -``` - -Do not add a public `executionMode`, `planningStrategy`, or `conflictPolicy` -adapter field. Do not add a CLI flag. - -- [ ] **Step 4: Run isolated regression tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. The trace file assertions must prove that the run records input -snapshot, routing decision, WorkUnit child creation, patch collection, patch -application, semantic gate result, rollback/conflict events for failing cases, -and final run outcome. - -- [ ] **Step 5: Commit** - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/local-bundle-runtime.ts -git commit -m "test: cover isolated diff ingestion regressions" -``` - ---- - -### Task 9: Final verification and observability acceptance - -**Files:** -- Modify: no source files unless checks identify issues. - -- [ ] **Step 1: Run focused context tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/core/git.service.patch.test.ts \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run package tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. If this produces too much output, capture it: - -```bash -pnpm --filter @ktx/context run test 2>&1 | tee /tmp/ktx-context-isolated-diff-tests.log -``` - -Then inspect the failing section in `/tmp/ktx-context-isolated-diff-tests.log`. - -- [ ] **Step 4: Run dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. Investigate any new Knip findings before adding ignores. - -- [ ] **Step 5: Run pre-commit for changed TypeScript files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/core/git.service.ts \ - packages/context/src/core/git.service.patch.test.ts \ - packages/context/src/ingest/ingest-trace.ts \ - packages/context/src/ingest/ingest-trace.test.ts \ - packages/context/src/ingest/wiki-body-refs.ts \ - packages/context/src/ingest/wiki-body-refs.test.ts \ - packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts \ - packages/context/src/ingest/isolated-diff/git-patch.ts \ - packages/context/src/ingest/isolated-diff/git-patch.test.ts \ - packages/context/src/ingest/isolated-diff/work-unit-executor.ts \ - packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/types.ts \ - packages/context/src/ingest/ports.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/reports.ts \ - packages/context/src/ingest/report-snapshot.ts \ - packages/context/src/ingest/index.ts \ - packages/cli/src/ingest.ts -``` - -Expected: PASS. If `pre-commit` is unavailable or the configured hook -environment cannot run, record the exact error and rely on the focused tests, -type-check, dead-code, and `git diff --check`. - -- [ ] **Step 6: Verify trace usefulness manually** - -Run one isolated regression and inspect the trace: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "rejects the Metabase stale-measure wiki body regression before squash" -``` - -Expected: PASS. Open the test-created -`.ktx/ingest-traces/job-1/trace.jsonl` path printed by the failed-run assertion -or test output. Confirm it includes these events: - -- `ingest_started` -- `input_snapshot` -- `isolated_diff_enabled` -- `work_unit_child_created` -- `work_unit_patch_collected` -- `patch_apply_started` -- `semantic_gate_failed` or `patch_semantic_conflict` -- `ingest_failed` - -The trace must include `jobId`, `runId`, `syncId`, `connectionId`, -`sourceKey`, `unitKey` where applicable, worktree paths, patch paths, touched -paths, durations, error messages, and final status. - -- [ ] **Step 7: Commit final fixes** - -```bash -git status --short -git add packages/context/src packages/cli/src -git commit -m "feat: add isolated diff ingestion v1 core" -``` - ---- - -## Self-review - -Spec coverage: - -- Per-WorkUnit child worktrees, patch proposals, deterministic integration, - `slDisallowed` integration rejection, and fail-fast textual or semantic - conflicts are covered by Tasks 2, 5, 6, 7, and 8. -- The Metabase stale `total_contract_arr_cents` regression is covered by - Task 8. -- Deterministic projection before child worktree creation is covered by Task 7 - and the hybrid projection test in Task 8. -- Final global wiki body, wiki `sl_refs`, semantic-layer, and provenance gates - are covered by Tasks 3, 4, 7, and 8. -- Persistent postmortem observability is covered by Task 1 and required in every - ingestion task's acceptance checks. Task 9 explicitly verifies trace - usefulness from logs alone. - -Placeholder scan: - -- The implementation tasks contain exact file paths, commands, expected - results, and concrete code snippets. -- Task 8 contains concrete regression assertions for the Metabase incident, - clean integration, textual conflict, hybrid projection, invalid `sl_refs`, and - `slDisallowed` rejection. - -Type consistency: - -- `IngestTraceWriter`, `IngestTraceLevel`, `ProjectionResult`, - `DeterministicProjectionContext`, `WorkUnitOutcome.patchPath`, - `patchTouchedPaths`, and `childWorktreePath` are introduced before later - tasks consume them. -- Report fields use `tracePath` and `isolatedDiff` consistently across - `reports.ts`, `report-snapshot.ts`, runner output, and CLI status output. diff --git a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-gates-and-trace-closure.md b/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-gates-and-trace-closure.md deleted file mode 100644 index 850b4dc7..00000000 --- a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-gates-and-trace-closure.md +++ /dev/null @@ -1,1786 +0,0 @@ -# Isolated Diff Ingestion V1 Gates and Trace Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking isolated-diff ingestion gaps so the -actual final integration tree is globally gated and every failed run leaves a -persistent trace and stored failure report that are useful for postmortems. - -**Architecture:** Keep the isolated-diff runner private to the runner-owned -source allowlist, but make its safety boundary match the design: per-patch -gates still run during integration, reconciliation and follow-on deterministic -mutations are diffed, and one final global artifact gate runs after every -mutating integration-stage operation and before squash. Persistent JSONL traces -become the operational source of truth for postmortems, with start/finish/fail -events, timings, state snapshots, conflict classification, and a stored failure -report that lets `ktx ingest status ` surface the trace path even -when the run fails before the normal success report. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, simple-git, existing -`IngestBundleRunner`, `GitService`, `SessionWorktreeService`, -`SemanticLayerService`, `KnowledgeWikiService`, ingest report schemas, and CLI -status rendering. - ---- - -## Audit Summary - -The latest plan and commits implemented the first isolated-diff path and the -focused tests pass: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Current result: `7 passed`, `20 passed`. - -The remaining gaps below are v1-blocking: - -- The isolated branch runs `final_artifact_gates` immediately after accepted - WorkUnit patches, but reconciliation, post-processors, and wiki `sl_refs` - repair can still mutate the integration worktree afterward. The tree that is - squashed is therefore not globally gated after every mutating stage. -- Reconciliation changes are not captured as a diff against the - pre-reconciliation integration `HEAD`, and reconciliation-touched artifacts - are not included in a post-reconciliation artifact gate. -- Wiki frontmatter `sl_refs` validation checks only source existence. It does - not validate measure-level references such as - `mart_account_segments.total_contract_arr_cents`. -- Wiki body reference parsing treats every two-part inline-code token as a - semantic-layer reference, even when the left side is not a visible source. The - spec says those tokens must be ignored unless they name a visible source. -- Semantic-layer final gates validate only touched sources. They do not expand - the touched set to direct declared-join neighbors, including sources joined - from touched sources and sources that join to touched sources. -- `slDisallowed` and patch policy rejections can throw before integration emits - a structured conflict event or stored failure report. -- Failed runs before success-report creation do not leave a stored ingest - report, so `ktx ingest status ` cannot surface the trace path. -- Trace coverage does not yet cover fetch/stage/detect/planning decisions, - reconciliation, post-processing, wiki repair, provenance validation and - insertion, squash, report creation, and failure-report creation with timings - and state needed for postmortem reconstruction. -- Failed child WorkUnit worktrees are preserved with `cleanup('crash')`. The - spec requires child worktrees to be cleaned up after diff, transcript, and - outcome metadata are persisted. Only the integration worktree should be - preserved for version-one resolver conflicts. - -Non-blocking gaps remain after this plan: - -- Migrating Notion, LookML, Looker, dbt, MetricFlow, and historic-SQL direct - durable writes to the isolated path. -- Promoting isolated diffs as the default for all connectors. -- Removing the old shared-worktree WorkUnit execution path. -- Interactive, CLI, or agent-driven conflict resolution. -- Auto-merging semantic conflicts that cannot be proven correct. -- Transitive SQL-projection dependency expansion beyond direct declared joins. -- Moving provenance rows to worktree files. -- Public connector knobs such as `executionMode`, `planningStrategy`, or - `conflictPolicy`. - -## File Structure - -- Modify `packages/context/src/ingest/wiki-body-refs.ts`. - Fix inline-code grammar so unknown two-part tokens are ignored, while - explicit `source:` and `table:` references remain validated. -- Modify `packages/context/src/ingest/wiki-body-refs.test.ts`. - Add regression coverage for ignored non-source two-part tokens. -- Modify `packages/context/src/ingest/artifact-gates.ts`. - Add source/entity frontmatter validation, direct join-neighbor expansion, and - reusable gate-scope helpers. -- Modify `packages/context/src/ingest/artifact-gates.test.ts`. - Cover measure-level `sl_refs`, direct dependency validation, and final body - ref behavior. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Move the final global gate after reconciliation, post-processing, and wiki - ref repair. Add trace events around every meaningful phase, create stored - failure reports, and preserve only the integration worktree on conflicts. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Add regressions for reconciliation-created stale refs, failed-run report - trace surfacing, and trace event completeness. -- Modify `packages/context/src/ingest/isolated-diff/work-unit-executor.ts`. - Stop enforcing patch policy during collection, record patch metadata only, - and always remove child worktrees after outcome metadata is emitted. -- Modify `packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts`. - Cover cleanup on failed WorkUnits. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.ts`. - Classify patch policy rejections as structured textual conflicts and emit - trace events before returning. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`. - Cover `slDisallowed` policy rejection as a traced textual conflict. -- Modify `packages/context/src/ingest/reports.ts`. - Add report-level `status` and `failure` fields. -- Modify `packages/context/src/ingest/report-snapshot.ts`. - Parse the new failure report fields while preserving old reports. -- Modify `packages/context/src/ingest/report-snapshot.test.ts`. - Cover failed report parsing. -- Modify `packages/cli/src/ingest.ts`. - Render failed stored reports as `Status: error` even when no WorkUnit failed, - and keep the trace path near run identifiers. -- Modify `packages/cli/src/ingest.test.ts`. - Cover status output for a failed report with a trace path. -- Modify `docs-site/content/docs/cli-reference/ktx-ingest.mdx`. - Document that failed runs also write stored reports and that trace events - include phase timings, state snapshots, decisions, errors, and final outcome. - ---- - -### Task 1: Correct artifact gate semantics - -**Files:** -- Modify: `packages/context/src/ingest/wiki-body-refs.test.ts` -- Modify: `packages/context/src/ingest/wiki-body-refs.ts` -- Modify: `packages/context/src/ingest/artifact-gates.test.ts` -- Modify: `packages/context/src/ingest/artifact-gates.ts` - -- [ ] **Step 1: Write failing wiki body grammar tests** - -Append these tests inside the existing `describe('wiki body refs', ...)` block -in `packages/context/src/ingest/wiki-body-refs.test.ts`: - -```ts - it('ignores two-part inline code when the source is not visible', async () => { - const invalid = await findInvalidWikiBodyRefs({ - pageKey: 'engineering-notes', - body: [ - 'A version token like `node.v22` is not a semantic-layer reference.', - 'A raw table must use `table:analytics.mart_account_segments`.', - ].join('\n'), - visibleConnectionIds: ['warehouse'], - loadSources: async () => sources, - tableExists: async (_connectionId, tableRef) => tableRef === 'analytics.mart_account_segments', - }); - - expect(invalid).toEqual([]); - }); - - it('still rejects explicit missing source and table references', async () => { - const invalid = await findInvalidWikiBodyRefs({ - pageKey: 'account-segments', - body: [ - '`source:missing_source`', - '`warehouse/source:missing_source`', - '`table:analytics.missing_table`', - ].join('\n'), - visibleConnectionIds: ['warehouse'], - loadSources: async () => sources, - tableExists: async () => false, - }); - - expect(invalid).toEqual([ - 'account-segments: unknown semantic-layer source missing_source', - 'account-segments: unknown semantic-layer source warehouse/missing_source', - 'account-segments: unknown raw table analytics.missing_table', - ]); - }); -``` - -- [ ] **Step 2: Run wiki body tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/wiki-body-refs.test.ts -``` - -Expected: FAIL because `node.v22` is treated as an unknown semantic-layer -source. - -- [ ] **Step 3: Implement the wiki body grammar fix** - -In `packages/context/src/ingest/wiki-body-refs.ts`, replace -`findInvalidWikiBodyRefs()` with this implementation: - -```ts -export async function findInvalidWikiBodyRefs(input: WikiBodyRefValidationInput): Promise { - const errors: string[] = []; - const sourceCache = new Map(); - const loadSources = async (connectionId: string): Promise => { - const cached = sourceCache.get(connectionId); - if (cached) { - return cached; - } - const sources = await input.loadSources(connectionId); - sourceCache.set(connectionId, sources); - return sources; - }; - - const findSource = async ( - connectionIds: string[], - sourceName: string, - ): Promise<{ connectionId: string; source: SemanticLayerSource } | null> => { - for (const connectionId of connectionIds) { - const source = (await loadSources(connectionId)).find((candidate) => candidate.name === sourceName); - if (source) { - return { connectionId, source }; - } - } - return null; - }; - - for (const ref of parseWikiBodyRefs(input.body)) { - const connectionIds = ref.connectionId ? [ref.connectionId] : input.visibleConnectionIds; - if (ref.kind === 'table') { - const found = await Promise.all(connectionIds.map((connectionId) => input.tableExists(connectionId, ref.tableRef))); - if (!found.some(Boolean)) { - errors.push(`${input.pageKey}: unknown raw table ${ref.connectionId ? `${ref.connectionId}/` : ''}${ref.tableRef}`); - } - continue; - } - - const found = await findSource(connectionIds, ref.sourceName); - if (!found) { - if (ref.kind === 'sl_source') { - errors.push( - `${input.pageKey}: unknown semantic-layer source ${ref.connectionId ? `${ref.connectionId}/` : ''}${ref.sourceName}`, - ); - } - continue; - } - - if (ref.kind === 'sl_entity' && !entityNames(found.source).has(ref.entityName)) { - errors.push(`${input.pageKey}: unknown semantic-layer entity ${ref.sourceName}.${ref.entityName}`); - } - } - - return errors; -} -``` - -- [ ] **Step 4: Run wiki body tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/wiki-body-refs.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing artifact gate tests** - -Append these tests inside `describe('artifact gates', ...)` in -`packages/context/src/ingest/artifact-gates.test.ts`: - -```ts - it('fails measure-level wiki frontmatter sl_refs that point at missing entities', async () => { - const wikiService = { - readPage: vi.fn().mockResolvedValue({ - pageKey: 'account-segments', - frontmatter: { - summary: 'Account segments', - usage_mode: 'auto', - sl_refs: ['mart_account_segments.total_contract_arr_cents'], - }, - content: 'ARR uses a renamed measure.', - }), - }; - const semanticLayerService = { - loadAllSources: vi.fn().mockResolvedValue({ - sources: [ - { - name: 'mart_account_segments', - grain: ['account_id'], - columns: [{ name: 'account_id', type: 'string' }], - joins: [], - measures: [{ name: 'total_contract_arr', expr: 'sum(contract_arr)' }], - table: 'analytics.mart_account_segments', - }, - ], - loadErrors: [], - }), - }; - - await expect( - validateFinalIngestArtifacts({ - connectionIds: ['warehouse'], - changedWikiPageKeys: ['account-segments'], - touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'mart_account_segments' }], - wikiService: wikiService as never, - semanticLayerService: semanticLayerService as never, - validateTouchedSources: async () => ({ invalidSources: [], validSources: ['warehouse:mart_account_segments'] }), - tableExists: async () => true, - }), - ).rejects.toThrow(/unknown sl_refs entity mart_account_segments\.total_contract_arr_cents/); - }); - - it('validates direct declared-join neighbors of touched semantic-layer sources', async () => { - const semanticLayerService = { - loadAllSources: vi.fn().mockResolvedValue({ - sources: [ - { - name: 'orders', - grain: ['order_id'], - columns: [{ name: 'order_id', type: 'string' }, { name: 'account_id', type: 'string' }], - joins: [{ to: 'accounts', on: 'orders.account_id = accounts.account_id', relationship: 'many_to_one' }], - measures: [{ name: 'order_count', expr: 'count(*)' }], - }, - { - name: 'accounts', - grain: ['account_id'], - columns: [{ name: 'account_id', type: 'string' }], - joins: [], - measures: [{ name: 'account_count', expr: 'count(*)' }], - }, - { - name: 'segments', - grain: ['segment_id'], - columns: [{ name: 'segment_id', type: 'string' }, { name: 'account_id', type: 'string' }], - joins: [{ to: 'accounts', on: 'segments.account_id = accounts.account_id', relationship: 'many_to_one' }], - measures: [], - }, - ], - loadErrors: [], - }), - }; - const validateTouchedSources = vi.fn().mockResolvedValue({ invalidSources: [], validSources: [] }); - - await validateFinalIngestArtifacts({ - connectionIds: ['warehouse'], - changedWikiPageKeys: [], - touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'accounts' }], - wikiService: { readPage: vi.fn() } as never, - semanticLayerService: semanticLayerService as never, - validateTouchedSources, - tableExists: async () => true, - }); - - expect(validateTouchedSources).toHaveBeenCalledWith([ - { connectionId: 'warehouse', sourceName: 'accounts' }, - { connectionId: 'warehouse', sourceName: 'orders' }, - { connectionId: 'warehouse', sourceName: 'segments' }, - ]); - }); -``` - -- [ ] **Step 6: Run artifact gate tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -``` - -Expected: FAIL because frontmatter entity refs and join-neighbor expansion are -not implemented. - -- [ ] **Step 7: Implement frontmatter entity refs and direct dependency expansion** - -In `packages/context/src/ingest/artifact-gates.ts`, replace the existing -`bareSlRef()` helper and `validateWikiSlRefs()` with this code, then update -`validateFinalIngestArtifacts()` as shown below: - -```ts -function parseSlRef(ref: string): { connectionId: string | null; sourceName: string; entityName: string | null } { - const withoutConnection = ref.includes('/') ? ref.slice(ref.indexOf('/') + 1) : ref; - const connectionId = ref.includes('/') ? ref.slice(0, ref.indexOf('/')) : null; - const [sourceName = '', entityName = null] = withoutConnection.split('.', 2); - return { connectionId, sourceName, entityName }; -} - -function slEntityNames(source: Awaited>['sources'][number]): Set { - return new Set([ - ...(source.measures ?? []).map((measure) => measure.name), - ...(source.columns ?? []).map((column) => column.name), - ...(source.segments ?? []).map((segment) => segment.name), - ]); -} - -function uniqueTouchedSources(sources: TouchedSlSource[]): TouchedSlSource[] { - const seen = new Set(); - const unique: TouchedSlSource[] = []; - for (const source of sources) { - const key = `${source.connectionId}:${source.sourceName}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - unique.push(source); - } - return unique.sort((left, right) => { - const byConnection = left.connectionId.localeCompare(right.connectionId); - return byConnection === 0 ? left.sourceName.localeCompare(right.sourceName) : byConnection; - }); -} - -async function expandTouchedSlSourcesWithDirectJoinNeighbors(input: FinalArtifactGateInput): Promise { - const expanded = [...input.touchedSlSources]; - const touchedByConnection = new Map>(); - for (const source of input.touchedSlSources) { - const bucket = touchedByConnection.get(source.connectionId) ?? new Set(); - bucket.add(source.sourceName); - touchedByConnection.set(source.connectionId, bucket); - } - - for (const connectionId of input.connectionIds) { - const touched = touchedByConnection.get(connectionId); - if (!touched || touched.size === 0) { - continue; - } - const { sources } = await input.semanticLayerService.loadAllSources(connectionId); - for (const source of sources) { - const sourceIsTouched = touched.has(source.name); - if (sourceIsTouched) { - for (const join of source.joins ?? []) { - expanded.push({ connectionId, sourceName: join.to }); - } - } - if ((source.joins ?? []).some((join) => touched.has(join.to))) { - expanded.push({ connectionId, sourceName: source.name }); - } - } - } - - return uniqueTouchedSources(expanded); -} - -async function validateWikiSlRefs(input: FinalArtifactGateInput): Promise { - const errors: string[] = []; - const sourcesByConnection = new Map>['sources']>(); - for (const connectionId of input.connectionIds) { - const { sources } = await input.semanticLayerService.loadAllSources(connectionId); - sourcesByConnection.set(connectionId, sources); - } - - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - for (const ref of page.frontmatter.sl_refs ?? []) { - const parsed = parseSlRef(ref); - const candidateConnections = parsed.connectionId ? [parsed.connectionId] : input.connectionIds; - let source: Awaited>['sources'][number] | undefined; - for (const connectionId of candidateConnections) { - source = sourcesByConnection.get(connectionId)?.find((candidate) => candidate.name === parsed.sourceName); - if (source) { - break; - } - } - if (!source) { - errors.push(`${pageKey}: unknown sl_refs entry ${ref}`); - continue; - } - if (parsed.entityName && !slEntityNames(source).has(parsed.entityName)) { - errors.push(`${pageKey}: unknown sl_refs entity ${ref}`); - } - } - } - return errors; -} -``` - -Then replace the first two lines inside `validateFinalIngestArtifacts()` with: - -```ts - const touchedWithDependencies = await expandTouchedSlSourcesWithDirectJoinNeighbors(input); - const validation = await input.validateTouchedSources(touchedWithDependencies); -``` - -- [ ] **Step 8: Run artifact gate tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts src/ingest/wiki-body-refs.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Commit artifact gate fixes** - -```bash -git add packages/context/src/ingest/wiki-body-refs.ts \ - packages/context/src/ingest/wiki-body-refs.test.ts \ - packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts -git commit -m "fix(ingest): tighten final artifact gates" -``` - -### Task 2: Gate the actual final integration tree - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` - -- [ ] **Step 1: Write failing reconciliation stale-reference regression** - -Append this test to `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -inside the existing `describe('IngestBundleRunner isolated diff path', ...)` -block: - -```ts - it('runs final artifact gates after reconciliation mutates the integration tree', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - if (params.telemetryTags.operationName === 'ingest-bundle-wu') { - await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'mart_account_segments', - detail: 'Source with renamed ARR measure', - targetConnectionId: 'warehouse', - rawPaths: ['cards/source.json'], - }); - await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source', 'KTX Test', 'system@ktx.local'); - } else { - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nReconcile wrote stale ARR `mart_account_segments.total_contract_arr_cents`.\n', - ); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'account-segments', - detail: 'Stale reconcile wiki page', - rawPaths: ['cards/source.json'], - }); - await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'reconcile wiki', 'KTX Test', 'system@ktx.local'); - } - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]); - - await expect( - runner.run({ jobId: 'job-reconcile-stale', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/total_contract_arr_cents/); - - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-reconcile-stale/trace.jsonl'), 'utf-8'); - expect(trace).toContain('reconciliation_finished'); - expect(trace).toContain('final_artifact_gates_failed'); - expect(trace).toContain('ingest_failed'); - expect(await runtime.git.revParseHead()).not.toContain('reconcile wiki'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Run the failing reconciliation regression** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "after reconciliation" -``` - -Expected: FAIL because the current runner gates before reconciliation and then -squashes the invalid reconciled page. - -- [ ] **Step 3: Add final gate scope helpers to the runner** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add these private -helpers after `touchedSlSourcesFromPaths()`: - -```ts - private touchedSlSourcesFromActions(actions: MemoryAction[], fallbackConnectionId: string): TouchedSlSource[] { - return actions - .filter((action) => action.target === 'sl') - .map((action) => ({ - connectionId: actionTargetConnectionId(action, fallbackConnectionId), - sourceName: action.key, - })); - } - - private wikiPageKeysFromActions(actions: MemoryAction[]): string[] { - return actions.filter((action) => action.target === 'wiki').map((action) => action.key); - } - - private uniqueWikiPageKeys(keys: string[]): string[] { - return [...new Set(keys.filter((key) => key.length > 0))].sort(); - } - - private uniqueTouchedSlSources(sources: TouchedSlSource[]): TouchedSlSource[] { - const seen = new Set(); - const unique: TouchedSlSource[] = []; - for (const source of sources) { - const key = `${source.connectionId}:${source.sourceName}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - unique.push(source); - } - return unique.sort((left, right) => { - const byConnection = left.connectionId.localeCompare(right.connectionId); - return byConnection === 0 ? left.sourceName.localeCompare(right.sourceName) : byConnection; - }); - } -``` - -- [ ] **Step 4: Track integration mutations after WorkUnit patch integration** - -In `runInner()` in `packages/context/src/ingest/ingest-bundle.runner.ts`, add -these variables before the Stage 4 reconciliation block: - -```ts - const preReconciliationSha = await sessionWorktree.git.revParseHead(); -``` - -Remove the isolated-branch `traceTimed(... 'final_artifact_gates' ...)` block -that currently runs before the `else if (!overrideReport)` branch ends. Keep -per-patch `validateAppliedTree` in `integrateWorkUnitPatch()` unchanged. - -- [ ] **Step 5: Run the final global gate after reconciliation, post-processing, and repair** - -In `runInner()`, immediately after `wikiSlRefRepairResult = await -repairWikiSlRefs(...)` and before Stage 6 starts, add this block: - -```ts - const postReconciliationSha = await sessionWorktree.git.revParseHead(); - const postReconciliationPaths = - preReconciliationSha && postReconciliationSha && preReconciliationSha !== postReconciliationSha - ? (await sessionWorktree.git.diffNameStatus(preReconciliationSha, postReconciliationSha)).map((entry) => entry.path) - : []; - const finalChangedWikiPageKeys = this.uniqueWikiPageKeys([ - ...(isolatedDiffEnabled ? projectionChangedWikiPageKeys : []), - ...workUnitOutcomes - .flatMap((outcome) => outcome.patchTouchedPaths ?? []) - .flatMap((path) => this.wikiPageKeysFromPaths([path])), - ...this.wikiPageKeysFromActions(reconcileActions), - ...postReconciliationPaths.flatMap((path) => this.wikiPageKeysFromPaths([path])), - ...wikiSlRefRepairResult.repairs - .filter((repair) => repair.scope === 'GLOBAL') - .map((repair) => repair.pageKey), - ]); - const finalTouchedSlSources = this.uniqueTouchedSlSources([ - ...(isolatedDiffEnabled ? projectionTouchedSources : []), - ...workUnitOutcomes.flatMap((outcome) => outcome.touchedSlSources), - ...this.touchedSlSourcesFromActions(reconcileActions, job.connectionId), - ...this.touchedSlSourcesFromPaths(postReconciliationPaths), - ...(postProcessorOutcome?.touchedSources ?? []), - ]); - - await traceTimed( - runTrace, - 'final_gates', - 'final_artifact_gates', - { - changedWikiPageKeys: finalChangedWikiPageKeys, - touchedSlSources: finalTouchedSlSources, - preReconciliationSha, - postReconciliationSha, - postReconciliationPaths, - reconciliationActionCount: reconcileActions.length, - wikiSlRefRepairCount: wikiSlRefRepairResult.repairs.length, - }, - async () => { - await validateFinalIngestArtifacts({ - connectionIds: repairConnectionIds, - changedWikiPageKeys: finalChangedWikiPageKeys, - touchedSlSources: finalTouchedSlSources, - wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - validateTouchedSources: (touched) => - validateWuTouchedSources( - { - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - connections: this.deps.connections, - configService: sessionWorktree.config, - gitService: sessionWorktree.git, - slSourcesRepository: this.deps.slSourcesRepository, - probeRowCount: this.deps.settings.probeRowCount, - slValidator: this.deps.slValidator, - }, - touched, - ), - tableExists: (connectionId, tableRef) => - this.tableRefExistsInSemanticLayer( - this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - [connectionId], - tableRef, - ), - }); - }, - ); -``` - -Use the existing `projectionTouchedSources` and `projectionChangedWikiPageKeys` -variables from the isolated branch by declaring them before the branch instead -of inside it: - -```ts - let projectionTouchedSources: TouchedSlSource[] = []; - let projectionChangedWikiPageKeys: string[] = []; -``` - -- [ ] **Step 6: Run the reconciliation regression** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "after reconciliation" -``` - -Expected: PASS. - -- [ ] **Step 7: Run isolated runner tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit final gate ordering** - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "fix(ingest): gate isolated final integration tree" -``` - -### Task 3: Complete persistent traces and failed-run surfacing - -**Files:** -- Modify: `packages/context/src/ingest/reports.ts` -- Modify: `packages/context/src/ingest/report-snapshot.ts` -- Modify: `packages/context/src/ingest/report-snapshot.test.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Modify: `packages/cli/src/ingest.ts` -- Modify: `packages/cli/src/ingest.test.ts` -- Modify: `docs-site/content/docs/cli-reference/ktx-ingest.mdx` - -- [ ] **Step 1: Add failing report schema coverage for failed runs** - -Append this test to `packages/context/src/ingest/report-snapshot.test.ts`: - -```ts - it('parses failed ingest reports with trace and failure details', () => { - const snapshot = parseIngestReportSnapshot({ - id: 'report-failed', - runId: 'run-failed', - jobId: 'job-failed', - connectionId: 'warehouse', - sourceKey: 'metabase', - createdAt: '2026-05-17T12:00:00.000Z', - body: { - status: 'failed', - syncId: 'sync-failed', - diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 }, - commitSha: null, - tracePath: '/project/.ktx/ingest-traces/job-failed/trace.jsonl', - failure: { - phase: 'final_gates', - message: 'final artifact gates failed', - }, - workUnits: [], - failedWorkUnits: [], - reconciliationSkipped: true, - conflictsResolved: [], - evictionsApplied: [], - unmappedFallbacks: [], - evictionInputs: [], - unresolvedCards: [], - supersededBy: null, - overrideOf: null, - provenanceRows: [], - toolTranscripts: [], - }, - }); - - expect(snapshot.body.status).toBe('failed'); - expect(snapshot.body.failure).toEqual({ - phase: 'final_gates', - message: 'final artifact gates failed', - }); - expect(snapshot.body.tracePath).toContain('trace.jsonl'); - }); -``` - -- [ ] **Step 2: Run report snapshot test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/report-snapshot.test.ts -t "failed ingest reports" -``` - -Expected: FAIL because `status` and `failure` are not typed or parsed. - -- [ ] **Step 3: Add report status and failure fields** - -In `packages/context/src/ingest/reports.ts`, add this interface after -`IngestReportPostProcessorOutcome`: - -```ts -export interface IngestReportFailure { - phase: string; - message: string; -} -``` - -Then add these fields to `IngestReportBody`: - -```ts - status?: 'completed' | 'failed'; - failure?: IngestReportFailure; -``` - -In `packages/context/src/ingest/report-snapshot.ts`, add this schema near the -other body field schemas: - -```ts -const ingestReportFailureSchema = z.object({ - phase: z.string().min(1), - message: z.string().min(1), -}); -``` - -Then add these fields to the `body` object schema: - -```ts - status: z.enum(['completed', 'failed']).optional(), - failure: ingestReportFailureSchema.optional(), -``` - -- [ ] **Step 4: Run report snapshot tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Write failing CLI status test for failed reports** - -In `packages/cli/src/ingest.test.ts`, add a test near the existing ingest -status tests: - -```ts - it('prints trace path and error status for stored failed ingest reports', async () => { - const io = makeIo(); - const report = { - id: 'report-failed', - runId: 'run-failed', - jobId: 'job-failed', - connectionId: 'warehouse', - sourceKey: 'metabase', - createdAt: '2026-05-17T12:00:00.000Z', - body: { - status: 'failed', - syncId: 'sync-failed', - diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 }, - commitSha: null, - tracePath: '/project/.ktx/ingest-traces/job-failed/trace.jsonl', - failure: { phase: 'final_gates', message: 'final artifact gates failed' }, - workUnits: [], - failedWorkUnits: [], - reconciliationSkipped: true, - conflictsResolved: [], - evictionsApplied: [], - unmappedFallbacks: [], - evictionInputs: [], - unresolvedCards: [], - supersededBy: null, - overrideOf: null, - provenanceRows: [], - toolTranscripts: [], - }, - }; - - await runKtxIngest( - { command: 'status', projectDir: '/project', runId: 'run-failed', outputMode: 'plain', inputMode: 'disabled' }, - { - loadProject: vi.fn().mockResolvedValue({ projectDir: '/project' }), - getLocalIngestStatus: vi.fn().mockResolvedValue(report), - } as never, - io, - ); - - expect(io.stdout()).toContain('Trace: /project/.ktx/ingest-traces/job-failed/trace.jsonl'); - expect(io.stdout()).toContain('Status: error'); - expect(io.stdout()).toContain('Error: final artifact gates failed'); - }); -``` - -Use the actual local test helpers in `packages/cli/src/ingest.test.ts`. If the -file names the command function or IO helper differently, keep the assertions -exactly as written and adapt only the helper calls. - -- [ ] **Step 6: Update CLI rendering** - -In `packages/cli/src/ingest.ts`, replace `reportStatus()` with: - -```ts -function reportStatus(report: IngestReportSnapshot): 'done' | 'error' { - return report.body.status === 'failed' || report.body.failedWorkUnits.length > 0 ? 'error' : 'done'; -} -``` - -In `failedReportMessage()`, add this block before reading `failedCount`: - -```ts - if (report.body.status === 'failed' && report.body.failure?.message) { - return sanitizeMemoryFlowError(report.body.failure.message); - } -``` - -- [ ] **Step 7: Add failed-run report creation state to the runner** - -In `runInner()` in `packages/context/src/ingest/ingest-bundle.runner.ts`, add -these helpers near `createTrace()`: - -```ts - private errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); - } -``` - -Inside `runInner()`, immediately after `const trace = this.createTrace(job);`, -add: - -```ts - let activeTrace: IngestTraceWriter = trace; - let activePhase = 'run'; - let runRow: Awaited | null = null; - let latestDiffSummary: IngestDiffSummary = { added: 0, modified: 0, deleted: 0, unchanged: 0 }; - let latestWorkUnits: WorkUnitOutcome[] = []; - let latestFailedWorkUnits: string[] = []; - let latestReconciliationSkipped = true; - let latestIsolatedDiffSummary: - | { - enabled: boolean; - integrationWorktreePath?: string; - ingestionBaseSha?: string; - projectionSha?: string | null; - acceptedPatches: number; - textualConflicts: number; - semanticConflicts: number; - } - | undefined; -``` - -Replace the existing inner `const runRow = await this.deps.runs.create(...)` -with: - -```ts - runRow = await this.deps.runs.create({ - jobId: job.jobId, - connectionId: job.connectionId, - sourceKey: job.sourceKey, - syncId, - trigger: job.trigger, - scopeFingerprint: scopeDescriptor?.fingerprint ?? null, - }); -``` - -After creating `runTrace`, set: - -```ts - activeTrace = runTrace; -``` - -After computing `diffSummary`, set: - -```ts - latestDiffSummary = diffSummary; -``` - -After `workUnitOutcomes.push(...)`, set: - -```ts - latestWorkUnits = workUnitOutcomes; - latestFailedWorkUnits = failedWorkUnits; -``` - -After `isolatedDiffSummary` is created, set: - -```ts - latestIsolatedDiffSummary = isolatedDiffSummary; -``` - -After reconciliation finishes, set: - -```ts - latestReconciliationSkipped = reconcileOutcome.skipped; -``` - -In the success `reportBody`, add: - -```ts - status: 'completed' as const, -``` - -In the outer `catch`, replace the existing trace event with: - -```ts - await activeTrace.event( - 'error', - 'run', - 'ingest_failed', - { - tracePath: activeTrace.tracePath, - phase: activePhase, - runId: runRow?.id ?? null, - syncId, - }, - error, - ); - if (runRow) { - await this.deps.reports.create({ - runId: runRow.id, - jobId: job.jobId, - connectionId: job.connectionId, - sourceKey: job.sourceKey, - body: { - status: 'failed' as const, - syncId, - diffSummary: latestDiffSummary, - commitSha: null, - tracePath: activeTrace.tracePath, - isolatedDiff: latestIsolatedDiffSummary, - failure: { - phase: activePhase, - message: this.errorMessage(error), - }, - workUnits: latestWorkUnits.map((wu) => ({ - unitKey: wu.unitKey, - rawFiles: [], - status: wu.status, - reason: wu.reason, - actions: wu.actions, - touchedSlSources: wu.touchedSlSources, - slDisallowed: wu.slDisallowed, - slDisallowedReason: wu.slDisallowedReason, - })), - failedWorkUnits: latestFailedWorkUnits, - reconciliationSkipped: latestReconciliationSkipped, - conflictsResolved: [], - evictionsApplied: [], - unmappedFallbacks: [], - artifactResolutions: [], - evictionInputs: [], - reconciliationActions: [], - evictionDecisions: [], - unresolvedCards: [], - supersededBy: null, - overrideOf: null, - provenanceRows: [], - toolTranscripts: Array.from(transcriptSummaries.values()).map((summary) => ({ - unitKey: summary.unitKey, - path: summary.path, - toolCallCount: summary.toolCallCount, - errorCount: summary.errorCount, - toolNames: Array.from(summary.toolNames).sort(), - })), - }, - }); - await activeTrace.event('info', 'report', 'failure_report_created', { - runId: runRow.id, - jobId: job.jobId, - tracePath: activeTrace.tracePath, - }); - } - throw error; -``` - -At each major phase, assign `activePhase` before work begins: - -```ts - activePhase = 'fetch'; - activePhase = 'stage_raw_files'; - activePhase = 'diff'; - activePhase = 'detect'; - activePhase = 'planning'; - activePhase = 'work_units'; - activePhase = 'integration'; - activePhase = 'reconciliation'; - activePhase = 'post_processor'; - activePhase = 'wiki_sl_ref_repair'; - activePhase = 'final_gates'; - activePhase = 'squash'; - activePhase = 'provenance'; - activePhase = 'report'; -``` - -- [ ] **Step 8: Add trace timing and decision events for missing phases** - -Wrap these existing operations in `traceTimed()` and include the listed data: - -```ts - activePhase = 'fetch'; - const stagedDir = await traceTimed(trace, 'fetch', 'resolve_staged_dir', { - bundleRefKind: job.bundleRef.kind, - sourceKey: job.sourceKey, - }, () => - overrideReport - ? this.materializeOverrideSnapshot(overrideReport, { - connectionId: job.connectionId, - sourceKey: job.sourceKey, - jobId: job.jobId, - }) - : this.resolveStagedDir(job.bundleRef, { - connectionId: job.connectionId, - sourceKey: job.sourceKey, - jobId: job.jobId, - }), - ); -``` - -Add explicit events after decisions: - -```ts - await runTrace.event('debug', 'detect', 'adapter_detected', { detected }); - await runTrace.event('debug', 'planning', 'work_units_planned', { - workUnitCount: workUnits.length, - evictionCount: eviction?.deletedRawPaths.length ?? 0, - unresolvedCardCount: unresolvedCards?.length ?? 0, - triageEnabled: triageResult?.enabled ?? false, - }); - await runTrace.event('debug', 'planning', 'target_connections_resolved', { - connectionIds: slConnectionIds, - }); - await runTrace.event('debug', 'reconciliation', 'reconciliation_finished', { - skipped: reconcileOutcome.skipped, - stopReason: reconcileOutcome.stopReason ?? null, - actionCount: reconcileActions.length, - conflictCount: stageIndex.conflictsResolved.length, - fallbackCount: stageIndex.unmappedFallbacks.length, - artifactResolutionCount: stageIndex.artifactResolutions?.length ?? 0, - }); - await runTrace.event('debug', 'post_processor', 'post_processor_finished', { - sourceKey: job.sourceKey, - status: postProcessorOutcome?.status ?? 'skipped', - touchedSources: postProcessorOutcome?.touchedSources ?? [], - warnings: postProcessorOutcome?.warnings ?? [], - }); - await runTrace.event('debug', 'wiki_sl_ref_repair', 'wiki_sl_refs_repaired', { - repairCount: wikiSlRefRepairResult.repairs.length, - repairs: wikiSlRefRepairResult.repairs, - warnings: wikiSlRefRepairResult.warnings, - }); - await runTrace.event('debug', 'provenance', 'provenance_rows_validated', { - rowCount: provenanceRows.length, - }); - await runTrace.event('debug', 'squash', 'squash_finished', { - commitSha, - touchedPaths: mergeResult.touchedPaths, - }); - await runTrace.event('debug', 'report', 'success_report_created', { - reportId, - runId: runRow.id, - tracePath: runTrace.tracePath, - }); -``` - -Acceptance criteria for this step: - -- A successful isolated run trace contains phase events for `fetch`, - `snapshot`, `routing`, `planning`, `work_unit`, `integration`, - `reconciliation`, `final_gates`, `squash`, `provenance`, `report`, and - `run`. -- A failed isolated run trace contains an `ingest_failed` event with `runId`, - `syncId`, `phase`, `tracePath`, and serialized error details. -- Failed runs after `runRow` creation have a stored report whose body includes - `status: "failed"`, `failure.phase`, `failure.message`, and `tracePath`. - -- [ ] **Step 9: Add isolated trace completeness test** - -Append this test to `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`: - -```ts - it('stores a failure report and postmortem trace for final gate failures', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - const createdReports: any[] = []; - deps.reports.create = vi.fn(async (args: any) => { - createdReports.push(args); - return { id: `report-${createdReports.length}` }; - }); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { unitKey: 'card-wiki', rawFiles: ['cards/wiki.json'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }, - ], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - if (params.telemetryTags.unitKey === 'card-wiki') { - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nARR is `mart_account_segments.total_contract_arr_cents`.\n', - ); - currentSession.actions.push({ target: 'wiki', type: 'created', key: 'account-segments', detail: 'Account segments', rawPaths: ['cards/wiki.json'] }); - await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'wu wiki', 'KTX Test', 'system@ktx.local'); - } - if (params.telemetryTags.unitKey === 'card-source') { - await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ target: 'sl', type: 'created', key: 'mart_account_segments', detail: 'Dollar measure', targetConnectionId: 'warehouse', rawPaths: ['cards/source.json'] }); - await currentSession.gitService.commitFiles(['semantic-layer/warehouse/mart_account_segments.yaml'], 'wu source', 'KTX Test', 'system@ktx.local'); - } - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [ - ['cards/wiki.json', 'h1'], - ['cards/source.json', 'h2'], - ]); - - await expect( - runner.run({ jobId: 'job-trace-failure', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/total_contract_arr_cents/); - - const failureReport = createdReports.find((report) => report.body.status === 'failed'); - expect(failureReport.body.tracePath).toContain('job-trace-failure/trace.jsonl'); - expect(failureReport.body.failure).toMatchObject({ phase: 'final_gates' }); - - const events = (await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-trace-failure/trace.jsonl'), 'utf-8')) - .trim() - .split('\n') - .map((line) => JSON.parse(line)); - expect(events.map((event) => event.event)).toEqual(expect.arrayContaining([ - 'ingest_started', - 'input_snapshot', - 'work_units_planned', - 'isolated_diff_enabled', - 'work_unit_child_created', - 'work_unit_patch_collected', - 'patch_apply_started', - 'patch_accepted', - 'reconciliation_finished', - 'final_artifact_gates_failed', - 'ingest_failed', - 'failure_report_created', - ])); - const failed = events.find((event) => event.event === 'ingest_failed'); - expect(failed).toMatchObject({ - runId: 'run-1', - syncId: expect.any(String), - data: { phase: 'final_gates', tracePath: expect.stringContaining('trace.jsonl') }, - error: { message: expect.stringContaining('total_contract_arr_cents') }, - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 10: Run context and CLI trace tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/report-snapshot.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts -t "failed ingest reports" -``` - -Expected: PASS. - -- [ ] **Step 11: Update trace inspection docs** - -In `docs-site/content/docs/cli-reference/ktx-ingest.mdx`, replace the paragraph -under "Inspect source ingest traces" that starts with "Each line is a JSON -event" with: - -```mdx -The trace file lives under the project directory at -`.ktx/ingest-traces//trace.jsonl`. Each line is a JSON event with the -job id, run id, sync id, connection id, source key, phase, event name, timing, -state snapshot, decision context, and error details. Failed runs also write a -stored ingest report with `status: "failed"`, `failure.phase`, -`failure.message`, and the same trace path, so `ktx ingest status ` can -point you to the postmortem trace. -``` - -- [ ] **Step 12: Commit trace and failure report work** - -```bash -git add packages/context/src/ingest/reports.ts \ - packages/context/src/ingest/report-snapshot.ts \ - packages/context/src/ingest/report-snapshot.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/cli/src/ingest.ts \ - packages/cli/src/ingest.test.ts \ - docs-site/content/docs/cli-reference/ktx-ingest.mdx -git commit -m "fix(ingest): persist postmortem failure traces" -``` - -### Task 4: Structured policy conflicts and child cleanup - -**Files:** -- Modify: `packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts` -- Modify: `packages/context/src/ingest/isolated-diff/work-unit-executor.ts` -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Add failing child cleanup test** - -Append this test to `packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts`: - -```ts - it('removes child worktrees after failed WorkUnit outcomes are traced', async () => { - const { homeDir, git, baseSha } = await makeGit(); - const childDir = join(homeDir, '.worktrees/session-job-1-wu-fail'); - const sessionWorktreeService = { - create: vi.fn(async (_key: string, startSha: string) => { - await mkdir(join(homeDir, '.worktrees'), { recursive: true }); - await git.addWorktree(childDir, 'session/job-1-wu-fail', startSha); - return { - chatId: 'job-1-wu-fail', - workdir: childDir, - branch: 'session/job-1-wu-fail', - baseSha: startSha, - createdAt: new Date(), - git: git.forWorktree(childDir), - config: {}, - }; - }), - cleanup: vi.fn(async () => undefined), - }; - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-1/trace.jsonl'), - jobId: 'job-1', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await runIsolatedWorkUnit({ - unitIndex: 0, - ingestionBaseSha: baseSha, - sessionWorktreeService: sessionWorktreeService as never, - patchDir: join(homeDir, '.ktx/ingest-patches/job-1'), - trace, - run: async () => ({ - unitKey: 'wu-fail', - status: 'failed', - reason: 'agent loop errored', - preSha: baseSha, - postSha: baseSha, - actions: [], - touchedSlSources: [], - }), - workUnit: { unitKey: 'wu-fail', rawFiles: ['a.json'], peerFileIndex: [], dependencyPaths: [] }, - }); - - expect(result.status).toBe('failed'); - expect(sessionWorktreeService.cleanup).toHaveBeenCalledWith(expect.any(Object), 'success'); - }); -``` - -- [ ] **Step 2: Run child cleanup test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/work-unit-executor.test.ts -t "failed WorkUnit" -``` - -Expected: FAIL because failed WorkUnits call `cleanup(..., 'crash')`. - -- [ ] **Step 3: Cleanup child worktrees on failed outcomes and collect patch metadata only** - -In `packages/context/src/ingest/isolated-diff/work-unit-executor.ts`, replace -the import: - -```ts -import { assertPatchAllowedForWorkUnit } from './git-patch.js'; -``` - -with: - -```ts -import { parsePatchTouchedPaths } from './git-patch.js'; -``` - -Then replace this failed-outcome block: - -```ts - if (outcome.status !== 'success') { - cleanupOutcome = 'crash'; - await input.trace.event('error', 'work_unit', 'work_unit_failed_before_patch', { - unitKey: input.workUnit.unitKey, - reason: outcome.reason ?? 'unknown failure', - }); - return { ...outcome, childWorktreePath: child.workdir }; - } -``` - -with: - -```ts - if (outcome.status !== 'success') { - cleanupOutcome = 'success'; - await input.trace.event('error', 'work_unit', 'work_unit_failed_before_patch', { - unitKey: input.workUnit.unitKey, - reason: outcome.reason ?? 'unknown failure', - }); - return { ...outcome, childWorktreePath: child.workdir }; - } -``` - -Replace patch policy enforcement: - -```ts - const touched = assertPatchAllowedForWorkUnit({ - unitKey: input.workUnit.unitKey, - patch, - slDisallowed: input.workUnit.slDisallowed === true, - }); -``` - -with: - -```ts - const touched = parsePatchTouchedPaths(patch); -``` - -In the `catch` block, set `cleanupOutcome = 'success'` after the error is -traced: - -```ts - cleanupOutcome = 'success'; -``` - -- [ ] **Step 4: Run child cleanup tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/work-unit-executor.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Add failing policy rejection trace test** - -Append this test to `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`: - -```ts - it('classifies slDisallowed patch policy failures as traced textual conflicts', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - await mkdir(join(configDir, 'semantic-layer/c1'), { recursive: true }); - await git.commitFiles(['semantic-layer/c1'], 'empty sl dir', 'System User', 'system@example.com'); - const childDir = join(homeDir, 'child-policy'); - await git.addWorktree(childDir, 'child-policy', baseSha); - const childGit = git.forWorktree(childDir); - await mkdir(join(childDir, 'semantic-layer/c1'), { recursive: true }); - await writeFile(join(childDir, 'semantic-layer/c1/orders.yaml'), 'name: orders\ncolumns: []\njoins: []\nmeasures: []\n'); - await childGit.commitFiles(['semantic-layer/c1/orders.yaml'], 'forbidden sl', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/forbidden.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-policy/trace.jsonl'), - jobId: 'job-policy', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'lookml-mismatch', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree: vi.fn().mockResolvedValue(undefined), - slDisallowed: true, - }); - - expect(result).toMatchObject({ - status: 'textual_conflict', - touchedPaths: ['semantic-layer/c1/orders.yaml'], - }); - const rawTrace = await readFile(trace.tracePath, 'utf-8'); - expect(rawTrace).toContain('patch_policy_rejected'); - expect(rawTrace).toContain('slDisallowed WorkUnit lookml-mismatch touched semantic-layer/c1/orders.yaml'); - }); -``` - -- [ ] **Step 6: Run policy rejection test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -t "policy failures" -``` - -Expected: FAIL because policy rejection throws before a structured result. - -- [ ] **Step 7: Classify policy rejections in the integrator** - -In `packages/context/src/ingest/isolated-diff/patch-integrator.ts`, add -`parsePatchTouchedPaths` to the import from `git-patch.js`: - -```ts -import { assertPatchAllowedForWorkUnit, parsePatchTouchedPaths } from './git-patch.js'; -``` - -Replace lines that read and assert the patch with: - -```ts - const patch = await readFile(input.patchPath, 'utf-8'); - const touchedPaths = parsePatchTouchedPaths(patch).map((entry) => entry.path); - try { - assertPatchAllowedForWorkUnit({ - unitKey: input.unitKey, - patch, - slDisallowed: input.slDisallowed, - }); - } catch (error) { - await input.trace.event('error', 'integration', 'patch_policy_rejected', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason: errorMessage(error), - }); - return { - status: 'textual_conflict', - reason: errorMessage(error), - touchedPaths, - }; - } -``` - -Keep the existing `patch_apply`, `patch_textual_conflict`, -`semantic_gate`, and `patch_semantic_conflict` blocks unchanged. - -- [ ] **Step 8: Update isolated slDisallowed regression expectations** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -replace the `slDisallowed` rejection assertion with: - -```ts - await expect( - runner.run({ jobId: 'job-sl-disallowed', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/isolated diff textual conflict/); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-sl-disallowed/trace.jsonl'), 'utf-8'); - expect(trace).toContain('patch_policy_rejected'); - expect(trace).toContain('slDisallowed WorkUnit lookml-mismatch touched semantic-layer/warehouse/orders.yaml'); -``` - -- [ ] **Step 9: Run policy and isolated tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. - -- [ ] **Step 10: Commit policy and cleanup fixes** - -```bash -git add packages/context/src/ingest/isolated-diff/work-unit-executor.ts \ - packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "fix(ingest): trace policy conflicts and cleanup child worktrees" -``` - -### Task 5: Final verification - -**Files:** -- Verify: all files modified in Tasks 1-4 - -- [ ] **Step 1: Run focused context tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run focused CLI tests** - -Run: - -```bash -pnpm --filter @ktx/cli exec vitest run src/ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code check because TypeScript exports and report fields changed** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. - -- [ ] **Step 5: Run pre-commit for touched files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/ingest/wiki-body-refs.ts \ - packages/context/src/ingest/wiki-body-refs.test.ts \ - packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/isolated-diff/work-unit-executor.ts \ - packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/reports.ts \ - packages/context/src/ingest/report-snapshot.ts \ - packages/context/src/ingest/report-snapshot.test.ts \ - packages/cli/src/ingest.ts \ - packages/cli/src/ingest.test.ts \ - docs-site/content/docs/cli-reference/ktx-ingest.mdx -``` - -Expected: PASS. If the local `uv` version does not satisfy the repository pin, -record the version mismatch and run the focused `pnpm` checks above. - -- [ ] **Step 6: Inspect one failed trace manually** - -Run the final-gate failure test and inspect the trace: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - -t "postmortem trace" -``` - -Open the trace path printed in the assertion failure output or the test temp -directory if the test logs it. The trace must let a human reconstruct: - -- the job, run, sync, source, connection, and input snapshot; -- routing into isolated diffs; -- WorkUnit child creation, patch collection, patch application, and accepted - patch order; -- reconciliation status and action counts; -- final gate input scope and failure reason; -- failure report creation; and -- final `ingest_failed` event with phase and serialized error. - -- [ ] **Step 7: Inspect git status** - -Run: - -```bash -git status --short -``` - -Expected: only intended source, test, CLI, and docs files are modified before -the final commit. - -- [ ] **Step 8: Commit verification updates if any** - -If test or docs edits were needed during verification: - -```bash -git add packages/context/src/ingest packages/cli/src/ingest.ts packages/cli/src/ingest.test.ts docs-site/content/docs/cli-reference/ktx-ingest.mdx -git commit -m "test(ingest): verify isolated diff postmortem coverage" -``` - -If no files changed during verification, do not create an empty commit. - -## Self-Review - -Spec coverage: - -- Isolated WorkUnits and binary no-rename patches are already implemented in - the previous plan. Task 4 moves policy rejection to the integration layer and - keeps child cleanup aligned with the spec. -- Artifact-aware gates are completed by Task 1 for semantic-layer YAML, wiki - frontmatter source/entity refs, wiki body refs, and direct join dependencies. -- The final global gate moves to the correct point in Task 2, after - reconciliation, post-processing, and wiki repair, and before squash. -- Reconciliation mutation tracking is added in Task 2 through a diff from - pre-reconciliation `HEAD` to post-repair `HEAD`. -- Persistent postmortem observability is completed by Task 3 with trace events, - timings, state snapshots, stored failure reports, and CLI status surfacing. -- Version-one resolver behavior remains fail-fast and preserves the integration - worktree on conflicts. - -Placeholder scan: - -- The plan contains no placeholder tasks. -- Each code-changing step includes concrete code or exact replacement blocks. -- Verification commands and expected outcomes are explicit. - -Type consistency: - -- New report fields are named `status` and `failure` consistently in - `reports.ts`, `report-snapshot.ts`, runner report bodies, and CLI rendering. -- Final gate scope uses existing `TouchedSlSource`, `MemoryAction`, - `WorkUnitOutcome`, and `WikiSlRefRepairResult` types. -- Trace event names are stable and asserted by tests: - `reconciliation_finished`, `final_artifact_gates_failed`, - `failure_report_created`, `patch_policy_rejected`, and `ingest_failed`. diff --git a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-global-wiki-reference-gate-closure.md b/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-global-wiki-reference-gate-closure.md deleted file mode 100644 index 9ed083ec..00000000 --- a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-global-wiki-reference-gate-closure.md +++ /dev/null @@ -1,493 +0,0 @@ -# Isolated Diff Ingestion V1 Global Wiki Reference Gate Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Reject final trees where an isolated-diff run changes semantic-layer -sources or deletes wiki pages and leaves pre-existing wiki pages with stale -body, `sl_refs`, frontmatter `refs`, or inline `[[page-key]]` references. - -**Architecture:** Keep `artifact-gates.ts` validation-only. The runner expands -the final wiki gate scope before the existing final artifact gate: changed pages -are always validated, and all global wiki pages are validated when the run -changes any semantic-layer source or removes any wiki page. The final-gate trace -records the expanded scope and why it was expanded. - -**Tech Stack:** TypeScript, Vitest, pnpm workspace commands, existing -`IngestBundleRunner`, `KnowledgeWikiService`, and isolated-diff test fixtures. - ---- - -## Audit Summary - -The implemented isolated-diff plans cover the core v1 flow: child worktrees, -binary no-rename patch proposals, `git apply --3way --index`, policy rejection, -final gates after reconciliation and repair, pre-squash provenance raw-path -validation, target-connection enforcement, failed reports, and persistent JSONL -traces. - -One v1-blocking correctness gap remains. Final wiki gates currently validate -wiki pages changed by the run. They do not validate unchanged pages that become -invalid because the run changes a semantic-layer source or deletes a referenced -wiki page. Two concrete failures can therefore squash into main: - -- A pre-existing wiki page body contains - `` `mart_account_segments.total_contract_arr_cents` `` while the run updates - `semantic-layer/warehouse/mart_account_segments.yaml` to define only - `total_contract_arr`. -- A pre-existing wiki page has `refs: [source-page]` or `[[source-page]]` while - the run deletes `wiki/global/source-page.md`. - -This plan does not expand connector rollout, promote isolated diffs to the -default, add interactive resolution, add semantic auto-merge, remove the old -path, expand transitive semantic-layer dependencies, or move provenance into -files. - -## File Structure - -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Adds two failing end-to-end regressions for unchanged wiki pages made stale by - semantic-layer changes and wiki-page deletion. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Adds a final wiki gate scope helper, expands validation to all global wiki - pages when final state changes can invalidate unchanged references, and records - scope details in the final-gate trace and failed report. - ---- - -### Task 1: Add failing unchanged wiki regressions - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Add the stale existing wiki body regression** - -Insert this test inside `describe('IngestBundleRunner isolated diff path', ...)` -after the existing Metabase stale-measure regression: - -```ts - it('rejects unchanged wiki body refs made stale by isolated semantic-layer changes', async () => { - const runtime = await makeRealGitRuntime(); - try { - await mkdir(join(runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n', - ); - await writeFile( - join(runtime.configDir, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\n---\n\nExisting ARR uses `mart_account_segments.total_contract_arr_cents`.\n', - ); - await runtime.git.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], - 'seed existing wiki body ref', - 'KTX Test', - 'system@ktx.local', - ); - const preRunHead = await runtime.git.revParseHead(); - - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'source-only', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ - target: 'sl', - type: 'updated', - key: 'mart_account_segments', - detail: 'Rename ARR measure', - targetConnectionId: 'warehouse', - rawPaths: ['cards/source.json'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml'], - 'wu source rename', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]); - - await expect( - runner.run({ - jobId: 'job-existing-body-stale', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/total_contract_arr_cents/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-existing-body-stale/trace.jsonl'), 'utf-8'); - expect(trace).toContain('final_artifact_gates_failed'); - expect(trace).toContain('account-segments'); - expect(trace).toContain('semantic_layer_changed'); - expect(trace).toContain('ingest_failed'); - expect(trace).toContain('failure_report_created'); - expect(trace).not.toContain('squash_finished'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Add the stale existing wiki page-reference regression** - -Insert this test near the existing final wiki reference regression: - -```ts - it('rejects unchanged inbound wiki refs broken by an isolated wiki deletion', async () => { - const runtime = await makeRealGitRuntime(); - try { - await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(runtime.configDir, 'wiki/global/source-page.md'), - '---\nsummary: Source page\nusage_mode: auto\n---\n\nSource page\n', - ); - await writeFile( - join(runtime.configDir, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\nrefs:\n - source-page\n---\n\nSee [[source-page]].\n', - ); - await runtime.git.commitFiles( - ['wiki/global/source-page.md', 'wiki/global/account-segments.md'], - 'seed inbound wiki refs', - 'KTX Test', - 'system@ktx.local', - ); - const preRunHead = await runtime.git.revParseHead(); - - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'delete-target-page', rawFiles: ['pages/delete.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await rm(join(root, 'wiki/global/source-page.md'), { force: true }); - currentSession.actions.push({ - target: 'wiki', - type: 'removed', - key: 'source-page', - detail: 'Delete referenced page', - rawPaths: ['pages/delete.json'], - }); - await currentSession.gitService.commitFiles( - ['wiki/global/source-page.md'], - 'wu delete target page', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['pages/delete.json', 'h1']]); - - await expect( - runner.run({ - jobId: 'job-existing-wiki-ref-stale', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/wiki references target missing page\(s\): account-segments -> source-page/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-existing-wiki-ref-stale/trace.jsonl'), 'utf-8'); - expect(trace).toContain('final_artifact_gates_failed'); - expect(trace).toContain('account-segments -> source-page'); - expect(trace).toContain('wiki_page_removed'); - expect(trace).toContain('ingest_failed'); - expect(trace).toContain('failure_report_created'); - expect(trace).not.toContain('squash_finished'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 3: Run the focused regressions and verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "unchanged wiki body refs|unchanged inbound wiki refs" -``` - -Expected: FAIL. The stale body test currently squashes successfully because the -unchanged `account-segments` page is not in `finalChangedWikiPageKeys`. The -inbound wiki ref test currently squashes successfully because the deleted -`source-page` is validated as a missing changed page and skipped, while the -unchanged page that references it is never validated. - ---- - -### Task 2: Expand the final wiki validation scope - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` - -- [ ] **Step 1: Add final wiki gate scope helpers** - -Add these private methods after `uniqueTouchedSlSources()`: - -```ts - private removedWikiPageKeysFromActions(actions: MemoryAction[]): string[] { - return this.uniqueWikiPageKeys( - actions.filter((action) => action.target === 'wiki' && action.type === 'removed').map((action) => action.key), - ); - } - - private async wikiPageKeysForFinalGates(input: { - wikiService: ReturnType; - changedWikiPageKeys: string[]; - touchedSlSources: TouchedSlSource[]; - actions: MemoryAction[]; - }): Promise<{ - pageKeys: string[]; - trace: { - global: boolean; - reasons: string[]; - changedWikiPageKeys: string[]; - removedWikiPageKeys: string[]; - pageKeysValidated: string[]; - }; - }> { - const changedWikiPageKeys = this.uniqueWikiPageKeys(input.changedWikiPageKeys); - const removedWikiPageKeys = this.removedWikiPageKeysFromActions(input.actions); - const reasons: string[] = []; - if (input.touchedSlSources.length > 0) { - reasons.push('semantic_layer_changed'); - } - if (removedWikiPageKeys.length > 0) { - reasons.push('wiki_page_removed'); - } - - let pageKeys = changedWikiPageKeys; - if (reasons.length > 0) { - pageKeys = this.uniqueWikiPageKeys([ - ...changedWikiPageKeys, - ...(await input.wikiService.listPageKeys('GLOBAL', null)), - ]); - } - - return { - pageKeys, - trace: { - global: reasons.length > 0, - reasons, - changedWikiPageKeys, - removedWikiPageKeys, - pageKeysValidated: pageKeys, - }, - }; - } -``` - -- [ ] **Step 2: Use the expanded scope before final gates** - -In `runInner()`, replace the current `finalChangedWikiPageKeys` and -`finalTouchedSlSources` block with this code: - -```ts - const baseFinalChangedWikiPageKeys = this.uniqueWikiPageKeys([ - ...(isolatedDiffEnabled ? projectionChangedWikiPageKeys : []), - ...workUnitOutcomes - .flatMap((outcome) => outcome.patchTouchedPaths ?? []) - .flatMap((path) => this.wikiPageKeysFromPaths([path])), - ...this.wikiPageKeysFromActions(reconcileActions), - ...postReconciliationPaths.flatMap((path) => this.wikiPageKeysFromPaths([path])), - ...wikiSlRefRepairResult.repairs.filter((repair) => repair.scope === 'GLOBAL').map((repair) => repair.pageKey), - ]); - const finalTouchedSlSources = this.uniqueTouchedSlSources([ - ...(isolatedDiffEnabled ? projectionTouchedSources : []), - ...workUnitOutcomes.flatMap((outcome) => outcome.touchedSlSources), - ...this.touchedSlSourcesFromActions(reconcileActions, job.connectionId), - ...this.touchedSlSourcesFromPaths(postReconciliationPaths), - ...(postProcessorOutcome?.touchedSources ?? []), - ]); - const finalWikiGateScope = await this.wikiPageKeysForFinalGates({ - wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), - changedWikiPageKeys: baseFinalChangedWikiPageKeys, - touchedSlSources: finalTouchedSlSources, - actions: [...stageIndex.workUnits.flatMap((wu) => wu.actions), ...reconcileActions], - }); - const finalChangedWikiPageKeys = finalWikiGateScope.pageKeys; -``` - -This keeps the existing variable name used by `validateFinalIngestArtifacts()`, -but the value now means "wiki page keys to validate in final gates." - -- [ ] **Step 3: Add scope details to final-gate trace data** - -In the `finalArtifactGateTraceData` object, add the -`wikiReferenceGateScope` field: - -```ts - const finalArtifactGateTraceData = { - changedWikiPageKeys: finalChangedWikiPageKeys, - wikiReferenceGateScope: finalWikiGateScope.trace, - touchedSlSources: finalTouchedSlSources, - projectionTouchedPaths, - workUnitPatchTouchedPaths: workUnitOutcomes.flatMap((outcome) => outcome.patchTouchedPaths ?? []), - preReconciliationSha, - postReconciliationSha, - postReconciliationPaths, - reconciliationActionCount: reconcileActions.length, - wikiSlRefRepairCount: wikiSlRefRepairResult.repairs.length, - }; -``` - -The failure report already stores `activeFailureDetails`, so this trace data -also becomes persistent failed-report context when final gates fail. - -- [ ] **Step 4: Run the focused regressions and verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "unchanged wiki body refs|unchanged inbound wiki refs" -``` - -Expected: PASS. Both traces include `final_artifact_gates_failed`, -`failure_report_created`, no `squash_finished`, and -`wikiReferenceGateScope` with either `semantic_layer_changed` or -`wiki_page_removed`. - ---- - -### Task 3: Verification and commit - -**Files:** -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Run the isolated-diff focused suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/semantic-layer-target-policy.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/core/git.service.patch.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Type-check the context package** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run dead-code analysis** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only pre-existing findings unrelated to -`packages/context/src/ingest/ingest-bundle.runner.ts` and -`packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. -Investigate any new finding before committing. - -- [ ] **Step 4: Verify trace acceptance criteria** - -Open the traces produced by the two new failing-run tests and confirm these -events and fields exist: - -```text -job-existing-body-stale: -- final_artifact_gates_started -- final_artifact_gates_failed -- ingest_failed -- failure_report_created -- no squash_finished -- wikiReferenceGateScope.global is true -- wikiReferenceGateScope.reasons includes semantic_layer_changed -- wikiReferenceGateScope.pageKeysValidated includes account-segments -- error.message includes total_contract_arr_cents - -job-existing-wiki-ref-stale: -- final_artifact_gates_started -- final_artifact_gates_failed -- ingest_failed -- failure_report_created -- no squash_finished -- wikiReferenceGateScope.global is true -- wikiReferenceGateScope.reasons includes wiki_page_removed -- wikiReferenceGateScope.removedWikiPageKeys includes source-page -- error.message includes account-segments -> source-page -``` - -- [ ] **Step 5: Commit** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "fix(ingest): gate global wiki references" -``` - -Expected: one commit containing only the runner and isolated-diff runner test -changes. - ---- - -## Self-Review - -Spec coverage: -- Final global wiki body reference validation now covers unchanged wiki pages - when a run changes semantic-layer sources. -- Final global wiki page reference validation now covers unchanged inbound - references when a run deletes wiki pages. -- The plan keeps resolver behavior fail-fast and stops before squash. -- Persistent trace and failed-report acceptance criteria are explicit and tied - to the concrete failure modes. - -Non-blocking gaps unchanged: -- Broader connector rollout. -- Isolated-diff default promotion. -- Old shared-worktree path removal. -- Interactive conflict resolution. -- Semantic auto-merge. -- Transitive semantic-layer dependency expansion. -- Provenance-as-files. diff --git a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-provenance-gate-closure.md b/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-provenance-gate-closure.md deleted file mode 100644 index 8d2fb94d..00000000 --- a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-provenance-gate-closure.md +++ /dev/null @@ -1,494 +0,0 @@ -# Isolated Diff Ingestion V1 Provenance Gate Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Ensure invalid provenance raw paths are rejected before isolated-diff -ingestion squashes any integration worktree changes into the main project -worktree. - -**Architecture:** Keep provenance insertion after squash, but derive and -validate the planned provenance rows immediately after final artifact gates and -before the squash stage. This makes provenance validation part of the final -pre-main safety boundary while preserving the existing report and database -write shape. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, existing -`IngestBundleRunner`, `validateProvenanceRawPaths`, ingest reports, and -persistent ingest traces. - ---- - -## Audit Summary - -The implemented isolated-diff path now covers the core v1 safety surface: -child worktrees, binary no-rename patches, `git apply --3way --index`, patch -policy rejection, final wiki and semantic-layer gates after reconciliation and -post-processing, failure reports, and persistent JSONL traces. The focused -isolated-diff test suite passes: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Current result: `7 passed`, `28 passed`. - -One v1-blocking gap remains. `validateProvenanceRawPaths()` is called in -`packages/context/src/ingest/ingest-bundle.runner.ts` after -`squashMergeIntoMain()`. A work unit or reconciliation action can emit an -otherwise valid wiki or semantic-layer artifact whose `rawPaths` contain a path -outside the current raw snapshot and eviction set. Today the run fails during -provenance recording, but only after the invalidly-attributed artifacts have -already reached the main project worktree. That violates the spec requirement -that final global gates run before any changes reach main. - -Observability for the already-implemented phases is sufficient for postmortem -reconstruction: traces include input snapshots, routing, child worktree -creation and cleanup, patch collection and application, conflict -classification, reconciliation, final gates, failure reports, and run outcome. -This plan adds only the missing provenance validation failure trace because it -corresponds to a concrete pre-main failure mode, not cosmetic trace expansion. - -Non-blocking gaps that remain after this plan: - -- Migrating Notion, LookML, Looker, dbt, MetricFlow, and historic-SQL direct - durable writes to the isolated path. -- Promoting isolated diffs as the default for all connectors. -- Removing the old shared-worktree WorkUnit execution path. -- Interactive, CLI, or agent-driven conflict resolution. -- Auto-merging semantic conflicts that cannot be proven correct. -- Transitive SQL-projection dependency expansion beyond direct declared joins. -- Moving provenance rows to worktree files. -- Adding failure reports for failures that happen before an ingest run row - exists. The trace file is still written at the deterministic job path. - -## File Structure - -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Add a regression proving invalid provenance raw paths fail before squash, - leave main unchanged, skip SQLite provenance insertion, and emit a - postmortem-grade trace event. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Extract provenance row construction into private helpers, run provenance - raw-path validation before squash, trace validation success and failure, and - reuse the prevalidated rows for insertion and reports after squash. - ---- - -### Task 1: Add the pre-squash provenance regression - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Write the failing runner test** - -Append this test inside the existing -`describe('IngestBundleRunner isolated diff path', ...)` block in -`packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`: - -```ts - it('rejects invalid provenance raw paths before squash reaches main', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'card-valid-artifacts', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true }); - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - await writeFile( - join(root, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\nsl_refs:\n - mart_account_segments\n---\n\nARR is `mart_account_segments.total_contract_arr`.\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'mart_account_segments', - detail: 'Valid source', - targetConnectionId: 'warehouse', - rawPaths: ['cards/source.json'], - }); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'account-segments', - detail: 'Valid wiki with invalid provenance raw path', - rawPaths: ['cards/missing.json'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], - 'valid artifacts with invalid provenance', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]); - const preRunHead = await runtime.git.revParseHead(); - - await expect( - runner.run({ - jobId: 'job-invalid-provenance', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/provenance row references raw path outside this snapshot: cards\/missing\.json/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - expect(deps.provenance.insertMany).not.toHaveBeenCalled(); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-invalid-provenance/trace.jsonl'), 'utf-8'); - expect(trace).toContain('final_artifact_gates_finished'); - expect(trace).toContain('provenance_rows_validation_failed'); - expect(trace).toContain('cards/missing.json'); - expect(trace).toContain('ingest_failed'); - expect(trace).not.toContain('squash_finished'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Run the failing regression** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "invalid provenance raw paths" -``` - -Expected: FAIL because the current runner validates provenance after -`squashMergeIntoMain()`, so `runtime.git.revParseHead()` changes and the trace -does not contain `provenance_rows_validation_failed`. - -### Task 2: Move provenance validation into the pre-squash gate boundary - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` - -- [ ] **Step 1: Import the provenance report and insert types** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, update the imports. - -Replace this import block: - -```ts -import type { - ContextEvidenceIndexSummary, - IngestBundleRunnerDeps, - IngestProvenanceRow, - IngestRunsPort, - IngestSessionWorktree, - PageTriageRunResult, -} from './ports.js'; -``` - -With: - -```ts -import type { - ContextEvidenceIndexSummary, - IngestBundleRunnerDeps, - IngestProvenanceInsert, - IngestProvenanceRow, - IngestRunsPort, - IngestSessionWorktree, - PageTriageRunResult, -} from './ports.js'; -``` - -Replace this import block: - -```ts -import { - buildStageIndexFromReportBody, - postProcessorSavedMemoryCounts, - type IngestReportPostProcessorOutcome, - type IngestReportSnapshot, -} from './reports.js'; -``` - -With: - -```ts -import { - buildStageIndexFromReportBody, - postProcessorSavedMemoryCounts, - type IngestReportPostProcessorOutcome, - type IngestReportProvenanceDetail, - type IngestReportSnapshot, -} from './reports.js'; -``` - -- [ ] **Step 2: Add provenance row helpers** - -Add these private methods after `private errorMessage(error: unknown): string` -in `packages/context/src/ingest/ingest-bundle.runner.ts`: - -```ts - private buildProvenanceRows(input: { - job: IngestBundleJob; - syncId: string; - currentHashes: Map; - stageIndex: StageIndex; - reconcileActions: MemoryAction[]; - eviction?: EvictionUnit; - }): IngestProvenanceInsert[] { - const provenanceRows: IngestProvenanceInsert[] = []; - const actionToType = (action: MemoryAction): IngestProvenanceInsert['actionType'] => { - if (action.target === 'wiki') { - return 'wiki_written'; - } - return action.type === 'created' ? 'source_created' : 'measure_added'; - }; - const producedPaths = new Set(); - const pushActionProvenance = (rawPath: string, action: MemoryAction): void => { - const hash = input.currentHashes.get(rawPath) ?? ''; - provenanceRows.push({ - connectionId: input.job.connectionId, - sourceKey: input.job.sourceKey, - syncId: input.syncId, - rawPath, - rawContentHash: hash, - artifactKind: action.target, - artifactKey: action.key, - targetConnectionId: action.target === 'sl' ? actionTargetConnectionId(action, input.job.connectionId) : null, - artifactContentHash: null, - actionType: actionToType(action), - }); - producedPaths.add(rawPath); - }; - - for (const wu of input.stageIndex.workUnits) { - for (const action of wu.actions) { - for (const rawPath of rawPathsForAction(action, wu.rawFiles)) { - pushActionProvenance(rawPath, action); - } - } - } - for (const action of input.reconcileActions) { - for (const rawPath of action.rawPaths ?? []) { - pushActionProvenance(rawPath, action); - } - } - for (const resolution of input.stageIndex.artifactResolutions ?? []) { - const hash = input.currentHashes.get(resolution.rawPath) ?? ''; - provenanceRows.push({ - connectionId: input.job.connectionId, - sourceKey: input.job.sourceKey, - syncId: input.syncId, - rawPath: resolution.rawPath, - rawContentHash: hash, - artifactKind: resolution.artifactKind, - artifactKey: resolution.artifactKey, - targetConnectionId: null, - artifactContentHash: null, - actionType: resolution.actionType, - }); - producedPaths.add(resolution.rawPath); - } - for (const [rawPath, hash] of input.currentHashes) { - if (producedPaths.has(rawPath)) { - continue; - } - provenanceRows.push({ - connectionId: input.job.connectionId, - sourceKey: input.job.sourceKey, - syncId: input.syncId, - rawPath, - rawContentHash: hash, - artifactKind: null, - artifactKey: null, - targetConnectionId: null, - artifactContentHash: null, - actionType: 'skipped', - }); - } - - return provenanceRows; - } - - private toReportProvenanceRows(rows: IngestProvenanceInsert[]): IngestReportProvenanceDetail[] { - return rows.map(({ rawPath, artifactKind, artifactKey, actionType, targetConnectionId }) => ({ - rawPath, - artifactKind, - artifactKey, - targetConnectionId: targetConnectionId ?? null, - actionType, - })); - } -``` - -- [ ] **Step 3: Validate planned provenance rows before squash** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, find the code that -sets `activePhase = 'final_gates';` and runs `traceTimed(..., -'final_artifact_gates', ...)`. Immediately after that `await traceTimed(...)` -block and before the `// Stage 6 — squash commit` comment, insert: - -```ts - activePhase = 'provenance_validation'; - const provenanceRows = this.buildProvenanceRows({ - job, - syncId, - currentHashes, - stageIndex, - reconcileActions, - eviction, - }); - await traceTimed( - runTrace, - 'provenance', - 'provenance_rows_validation', - { - rowCount: provenanceRows.length, - currentRawPathCount: currentHashes.size, - deletedRawPathCount: eviction?.deletedRawPaths.length ?? 0, - }, - async () => { - validateProvenanceRawPaths({ - rows: provenanceRows, - currentRawPaths: new Set(currentHashes.keys()), - deletedRawPaths: new Set(eviction?.deletedRawPaths ?? []), - }); - }, - ); - const reportProvenanceRows = this.toReportProvenanceRows(provenanceRows); -``` - -- [ ] **Step 4: Replace the post-squash provenance construction block** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, in the -`activePhase = 'provenance';` section after squash, delete the current block -that starts with: - -```ts - // Provenance rows: per-artifact when the WU emitted actions, plus a `skipped` - // fallback for raw files that produced nothing so the next DiffSet still sees - // them. - const provenanceRows: Parameters[0] = []; -``` - -And ends with: - -```ts - await runTrace.event('debug', 'provenance', 'provenance_rows_validated', { - rowCount: provenanceRows.length, - }); -``` - -Do not delete the existing call to `await this.deps.provenance.insertMany(provenanceRows);`. -Immediately after that insertion call, add: - -```ts - await runTrace.event('debug', 'provenance', 'provenance_rows_inserted', { - rowCount: provenanceRows.length, - }); -``` - -Then delete the later `const reportProvenanceRows = provenanceRows.map(...)` -block because `reportProvenanceRows` is now created before squash from the -prevalidated rows. - -- [ ] **Step 5: Run the provenance regression** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "invalid provenance raw paths" -``` - -Expected: PASS. The trace contains `provenance_rows_validation_failed`, main -HEAD remains unchanged, and `provenance.insertMany` is not called. - -- [ ] **Step 6: Run the focused isolated-diff suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. - -### Task 3: Type-check, dead-code check, and commit - -**Files:** -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Run the context package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 2: Run the workspace dead-code check** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only existing unrelated Knip/Biome findings. Investigate -any new findings in the two modified files before continuing. - -- [ ] **Step 3: Commit the provenance gate closure** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "fix(ingest): gate provenance before isolated diff squash" -``` - -Expected: one commit containing only the runner and isolated-diff runner test -changes. - -## Self-Review - -Spec coverage: this plan closes the remaining violation of the design's final -global gate invariant by proving invalid provenance raw paths fail before -squash and by moving provenance validation into the pre-main gate boundary. - -Placeholder scan: no placeholder steps remain. Every implementation step names -the exact files, code, commands, and expected results. - -Type consistency: the plan uses existing `IngestProvenanceInsert`, -`IngestReportProvenanceDetail`, `MemoryAction`, `EvictionUnit`, `StageIndex`, -`rawPathsForAction()`, and `validateProvenanceRawPaths()` names. diff --git a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-reference-and-target-gate-closure.md b/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-reference-and-target-gate-closure.md deleted file mode 100644 index 0b7f3837..00000000 --- a/docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-reference-and-target-gate-closure.md +++ /dev/null @@ -1,1350 +0,0 @@ -# Isolated Diff Ingestion V1 Reference and Target Gate Closure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Close the remaining v1-blocking isolated-diff correctness gaps by -validating final wiki page references and enforcing allowed semantic-layer -target connections before any isolated-diff run can squash into main. - -**Architecture:** Extend the existing validation-only artifact gate rather than -adding a resolver. Wiki reference validation runs in the final composed -integration worktree for changed wiki pages, including frontmatter `refs` and -inline `[[page-key]]` references. Semantic-layer target authorization is -enforced in three places: SL write/edit tools reject out-of-scope connection -IDs, WorkUnit patch policy rejects unauthorized `semantic-layer//` -paths, and the runner checks projection, reconciliation, post-processor, and -repair paths before final gates and squash. Target-policy failures emit -persistent JSONL trace events and failed reports with enough path and connection -context for postmortem reconstruction. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, simple-git, existing -`IngestBundleRunner`, `GitService`, `SlWriteSourceTool`, `SlEditSourceTool`, -`KnowledgeWikiService`, `findMissingWikiRefs`, ingest reports, and persistent -ingest traces. - ---- - -## Audit summary - -The implemented plans cover the main v1 isolated-diff flow: integration -worktree creation, child worktrees from the post-projection base, binary -no-rename patches, `git apply --3way --index`, final semantic-layer and wiki -SL/body gates after reconciliation, structured conflict classification, child -cleanup, failed reports, persistent JSONL traces, and pre-squash provenance raw -path validation. - -Two concrete v1-blocking gaps remain: - -- Final global gates do not validate wiki page references. Existing local - checks use `findDanglingWikiRefsForActions()`, but - `validateFinalIngestArtifacts()` validates only wiki `sl_refs` and body - semantic/table references. A WorkUnit can update a page that references an - existing page while another accepted WorkUnit deletes that target page. Both - local gates can pass, and the final tree can squash with a dangling - frontmatter `refs` or inline `[[page-key]]` reference. -- Allowed semantic-layer target connections are not enforced for SL write/edit - tools or integration diffs. The runner computes `slConnectionIds` from the - primary connection plus adapter-declared targets, but `sl_write_source` and - `sl_edit_source` ignore `session.allowedConnectionNames`, and patch policy - rejects only `slDisallowed` plus binary/mode violations. A buggy tool call or - bypassed tool can write `semantic-layer//...` and reach main if - the artifact is otherwise valid. - -Non-blocking gaps remain unchanged: - -- Migrating Notion, LookML, Looker, dbt, MetricFlow, and historic-SQL direct - durable writes to the isolated path. -- Promoting isolated diffs as the default for all connectors. -- Removing the old shared-worktree WorkUnit execution path. -- Interactive, CLI, or agent-driven conflict resolution. -- Auto-merging semantic conflicts that cannot be proven correct. -- Transitive SQL-projection dependency expansion beyond direct declared joins. -- Moving provenance rows to worktree files. -- Adding stored failure reports for failures before an ingest run row exists. - The deterministic trace file is still written for those early failures. - -## File structure - -- Create `packages/context/src/ingest/semantic-layer-target-policy.ts`. - Owns semantic-layer path-to-connection parsing and authorization errors. -- Create `packages/context/src/ingest/semantic-layer-target-policy.test.ts`. - Covers allowed paths, unauthorized paths, non-SL paths, and sorted errors. -- Modify `packages/context/src/ingest/artifact-gates.ts`. - Adds final wiki page reference validation for changed pages. -- Modify `packages/context/src/ingest/artifact-gates.test.ts`. - Adds dangling final wiki `refs` and `[[...]]` coverage and updates mocks with - `listPageKeys()`. -- Create `packages/context/src/tools/action-target-connection.ts`. - Adds session-level target connection validation shared by SL write/edit - tools. -- Modify `packages/context/src/tools/index.ts`. - Exports `validateActionTargetConnection()`. -- Modify `packages/context/src/sl/tools/sl-write-source.tool.ts`. - Rejects session-scoped writes to connections outside - `allowedConnectionNames`. -- Modify `packages/context/src/sl/tools/sl-write-source.tool.test.ts`. - Covers denied session-scoped writes. -- Modify `packages/context/src/sl/tools/sl-edit-source.tool.ts`. - Rejects session-scoped edits and deletes to connections outside - `allowedConnectionNames`. -- Modify `packages/context/src/sl/tools/sl-edit-source.tool.test.ts`. - Covers denied session-scoped edits. -- Modify `packages/context/src/ingest/isolated-diff/git-patch.ts`. - Adds allowed target connection checks to WorkUnit patch policy. -- Modify `packages/context/src/ingest/isolated-diff/git-patch.test.ts`. - Covers unauthorized semantic-layer paths in patches. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.ts`. - Accepts `allowedTargetConnectionIds` and includes it in policy rejection - traces. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`. - Covers traced unauthorized target rejection. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Passes allowed target sets to patch integration and runs a traced target - policy gate over final integration-stage paths before final artifact gates. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Adds cross-WorkUnit wiki-ref deletion, unauthorized WorkUnit patch, and - unauthorized reconciliation mutation regressions. -- Modify `packages/context/src/ingest/index.ts`. - Exports target-policy helpers for tests and future runner checks. - ---- - -### Task 1: Add final wiki reference validation - -**Files:** -- Modify: `packages/context/src/ingest/artifact-gates.test.ts` -- Modify: `packages/context/src/ingest/artifact-gates.ts` - -- [ ] **Step 1: Write failing final wiki reference tests** - -In `packages/context/src/ingest/artifact-gates.test.ts`, add this helper near -the top of the file after the imports: - -```ts -function wikiServiceWithPages(pages: Record) { - return { - listPageKeys: vi.fn().mockResolvedValue(Object.keys(pages)), - readPage: vi.fn().mockImplementation((_scope: string, _scopeId: string | null, pageKey: string) => { - const page = pages[pageKey]; - if (!page) { - return Promise.resolve(null); - } - return Promise.resolve({ - pageKey, - frontmatter: { - summary: pageKey, - usage_mode: 'auto', - refs: page.refs, - sl_refs: page.slRefs, - }, - content: page.content ?? '', - }); - }), - }; -} -``` - -Replace the three existing inline `wikiService = { readPage: ... }` mocks with -`wikiServiceWithPages(...)` so those tests expose `listPageKeys()`. Use these -exact replacements: - -```ts -const wikiService = wikiServiceWithPages({ - 'account-segments': { - slRefs: ['mart_account_segments'], - content: 'ARR is `mart_account_segments.total_contract_arr_cents`.', - }, -}); -``` - -```ts -const wikiService = wikiServiceWithPages({ - 'account-segments': { - slRefs: ['mart_account_segments.total_contract_arr_cents'], - content: 'ARR uses a renamed measure.', - }, -}); -``` - -```ts -const wikiService = wikiServiceWithPages({}); -``` - -Append this test inside `describe('artifact gates', ...)`: - -```ts - it('fails final gates when a changed wiki page references a missing wiki page', async () => { - const wikiService = wikiServiceWithPages({ - 'account-segments': { - refs: ['missing-frontmatter-page'], - content: 'See [[missing-inline-page]] for the related process.', - }, - }); - const semanticLayerService = { - loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }), - }; - - await expect( - validateFinalIngestArtifacts({ - connectionIds: ['warehouse'], - changedWikiPageKeys: ['account-segments'], - touchedSlSources: [], - wikiService: wikiService as never, - semanticLayerService: semanticLayerService as never, - validateTouchedSources: async () => ({ invalidSources: [], validSources: [] }), - tableExists: async () => true, - }), - ).rejects.toThrow(/wiki references target missing page\(s\): account-segments -> missing-frontmatter-page, account-segments -> missing-inline-page/); - }); -``` - -- [ ] **Step 2: Run the failing artifact-gate test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -t "missing wiki page" -``` - -Expected: FAIL because `validateFinalIngestArtifacts()` does not validate wiki -frontmatter `refs` or inline `[[...]]` references. - -- [ ] **Step 3: Implement final wiki reference validation** - -In `packages/context/src/ingest/artifact-gates.ts`, add this import: - -```ts -import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js'; -``` - -Add this helper after `validateWikiSlRefs()`: - -```ts -async function validateWikiRefs(input: FinalArtifactGateInput): Promise { - const dangling: string[] = []; - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - const missingRefs = await findMissingWikiRefs({ - wikiService: input.wikiService, - scope: 'GLOBAL', - scopeId: null, - pageKey, - refs: page.frontmatter.refs, - content: page.content, - }); - for (const missingRef of missingRefs) { - dangling.push(`${pageKey} -> ${missingRef}`); - } - } - return dangling; -} -``` - -In `validateFinalIngestArtifacts()`, immediately after this line: - -```ts - errors.push(...(await validateWikiSlRefs(input))); -``` - -add: - -```ts - const danglingWikiRefs = await validateWikiRefs(input); - if (danglingWikiRefs.length > 0) { - errors.push(`wiki references target missing page(s): ${danglingWikiRefs.join(', ')}`); - } -``` - -- [ ] **Step 4: Run artifact-gate tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit final wiki reference gate** - -Run: - -```bash -git add packages/context/src/ingest/artifact-gates.ts packages/context/src/ingest/artifact-gates.test.ts -git commit -m "fix(ingest): gate final wiki references" -``` - -### Task 2: Enforce target connections in SL tools and patch policy - -**Files:** -- Create: `packages/context/src/tools/action-target-connection.ts` -- Modify: `packages/context/src/tools/index.ts` -- Modify: `packages/context/src/sl/tools/sl-write-source.tool.ts` -- Modify: `packages/context/src/sl/tools/sl-write-source.tool.test.ts` -- Modify: `packages/context/src/sl/tools/sl-edit-source.tool.ts` -- Modify: `packages/context/src/sl/tools/sl-edit-source.tool.test.ts` -- Create: `packages/context/src/ingest/semantic-layer-target-policy.ts` -- Create: `packages/context/src/ingest/semantic-layer-target-policy.test.ts` -- Modify: `packages/context/src/ingest/isolated-diff/git-patch.ts` -- Modify: `packages/context/src/ingest/isolated-diff/git-patch.test.ts` - -- [ ] **Step 1: Write failing session target-connection tests** - -In `packages/context/src/sl/tools/sl-write-source.tool.test.ts`, append this -test inside `describe('SlWriteSourceTool — session gating', ...)`: - -```ts - it('rejects session-scoped writes outside allowed target connections', async () => { - const { tool } = makeTool(); - const session = makeSession({ - allowedConnectionNames: new Set(['warehouse']), - }); - const context: ToolContext = { ...baseContext, session }; - - const result = await tool.call( - { - connectionId: 'finance', - sourceName: 'finance_orders', - source: { - name: 'finance_orders', - table: 'public.orders', - grain: ['id'], - columns: [{ name: 'id', type: 'string' }], - measures: [], - joins: [], - } as any, - } as any, - context, - ); - - expect(result.structured.success).toBe(false); - expect(result.markdown).toContain('connectionId "finance" is outside this ingest session'); - expect(session.actions).toEqual([]); - }); -``` - -In `packages/context/src/sl/tools/sl-edit-source.tool.test.ts`, append this test -inside `describe('SlEditSourceTool — session gating', ...)`: - -```ts - it('rejects session-scoped edits outside allowed target connections', async () => { - const { tool } = makeTool(); - const session = makeSession({ - allowedConnectionNames: new Set(['warehouse']), - }); - const context: ToolContext = { ...baseContext, session }; - - const result = await tool.call( - { - connectionId: 'finance', - sourceName: 'orders', - yaml_edits: [{ oldText: 'measures: []', newText: 'measures: []' }], - } as any, - context, - ); - - expect(result.structured.success).toBe(false); - expect(result.markdown).toContain('connectionId "finance" is outside this ingest session'); - expect(session.actions).toEqual([]); - }); -``` - -- [ ] **Step 2: Run the failing SL tool tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/sl/tools/sl-write-source.tool.test.ts \ - src/sl/tools/sl-edit-source.tool.test.ts \ - -t "outside allowed target connections" -``` - -Expected: FAIL because the tools do not inspect -`session.allowedConnectionNames`. - -- [ ] **Step 3: Add shared session target validation** - -Create `packages/context/src/tools/action-target-connection.ts`: - -```ts -import type { ToolSession } from './tool-session.js'; - -type ActionTargetConnectionValidation = { ok: true } | { ok: false; error: string }; - -export function validateActionTargetConnection( - session: ToolSession | undefined, - connectionId: string, -): ActionTargetConnectionValidation { - const allowed = session?.allowedConnectionNames; - if (!allowed) { - return { ok: true }; - } - if (allowed.has(connectionId)) { - return { ok: true }; - } - const allowedList = [...allowed].sort(); - return { - ok: false, - error: `connectionId "${connectionId}" is outside this ingest session's allowed target connections: ${ - allowedList.length > 0 ? allowedList.join(', ') : '(none)' - }`, - }; -} -``` - -In `packages/context/src/tools/index.ts`, add this export next to -`validateActionRawPaths`: - -```ts -export { validateActionTargetConnection } from './action-target-connection.js'; -``` - -- [ ] **Step 4: Wire target validation into SL write/edit tools** - -In `packages/context/src/sl/tools/sl-write-source.tool.ts`, replace this import: - -```ts -import { addTouchedSlSource, type ToolContext, type ToolOutput, validateActionRawPaths } from '../../tools/index.js'; -``` - -with: - -```ts -import { - addTouchedSlSource, - type ToolContext, - type ToolOutput, - validateActionRawPaths, - validateActionTargetConnection, -} from '../../tools/index.js'; -``` - -In `SlWriteSourceTool.call()`, immediately after: - -```ts - const semanticLayerService = context.session?.semanticLayerService ?? this.semanticLayerService; - const skipIndex = context.session?.isWorktreeScoped === true; -``` - -add: - -```ts - const targetConnectionValidation = validateActionTargetConnection(context.session, connectionId); - if (!targetConnectionValidation.ok) { - return this.buildOutput(false, [targetConnectionValidation.error], sourceName); - } -``` - -In `packages/context/src/sl/tools/sl-edit-source.tool.ts`, replace this import: - -```ts -import { addTouchedSlSource, type ToolContext, type ToolOutput, validateActionRawPaths } from '../../tools/index.js'; -``` - -with: - -```ts -import { - addTouchedSlSource, - type ToolContext, - type ToolOutput, - validateActionRawPaths, - validateActionTargetConnection, -} from '../../tools/index.js'; -``` - -In `SlEditSourceTool.call()`, immediately after: - -```ts - const semanticLayerService = context.session?.semanticLayerService ?? this.semanticLayerService; - const skipIndex = context.session?.isWorktreeScoped === true; -``` - -add: - -```ts - const targetConnectionValidation = validateActionTargetConnection(context.session, connectionId); - if (!targetConnectionValidation.ok) { - return this.buildOutput(false, [targetConnectionValidation.error], sourceName); - } -``` - -- [ ] **Step 5: Write target-policy unit tests** - -Create `packages/context/src/ingest/semantic-layer-target-policy.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { - assertSemanticLayerTargetPathsAllowed, - findDisallowedSemanticLayerTargetPaths, - semanticLayerConnectionIdFromPath, -} from './semantic-layer-target-policy.js'; - -describe('semantic-layer target policy', () => { - it('extracts connection ids from semantic-layer paths', () => { - expect(semanticLayerConnectionIdFromPath('semantic-layer/warehouse/orders.yaml')).toBe('warehouse'); - expect(semanticLayerConnectionIdFromPath('a/semantic-layer/finance/orders.yaml')).toBe('finance'); - expect(semanticLayerConnectionIdFromPath('wiki/global/orders.md')).toBeNull(); - }); - - it('finds semantic-layer paths outside the allowed target connections', () => { - expect( - findDisallowedSemanticLayerTargetPaths({ - paths: [ - 'semantic-layer/warehouse/orders.yaml', - 'semantic-layer/finance/orders.yaml', - 'wiki/global/orders.md', - ], - allowedConnectionIds: new Set(['warehouse']), - }), - ).toEqual([{ path: 'semantic-layer/finance/orders.yaml', connectionId: 'finance' }]); - }); - - it('throws a deterministic error for unauthorized semantic-layer targets', () => { - expect(() => - assertSemanticLayerTargetPathsAllowed({ - paths: ['semantic-layer/finance/orders.yaml', 'semantic-layer/marketing/accounts.yaml'], - allowedConnectionIds: new Set(['warehouse']), - }), - ).toThrow( - /semantic-layer target connection not allowed: semantic-layer\/finance\/orders\.yaml \(finance\), semantic-layer\/marketing\/accounts\.yaml \(marketing\); allowed: warehouse/, - ); - }); -}); -``` - -- [ ] **Step 6: Implement target-policy helpers** - -Create `packages/context/src/ingest/semantic-layer-target-policy.ts`: - -```ts -export interface SemanticLayerTargetPolicyInput { - paths: readonly string[]; - allowedConnectionIds: ReadonlySet; -} - -export interface SemanticLayerTargetPolicyViolation { - path: string; - connectionId: string; -} - -export function semanticLayerConnectionIdFromPath(path: string): string | null { - const normalized = path.replace(/^[ab]\//, ''); - const match = /^semantic-layer\/([^/]+)\//.exec(normalized); - return match?.[1] ?? null; -} - -export function findDisallowedSemanticLayerTargetPaths( - input: SemanticLayerTargetPolicyInput, -): SemanticLayerTargetPolicyViolation[] { - return input.paths - .map((path) => ({ path, connectionId: semanticLayerConnectionIdFromPath(path) })) - .filter((entry): entry is SemanticLayerTargetPolicyViolation => { - return entry.connectionId !== null && !input.allowedConnectionIds.has(entry.connectionId); - }) - .sort((left, right) => { - const byConnection = left.connectionId.localeCompare(right.connectionId); - return byConnection === 0 ? left.path.localeCompare(right.path) : byConnection; - }); -} - -export function assertSemanticLayerTargetPathsAllowed(input: SemanticLayerTargetPolicyInput): void { - const violations = findDisallowedSemanticLayerTargetPaths(input); - if (violations.length === 0) { - return; - } - const allowed = [...input.allowedConnectionIds].sort(); - throw new Error( - `semantic-layer target connection not allowed: ${violations - .map((violation) => `${violation.path} (${violation.connectionId})`) - .join(', ')}; allowed: ${allowed.length > 0 ? allowed.join(', ') : '(none)'}`, - ); -} -``` - -- [ ] **Step 7: Add failing patch-policy test** - -In `packages/context/src/ingest/isolated-diff/git-patch.test.ts`, append this -test inside `describe('isolated diff patch contract', ...)`: - -```ts - it('rejects semantic-layer paths outside allowed target connections', () => { - const patch = - 'diff --git a/semantic-layer/finance/orders.yaml b/semantic-layer/finance/orders.yaml\nindex 1..2 100644\n'; - - expect(() => - assertPatchAllowedForWorkUnit({ - unitKey: 'wu-finance', - patch, - slDisallowed: false, - allowedTargetConnectionIds: new Set(['warehouse']), - }), - ).toThrow(/semantic-layer target connection not allowed: semantic-layer\/finance\/orders.yaml \(finance\); allowed: warehouse/); - }); -``` - -- [ ] **Step 8: Wire target policy into patch parsing** - -In `packages/context/src/ingest/isolated-diff/git-patch.ts`, add this import: - -```ts -import { assertSemanticLayerTargetPathsAllowed } from '../semantic-layer-target-policy.js'; -``` - -Update `PatchPolicyInput` to include allowed targets: - -```ts -export interface PatchPolicyInput { - unitKey: string; - patch: string; - slDisallowed: boolean; - allowedTargetConnectionIds?: ReadonlySet; -} -``` - -In `assertPatchAllowedForWorkUnit()`, after `const touched = -parsePatchTouchedPaths(input.patch);`, add: - -```ts - if (input.allowedTargetConnectionIds) { - assertSemanticLayerTargetPathsAllowed({ - paths: touched.map((entry) => entry.path), - allowedConnectionIds: input.allowedTargetConnectionIds, - }); - } -``` - -- [ ] **Step 9: Run policy and SL tool tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/sl/tools/sl-write-source.tool.test.ts \ - src/sl/tools/sl-edit-source.tool.test.ts \ - src/ingest/semantic-layer-target-policy.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts -``` - -Expected: PASS. - -- [ ] **Step 10: Commit target tool and patch policy** - -Run: - -```bash -git add \ - packages/context/src/tools/action-target-connection.ts \ - packages/context/src/tools/index.ts \ - packages/context/src/sl/tools/sl-write-source.tool.ts \ - packages/context/src/sl/tools/sl-write-source.tool.test.ts \ - packages/context/src/sl/tools/sl-edit-source.tool.ts \ - packages/context/src/sl/tools/sl-edit-source.tool.test.ts \ - packages/context/src/ingest/semantic-layer-target-policy.ts \ - packages/context/src/ingest/semantic-layer-target-policy.test.ts \ - packages/context/src/ingest/isolated-diff/git-patch.ts \ - packages/context/src/ingest/isolated-diff/git-patch.test.ts -git commit -m "fix(ingest): enforce SL target connection scope" -``` - -### Task 3: Wire target policy through integration and final runner gates - -**Files:** -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/index.ts` - -- [ ] **Step 1: Add traced patch-integrator target rejection coverage** - -In `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`, add -`allowedTargetConnectionIds: new Set(['c1']),` to every existing -`integrateWorkUnitPatch()` call. - -Append this test inside `describe('integrateWorkUnitPatch', ...)`: - -```ts - it('classifies unauthorized semantic-layer targets as traced textual conflicts', async () => { - const { homeDir, git, baseSha } = await makeRepo(); - const childDir = join(homeDir, 'child-target-policy'); - await git.addWorktree(childDir, 'child-target-policy', baseSha); - const childGit = git.forWorktree(childDir); - await mkdir(join(childDir, 'semantic-layer/finance'), { recursive: true }); - await writeFile( - join(childDir, 'semantic-layer/finance/orders.yaml'), - 'name: orders\ncolumns: []\njoins: []\nmeasures: []\n', - ); - await childGit.commitFiles(['semantic-layer/finance/orders.yaml'], 'unauthorized sl', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/unauthorized.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-target-policy/trace.jsonl'), - jobId: 'job-target-policy', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-finance', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree: vi.fn().mockResolvedValue(undefined), - slDisallowed: false, - allowedTargetConnectionIds: new Set(['warehouse']), - }); - - expect(result).toMatchObject({ - status: 'textual_conflict', - touchedPaths: ['semantic-layer/finance/orders.yaml'], - }); - const rawTrace = await readFile(trace.tracePath, 'utf-8'); - expect(rawTrace).toContain('patch_policy_rejected'); - expect(rawTrace).toContain('semantic-layer target connection not allowed'); - expect(rawTrace).toContain('allowedTargetConnectionIds'); - }); -``` - -- [ ] **Step 2: Run the failing patch-integrator test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -t "unauthorized semantic-layer targets" -``` - -Expected: FAIL because `IntegrateWorkUnitPatchInput` does not accept or pass -allowed target connections to patch policy. - -- [ ] **Step 3: Implement patch-integrator target policy wiring** - -In `packages/context/src/ingest/isolated-diff/patch-integrator.ts`, add this -field to `IntegrateWorkUnitPatchInput`: - -```ts - allowedTargetConnectionIds: ReadonlySet; -``` - -In the `assertPatchAllowedForWorkUnit()` call, add: - -```ts - allowedTargetConnectionIds: input.allowedTargetConnectionIds, -``` - -In the `patch_policy_rejected` trace data, add: - -```ts - allowedTargetConnectionIds: [...input.allowedTargetConnectionIds].sort(), -``` - -- [ ] **Step 4: Wire WorkUnit target sets and final target gate in the runner** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add this import: - -```ts -import { assertSemanticLayerTargetPathsAllowed } from './semantic-layer-target-policy.js'; -``` - -Near the existing projection state: - -```ts - let projectionTouchedSources: TouchedSlSource[] = []; - let projectionChangedWikiPageKeys: string[] = []; -``` - -add: - -```ts - let projectionTouchedPaths: string[] = []; -``` - -Inside the `adapter.project` block, immediately after `const projectionPaths = -[...]`, add: - -```ts - projectionTouchedPaths = projectionPaths; -``` - -In the `integrateWorkUnitPatch()` call, add: - -```ts - allowedTargetConnectionIds: new Set(slConnectionIds), -``` - -After `const finalTouchedSlSources = this.uniqueTouchedSlSources([...]);` and -before `activePhase = 'final_gates';`, add this traced policy gate: - -```ts - const finalTargetPolicyPaths = [ - ...projectionTouchedPaths, - ...workUnitOutcomes.flatMap((outcome) => outcome.patchTouchedPaths ?? []), - ...postReconciliationPaths, - ...(postProcessorOutcome?.touchedSources ?? []).map( - (source) => `semantic-layer/${source.connectionId}/${source.sourceName}.yaml`, - ), - ]; - const targetPolicyTraceData = { - allowedTargetConnectionIds: slConnectionIds, - touchedPaths: [...new Set(finalTargetPolicyPaths)].sort(), - }; - activePhase = 'target_policy'; - activeFailureDetails = targetPolicyTraceData; - await traceTimed(runTrace, 'target_policy', 'semantic_layer_target_policy', targetPolicyTraceData, async () => { - assertSemanticLayerTargetPathsAllowed({ - paths: finalTargetPolicyPaths, - allowedConnectionIds: new Set(slConnectionIds), - }); - }); - activeFailureDetails = undefined; -``` - -In `packages/context/src/ingest/index.ts`, export the target policy helpers: - -```ts -export { - assertSemanticLayerTargetPathsAllowed, - findDisallowedSemanticLayerTargetPaths, - semanticLayerConnectionIdFromPath, -} from './semantic-layer-target-policy.js'; -``` - -- [ ] **Step 5: Run patch-integrator tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit integration target-policy wiring** - -Run: - -```bash -git add \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/index.ts -git commit -m "fix(ingest): trace isolated SL target policy gates" -``` - -### Task 4: Add end-to-end isolated-diff regressions - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Update the runner test wiki helper** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -replace `makeWikiService()` with this implementation: - -```ts -async function listGlobalWikiPageKeys(root: string): Promise { - const dir = join(root, 'wiki/global'); - const entries = await readdir(dir).catch(() => []); - return entries - .filter((entry) => entry.endsWith('.md')) - .map((entry) => entry.slice(0, -'.md'.length)) - .sort(); -} - -function frontmatterList(yaml: string, key: string): string[] { - const pattern = new RegExp(`${key}:\\n((?: - .+\\n?)*)`); - return ( - pattern - .exec(yaml)?.[1] - ?.split('\n') - .map((line) => line.trim().replace(/^- /, '')) - .filter(Boolean) ?? [] - ); -} - -function makeWikiService(root: string) { - return { - listPageKeys: vi.fn(async (scope: string) => (scope === 'GLOBAL' ? listGlobalWikiPageKeys(root) : [])), - readPage: vi.fn(async (_scope: string, _scopeId: string | null, key: string) => { - const path = join(root, 'wiki/global', `${key}.md`); - const raw = await readFile(path, 'utf-8').catch(() => null); - if (!raw) { - return null; - } - const [, yaml = '', content = ''] = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(raw) ?? []; - return { - pageKey: key, - frontmatter: { - summary: key, - usage_mode: 'auto', - refs: frontmatterList(yaml, 'refs'), - sl_refs: frontmatterList(yaml, 'sl_refs'), - }, - content: content.trim(), - }; - }), - syncFromCommit: vi.fn(), - }; -} -``` - -Add `readdir` to the first import from `node:fs/promises`: - -```ts -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; -``` - -- [ ] **Step 2: Add failing cross-WorkUnit wiki ref regression** - -Append this test inside -`describe('IngestBundleRunner isolated diff path', ...)`: - -```ts - it('rejects final wiki refs broken by another accepted WorkUnit before squash', async () => { - const runtime = await makeRealGitRuntime(); - try { - await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(runtime.configDir, 'wiki/global/source-page.md'), - '---\nsummary: Source page\nusage_mode: auto\n---\n\nSource page\n', - ); - await runtime.git.commitFiles(['wiki/global/source-page.md'], 'seed source page', 'KTX Test', 'system@ktx.local'); - const preRunHead = await runtime.git.revParseHead(); - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { unitKey: 'page-ref', rawFiles: ['pages/ref.json'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'page-delete', rawFiles: ['pages/delete.json'], peerFileIndex: [], dependencyPaths: [] }, - ], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - if (params.telemetryTags.unitKey === 'page-ref') { - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\nrefs:\n - source-page\n---\n\nSee [[source-page]].\n', - ); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'account-segments', - detail: 'Page with wiki ref', - rawPaths: ['pages/ref.json'], - }); - await currentSession.gitService.commitFiles(['wiki/global/account-segments.md'], 'wu page ref', 'KTX Test', 'system@ktx.local'); - } - if (params.telemetryTags.unitKey === 'page-delete') { - await rm(join(root, 'wiki/global/source-page.md'), { force: true }); - currentSession.actions.push({ - target: 'wiki', - type: 'removed', - key: 'source-page', - detail: 'Delete referenced page', - rawPaths: ['pages/delete.json'], - }); - await currentSession.gitService.commitFiles(['wiki/global/source-page.md'], 'wu delete source page', 'KTX Test', 'system@ktx.local'); - } - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [ - ['pages/ref.json', 'h1'], - ['pages/delete.json', 'h2'], - ]); - - await expect( - runner.run({ - jobId: 'job-wiki-ref-conflict', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/wiki references target missing page\(s\): account-segments -> source-page/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-wiki-ref-conflict/trace.jsonl'), 'utf-8'); - expect(trace).toContain('final_artifact_gates_failed'); - expect(trace).toContain('account-segments -> source-page'); - expect(trace).toContain('ingest_failed'); - expect(trace).not.toContain('squash_finished'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 3: Add failing unauthorized WorkUnit patch regression** - -Append this test inside the same `describe(...)` block: - -```ts - it('rejects WorkUnit patches that touch unauthorized semantic-layer target connections', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'finance-source', rawFiles: ['cards/finance.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async () => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'semantic-layer/finance'), { recursive: true }); - await writeFile( - join(root, 'semantic-layer/finance/orders.yaml'), - 'name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures: []\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'finance', 'orders'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'orders', - detail: 'Unauthorized target', - targetConnectionId: 'finance', - rawPaths: ['cards/finance.json'], - }); - await currentSession.gitService.commitFiles(['semantic-layer/finance/orders.yaml'], 'wu unauthorized target', 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['cards/finance.json', 'h1']]); - const preRunHead = await runtime.git.revParseHead(); - - await expect( - runner.run({ - jobId: 'job-unauthorized-wu-target', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/isolated diff textual conflict.*semantic-layer target connection not allowed/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-unauthorized-wu-target/trace.jsonl'), 'utf-8'); - expect(trace).toContain('patch_policy_rejected'); - expect(trace).toContain('semantic-layer/finance/orders.yaml'); - expect(trace).toContain('allowedTargetConnectionIds'); - expect(trace).not.toContain('squash_finished'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 4: Add failing unauthorized reconciliation regression** - -Append this test inside the same `describe(...)` block: - -```ts - it('rejects reconciliation mutations that touch unauthorized semantic-layer target connections before squash', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'valid-page', rawFiles: ['pages/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - if (params.telemetryTags.operationName === 'ingest-bundle-wu') { - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile(join(root, 'wiki/global/valid-page.md'), '---\nsummary: Valid page\nusage_mode: auto\n---\n\nValid\n'); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'valid-page', - detail: 'Valid page', - rawPaths: ['pages/source.json'], - }); - await currentSession.gitService.commitFiles(['wiki/global/valid-page.md'], 'wu valid page', 'KTX Test', 'system@ktx.local'); - } else { - await mkdir(join(root, 'semantic-layer/finance'), { recursive: true }); - await writeFile( - join(root, 'semantic-layer/finance/reconcile_orders.yaml'), - 'name: reconcile_orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures: []\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'finance', 'reconcile_orders'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'reconcile_orders', - detail: 'Unauthorized reconcile target', - targetConnectionId: 'finance', - rawPaths: ['pages/source.json'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/finance/reconcile_orders.yaml'], - 'reconcile unauthorized target', - 'KTX Test', - 'system@ktx.local', - ); - } - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['pages/source.json', 'h1']]); - const preRunHead = await runtime.git.revParseHead(); - - await expect( - runner.run({ - jobId: 'job-unauthorized-reconcile-target', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/semantic-layer target connection not allowed/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-unauthorized-reconcile-target/trace.jsonl'), 'utf-8'); - expect(trace).toContain('semantic_layer_target_policy_failed'); - expect(trace).toContain('semantic-layer/finance/reconcile_orders.yaml'); - expect(trace).toContain('ingest_failed'); - expect(trace).not.toContain('squash_finished'); - const failureReport = (deps.reports.create as any).mock.calls - .map((call: any[]) => call[0]) - .find((report: any) => report.body.status === 'failed'); - expect(failureReport.body.failure).toMatchObject({ - phase: 'target_policy', - message: expect.stringContaining('semantic-layer target connection not allowed'), - }); - expect(failureReport.body.failure.details).toMatchObject({ - allowedTargetConnectionIds: ['warehouse'], - touchedPaths: expect.arrayContaining(['semantic-layer/finance/reconcile_orders.yaml']), - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 5: Run failing runner regressions** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - -t "wiki refs broken|unauthorized semantic-layer target" -``` - -Expected before Tasks 1-3 are complete: FAIL. Expected after Tasks 1-3 are -complete: PASS. - -- [ ] **Step 6: Commit runner regressions** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "test(ingest): cover isolated diff reference and target gates" -``` - -### Task 5: Verification and trace acceptance - -**Files:** -- Verify: `packages/context/src/ingest/*` -- Verify: `packages/context/src/ingest/isolated-diff/*` -- Verify: `packages/context/src/sl/tools/*` -- Verify: `packages/context/src/tools/*` - -- [ ] **Step 1: Run the focused isolated-diff and tool suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/semantic-layer-target-policy.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/sl/tools/sl-write-source.tool.test.ts \ - src/sl/tools/sl-edit-source.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run dead-code check for TypeScript changes** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only pre-existing findings unrelated to the files in this -plan. If there are unrelated pre-existing findings, capture the exact output in -the final handoff. - -- [ ] **Step 4: Run pre-commit for changed files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts \ - packages/context/src/ingest/semantic-layer-target-policy.ts \ - packages/context/src/ingest/semantic-layer-target-policy.test.ts \ - packages/context/src/ingest/isolated-diff/git-patch.ts \ - packages/context/src/ingest/isolated-diff/git-patch.test.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/index.ts \ - packages/context/src/tools/action-target-connection.ts \ - packages/context/src/tools/index.ts \ - packages/context/src/sl/tools/sl-write-source.tool.ts \ - packages/context/src/sl/tools/sl-write-source.tool.test.ts \ - packages/context/src/sl/tools/sl-edit-source.tool.ts \ - packages/context/src/sl/tools/sl-edit-source.tool.test.ts -``` - -Expected: PASS. If the repository has no usable pre-commit configuration or the -local `uv` version cannot satisfy the project pin, report the exact failure and -run `pnpm --filter @ktx/context run type-check` plus the Vitest suite above. - -- [ ] **Step 5: Verify persistent trace acceptance criteria** - -Inspect the traces produced by the two new runner failures. The trace must -include these events and fields: - -```text -job-wiki-ref-conflict: -- final_artifact_gates_failed -- ingest_failed -- failure_report_created -- no squash_finished event -- error.message includes "account-segments -> source-page" - -job-unauthorized-wu-target: -- patch_policy_rejected -- ingest_failed -- failure_report_created -- no squash_finished event -- data.allowedTargetConnectionIds includes "warehouse" -- data.touchedPaths includes "semantic-layer/finance/orders.yaml" - -job-unauthorized-reconcile-target: -- semantic_layer_target_policy_started -- semantic_layer_target_policy_failed -- ingest_failed -- failure_report_created -- no squash_finished event -- data.allowedTargetConnectionIds includes "warehouse" -- data.touchedPaths includes "semantic-layer/finance/reconcile_orders.yaml" -- error.message includes "semantic-layer target connection not allowed" -``` - -The failed stored reports for the two target-policy regressions must include: - -```text -failure.phase: -- "integration" for WorkUnit patch policy rejection -- "target_policy" for reconciliation or integration-stage mutation rejection - -failure.details: -- allowedTargetConnectionIds -- touchedPaths -- invalid path and connection in the error message -``` - -- [ ] **Step 6: Commit verification-only fixes if needed** - -If verification exposes formatting, type, or test issues in the files changed -by this plan, fix them and commit: - -```bash -git add \ - packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts \ - packages/context/src/ingest/semantic-layer-target-policy.ts \ - packages/context/src/ingest/semantic-layer-target-policy.test.ts \ - packages/context/src/ingest/isolated-diff/git-patch.ts \ - packages/context/src/ingest/isolated-diff/git-patch.test.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/index.ts \ - packages/context/src/tools/action-target-connection.ts \ - packages/context/src/tools/index.ts \ - packages/context/src/sl/tools/sl-write-source.tool.ts \ - packages/context/src/sl/tools/sl-write-source.tool.test.ts \ - packages/context/src/sl/tools/sl-edit-source.tool.ts \ - packages/context/src/sl/tools/sl-edit-source.tool.test.ts -git commit -m "chore(ingest): verify isolated diff gate closure" -``` - -If verification passes without edits, do not create an empty commit. - -## Self-review - -Spec coverage: - -- Wiki `refs` and inline `[[...]]` validation is added to the final global gate - for changed wiki pages in the composed integration tree. -- WorkUnit patch integration rejects unauthorized semantic-layer target - connections before patch application can commit into the integration tree. -- Reconciliation and other integration-stage mutations are checked with a - traced target-policy gate before final artifact gates and before squash. -- SL write/edit tools reject out-of-scope target connections during - session-scoped ingest tool calls. -- Failure traces and failed reports include explicit target-policy context, - rejected paths, allowed connection IDs, failure phase, and no `squash_finished` - event when the run stops before main. - -Placeholder scan: - -- The plan contains no placeholder tokens, deferred implementation notes, or - unspecified edge-case instructions. - -Type consistency: - -- `allowedTargetConnectionIds` is the patch-policy and patch-integrator field. -- `allowedConnectionNames` remains the existing `ToolSession` field. -- `semantic_layer_target_policy_*` is the trace event prefix from `traceTimed()`. -- `refs` is the existing wiki frontmatter field that implements the spec's - wiki-reference gate. diff --git a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-connector-migration.md b/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-connector-migration.md deleted file mode 100644 index cb7b7f53..00000000 --- a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-connector-migration.md +++ /dev/null @@ -1,1051 +0,0 @@ -# Isolated Diff Ingestion V1 Connector Migration Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Route Notion, LookML, Looker, dbt, and MetricFlow direct durable-write -ingest through the isolated-diff runner path. - -**Architecture:** Keep isolated-diff routing private and runner-owned by -centralizing the default source-key list outside adapters and public -configuration. The shared runner continues to own per-work-unit child -worktrees, patch integration, gates, repair, traces, and reports. MetricFlow -also gets its deterministic semantic-model import moved into the adapter -projector hook so those authoritative writes land in the integration worktree -before child worktrees are created. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, simple-git, existing -`IngestBundleRunner`, `SessionWorktreeService`, `MetricflowSourceAdapter`, -`importMetricflowSemanticModels()`, and local ingest runtime wiring. - ---- - -## Audit summary - -This audit read -`docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md`, all -implemented isolated-diff plans from May 17 and May 18, and the current runner -and adapter code under `packages/context/src/ingest/`. - -Implemented v1 safety plans: - -- `2026-05-17-isolated-diff-ingestion-v1-core.md`: core isolated worktrees, - patch proposals, integration, trace storage, body-reference parsing, and the - Metabase stale-measure regression exist in code. -- `2026-05-17-isolated-diff-ingestion-v1-gates-and-trace-closure.md`: final - gates run after reconciliation and later mutating stages, child worktrees are - cleaned up, failed reports are stored, and traces cover postmortem phases. -- `2026-05-17-isolated-diff-ingestion-v1-provenance-gate-closure.md`: - provenance validation runs before squash. -- `2026-05-17-isolated-diff-ingestion-v1-reference-and-target-gate-closure.md`: - final wiki reference gates, semantic-layer target policy, and patch target - checks exist. -- `2026-05-17-isolated-diff-ingestion-v1-global-wiki-reference-gate-closure.md`: - global wiki reference scope expands when semantic-layer sources change or - wiki pages are removed. -- `2026-05-18-isolated-diff-ingestion-v1-textual-conflict-resolver.md`: - bounded textual conflict repair exists and is wired into patch integration. -- `2026-05-18-isolated-diff-ingestion-v1-gate-repair.md`: bounded repair for - cleanly applied patch and final artifact gate failures exists. - -Current v1-blocking gaps: - -- `packages/context/src/ingest/local-bundle-runtime.ts` still sets - `isolatedDiffSourceKeys: ['metabase']`, so Notion, LookML, Looker, dbt, and - MetricFlow still use the old shared-worktree WorkUnit path by default. -- `packages/context/src/ingest/ingest-bundle.runner.ts` still contains the - shared-worktree fallback branch. That branch must remain until connector - migration and default promotion finish, but the other direct durable-write - connectors must stop taking it. -- There is no regression matrix proving the five non-Metabase connector source - keys route through child worktrees and produce `isolatedDiff` report data. -- MetricFlow has `importMetricflowSemanticModels()` but - `MetricflowSourceAdapter` does not expose it as `project()`. The spec says - MetricFlow's deterministic semantic-model import becomes an ingestion - projector, not a post-WorkUnit shared-worktree write. - -Later v1-blocking gaps after this plan: - -- Promote isolated diffs to the default once the Metabase regression and at - least one non-Metabase connector pass are green. -- Remove the old shared-worktree WorkUnit execution path after the default path - is promoted. - -Non-blocking gaps: - -- Deterministic semantic merge helpers from rollout step 9. -- Transitive SQL-projection dependency expansion beyond direct declared joins. -- Moving provenance rows into worktree files. -- Public connector knobs such as `executionMode`, `planningStrategy`, or - `conflictPolicy`. -- Resolver context expansion to include richer transcript excerpts and every - previously applied overlapping patch. - -## File structure - -- Create `packages/context/src/ingest/isolated-diff/source-routing.ts`. - Owns the private runner default source-key list for direct durable-write - connectors. -- Create `packages/context/src/ingest/isolated-diff/source-routing.test.ts`. - Locks the internal list to Metabase plus the five migrated connectors. -- Modify `packages/context/src/ingest/local-bundle-runtime.ts`. - Uses the centralized isolated-diff source-key list instead of the Metabase-only - inline array. -- Modify `packages/context/src/ingest/local-bundle-runtime.test.ts`. - Verifies local ingest runtime deps enable isolated routing for the migrated - connector list. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Adds a non-Metabase source-key routing matrix that proves direct writes run in - isolated child worktrees and report `isolatedDiff` metadata. -- Modify `packages/context/src/ingest/types.ts`. - Adds the semantic-layer service to `DeterministicProjectionContext` so - adapter projectors can write to the integration worktree. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Passes the semantic-layer service into adapter projectors. -- Create `packages/context/src/ingest/adapters/metricflow/projection-config.ts`. - Persists and reads MetricFlow projection metadata from the staged snapshot and - converts parsed target-table mappings into importer host-table inputs. -- Modify `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts`. - Writes projection metadata during fetch and implements `project()` via - `importMetricflowSemanticModels()`. -- Modify `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts`. - Covers projection metadata persistence and the adapter projector. -- Modify `packages/context/src/ingest/local-bundle-ingest.test.ts`. - Verifies local MetricFlow ingest takes the isolated path and records a - projection commit. - ---- - -### Task 1: Centralize runner-owned connector routing - -**Files:** -- Create: `packages/context/src/ingest/isolated-diff/source-routing.ts` -- Create: `packages/context/src/ingest/isolated-diff/source-routing.test.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` - -- [ ] **Step 1: Write the failing routing tests** - -Create `packages/context/src/ingest/isolated-diff/source-routing.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { - defaultIsolatedDiffSourceKeys, - isIsolatedDiffDirectWriteSourceKey, - ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS, -} from './source-routing.js'; - -describe('isolated-diff source routing', () => { - it('keeps the runner-owned direct-write connector list explicit', () => { - expect(ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS).toEqual([ - 'metabase', - 'notion', - 'lookml', - 'looker', - 'dbt', - 'metricflow', - ]); - }); - - it('returns a mutable copy for runtime settings', () => { - const keys = defaultIsolatedDiffSourceKeys(); - keys.push('fake'); - - expect(defaultIsolatedDiffSourceKeys()).toEqual([ - 'metabase', - 'notion', - 'lookml', - 'looker', - 'dbt', - 'metricflow', - ]); - }); - - it('recognizes migrated connector source keys only', () => { - expect(isIsolatedDiffDirectWriteSourceKey('notion')).toBe(true); - expect(isIsolatedDiffDirectWriteSourceKey('metricflow')).toBe(true); - expect(isIsolatedDiffDirectWriteSourceKey('historic-sql')).toBe(false); - expect(isIsolatedDiffDirectWriteSourceKey('live-database')).toBe(false); - }); -}); -``` - -In `packages/context/src/ingest/local-bundle-runtime.test.ts`, add this helper -type near the existing runtime helper types: - -```ts -type RuntimeWithSettingsDeps = { - deps: { - settings: { - isolatedDiffSourceKeys?: string[]; - }; - }; -}; -``` - -Then append this test inside `describe('createLocalBundleIngestRuntime', ...)`: - -```ts - it('enables isolated-diff routing for direct durable-write connectors', () => { - const runtime = createLocalBundleIngestRuntime({ - project, - adapters: [new FakeSourceAdapter()], - agentRunner: testAgentRunner(), - }); - - const settings = (runtime.runner as unknown as RuntimeWithSettingsDeps).deps.settings; - - expect(settings.isolatedDiffSourceKeys).toEqual([ - 'metabase', - 'notion', - 'lookml', - 'looker', - 'dbt', - 'metricflow', - ]); - }); -``` - -- [ ] **Step 2: Run the failing routing tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/source-routing.test.ts src/ingest/local-bundle-runtime.test.ts -t "isolated-diff source routing|direct durable-write connectors" -``` - -Expected: FAIL because `source-routing.ts` does not exist and local runtime -still uses only `['metabase']`. - -- [ ] **Step 3: Add centralized routing code** - -Create `packages/context/src/ingest/isolated-diff/source-routing.ts`: - -```ts -export const ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS = [ - 'metabase', - 'notion', - 'lookml', - 'looker', - 'dbt', - 'metricflow', -] as const; - -export type IsolatedDiffDirectWriteSourceKey = (typeof ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS)[number]; - -const ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEY_SET = new Set(ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS); - -export function defaultIsolatedDiffSourceKeys(): string[] { - return [...ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS]; -} - -export function isIsolatedDiffDirectWriteSourceKey( - sourceKey: string, -): sourceKey is IsolatedDiffDirectWriteSourceKey { - return ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEY_SET.has(sourceKey); -} -``` - -In `packages/context/src/ingest/local-bundle-runtime.ts`, add this import: - -```ts -import { defaultIsolatedDiffSourceKeys } from './isolated-diff/source-routing.js'; -``` - -Then replace the settings value: - -```ts - isolatedDiffSourceKeys: ['metabase'], -``` - -with: - -```ts - isolatedDiffSourceKeys: defaultIsolatedDiffSourceKeys(), -``` - -- [ ] **Step 4: Run the routing tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/source-routing.test.ts src/ingest/local-bundle-runtime.test.ts -t "isolated-diff source routing|direct durable-write connectors" -``` - -Expected: PASS. - -- [ ] **Step 5: Commit routing changes** - -Run: - -```bash -git add packages/context/src/ingest/isolated-diff/source-routing.ts \ - packages/context/src/ingest/isolated-diff/source-routing.test.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/local-bundle-runtime.test.ts -git commit -m "feat(ingest): route direct-write connectors through isolated diffs" -``` - -Expected: commit is created with only the routing files. - ---- - -### Task 2: Add non-Metabase isolated routing regressions - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Write the failing non-Metabase routing matrix** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -add this import: - -```ts -import { defaultIsolatedDiffSourceKeys } from './isolated-diff/source-routing.js'; -``` - -Change `makeDeps()` to accept a source key: - -```ts -function makeDeps(runtime: Awaited>, sourceKey = 'metabase') { - const adapter: any = { - source: sourceKey, - skillNames: [], - detect: vi.fn().mockResolvedValue(true), - chunk: vi.fn().mockResolvedValue({ - workUnits: [ - { unitKey: 'card-wiki', rawFiles: ['cards/wiki.json'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'card-source', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }, - ], - }), - }; -``` - -In the same helper, replace the settings block with: - -```ts - settings: { - memoryIngestionModel: 'test', - probeRowCount: 1, - isolatedDiffSourceKeys: defaultIsolatedDiffSourceKeys(), - ingestTraceLevel: 'trace', - }, -``` - -Change `mockStageRawFiles()` to accept the source key: - -```ts -async function mockStageRawFiles( - runner: IngestBundleRunner, - runtime: Awaited>, - hashes: [string, string][], - sourceKey = 'metabase', -) { - (runner as any).resolveStagedDir = vi.fn().mockResolvedValue(join(runtime.homeDir, 'stage')); - (runner as any).stageRawFilesStage1 = vi.fn(async ({ worktreeRoot }: any) => { - const rawDir = join(worktreeRoot, 'raw-sources/warehouse', sourceKey, 's'); - await mkdir(rawDir, { recursive: true }); - for (const [rawPath] of hashes) { - await mkdir(join(rawDir, rawPath.split('/').slice(0, -1).join('/')), { recursive: true }); - await writeFile(join(rawDir, rawPath), '{}'); - } - return { currentHashes: new Map(hashes), rawDirInWorktree: `raw-sources/warehouse/${sourceKey}/s` }; - }); -} -``` - -Append this test inside `describe('IngestBundleRunner isolated diff path', ...)`: - -```ts - it.each(['notion', 'lookml', 'looker', 'dbt', 'metricflow'] as const)( - 'routes %s direct writes through isolated child worktrees', - async (sourceKey) => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime, sourceKey); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { - unitKey: `${sourceKey}-wiki`, - rawFiles: [`${sourceKey}/page.json`], - peerFileIndex: [], - dependencyPaths: [], - }, - ], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global', `${sourceKey}-isolated.md`), - `---\nsummary: ${sourceKey} isolated write\nusage_mode: auto\n---\n\nIsolated ${sourceKey} write.\n`, - 'utf-8', - ); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: `${sourceKey}-isolated`, - detail: `${sourceKey} isolated write`, - rawPaths: [`${sourceKey}/page.json`], - }); - await currentSession.gitService.commitFiles( - [`wiki/global/${sourceKey}-isolated.md`], - `${sourceKey} wiki`, - 'KTX Test', - 'system@ktx.local', - ); - - expect(params.telemetryTags).toMatchObject({ - operationName: 'ingest-bundle-wu', - source: sourceKey, - unitKey: `${sourceKey}-wiki`, - }); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [[`${sourceKey}/page.json`, 'h1']], sourceKey); - - await expect( - runner.run({ - jobId: `job-${sourceKey}`, - connectionId: 'warehouse', - sourceKey, - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).resolves.toMatchObject({ - jobId: `job-${sourceKey}`, - failedWorkUnits: [], - workUnitCount: 1, - }); - - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces', `job-${sourceKey}`, 'trace.jsonl'), 'utf-8'); - expect(trace).toContain('isolated_diff_enabled'); - expect(trace).toContain('work_unit_child_created'); - expect(trace).toContain('work_unit_patch_collected'); - expect(trace).toContain('patch_apply_started'); - expect(trace).not.toContain('shared_worktree_path_enabled'); - - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0]; - expect(reportCreate?.body.isolatedDiff).toMatchObject({ - enabled: true, - acceptedPatches: 1, - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }, - ); -``` - -- [ ] **Step 2: Run the non-Metabase routing matrix** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "routes .* direct writes" -``` - -Expected: PASS after Task 1. If it fails, the failure must point to one of -these concrete problems: settings do not include the source key, the shared path -still runs, or the final report lacks `isolatedDiff`. - -- [ ] **Step 3: Commit runner regression coverage** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "test(ingest): cover non-metabase isolated diff routing" -``` - -Expected: commit contains only the isolated runner regression file. - ---- - -### Task 3: Move MetricFlow deterministic import into projection - -**Files:** -- Modify: `packages/context/src/ingest/types.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Create: `packages/context/src/ingest/adapters/metricflow/projection-config.ts` -- Modify: `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts` -- Modify: `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts` - -- [ ] **Step 1: Write failing MetricFlow projector tests** - -In `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts`, -add these imports: - -```ts -import type { MetricFlowParseResult } from './deep-parse.js'; -import { readMetricflowProjectionConfig, writeMetricflowProjectionConfig } from './projection-config.js'; -``` - -Add this helper near the top of the file: - -```ts -function metricflowParseResult(): MetricFlowParseResult { - return { - semanticModels: [ - { - name: 'orders', - description: 'Orders', - modelRef: 'orders', - dimensions: [{ name: 'status', column: 'status', type: 'string', label: 'Status' }], - measures: [{ type: 'simple', name: 'order_count', column: 'id', aggregation: 'count' }], - entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id' }], - defaultTimeDimension: null, - }, - ], - crossModelMetrics: [], - relationships: [], - warnings: ['parser warning'], - }; -} -``` - -Append these tests inside `describe('MetricflowSourceAdapter', ...)`: - -```ts - it('persists parsed target tables for deterministic projection during fetch', async () => { - const repo = await makeRepo(tmpRoot, { - 'dbt_project.yml': 'name: analytics\n', - 'models/orders.yml': 'semantic_models:\n - name: orders\n model: ref("orders")\n', - }); - - await adapter.fetch?.( - { - repoUrl: repo.repoUrl, - branch: 'main', - path: null, - authToken: null, - parsedTargetTables: { - orders: { - ok: true, - catalog: null, - schema: 'analytics', - name: 'orders', - canonicalTable: 'analytics.orders', - }, - }, - }, - stagedDir, - { connectionId: 'warehouse-1', sourceKey: 'metricflow' }, - ); - - await expect(readMetricflowProjectionConfig(stagedDir)).resolves.toMatchObject({ - parsedTargetTables: { - orders: { - ok: true, - schema: 'analytics', - name: 'orders', - }, - }, - }); - }); - - it('projects parsed MetricFlow semantic models in the integration worktree', async () => { - await writeMetricflowProjectionConfig(stagedDir, { - parsedTargetTables: { - orders: { - ok: true, - catalog: null, - schema: 'analytics', - name: 'orders', - canonicalTable: 'analytics.orders', - }, - }, - }); - const scoped = { - getManifestEntry: vi.fn().mockResolvedValue(null), - isManifestBacked: vi.fn().mockResolvedValue(false), - loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }), - loadSource: vi.fn().mockResolvedValue(null), - writeSource: vi.fn().mockResolvedValue({ warnings: [] }), - }; - const semanticLayerService = { - forWorktree: vi.fn().mockReturnValue(scoped), - getManifestEntry: vi.fn(), - isManifestBacked: vi.fn(), - loadAllSources: vi.fn(), - loadSource: vi.fn(), - writeSource: vi.fn(), - }; - - const result = await adapter.project?.({ - connectionId: 'warehouse-1', - sourceKey: 'metricflow', - syncId: 'sync-1', - jobId: 'job-1', - runId: 'run-1', - stagedDir, - workdir: '/tmp/metricflow-integration', - parseArtifacts: metricflowParseResult(), - semanticLayerService: semanticLayerService as never, - }); - - expect(semanticLayerService.forWorktree).toHaveBeenCalledWith('/tmp/metricflow-integration'); - expect(scoped.writeSource).toHaveBeenCalledWith( - 'warehouse-1', - expect.objectContaining({ name: 'orders' }), - 'dbt MetricFlow', - expect.any(String), - 'dbt MetricFlow sync: create source orders', - { skipValidation: true }, - ); - expect(result).toMatchObject({ - warnings: ['parser warning'], - errors: [], - touchedSources: [{ connectionId: 'warehouse-1', sourceName: 'orders' }], - changedWikiPageKeys: [], - }); - }); - - it('returns a projection error when parse artifacts are missing', async () => { - const result = await adapter.project?.({ - connectionId: 'warehouse-1', - sourceKey: 'metricflow', - syncId: 'sync-1', - jobId: 'job-1', - runId: 'run-1', - stagedDir, - workdir: '/tmp/metricflow-integration', - parseArtifacts: undefined, - semanticLayerService: {} as never, - }); - - expect(result).toMatchObject({ - warnings: [], - errors: ['MetricFlow deterministic projection requires parseArtifacts from chunk()'], - touchedSources: [], - changedWikiPageKeys: [], - }); - }); -``` - -- [ ] **Step 2: Run the failing MetricFlow projector tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/metricflow/metricflow.adapter.test.ts -t "deterministic projection|projects parsed|parse artifacts" -``` - -Expected: FAIL because `projection-config.ts` and `adapter.project()` do not -exist. - -- [ ] **Step 3: Add projector service context** - -In `packages/context/src/ingest/types.ts`, add this import: - -```ts -import type { SemanticLayerService } from '../sl/index.js'; -``` - -Then extend `DeterministicProjectionContext`: - -```ts -export interface DeterministicProjectionContext { - connectionId: string; - sourceKey: string; - syncId: string; - jobId: string; - runId: string; - stagedDir: string; - workdir: string; - parseArtifacts?: unknown; - semanticLayerService: SemanticLayerService; -} -``` - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add this property to -the `adapter.project!({ ... })` call: - -```ts - semanticLayerService: this.deps.semanticLayerService, -``` - -- [ ] **Step 4: Add MetricFlow projection config helpers** - -Create `packages/context/src/ingest/adapters/metricflow/projection-config.ts`: - -```ts -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { z } from 'zod'; -import { parsedTargetTableSchema, type ParsedTargetTable } from '../../parsed-target-table.js'; -import type { MetricflowHostTable } from './semantic-models.js'; - -export const METRICFLOW_PROJECTION_CONFIG_FILE = 'sync-config.json'; - -export const metricflowProjectionConfigSchema = z.object({ - parsedTargetTables: z.record(z.string(), parsedTargetTableSchema).default({}), -}); - -export type MetricflowProjectionConfig = z.infer; - -export async function writeMetricflowProjectionConfig( - stagedDir: string, - config: MetricflowProjectionConfig, -): Promise { - const parsed = metricflowProjectionConfigSchema.parse(config); - await writeFile(join(stagedDir, METRICFLOW_PROJECTION_CONFIG_FILE), `${JSON.stringify(parsed, null, 2)}\n`, 'utf-8'); -} - -export async function readMetricflowProjectionConfig(stagedDir: string): Promise { - const path = join(stagedDir, METRICFLOW_PROJECTION_CONFIG_FILE); - try { - return metricflowProjectionConfigSchema.parse(JSON.parse(await readFile(path, 'utf-8'))); - } catch (error) { - if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') { - return { parsedTargetTables: {} }; - } - throw error; - } -} - -export function metricflowHostTablesFromParsedTargets( - parsedTargetTables: Record, -): MetricflowHostTable[] { - return Object.entries(parsedTargetTables) - .flatMap(([id, table]) => - table.ok - ? [ - { - id, - name: table.name, - catalog: table.catalog, - db: table.schema, - columns: [], - }, - ] - : [], - ) - .sort((left, right) => left.id.localeCompare(right.id)); -} -``` - -- [ ] **Step 5: Implement MetricFlow adapter projection** - -In `packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts`, -replace the type import with: - -```ts -import type { - ChunkResult, - DeterministicProjectionContext, - DiffSet, - FetchContext, - ProjectionResult, - SourceAdapter, -} from '../../types.js'; -``` - -Add these imports: - -```ts -import { importMetricflowSemanticModels } from './import-semantic-models.js'; -import { - metricflowHostTablesFromParsedTargets, - readMetricflowProjectionConfig, - writeMetricflowProjectionConfig, -} from './projection-config.js'; -``` - -After `await fetchMetricflowRepo({ ... })` in `fetch()`, persist projection -metadata: - -```ts - await writeMetricflowProjectionConfig(stagedDir, { - parsedTargetTables: config.parsedTargetTables, - }); -``` - -Add this method to `MetricflowSourceAdapter`: - -```ts - async project(ctx: DeterministicProjectionContext): Promise { - if (!isMetricFlowParseResult(ctx.parseArtifacts)) { - return { - warnings: [], - errors: ['MetricFlow deterministic projection requires parseArtifacts from chunk()'], - touchedSources: [], - changedWikiPageKeys: [], - }; - } - - const projectionConfig = await readMetricflowProjectionConfig(ctx.stagedDir); - const result = await importMetricflowSemanticModels( - { semanticLayerService: ctx.semanticLayerService }, - { - connectionId: ctx.connectionId, - parseResult: ctx.parseArtifacts, - targetSchema: null, - hostTables: metricflowHostTablesFromParsedTargets(projectionConfig.parsedTargetTables), - workdir: ctx.workdir, - }, - ); - - return { - result, - warnings: result.warnings, - errors: result.errors, - touchedSources: result.touchedSources, - changedWikiPageKeys: [], - }; - } -``` - -Add this helper below `parseMetricflowStagedDirForImport()`: - -```ts -function isMetricFlowParseResult(value: unknown): value is MetricFlowParseResult { - if (!value || typeof value !== 'object') { - return false; - } - const candidate = value as Partial; - return ( - Array.isArray(candidate.semanticModels) && - Array.isArray(candidate.crossModelMetrics) && - Array.isArray(candidate.relationships) && - Array.isArray(candidate.warnings) - ); -} -``` - -- [ ] **Step 6: Run the MetricFlow projector tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/adapters/metricflow/metricflow.adapter.test.ts -t "deterministic projection|projects parsed|parse artifacts" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit MetricFlow projection changes** - -Run: - -```bash -git add packages/context/src/ingest/types.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/adapters/metricflow/projection-config.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts -git commit -m "feat(ingest): project metricflow semantic models before work units" -``` - -Expected: commit contains only MetricFlow projector and projector context files. - ---- - -### Task 4: Verify MetricFlow takes the isolated path locally - -**Files:** -- Modify: `packages/context/src/ingest/local-bundle-ingest.test.ts` - -- [ ] **Step 1: Add local MetricFlow isolated projection assertions** - -In -`packages/context/src/ingest/local-bundle-ingest.test.ts`, update the existing -`runs full MetricFlow local ingest from a dbt repo fixture through the canonical -runner` test after the report assertions: - -```ts - expect(result.report.body.isolatedDiff).toMatchObject({ - enabled: true, - acceptedPatches: 0, - projectionSha: expect.any(String), - }); - - const projectedSourcePath = join(metricflowProject.projectDir, 'semantic-layer/warehouse/orders.yaml'); - await expect(readFile(projectedSourcePath, 'utf-8')).resolves.toContain('name: orders'); -``` - -Keep the existing `expect(agentRunner.runLoop).toHaveBeenCalledTimes(1);` -assertion. It proves the connector remains hybrid: deterministic projection -runs first, then the MetricFlow WorkUnit still runs for agent-authored wiki or -enrichment work. - -- [ ] **Step 2: Run the local MetricFlow acceptance test** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-ingest.test.ts -t "runs full MetricFlow local ingest" -``` - -Expected: PASS. The report body must include `isolatedDiff.enabled: true`, and -the final project must contain `semantic-layer/warehouse/orders.yaml`. - -- [ ] **Step 3: Commit local acceptance coverage** - -Run: - -```bash -git add packages/context/src/ingest/local-bundle-ingest.test.ts -git commit -m "test(ingest): verify metricflow isolated projection path" -``` - -Expected: commit contains only the local bundle ingest acceptance test. - ---- - -### Task 5: Final verification - -**Files:** -- Verify: `packages/context/src/ingest/isolated-diff/source-routing.ts` -- Verify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/types.ts` -- Verify: `packages/context/src/ingest/adapters/metricflow/*` -- Verify: `packages/context/src/ingest/*.test.ts` - -- [ ] **Step 1: Run focused connector migration tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/isolated-diff/source-routing.test.ts \ - src/ingest/local-bundle-runtime.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ - src/ingest/local-bundle-ingest.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the isolated-diff safety suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/semantic-layer-target-policy.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/isolated-diff/textual-conflict-resolver.test.ts \ - src/ingest/isolated-diff/source-routing.test.ts \ - src/ingest/final-gate-repair.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/report-snapshot.test.ts \ - src/sl/tools/sl-write-source.tool.test.ts \ - src/sl/tools/sl-edit-source.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only pre-existing findings unrelated to this connector -migration. - -- [ ] **Step 5: Run formatting and diff checks** - -Run: - -```bash -pnpm exec biome check \ - packages/context/src/ingest/isolated-diff/source-routing.ts \ - packages/context/src/ingest/isolated-diff/source-routing.test.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/local-bundle-runtime.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/types.ts \ - packages/context/src/ingest/adapters/metricflow/projection-config.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts \ - packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts \ - packages/context/src/ingest/local-bundle-ingest.test.ts -git diff --check -``` - -Expected: PASS. - -- [ ] **Step 6: Decide docs-site impact** - -No `docs-site/content/docs/` update is required for this plan because it -changes an internal ingest correctness route and does not add, remove, or rename -public CLI commands, flags, config fields, or connector setup instructions. - -- [ ] **Step 7: Commit verification fixes only when files changed** - -If verification required formatting or type-only edits, run: - -```bash -git add packages/context/src/ingest docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-connector-migration.md -git commit -m "chore(ingest): verify isolated diff connector migration" -``` - -Expected: no empty commit. If no files changed during verification, leave the -branch at the previous task commit. - -## Self-review - -Spec coverage: - -- Rollout step 8 is covered for Notion, LookML, Looker, dbt, and MetricFlow by - the centralized source-key routing and the non-Metabase isolated runner - regression matrix. -- The connector migration notes remain source-shaped: adapters keep fetch, - chunk, clustering, target resolution, and domain rules; the runner owns - execution isolation and gates. -- MetricFlow's existing deterministic semantic-model import moves into - `project()`, so its authoritative writes happen in the integration worktree - before child worktrees are created. -- Notion clustering remains adapter logic; the routing change only changes where - WorkUnits execute. -- LookML `slDisallowed` remains adapter-scoped and continues to be enforced by - existing scoped tools and integration patch policy. -- Default promotion and old shared-worktree path removal remain later rollout - steps and are not implemented by this plan. - -Placeholder scan: - -- No deferred implementation markers remain. -- Every code-changing step includes exact paths, commands, expected outcomes, - and concrete code or insertion snippets. - -Type consistency: - -- The routing helper names are `ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS`, - `defaultIsolatedDiffSourceKeys()`, and - `isIsolatedDiffDirectWriteSourceKey()` across code and tests. -- The MetricFlow projection config helper names are - `writeMetricflowProjectionConfig()`, `readMetricflowProjectionConfig()`, and - `metricflowHostTablesFromParsedTargets()`. -- `DeterministicProjectionContext.semanticLayerService` is passed by - `IngestBundleRunner` and consumed by `MetricflowSourceAdapter.project()`. diff --git a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-default-promotion.md b/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-default-promotion.md deleted file mode 100644 index dc993557..00000000 --- a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-default-promotion.md +++ /dev/null @@ -1,754 +0,0 @@ -# Isolated Diff Ingestion V1 Default Promotion Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Promote isolated-diff WorkUnit execution to the default ingest runner -path while keeping the old shared-worktree branch reachable by an explicit -private fallback setting for the final cleanup rollout. - -**Architecture:** The runner stops asking whether a source is on an -isolated-diff allowlist. Instead, non-override bundle ingests use isolated -diffs unless the private settings object lists the source in -`sharedWorktreeSourceKeys`. Local runtime defaults that fallback list to empty, -and tests keep the old path covered with an explicit legacy source setting so -rollout step 11 can delete it safely. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, pnpm workspace commands, -existing `IngestBundleRunner`, `IngestSettingsPort`, local ingest runtime, and -isolated-diff runner tests. - ---- - -## Audit summary - -This audit read the original spec at -`docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md`, all -plans matching -`docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-*.md` and -`docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-*.md`, and the -current ingest runner code under `packages/context/src/ingest/`. - -Implemented v1 rollout coverage: - -- Rollout steps 1 and 2 are implemented by the core plan: child worktrees, - binary no-rename patch proposals, and `git apply --3way --index` - integration exist. -- Rollout step 3 is implemented by the textual conflict resolver plan: - `textual-conflict-resolver.ts` is wired through `patch-integrator.ts`. -- Rollout steps 4, 5, and 6 are implemented by the gates, provenance, - reference, global wiki, and gate-repair plans: final gates, persistent traces, - failure reports, provenance validation, target policy, and repair counters - exist. -- Rollout step 7 is implemented by the core and follow-up plans: Metabase has - isolated-diff stale-reference regression coverage. -- Rollout step 8 is implemented by - `2026-05-18-isolated-diff-ingestion-v1-connector-migration.md` and the - follow-up commits: Notion, LookML, Looker, dbt, and MetricFlow route through - isolated child worktrees, and MetricFlow projection runs before WorkUnits. - -Current v1-blocking gaps: - -- Rollout step 10 is not complete. `IngestBundleRunner.isIsolatedDiffEnabled()` - still checks `settings.isolatedDiffSourceKeys`, and - `local-bundle-runtime.ts` still installs the internal allowlist returned by - `defaultIsolatedDiffSourceKeys()`. -- Rollout step 11 remains blocked until step 10 lands. The old - shared-worktree WorkUnit branch is still present and must stay reachable in - this plan for final cleanup validation. - -Non-blocking gaps: - -- Rollout step 9 deterministic semantic merge helpers remain intentionally - deferred until v1 resolver metrics show frequent mechanical repairs. -- Transitive SQL-projection dependency expansion remains outside v1; current - gates cover direct declared join neighbors. -- Moving provenance into worktree files remains outside v1; the implemented - source of truth is the ingest provenance store and report body. -- Public connector knobs such as `executionMode`, `planningStrategy`, and - `conflictPolicy` remain non-goals and must not be added. -- Richer resolver context, such as full transcript excerpts for every - overlapping patch, can be evaluated after the default path has production - traces. - -## File structure - -- Modify `packages/context/src/ingest/isolated-diff/source-routing.ts`. - Replace the isolated-diff direct-write allowlist with an empty default - shared-worktree fallback list. -- Modify `packages/context/src/ingest/isolated-diff/source-routing.test.ts`. - Lock the fallback list semantics and remove direct-write allowlist - assertions. -- Modify `packages/context/src/ingest/ports.ts`. - Replace `isolatedDiffSourceKeys?: string[]` with - `sharedWorktreeSourceKeys?: string[]` on the private runner settings port. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Make isolated diff the default for non-override runs and route to the old - shared branch only when `sharedWorktreeSourceKeys` contains the source. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Prove an unlisted source uses isolated diffs by default and prove an - explicit fallback source can still reach the shared-worktree branch. -- Modify `packages/context/src/ingest/local-bundle-runtime.ts`. - Install the new empty fallback list instead of the old isolated-diff - allowlist. -- Modify `packages/context/src/ingest/local-bundle-runtime.test.ts`. - Assert local runtime settings do not expose `isolatedDiffSourceKeys` and do - default `sharedWorktreeSourceKeys` to `[]`. - ---- - -### Task 1: Replace source routing semantics - -**Files:** -- Modify: `packages/context/src/ingest/isolated-diff/source-routing.test.ts` -- Modify: `packages/context/src/ingest/isolated-diff/source-routing.ts` -- Modify: `packages/context/src/ingest/ports.ts` - -- [ ] **Step 1: Write the failing source-routing tests** - -Replace `packages/context/src/ingest/isolated-diff/source-routing.test.ts` with: - -```ts -import { describe, expect, it } from 'vitest'; -import { defaultSharedWorktreeSourceKeys, isSharedWorktreeFallbackSourceKey } from './source-routing.js'; - -describe('isolated-diff source routing', () => { - it('defaults every non-override source to isolated diffs', () => { - expect(defaultSharedWorktreeSourceKeys()).toEqual([]); - }); - - it('returns a mutable copy for runtime settings', () => { - const keys = defaultSharedWorktreeSourceKeys(); - keys.push('legacy-source'); - - expect(defaultSharedWorktreeSourceKeys()).toEqual([]); - }); - - it('recognizes only explicitly configured shared-worktree fallback sources', () => { - expect(isSharedWorktreeFallbackSourceKey('notion', [])).toBe(false); - expect(isSharedWorktreeFallbackSourceKey('metricflow', [])).toBe(false); - expect(isSharedWorktreeFallbackSourceKey('legacy-source', ['legacy-source'])).toBe(true); - expect(isSharedWorktreeFallbackSourceKey('other-source', ['legacy-source'])).toBe(false); - }); -}); -``` - -- [ ] **Step 2: Run the source-routing tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/source-routing.test.ts -``` - -Expected: FAIL because `defaultSharedWorktreeSourceKeys()` and -`isSharedWorktreeFallbackSourceKey()` are not exported yet. - -- [ ] **Step 3: Rewrite the routing helper** - -Replace `packages/context/src/ingest/isolated-diff/source-routing.ts` with: - -```ts -const DEFAULT_SHARED_WORKTREE_SOURCE_KEYS: readonly string[] = []; - -export function defaultSharedWorktreeSourceKeys(): string[] { - return [...DEFAULT_SHARED_WORKTREE_SOURCE_KEYS]; -} - -export function isSharedWorktreeFallbackSourceKey( - sourceKey: string, - sharedWorktreeSourceKeys: readonly string[] = DEFAULT_SHARED_WORKTREE_SOURCE_KEYS, -): boolean { - return sharedWorktreeSourceKeys.includes(sourceKey); -} -``` - -- [ ] **Step 4: Rename the private settings field** - -In `packages/context/src/ingest/ports.ts`, replace the -`IngestSettingsPort` interface with: - -```ts -export interface IngestSettingsPort { - memoryIngestionModel: string; - probeRowCount: number; - workUnitMaxConcurrency?: number; - workUnitStepBudget?: number; - workUnitFailureMode?: 'abort' | 'continue'; - sharedWorktreeSourceKeys?: string[]; - ingestTraceLevel?: IngestTraceLevel; -} -``` - -- [ ] **Step 5: Run the source-routing tests again** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/source-routing.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit routing semantics** - -Run: - -```bash -git add packages/context/src/ingest/isolated-diff/source-routing.ts \ - packages/context/src/ingest/isolated-diff/source-routing.test.ts \ - packages/context/src/ingest/ports.ts -git commit -m "feat(ingest): make isolated diff routing the private default" -``` - -### Task 2: Promote the runner default - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` - -- [ ] **Step 1: Update the isolated runner test imports and harness** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -replace the source-routing import with: - -```ts -import { defaultSharedWorktreeSourceKeys } from './isolated-diff/source-routing.js'; -``` - -Then change the `makeDeps()` signature and `settings` block to: - -```ts -function makeDeps( - runtime: Awaited>, - sourceKey = 'metabase', - settings: Partial = {}, -) { -``` - -```ts - settings: { - memoryIngestionModel: 'test', - probeRowCount: 1, - sharedWorktreeSourceKeys: defaultSharedWorktreeSourceKeys(), - ingestTraceLevel: 'trace', - ...settings, - }, -``` - -- [ ] **Step 2: Add the default-promotion regression tests** - -Insert these tests inside -`describe('IngestBundleRunner isolated diff path', ...)`, before the existing -non-Metabase routing matrix: - -```ts - it('routes an unlisted direct-writing source through isolated diffs by default', async () => { - const runtime = await makeRealGitRuntime(); - try { - const sourceKey = 'custom-direct-source'; - const { deps, adapter } = makeDeps(runtime, sourceKey); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { - unitKey: 'custom-wiki', - rawFiles: ['custom/page.json'], - peerFileIndex: [], - dependencyPaths: [], - }, - ], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName !== 'ingest-bundle-wu') { - return { stopReason: 'natural' }; - } - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global/custom-isolated.md'), - '---\nsummary: Custom isolated write\nusage_mode: auto\n---\n\nCustom isolated write.\n', - 'utf-8', - ); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'custom-isolated', - detail: 'Custom isolated write', - rawPaths: ['custom/page.json'], - }); - await currentSession.gitService.commitFiles( - ['wiki/global/custom-isolated.md'], - 'custom wiki', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['custom/page.json', 'h1']], sourceKey); - - await expect( - runner.run({ - jobId: 'job-custom-default', - connectionId: 'warehouse', - sourceKey, - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).resolves.toMatchObject({ - jobId: 'job-custom-default', - failedWorkUnits: [], - workUnitCount: 1, - }); - - const trace = await readFile( - join(runtime.configDir, '.ktx/ingest-traces/job-custom-default/trace.jsonl'), - 'utf-8', - ); - expect(trace).toContain('isolated_diff_enabled'); - expect(trace).toContain('work_unit_child_created'); - expect(trace).not.toContain('shared_worktree_path_enabled'); - - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0]; - const reportBody = reportCreate?.body as { isolatedDiff?: unknown } | undefined; - expect(reportBody?.isolatedDiff).toMatchObject({ - enabled: true, - acceptedPatches: 1, - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); - - it('keeps the shared-worktree path reachable through explicit private fallback settings', async () => { - const runtime = await makeRealGitRuntime(); - try { - const sourceKey = 'legacy-source'; - const { deps, adapter } = makeDeps(runtime, sourceKey, { - sharedWorktreeSourceKeys: ['legacy-source'], - }); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { - unitKey: 'legacy-wiki', - rawFiles: ['legacy/page.json'], - peerFileIndex: [], - dependencyPaths: [], - }, - ], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName !== 'ingest-bundle-wu') { - return { stopReason: 'natural' }; - } - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global/legacy-shared.md'), - '---\nsummary: Legacy shared write\nusage_mode: auto\n---\n\nLegacy shared write.\n', - 'utf-8', - ); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'legacy-shared', - detail: 'Legacy shared write', - rawPaths: ['legacy/page.json'], - }); - await currentSession.gitService.commitFiles( - ['wiki/global/legacy-shared.md'], - 'legacy wiki', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['legacy/page.json', 'h1']], sourceKey); - - await expect( - runner.run({ - jobId: 'job-legacy-shared', - connectionId: 'warehouse', - sourceKey, - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).resolves.toMatchObject({ - jobId: 'job-legacy-shared', - failedWorkUnits: [], - workUnitCount: 1, - }); - - const trace = await readFile( - join(runtime.configDir, '.ktx/ingest-traces/job-legacy-shared/trace.jsonl'), - 'utf-8', - ); - expect(trace).toContain('shared_worktree_path_enabled'); - expect(trace).not.toContain('work_unit_child_created'); - - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0]; - const reportBody = reportCreate?.body as { isolatedDiff?: unknown } | undefined; - expect(reportBody?.isolatedDiff).toMatchObject({ - enabled: false, - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 3: Run the new runner tests to verify the default test fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "unlisted direct-writing source|shared-worktree path reachable" -``` - -Expected: FAIL. The unlisted source still enters the old shared-worktree path -because the runner checks `isolatedDiffSourceKeys`. - -- [ ] **Step 4: Change the runner routing decision** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, replace -`isIsolatedDiffEnabled()` with: - -```ts - private isSharedWorktreeFallbackEnabled(sourceKey: string): boolean { - return (this.deps.settings.sharedWorktreeSourceKeys ?? []).includes(sourceKey); - } -``` - -Then replace the isolated-diff routing line with: - -```ts - const isolatedDiffEnabled = !overrideReport && !this.isSharedWorktreeFallbackEnabled(job.sourceKey); -``` - -Finally, replace the shared-path trace event with: - -```ts - await runTrace.event('info', 'routing', 'shared_worktree_path_enabled', { - sourceKey: job.sourceKey, - reason: 'explicit_private_fallback', - }); -``` - -- [ ] **Step 5: Run the new runner tests again** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "unlisted direct-writing source|shared-worktree path reachable" -``` - -Expected: PASS. - -- [ ] **Step 6: Commit runner default promotion** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "feat(ingest): promote isolated diff to default runner path" -``` - -### Task 3: Update local runtime defaults - -**Files:** -- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` - -- [ ] **Step 1: Update the local runtime settings test type** - -In `packages/context/src/ingest/local-bundle-runtime.test.ts`, replace -`RuntimeWithSettingsDeps` with: - -```ts -type RuntimeWithSettingsDeps = { - deps: { - settings: { - sharedWorktreeSourceKeys?: string[]; - isolatedDiffSourceKeys?: string[]; - }; - }; -}; -``` - -- [ ] **Step 2: Replace the local runtime settings assertion** - -Replace the test named -`enables isolated-diff routing for direct durable-write connectors` with: - -```ts - it('defaults local bundle ingest to isolated diffs without an allowlist', () => { - const runtime = createLocalBundleIngestRuntime({ - project, - adapters: [new FakeSourceAdapter()], - agentRunner: testAgentRunner(), - }); - - const settings = (runtime.runner as unknown as RuntimeWithSettingsDeps).deps.settings; - - expect(settings.sharedWorktreeSourceKeys).toEqual([]); - expect('isolatedDiffSourceKeys' in settings).toBe(false); - }); -``` - -- [ ] **Step 3: Run the local runtime settings test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -t "defaults local bundle ingest" -``` - -Expected: FAIL because `local-bundle-runtime.ts` still sets -`isolatedDiffSourceKeys`. - -- [ ] **Step 4: Update local runtime imports and settings** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, replace the -source-routing import with: - -```ts -import { defaultSharedWorktreeSourceKeys } from './isolated-diff/source-routing.js'; -``` - -Then replace the settings field: - -```ts - isolatedDiffSourceKeys: defaultIsolatedDiffSourceKeys(), -``` - -with: - -```ts - sharedWorktreeSourceKeys: defaultSharedWorktreeSourceKeys(), -``` - -- [ ] **Step 5: Run the local runtime settings test again** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-runtime.test.ts -t "defaults local bundle ingest" -``` - -Expected: PASS. - -- [ ] **Step 6: Commit local runtime defaults** - -Run: - -```bash -git add packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/local-bundle-runtime.test.ts -git commit -m "feat(ingest): default local ingest to isolated diffs" -``` - -### Task 4: Remove stale allowlist references - -**Files:** -- Verify: `packages/context/src/ingest/isolated-diff/source-routing.ts` -- Verify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/ports.ts` -- Verify: `packages/context/src/ingest/**/*.test.ts` - -- [ ] **Step 1: Search for old allowlist names** - -Run: - -```bash -rg -n "isolatedDiffSourceKeys|defaultIsolatedDiffSourceKeys|ISOLATED_DIFF_DIRECT_WRITE_SOURCE_KEYS|isIsolatedDiffDirectWriteSourceKey" packages/context/src -``` - -Expected: no matches. - -- [ ] **Step 2: Search for the new fallback setting** - -Run: - -```bash -rg -n "sharedWorktreeSourceKeys|defaultSharedWorktreeSourceKeys|isSharedWorktreeFallbackSourceKey" packages/context/src -``` - -Expected: matches only in these files: - -```text -packages/context/src/ingest/ports.ts -packages/context/src/ingest/ingest-bundle.runner.ts -packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -packages/context/src/ingest/isolated-diff/source-routing.ts -packages/context/src/ingest/isolated-diff/source-routing.test.ts -packages/context/src/ingest/local-bundle-runtime.ts -packages/context/src/ingest/local-bundle-runtime.test.ts -``` - -- [ ] **Step 3: Run a focused no-allowlist regression suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/isolated-diff/source-routing.test.ts \ - src/ingest/local-bundle-runtime.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - -t "source routing|defaults local bundle ingest|unlisted direct-writing source|shared-worktree path reachable|routes notion|routes lookml|routes looker|routes dbt|routes metricflow" -``` - -Expected: PASS. - -- [ ] **Step 4: Commit stale-reference cleanup if needed** - -If Step 1 or Step 2 required any edits, run: - -```bash -git add packages/context/src/ingest -git commit -m "chore(ingest): remove isolated diff allowlist references" -``` - -If no files changed, record that no cleanup commit was needed in the execution -notes for this task. - -### Task 5: Final verification - -**Files:** -- Verify: `packages/context/src/ingest/isolated-diff/source-routing.ts` -- Verify: `packages/context/src/ingest/isolated-diff/source-routing.test.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Verify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Verify: `packages/context/src/ingest/local-bundle-runtime.test.ts` -- Verify: `packages/context/src/ingest/ports.ts` -- Verify: `docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-default-promotion.md` - -- [ ] **Step 1: Run the full isolated-diff focused suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/semantic-layer-target-policy.test.ts \ - src/ingest/isolated-diff/source-routing.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/isolated-diff/textual-conflict-resolver.test.ts \ - src/ingest/final-gate-repair.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/report-snapshot.test.ts \ - src/ingest/local-bundle-runtime.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the MetricFlow local ingest regression** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/local-bundle-ingest.test.ts -t "runs full MetricFlow local ingest" -``` - -Expected: PASS. The report body includes `isolatedDiff.enabled: true`, -`acceptedPatches: 0`, and a string `projectionSha`. - -- [ ] **Step 3: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run package tests** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. - -- [ ] **Step 5: Run TypeScript dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only pre-existing findings unrelated to the files changed -by this plan. Investigate any finding that names `source-routing.ts`, -`ports.ts`, `local-bundle-runtime.ts`, or `ingest-bundle.runner.ts`. - -- [ ] **Step 6: Decide whether docs-site needs an update** - -No `docs-site/content/docs/` change is expected for this plan because the -change is an internal runner rollout switch and does not add or remove public -CLI commands, flags, config fields, connector setup steps, or user-facing -documentation concepts. - -- [ ] **Step 7: Commit final verification notes** - -Run: - -```bash -git status --short -git add docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-default-promotion.md -git commit -m "docs: add isolated diff default promotion plan" -``` - -Only include the plan file in this commit if all implementation commits have -already captured their code changes. - -## Completion criteria - -This plan is complete when: - -- `packages/context/src/ingest/ports.ts` has - `sharedWorktreeSourceKeys?: string[]` and no `isolatedDiffSourceKeys` field. -- `IngestBundleRunner` uses isolated diffs for every non-override source unless - `sharedWorktreeSourceKeys` explicitly contains that source. -- The trace for a default-routed source contains `isolated_diff_enabled` and - not `shared_worktree_path_enabled`. -- The trace for an explicitly fallback-routed source contains - `shared_worktree_path_enabled` and not `work_unit_child_created`. -- Local runtime settings default `sharedWorktreeSourceKeys` to `[]`. -- No production or test code under `packages/context/src` references the old - isolated-diff allowlist names. -- The focused isolated-diff suite, MetricFlow local ingest regression, - `@ktx/context` type-check, `@ktx/context` tests, and dead-code checks pass. - -## Next rollout step - -After this plan is implemented and verified, the only remaining v1-blocking -rollout item from the spec is step 11: remove the old shared-worktree WorkUnit -execution path and delete the private `sharedWorktreeSourceKeys` fallback -setting. diff --git a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair-classification.md b/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair-classification.md deleted file mode 100644 index d211629c..00000000 --- a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair-classification.md +++ /dev/null @@ -1,1436 +0,0 @@ -# Isolated Diff Ingestion V1 Gate Repair Classification Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Prevent isolated-diff gate repair from automatically editing -high-risk semantic-layer or warehouse-validation failures while preserving -bounded repair for stale wiki reference drift. - -**Architecture:** Keep the existing final gate and repair-agent modules, but -make artifact gate failures structured. A small repair-policy module classifies -structured issues before `repairFinalGateFailure()` is invoked from patch -integration or final composed-tree gates. Unknown or high-risk gate failures -fail before repair and before squash. - -**Tech Stack:** TypeScript, Vitest, pnpm, existing KTX ingest runner, -`FinalArtifactGateFailure`, JSONL ingest traces. - ---- - -## Audit summary - -The implemented rollout covers isolated child worktrees, binary no-rename -patch proposals, `git apply --3way --index`, textual conflict repair, final -artifact gates, provenance pre-squash validation, connector migration, default -promotion, and old shared-worktree path removal. - -One v1-blocking gap remains in the spec's Gate repair stage. The spec requires -the runner to classify final gate failures before deciding whether to repair or -fail. Repairable failures include stale wiki body references and stale wiki -frontmatter references. High-risk failures, including missing warehouse tables -or columns and invalid SQL sources, must fail without automatic repair unless a -later implementation adds a stronger evidence contract. - -Current code calls `repairFinalGateFailure()` for every -`validateFinalIngestArtifacts()` error in both: - -- `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- `packages/context/src/ingest/ingest-bundle.runner.ts` - -That lets a repair agent edit semantic-layer files after a warehouse dry-run -failure. Rerunning gates is necessary, but not sufficient: the spec explicitly -forbids automatic repair when the repair would require choosing facts without -evidence. - -Non-blocking gaps after this plan: - -- Deterministic semantic merge helpers remain intentionally deferred as rollout - step 9. -- Semantic-layer dependency expansion remains direct declared joins only. -- Provenance remains in the ingest provenance store and report body. -- Resolver and repair prompts can later include richer transcript excerpts, - overlapping patch summaries, and raw evidence bundles. -- Failures before an ingest run row exists still have deterministic trace files - but no stored ingest report. - -## File structure - -- Modify `packages/context/src/ingest/wiki-body-refs.ts`. - Add structured wiki body reference issues while keeping the existing - `findInvalidWikiBodyRefs()` string API for current callers. -- Modify `packages/context/src/ingest/wiki-body-refs.test.ts`. - Cover structured issue codes for stale semantic-layer entities and missing - raw tables. -- Modify `packages/context/src/ingest/stages/validate-wu-sources.ts`. - Preserve validator error messages in `validateWuTouchedSources()` output. -- Modify `packages/context/src/ingest/stages/validate-wu-sources.test.ts`. - Cover the new `issues` payload while keeping existing `validSources` and - `invalidSources` behavior. -- Modify `packages/context/src/ingest/artifact-gates.ts`. - Throw `FinalArtifactGateFailure` with structured issues from semantic-layer, - wiki frontmatter, wiki page-reference, wiki body, and provenance-adjacent - artifact gates. -- Modify `packages/context/src/ingest/artifact-gates.test.ts`. - Assert structured issue codes for repairable and non-repairable gate - failures. -- Create `packages/context/src/ingest/gate-repair-policy.ts`. - Classify structured artifact gate failures as repairable or non-repairable. -- Create `packages/context/src/ingest/gate-repair-policy.test.ts`. - Lock the policy for stale wiki refs versus high-risk semantic/warehouse - errors. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.ts`. - Call the repair policy before patch-level semantic gate repair. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`. - Prove high-risk semantic gate failures do not invoke the repair callback. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Call the repair policy before final composed-tree gate repair and include - non-repairable issue metadata in failure reports. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Add a final-gate regression where reconciliation creates an invalid - semantic-layer source and the repair agent is not called. - -### Task 1: Preserve structured artifact gate issues - -**Files:** -- Modify: `packages/context/src/ingest/wiki-body-refs.ts` -- Modify: `packages/context/src/ingest/wiki-body-refs.test.ts` -- Modify: `packages/context/src/ingest/stages/validate-wu-sources.ts` -- Modify: `packages/context/src/ingest/stages/validate-wu-sources.test.ts` - -- [ ] **Step 1: Add structured wiki body issue tests** - -In `packages/context/src/ingest/wiki-body-refs.test.ts`, extend the import and -append this test inside `describe('wiki body refs', ...)`: - -```ts -import { findInvalidWikiBodyRefIssues, findInvalidWikiBodyRefs, parseWikiBodyRefs } from './wiki-body-refs.js'; -``` - -```ts - it('returns structured issue codes for body reference failures', async () => { - const invalid = await findInvalidWikiBodyRefIssues({ - pageKey: 'account-segments', - body: [ - '`mart_account_segments.total_contract_arr_cents`', - '`source:missing_source`', - '`table:analytics.missing_table`', - ].join('\n'), - visibleConnectionIds: ['warehouse'], - loadSources: async () => sources, - tableExists: async () => false, - }); - - expect(invalid).toEqual([ - { - code: 'wiki_body_unknown_sl_entity', - message: 'account-segments: unknown semantic-layer entity mart_account_segments.total_contract_arr_cents', - pageKey: 'account-segments', - ref: 'mart_account_segments.total_contract_arr_cents', - sourceName: 'mart_account_segments', - entityName: 'total_contract_arr_cents', - connectionId: null, - }, - { - code: 'wiki_body_unknown_sl_source', - message: 'account-segments: unknown semantic-layer source missing_source', - pageKey: 'account-segments', - ref: 'source:missing_source', - sourceName: 'missing_source', - connectionId: null, - }, - { - code: 'wiki_body_unknown_raw_table', - message: 'account-segments: unknown raw table analytics.missing_table', - pageKey: 'account-segments', - ref: 'table:analytics.missing_table', - tableRef: 'analytics.missing_table', - connectionId: null, - }, - ]); - }); -``` - -- [ ] **Step 2: Run the wiki body issue test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/wiki-body-refs.test.ts -t "structured issue codes" -``` - -Expected: FAIL with an export error for `findInvalidWikiBodyRefIssues`. - -- [ ] **Step 3: Implement structured wiki body issues** - -In `packages/context/src/ingest/wiki-body-refs.ts`, add this type after -`WikiBodyRefValidationInput`: - -```ts -export type WikiBodyRefIssue = - | { - code: 'wiki_body_unknown_sl_source'; - message: string; - pageKey: string; - ref: string; - sourceName: string; - connectionId: string | null; - } - | { - code: 'wiki_body_unknown_sl_entity'; - message: string; - pageKey: string; - ref: string; - sourceName: string; - entityName: string; - connectionId: string | null; - } - | { - code: 'wiki_body_unknown_raw_table'; - message: string; - pageKey: string; - ref: string; - tableRef: string; - connectionId: string | null; - }; - -function renderConnectionScopedRef(connectionId: string | null, body: string): string { - return connectionId ? `${connectionId}/${body}` : body; -} -``` - -Replace `findInvalidWikiBodyRefs()` with these two functions: - -```ts -export async function findInvalidWikiBodyRefIssues(input: WikiBodyRefValidationInput): Promise { - const issues: WikiBodyRefIssue[] = []; - const sourceCache = new Map(); - const loadSources = async (connectionId: string): Promise => { - const cached = sourceCache.get(connectionId); - if (cached) { - return cached; - } - const sources = await input.loadSources(connectionId); - sourceCache.set(connectionId, sources); - return sources; - }; - - const findSource = async ( - connectionIds: string[], - sourceName: string, - ): Promise<{ connectionId: string; source: SemanticLayerSource } | null> => { - for (const connectionId of connectionIds) { - const source = (await loadSources(connectionId)).find((candidate) => candidate.name === sourceName); - if (source) { - return { connectionId, source }; - } - } - return null; - }; - - for (const ref of parseWikiBodyRefs(input.body)) { - const connectionIds = ref.connectionId ? [ref.connectionId] : input.visibleConnectionIds; - if (ref.kind === 'table') { - const found = await Promise.all(connectionIds.map((connectionId) => input.tableExists(connectionId, ref.tableRef))); - if (!found.some(Boolean)) { - const renderedRef = renderConnectionScopedRef(ref.connectionId, `table:${ref.tableRef}`); - issues.push({ - code: 'wiki_body_unknown_raw_table', - message: `${input.pageKey}: unknown raw table ${renderConnectionScopedRef(ref.connectionId, ref.tableRef)}`, - pageKey: input.pageKey, - ref: renderedRef, - tableRef: ref.tableRef, - connectionId: ref.connectionId, - }); - } - continue; - } - - const found = await findSource(connectionIds, ref.sourceName); - if (!found) { - if (ref.kind === 'sl_source') { - const renderedRef = renderConnectionScopedRef(ref.connectionId, `source:${ref.sourceName}`); - issues.push({ - code: 'wiki_body_unknown_sl_source', - message: `${input.pageKey}: unknown semantic-layer source ${renderConnectionScopedRef(ref.connectionId, ref.sourceName)}`, - pageKey: input.pageKey, - ref: renderedRef, - sourceName: ref.sourceName, - connectionId: ref.connectionId, - }); - } - continue; - } - if (ref.kind === 'sl_entity' && !entityNames(found.source).has(ref.entityName)) { - issues.push({ - code: 'wiki_body_unknown_sl_entity', - message: `${input.pageKey}: unknown semantic-layer entity ${ref.sourceName}.${ref.entityName}`, - pageKey: input.pageKey, - ref: renderConnectionScopedRef(ref.connectionId, `${ref.sourceName}.${ref.entityName}`), - sourceName: ref.sourceName, - entityName: ref.entityName, - connectionId: ref.connectionId, - }); - } - } - - return issues; -} - -export async function findInvalidWikiBodyRefs(input: WikiBodyRefValidationInput): Promise { - return (await findInvalidWikiBodyRefIssues(input)).map((issue) => issue.message); -} -``` - -- [ ] **Step 4: Run the wiki body issue test to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/wiki-body-refs.test.ts -t "structured issue codes" -``` - -Expected: PASS. - -- [ ] **Step 5: Add validator detail tests** - -In `packages/context/src/ingest/stages/validate-wu-sources.test.ts`, replace -the first test's final expectation block with: - -```ts - expect(result).toEqual({ - validSources: ['warehouse-a:good'], - invalidSources: ['warehouse-b:bad'], - issues: [ - { - connectionId: 'warehouse-b', - sourceName: 'bad', - sourceId: 'warehouse-b:bad', - errors: ['bad.yaml: measure "revenue" dry-run failed.\n Error: column missing_revenue does not exist'], - warnings: ['bad.yaml: warehouse warning'], - }, - ], - }); -``` - -Replace the mocked validator in that same test with: - -```ts - const validateSingleSource = vi - .fn() - .mockResolvedValueOnce({ errors: [], warnings: [] }) - .mockResolvedValueOnce({ - errors: ['bad.yaml: measure "revenue" dry-run failed.\n Error: column missing_revenue does not exist'], - warnings: ['bad.yaml: warehouse warning'], - }); -``` - -- [ ] **Step 6: Run the validator detail test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/stages/validate-wu-sources.test.ts -``` - -Expected: FAIL because `issues` is missing from the returned object. - -- [ ] **Step 7: Preserve validator error details** - -In `packages/context/src/ingest/stages/validate-wu-sources.ts`, replace the -interfaces and function with: - -```ts -export interface WuValidationIssue { - connectionId: string; - sourceName: string; - sourceId: string; - errors: string[]; - warnings: string[]; -} - -export interface WuValidationResult { - validSources: string[]; - invalidSources: string[]; - issues: WuValidationIssue[]; -} - -export async function validateWuTouchedSources( - deps: SlValidationDeps & { slValidator: SlValidatorPort }, - touched: TouchedSlSource[], -): Promise { - const valid: string[] = []; - const invalid: string[] = []; - const issues: WuValidationIssue[] = []; - for (const source of touched) { - const sourceId = `${source.connectionId}:${source.sourceName}`; - const result = await deps.slValidator.validateSingleSource(deps, source.connectionId, source.sourceName); - if (result.errors.length === 0) { - valid.push(sourceId); - } else { - invalid.push(sourceId); - issues.push({ - connectionId: source.connectionId, - sourceName: source.sourceName, - sourceId, - errors: result.errors, - warnings: result.warnings, - }); - } - } - return { validSources: valid, invalidSources: invalid, issues }; -} -``` - -- [ ] **Step 8: Run the validator tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/stages/validate-wu-sources.test.ts -``` - -Expected: PASS. - -- [ ] **Step 9: Commit structured issue foundations** - -```bash -git add packages/context/src/ingest/wiki-body-refs.ts \ - packages/context/src/ingest/wiki-body-refs.test.ts \ - packages/context/src/ingest/stages/validate-wu-sources.ts \ - packages/context/src/ingest/stages/validate-wu-sources.test.ts -git commit -m "feat(ingest): preserve structured gate issue details" -``` - -### Task 2: Throw structured final artifact gate failures - -**Files:** -- Modify: `packages/context/src/ingest/artifact-gates.ts` -- Modify: `packages/context/src/ingest/artifact-gates.test.ts` - -- [ ] **Step 1: Add structured failure tests** - -In `packages/context/src/ingest/artifact-gates.test.ts`, extend the import: - -```ts -import { FinalArtifactGateFailure, validateFinalIngestArtifacts, validateProvenanceRawPaths } from './artifact-gates.js'; -``` - -Append this test inside `describe('artifact gates', ...)`: - -```ts - it('throws structured final artifact gate issues', async () => { - const wikiService = wikiServiceWithPages({ - 'account-segments': { - refs: ['missing-page'], - slRefs: ['mart_account_segments.total_contract_arr_cents'], - content: [ - 'ARR is `mart_account_segments.total_contract_arr_cents`.', - 'Warehouse table `table:analytics.missing_table`.', - ].join('\n'), - }, - }); - const semanticLayerService = { - loadAllSources: vi.fn().mockResolvedValue({ - sources: [ - { - name: 'mart_account_segments', - grain: ['account_id'], - columns: [{ name: 'account_id', type: 'string' }], - joins: [], - measures: [{ name: 'total_contract_arr', expr: 'sum(contract_arr)' }], - table: 'analytics.mart_account_segments', - }, - ], - loadErrors: [], - }), - }; - - await expect( - validateFinalIngestArtifacts({ - connectionIds: ['warehouse'], - changedWikiPageKeys: ['account-segments'], - touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'mart_account_segments' }], - wikiService: wikiService as never, - semanticLayerService: semanticLayerService as never, - validateTouchedSources: async () => ({ - invalidSources: ['warehouse:mart_account_segments'], - validSources: [], - issues: [ - { - connectionId: 'warehouse', - sourceName: 'mart_account_segments', - sourceId: 'warehouse:mart_account_segments', - errors: ['mart_account_segments.yaml: measure "total_contract_arr" dry-run failed.\n Error: column missing_arr does not exist'], - warnings: [], - }, - ], - }), - tableExists: async () => false, - }), - ).rejects.toMatchObject({ - issues: expect.arrayContaining([ - expect.objectContaining({ code: 'semantic_layer_validation_failed', sourceId: 'warehouse:mart_account_segments' }), - expect.objectContaining({ code: 'wiki_sl_ref_unknown_entity', pageKey: 'account-segments' }), - expect.objectContaining({ code: 'wiki_ref_missing_page', pageKey: 'account-segments', missingRef: 'missing-page' }), - expect.objectContaining({ code: 'wiki_body_unknown_sl_entity', pageKey: 'account-segments' }), - expect.objectContaining({ code: 'wiki_body_unknown_raw_table', pageKey: 'account-segments' }), - ]), - }); - }); -``` - -- [ ] **Step 2: Run the structured failure test to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -t "structured final artifact gate issues" -``` - -Expected: FAIL with an export error for `FinalArtifactGateFailure` or missing -`issues`. - -- [ ] **Step 3: Add structured issue types and failure class** - -In `packages/context/src/ingest/artifact-gates.ts`, change the wiki body import: - -```ts -import { findInvalidWikiBodyRefIssues, type WikiBodyRefIssue } from './wiki-body-refs.js'; -``` - -Replace the existing `TouchedValidationResult` interface with these exported -types: - -```ts -export interface TouchedSourceValidationIssue { - connectionId: string; - sourceName: string; - sourceId: string; - errors: string[]; - warnings?: string[]; -} - -export interface TouchedValidationResult { - invalidSources: string[]; - validSources: string[]; - issues?: TouchedSourceValidationIssue[]; -} - -export type FinalArtifactGateIssue = - | { - code: 'semantic_layer_validation_failed'; - message: string; - connectionId: string | null; - sourceName: string; - sourceId: string; - sourceErrors: string[]; - } - | { - code: 'wiki_sl_ref_unknown_source'; - message: string; - pageKey: string; - ref: string; - sourceName: string; - connectionId: string | null; - } - | { - code: 'wiki_sl_ref_unknown_entity'; - message: string; - pageKey: string; - ref: string; - sourceName: string; - entityName: string; - connectionId: string | null; - } - | { - code: 'wiki_ref_missing_page'; - message: string; - pageKey: string; - missingRef: string; - } - | WikiBodyRefIssue; - -export class FinalArtifactGateFailure extends Error { - readonly issues: FinalArtifactGateIssue[]; - - constructor(issues: FinalArtifactGateIssue[]) { - super(`final artifact gates failed:\n${issues.map((issue) => issue.message).join('\n')}`); - this.name = 'FinalArtifactGateFailure'; - this.issues = issues; - } -} -``` - -- [ ] **Step 4: Return structured wiki frontmatter issues** - -Replace `validateWikiSlRefs()` with: - -```ts -async function validateWikiSlRefs(input: FinalArtifactGateInput): Promise { - const issues: FinalArtifactGateIssue[] = []; - const sourcesByConnection = new Map>['sources']>(); - for (const connectionId of input.connectionIds) { - const { sources } = await input.semanticLayerService.loadAllSources(connectionId); - sourcesByConnection.set(connectionId, sources); - } - - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - for (const ref of page.frontmatter.sl_refs ?? []) { - const parsed = parseSlRef(ref); - const candidateConnections = parsed.connectionId ? [parsed.connectionId] : input.connectionIds; - let source: Awaited>['sources'][number] | undefined; - for (const connectionId of candidateConnections) { - source = sourcesByConnection.get(connectionId)?.find((candidate) => candidate.name === parsed.sourceName); - if (source) { - break; - } - } - if (!source) { - issues.push({ - code: 'wiki_sl_ref_unknown_source', - message: `${pageKey}: unknown sl_refs entry ${ref}`, - pageKey, - ref, - sourceName: parsed.sourceName, - connectionId: parsed.connectionId, - }); - continue; - } - if (parsed.entityName && !slEntityNames(source).has(parsed.entityName)) { - issues.push({ - code: 'wiki_sl_ref_unknown_entity', - message: `${pageKey}: unknown sl_refs entity ${ref}`, - pageKey, - ref, - sourceName: parsed.sourceName, - entityName: parsed.entityName, - connectionId: parsed.connectionId, - }); - } - } - } - return issues; -} -``` - -- [ ] **Step 5: Return structured wiki page reference issues** - -Replace `validateWikiRefs()` with: - -```ts -async function validateWikiRefs(input: FinalArtifactGateInput): Promise { - const issues: FinalArtifactGateIssue[] = []; - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - const missingRefs = await findMissingWikiRefs({ - wikiService: input.wikiService, - scope: 'GLOBAL', - scopeId: null, - pageKey, - refs: page.frontmatter.refs, - content: page.content, - }); - for (const missingRef of missingRefs) { - issues.push({ - code: 'wiki_ref_missing_page', - message: `${pageKey} -> ${missingRef}`, - pageKey, - missingRef, - }); - } - } - return issues; -} -``` - -- [ ] **Step 6: Throw `FinalArtifactGateFailure` from final gates** - -Replace `validateFinalIngestArtifacts()` with: - -```ts -export async function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise { - const touchedWithDependencies = await expandTouchedSlSourcesWithDirectJoinNeighbors(input); - const validation = await input.validateTouchedSources(touchedWithDependencies); - const issues: FinalArtifactGateIssue[] = []; - const validationIssues = - validation.issues ?? - validation.invalidSources.map((sourceId) => { - const [connectionId, sourceName] = sourceId.includes(':') ? sourceId.split(':', 2) : [null, sourceId]; - return { - connectionId, - sourceName: sourceName ?? sourceId, - sourceId, - errors: [`semantic-layer validation failed for ${sourceId}`], - warnings: [], - }; - }); - for (const issue of validationIssues) { - issues.push({ - code: 'semantic_layer_validation_failed', - message: `semantic-layer validation failed for ${issue.sourceId}`, - connectionId: issue.connectionId, - sourceName: issue.sourceName, - sourceId: issue.sourceId, - sourceErrors: issue.errors, - }); - } - - issues.push(...(await validateWikiSlRefs(input))); - const danglingWikiRefs = await validateWikiRefs(input); - if (danglingWikiRefs.length > 0) { - const combined = danglingWikiRefs.map((issue) => issue.message).join(', '); - issues.push({ - code: 'wiki_ref_missing_page', - message: `wiki references target missing page(s): ${combined}`, - pageKey: danglingWikiRefs[0].pageKey, - missingRef: danglingWikiRefs[0].missingRef, - }); - } - - for (const pageKey of input.changedWikiPageKeys) { - const page = await input.wikiService.readPage('GLOBAL', null, pageKey); - if (!page) { - continue; - } - issues.push( - ...(await findInvalidWikiBodyRefIssues({ - pageKey, - body: page.content, - visibleConnectionIds: input.connectionIds, - loadSources: async (connectionId) => { - const { sources } = await input.semanticLayerService.loadAllSources(connectionId); - return sources; - }, - tableExists: input.tableExists, - })), - ); - } - - if (issues.length > 0) { - throw new FinalArtifactGateFailure(issues); - } -} -``` - -- [ ] **Step 7: Run artifact gate tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/artifact-gates.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit structured final gate failures** - -```bash -git add packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts -git commit -m "feat(ingest): structure final artifact gate failures" -``` - -### Task 3: Add gate repair policy - -**Files:** -- Create: `packages/context/src/ingest/gate-repair-policy.ts` -- Create: `packages/context/src/ingest/gate-repair-policy.test.ts` - -- [ ] **Step 1: Add policy tests** - -Create `packages/context/src/ingest/gate-repair-policy.test.ts`: - -```ts -import { describe, expect, it } from 'vitest'; -import { FinalArtifactGateFailure, type FinalArtifactGateIssue } from './artifact-gates.js'; -import { classifyFinalGateRepair } from './gate-repair-policy.js'; - -function failure(...issues: FinalArtifactGateIssue[]): FinalArtifactGateFailure { - return new FinalArtifactGateFailure(issues); -} - -describe('classifyFinalGateRepair', () => { - it('allows stale wiki reference drift to use the repair agent', () => { - const decision = classifyFinalGateRepair( - failure({ - code: 'wiki_body_unknown_sl_entity', - message: 'account-segments: unknown semantic-layer entity mart_account_segments.total_contract_arr_cents', - pageKey: 'account-segments', - ref: 'mart_account_segments.total_contract_arr_cents', - sourceName: 'mart_account_segments', - entityName: 'total_contract_arr_cents', - connectionId: null, - }), - ); - - expect(decision).toEqual({ - repairable: true, - issueCodes: ['wiki_body_unknown_sl_entity'], - }); - }); - - it('blocks semantic-layer validation failures from automatic repair', () => { - const decision = classifyFinalGateRepair( - failure({ - code: 'semantic_layer_validation_failed', - message: 'semantic-layer validation failed for warehouse:orders', - connectionId: 'warehouse', - sourceName: 'orders', - sourceId: 'warehouse:orders', - sourceErrors: ['orders.yaml: measure "revenue" dry-run failed.\n Error: column missing_revenue does not exist'], - }), - ); - - expect(decision).toEqual({ - repairable: false, - reason: 'non-repairable artifact gate issue(s): semantic_layer_validation_failed', - issueCodes: ['semantic_layer_validation_failed'], - }); - }); - - it('blocks missing raw table body references from automatic repair', () => { - const decision = classifyFinalGateRepair( - failure({ - code: 'wiki_body_unknown_raw_table', - message: 'account-segments: unknown raw table analytics.missing_table', - pageKey: 'account-segments', - ref: 'table:analytics.missing_table', - tableRef: 'analytics.missing_table', - connectionId: null, - }), - ); - - expect(decision).toEqual({ - repairable: false, - reason: 'non-repairable artifact gate issue(s): wiki_body_unknown_raw_table', - issueCodes: ['wiki_body_unknown_raw_table'], - }); - }); - - it('blocks unstructured errors by default', () => { - expect(classifyFinalGateRepair(new Error('plain gate failure'))).toEqual({ - repairable: false, - reason: 'unclassified artifact gate failure', - issueCodes: [], - }); - }); -}); -``` - -- [ ] **Step 2: Run policy tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/gate-repair-policy.test.ts -``` - -Expected: FAIL because `gate-repair-policy.ts` does not exist. - -- [ ] **Step 3: Implement the policy module** - -Create `packages/context/src/ingest/gate-repair-policy.ts`: - -```ts -import { FinalArtifactGateFailure, type FinalArtifactGateIssue } from './artifact-gates.js'; - -export type GateRepairDecision = - | { repairable: true; issueCodes: string[] } - | { repairable: false; reason: string; issueCodes: string[] }; - -const repairableIssueCodes = new Set([ - 'wiki_body_unknown_sl_entity', - 'wiki_body_unknown_sl_source', - 'wiki_sl_ref_unknown_entity', - 'wiki_ref_missing_page', -]); - -export function artifactGateIssueSummary(error: unknown): { message: string; issues: FinalArtifactGateIssue[] } { - if (error instanceof FinalArtifactGateFailure) { - return { message: error.message, issues: error.issues }; - } - return { message: error instanceof Error ? error.message : String(error), issues: [] }; -} - -export function classifyFinalGateRepair(error: unknown): GateRepairDecision { - const { issues } = artifactGateIssueSummary(error); - if (issues.length === 0) { - return { - repairable: false, - reason: 'unclassified artifact gate failure', - issueCodes: [], - }; - } - - const issueCodes = [...new Set(issues.map((issue) => issue.code))].sort(); - const nonRepairableCodes = issueCodes.filter( - (code): code is FinalArtifactGateIssue['code'] => !repairableIssueCodes.has(code as FinalArtifactGateIssue['code']), - ); - if (nonRepairableCodes.length > 0) { - return { - repairable: false, - reason: `non-repairable artifact gate issue(s): ${nonRepairableCodes.join(', ')}`, - issueCodes, - }; - } - - return { repairable: true, issueCodes }; -} -``` - -- [ ] **Step 4: Run policy tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/gate-repair-policy.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the policy module** - -```bash -git add packages/context/src/ingest/gate-repair-policy.ts \ - packages/context/src/ingest/gate-repair-policy.test.ts -git commit -m "feat(ingest): classify final gate repair safety" -``` - -### Task 4: Block non-repairable patch-level gate failures - -**Files:** -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` - -- [ ] **Step 1: Add patch-level non-repairable regression** - -In `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`, -extend the imports: - -```ts -import { FinalArtifactGateFailure } from '../artifact-gates.js'; -``` - -Append this test inside `describe('integrateWorkUnitPatch', ...)`: - -```ts - it('does not invoke gate repair for non-repairable semantic validation failures', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - const childDir = join(homeDir, 'child-semantic-high-risk'); - await git.addWorktree(childDir, 'child-semantic-high-risk', baseSha); - const childGit = git.forWorktree(childDir); - await mkdir(join(childDir, 'semantic-layer/c1'), { recursive: true }); - await writeFile( - join(childDir, 'semantic-layer/c1/orders.yaml'), - 'name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures:\n - name: revenue\n expr: sum(missing_revenue)\n', - ); - await childGit.commitFiles(['semantic-layer/c1/orders.yaml'], 'invalid semantic edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/semantic-high-risk.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-semantic-high-risk/trace.jsonl'), - jobId: 'job-semantic-high-risk', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - const repairGateFailure = vi.fn(async () => { - throw new Error('repair must not run for high-risk semantic validation failures'); - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-high-risk', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree: vi.fn().mockRejectedValue( - new FinalArtifactGateFailure([ - { - code: 'semantic_layer_validation_failed', - message: 'semantic-layer validation failed for c1:orders', - connectionId: 'c1', - sourceName: 'orders', - sourceId: 'c1:orders', - sourceErrors: ['orders.yaml: measure "revenue" dry-run failed.\n Error: column missing_revenue does not exist'], - }, - ]), - ), - slDisallowed: false, - allowedTargetConnectionIds: new Set(['c1']), - repairGateFailure, - }); - - expect(result).toMatchObject({ - status: 'semantic_conflict', - reason: expect.stringContaining('semantic-layer validation failed for c1:orders'), - }); - expect(repairGateFailure).not.toHaveBeenCalled(); - await expect(readFile(join(configDir, 'semantic-layer/c1/orders.yaml'), 'utf-8')).rejects.toThrow(); - await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('patch_semantic_conflict_not_repairable'); - }); -``` - -- [ ] **Step 2: Run the patch integrator regression to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -t "non-repairable semantic validation failures" -``` - -Expected: FAIL because `repairGateFailure` is called. - -- [ ] **Step 3: Wire repair policy into patch integration** - -In `packages/context/src/ingest/isolated-diff/patch-integrator.ts`, add: - -```ts -import { artifactGateIssueSummary, classifyFinalGateRepair } from '../gate-repair-policy.js'; -``` - -Inside the `catch (error)` block after the clean patch applies and -`validateAppliedTree(touchedPaths)` rejects, replace: - -```ts - const reason = errorMessage(error); -``` - -with: - -```ts - const gateFailure = artifactGateIssueSummary(error); - const reason = gateFailure.message; - const repairDecision = classifyFinalGateRepair(error); -``` - -Immediately after the existing `patch_semantic_conflict` trace event, insert: - -```ts - if (!repairDecision.repairable) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - await input.trace.event('error', 'integration', 'patch_semantic_conflict_not_repairable', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason: repairDecision.reason, - issueCodes: repairDecision.issueCodes, - }); - return { - status: 'semantic_conflict', - reason, - touchedPaths, - }; - } -``` - -Then keep the existing `if (input.repairGateFailure) { ... }` block unchanged. -This means gate repair runs only when `repairDecision.repairable` is true. - -- [ ] **Step 4: Convert existing semantic repair tests to structured repairable failures** - -In `patch-integrator.test.ts`, change the repairable semantic-gate test's -mock rejection from: - -```ts - .mockRejectedValueOnce(new Error('final artifact gates failed:\na: unknown semantic-layer entity')) -``` - -to: - -```ts - .mockRejectedValueOnce( - new FinalArtifactGateFailure([ - { - code: 'wiki_body_unknown_sl_entity', - message: 'a: unknown semantic-layer entity mart_account_segments.total_contract_arr_cents', - pageKey: 'a', - ref: 'mart_account_segments.total_contract_arr_cents', - sourceName: 'mart_account_segments', - entityName: 'total_contract_arr_cents', - connectionId: null, - }, - ]), - ) -``` - -- [ ] **Step 5: Run patch integrator tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: PASS. - -- [ ] **Step 6: Commit patch-level policy wiring** - -```bash -git add packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts -git commit -m "fix(ingest): block high-risk patch gate repair" -``` - -### Task 5: Block non-repairable final composed-tree gate failures - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Add final-gate non-repairable regression** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -append this test inside `describe('IngestBundleRunner isolated diff path', ...)` -before the final gate repair success test: - -```ts - it('does not invoke final gate repair for semantic-layer warehouse validation failures', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'valid-page', rawFiles: ['pages/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - deps.slValidator.validateSingleSource = vi.fn().mockResolvedValue({ - errors: ['orders.yaml: measure "revenue" dry-run failed.\n Error: column missing_revenue does not exist'], - warnings: [], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName === 'ingest-isolated-diff-gate-repair') { - throw new Error('gate repair must not run for semantic-layer validation failures'); - } - - const root = rootOfConfig(currentSession.configService, runtime.configDir); - if (params.telemetryTags.operationName === 'ingest-bundle-wu') { - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile(join(root, 'wiki/global/valid-page.md'), '---\nsummary: Valid page\nusage_mode: auto\n---\n\nValid\n'); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'valid-page', - detail: 'Valid page', - rawPaths: ['pages/source.json'], - }); - await currentSession.gitService.commitFiles(['wiki/global/valid-page.md'], 'wu valid page', 'KTX Test', 'system@ktx.local'); - return { stopReason: 'natural' as const }; - } - - await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(root, 'semantic-layer/warehouse/orders.yaml'), - 'name: orders\ngrain: [id]\ncolumns: [{name: id, type: string}]\njoins: []\nmeasures:\n - name: revenue\n expr: sum(missing_revenue)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'orders'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'orders', - detail: 'Invalid source from reconciliation', - targetConnectionId: 'warehouse', - rawPaths: ['pages/source.json'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/orders.yaml'], - 'reconcile invalid semantic source', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' as const }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['pages/source.json', 'h1']]); - const preRunHead = await runtime.git.revParseHead(); - - await expect( - runner.run({ - jobId: 'job-final-high-risk-semantic', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/semantic-layer validation failed for warehouse:orders/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - expect(deps.agentRunner.runLoop).not.toHaveBeenCalledWith( - expect.objectContaining({ - telemetryTags: expect.objectContaining({ - operationName: 'ingest-isolated-diff-gate-repair', - }), - }), - ); - const trace = await readFile( - join(runtime.configDir, '.ktx/ingest-traces/job-final-high-risk-semantic/trace.jsonl'), - 'utf-8', - ); - expect(trace).toContain('final_artifact_gates_failed'); - expect(trace).toContain('final_artifact_gates_not_repairable'); - expect(trace).toContain('semantic_layer_validation_failed'); - expect(trace).not.toContain('gate_repair_started'); - expect(trace).not.toContain('squash_finished'); - - const failureReport = (deps.reports.create as any).mock.calls - .map((call: any[]) => call[0]) - .find((report: any) => report.body.status === 'failed'); - expect(failureReport.body.failure).toMatchObject({ - phase: 'final_gates', - message: expect.stringContaining('semantic-layer validation failed for warehouse:orders'), - details: expect.objectContaining({ - gateRepairDecision: { - repairable: false, - reason: 'non-repairable artifact gate issue(s): semantic_layer_validation_failed', - issueCodes: ['semantic_layer_validation_failed'], - }, - }), - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Run the final-gate regression to verify it fails** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "semantic-layer warehouse validation failures" -``` - -Expected: FAIL because final gate repair is invoked for the semantic-layer -validation failure. - -- [ ] **Step 3: Wire repair policy into the runner** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add: - -```ts -import { classifyFinalGateRepair } from './gate-repair-policy.js'; -``` - -Inside the `catch (error)` block around final `validateFinalIngestArtifacts()`, -immediately after: - -```ts - const gateError = this.errorMessage(error); -``` - -insert: - -```ts - const gateRepairDecision = classifyFinalGateRepair(error); - if (!gateRepairDecision.repairable) { - activeFailureDetails = { - ...finalArtifactGateTraceData, - gateRepairDecision, - }; - await runTrace.event('error', 'final_gates', 'final_artifact_gates_not_repairable', { - ...finalArtifactGateTraceData, - gateRepairDecision, - }); - throw error; - } -``` - -Leave the existing repair path unchanged after this insertion. It will run only -for repairable structured wiki-reference failures. - -- [ ] **Step 4: Run the final-gate regression to verify it passes** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "semantic-layer warehouse validation failures" -``` - -Expected: PASS. - -- [ ] **Step 5: Update the invalid `sl_refs` regression** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -find the test named -`rejects Notion-style changed wiki pages with invalid sl_refs`. Replace the -final assertion with: - -```ts - await expect( - runner.run({ jobId: 'job-invalid-slrefs', connectionId: 'warehouse', sourceKey: 'metabase', trigger: 'upload', bundleRef: { kind: 'upload', uploadId: 'upload' } }), - ).rejects.toThrow(/unknown sl_refs entry missing_source/); - - expect(deps.agentRunner.runLoop).not.toHaveBeenCalledWith( - expect.objectContaining({ - telemetryTags: expect.objectContaining({ - operationName: 'ingest-isolated-diff-gate-repair', - }), - }), - ); -``` - -Missing source-level `sl_refs` are non-repairable in v1 because selecting a -replacement source without evidence can invent semantic context. - -- [ ] **Step 6: Run existing gate repair regressions** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "repairs final wiki body refs|fails before squash when final gate repair makes no edit|invalid sl_refs" -``` - -Expected: PASS. - -- [ ] **Step 7: Commit runner policy wiring** - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "fix(ingest): block high-risk final gate repair" -``` - -### Task 6: Verify the v1 closure - -**Files:** -- Verify: `packages/context/src/ingest/**/*.ts` -- Verify: `packages/context/src/ingest/**/*.test.ts` - -- [ ] **Step 1: Run the focused gate repair suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/gate-repair-policy.test.ts \ - src/ingest/final-gate-repair.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/stages/validate-wu-sources.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 3: Run the context test suite** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code analysis** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS or only pre-existing findings unrelated to these files. If -there are findings in files changed by this plan, remove the dead code and run -the command again. - -- [ ] **Step 5: Run pre-commit on changed TypeScript and plan files** - -Run: - -```bash -uv run pre-commit run --files \ - packages/context/src/ingest/wiki-body-refs.ts \ - packages/context/src/ingest/wiki-body-refs.test.ts \ - packages/context/src/ingest/stages/validate-wu-sources.ts \ - packages/context/src/ingest/stages/validate-wu-sources.test.ts \ - packages/context/src/ingest/artifact-gates.ts \ - packages/context/src/ingest/artifact-gates.test.ts \ - packages/context/src/ingest/gate-repair-policy.ts \ - packages/context/src/ingest/gate-repair-policy.test.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.ts \ - packages/context/src/ingest/isolated-diff/patch-integrator.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair-classification.md -``` - -Expected: PASS. If this repository does not have a pre-commit configuration or -the local `uv` binary cannot satisfy the pinned project version, record that -explicitly in the implementation summary and keep the TypeScript checks above -as the authoritative verification. - -- [ ] **Step 6: Commit verification** - -```bash -git status --short -git commit --allow-empty -m "chore(ingest): verify gate repair classification" -``` - -## Self-review - -Spec coverage: - -- The Gate repair stage classification requirement is covered by Tasks 2 - through 5. High-risk semantic-layer validation failures and missing raw table - references are blocked before repair. -- Repairable stale wiki body references still run through bounded gate repair, - rerun final gates, and commit only after validation passes. -- Patch-level semantic gate failures and final composed-tree gate failures use - the same repair policy. -- The Global semantic gates section remains covered by the existing gates; - this plan preserves direct declared-join validation and the existing wiki - body grammar. -- Regression coverage now includes the spec's unrepairable final-gate failure - class without relying on the repair agent choosing not to edit. - -Remaining gaps: - -- No v1-blocking gaps remain after this plan is implemented and verified. -- Deterministic semantic merge helpers remain rollout step 9 and are - intentionally post-v1. -- Richer resolver and repair context can be added after v1 traces show the - frequent repair shapes. - -Placeholder scan: - -- The plan contains exact file paths, concrete test code, concrete - implementation snippets, commands, and expected outcomes. -- The plan contains no deferred implementation markers. - -Type consistency: - -- `FinalArtifactGateIssue`, `FinalArtifactGateFailure`, - `WikiBodyRefIssue`, `TouchedSourceValidationIssue`, - `classifyFinalGateRepair()`, and `artifactGateIssueSummary()` are introduced - before use. -- `GateRepairDecision` uses `repairable`, `reason`, and `issueCodes` - consistently in tests, traces, and failure-report details. - -Plan complete and saved to -`docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair-classification.md`. diff --git a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair.md b/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair.md deleted file mode 100644 index 62a174f5..00000000 --- a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair.md +++ /dev/null @@ -1,1438 +0,0 @@ -# Isolated Diff Ingestion V1 Gate Repair Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add bounded repair-agent handling for isolated-diff artifact gate -failures so cleanly applied integration trees get one scoped repair attempt -before the ingest fails. - -**Architecture:** Reuse the existing isolated-diff integration worktree, -trace writer, and `AgentRunnerPort`. A new `final-gate-repair` module exposes -scoped read/write tools over the exact wiki and semantic-layer files involved -in the failed gate. Patch-level semantic conflicts and final composed-tree gate -failures both call this repair module, rerun artifact gates, commit repaired -files only after gates pass, and record repair counters in ingest reports. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, zod, Node `fs/promises`, -existing `IngestBundleRunner`, `GitService`, `AgentRunnerPort`, -`IngestTraceWriter`, `integrateWorkUnitPatch`, and `validateFinalIngestArtifacts`. - ---- - -## Audit summary - -This audit read -`docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md`, searched -`docs/superpowers/plans/`, inspected the current isolated-diff implementation, -and ran the focused isolated-diff verification suite. - -Plans already based on the spec: - -| Plan | Implementation status | Evidence | -| --- | --- | --- | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-core.md` | Implemented | `packages/context/src/ingest/isolated-diff/git-patch.ts`, `work-unit-executor.ts`, `patch-integrator.ts`, `ingest-trace.ts`, `wiki-body-refs.ts`, and runner coverage exist. Git history includes `cae5c4b`, `1013bb6`, and `c481f1c`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-gates-and-trace-closure.md` | Implemented | Final gates run after reconciliation and follow-on mutations, child worktrees clean up, failed reports are stored, and trace coverage exists. Git history includes `656e584` and `87f1193`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-provenance-gate-closure.md` | Implemented | `validateProvenanceRawPaths()` runs before squash and has isolated-diff regression coverage. Git history includes `977a610`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-reference-and-target-gate-closure.md` | Implemented | Final wiki reference gates, SL write/edit target checks, patch target checks, and target-policy traces exist. Git history includes `5ec6396` and `c61c50b`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-global-wiki-reference-gate-closure.md` | Implemented | `wikiPageKeysForFinalGates()` expands to all global wiki pages when semantic-layer sources change or wiki pages are removed. Git history includes `ba534fb`. | -| `docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-textual-conflict-resolver.md` | Implemented | `textual-conflict-resolver.ts` exists, `patch-integrator.ts` invokes it after Git textual conflicts, `ingest-bundle.runner.ts` passes the callback, and report snapshots parse resolver counters. Git history includes `9f0abe5`, `529c6da`, `8784a47`, `aa8d59c`, and `3228843`. | - -Focused verification passed before writing this plan: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-trace.test.ts src/ingest/wiki-body-refs.test.ts src/ingest/artifact-gates.test.ts src/ingest/semantic-layer-target-policy.test.ts src/ingest/isolated-diff/git-patch.test.ts src/ingest/isolated-diff/work-unit-executor.test.ts src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/isolated-diff/textual-conflict-resolver.test.ts src/ingest/ingest-bundle.runner.isolated-diff.test.ts src/ingest/report-snapshot.test.ts src/sl/tools/sl-write-source.tool.test.ts src/sl/tools/sl-edit-source.tool.test.ts -``` - -Current result: `12 passed`, `73 passed`. - -One v1-essential design gap remains. The spec's gate repair stage says that -cleanly applied trees that fail semantic or wiki gates get a bounded repair -agent before the run fails. Current code still fails immediately in two places: - -- `packages/context/src/ingest/isolated-diff/patch-integrator.ts` returns - `semantic_conflict` as soon as `validateAppliedTree()` rejects after a patch - applies cleanly. -- `packages/context/src/ingest/ingest-bundle.runner.ts` calls - `validateFinalIngestArtifacts()` inside `traceTimed()` and lets the error - abort the run without a repair attempt. - -## Scope - -This plan implements bounded gate repair for artifact gate failures only: - -- semantic gate failures after a patch applies cleanly; -- final artifact gate failures after reconciliation, deterministic - post-processing, and wiki `sl_refs` repair; -- repair counters and traces for attempts, repairs, and failures. - -This plan does not repair patch policy failures, target-policy failures, -textual Git conflicts, provenance validation failures, squash conflicts, -connector rollout gaps, default-path promotion, semantic auto-merge helpers, or -removal of the shared-worktree fallback path. - -## File structure - -- Create `packages/context/src/ingest/final-gate-repair.ts`. - Owns bounded repair-agent execution, scoped repair tools, allowed path - derivation, prompt text, and result types. -- Create `packages/context/src/ingest/final-gate-repair.test.ts`. - Covers allowed-path derivation, scoped read/write enforcement, successful - repair, and no-edit failure. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.ts`. - Calls gate repair after clean patch application when artifact gates fail, - reruns gates, commits repaired files, and returns repair metadata. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`. - Adds semantic-gate repair success and failure coverage. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Wires final gate repair into the isolated-diff runner, commits repaired final - gate files before provenance validation, and updates isolated-diff counters. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Adds end-to-end coverage for repairable final wiki body references and a - failed no-edit repair. -- Modify `packages/context/src/ingest/reports.ts`. - Adds gate repair counters to `IngestReportBody.isolatedDiff`. -- Modify `packages/context/src/ingest/report-snapshot.ts`. - Parses gate repair counters from stored reports. -- Modify `packages/context/src/ingest/report-snapshot.test.ts`. - Covers stored gate repair counters. - ---- - -### Task 1: Add final gate repair unit tests - -**Files:** -- Create: `packages/context/src/ingest/final-gate-repair.test.ts` - -- [ ] **Step 1: Write the failing unit tests** - -Create `packages/context/src/ingest/final-gate-repair.test.ts`: - -```ts -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import { FileIngestTraceWriter } from './ingest-trace.js'; -import { finalGateRepairPaths, repairFinalGateFailure } from './final-gate-repair.js'; - -async function makeHarness() { - const root = await mkdtemp(join(tmpdir(), 'ktx-final-gate-repair-')); - const workdir = join(root, 'workdir'); - await mkdir(join(workdir, 'wiki/global'), { recursive: true }); - await mkdir(join(workdir, 'semantic-layer/warehouse'), { recursive: true }); - await writeFile( - join(workdir, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\n---\n\nARR uses `mart_account_segments.total_contract_arr_cents`.\n', - 'utf-8', - ); - await writeFile( - join(workdir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - 'utf-8', - ); - const trace = new FileIngestTraceWriter({ - tracePath: join(root, 'trace.jsonl'), - jobId: 'job-1', - connectionId: 'warehouse', - sourceKey: 'metabase', - runId: 'run-1', - syncId: 'sync-1', - level: 'trace', - }); - return { root, workdir, trace }; -} - -describe('finalGateRepairPaths', () => { - it('derives sorted wiki and semantic-layer file paths', () => { - expect( - finalGateRepairPaths({ - changedWikiPageKeys: ['account-segments', 'overview', 'account-segments'], - touchedSlSources: [ - { connectionId: 'warehouse', sourceName: 'mart_account_segments' }, - { connectionId: 'warehouse', sourceName: 'orders' }, - { connectionId: 'warehouse', sourceName: 'orders' }, - ], - }), - ).toEqual([ - 'semantic-layer/warehouse/mart_account_segments.yaml', - 'semantic-layer/warehouse/orders.yaml', - 'wiki/global/account-segments.md', - 'wiki/global/overview.md', - ]); - }); -}); - -describe('repairFinalGateFailure', () => { - it('lets the repair agent read gate errors and edit only allowed files', async () => { - const { workdir, trace } = await makeHarness(); - const agentRunner = { - runLoop: vi.fn(async (params: any) => { - const error = await params.toolSet.read_gate_error.execute({}); - expect(error.markdown).toContain('total_contract_arr_cents'); - - const page = await params.toolSet.read_repair_file.execute({ - path: 'wiki/global/account-segments.md', - }); - expect(page.markdown).toContain('total_contract_arr_cents'); - - await expect( - params.toolSet.write_repair_file.execute({ - path: 'wiki/global/other.md', - content: 'not allowed', - }), - ).rejects.toThrow(/gate repair path not allowed/); - - await params.toolSet.write_repair_file.execute({ - path: 'wiki/global/account-segments.md', - content: page.markdown.replace('total_contract_arr_cents', 'total_contract_arr'), - }); - return { stopReason: 'natural' as const }; - }), - }; - - const result = await repairFinalGateFailure({ - agentRunner, - workdir, - gateError: - 'final artifact gates failed:\naccount-segments: unknown semantic-layer entity mart_account_segments.total_contract_arr_cents', - allowedPaths: ['wiki/global/account-segments.md'], - trace, - repairKind: 'final_artifact_gate', - maxAttempts: 1, - stepBudget: 8, - }); - - expect(result).toEqual({ - status: 'repaired', - attempts: 1, - changedPaths: ['wiki/global/account-segments.md'], - }); - await expect(readFile(join(workdir, 'wiki/global/account-segments.md'), 'utf-8')).resolves.toContain( - 'total_contract_arr', - ); - await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('gate_repair_repaired'); - expect(agentRunner.runLoop).toHaveBeenCalledWith( - expect.objectContaining({ - modelRole: 'repair', - stepBudget: 8, - telemetryTags: expect.objectContaining({ - operationName: 'ingest-isolated-diff-gate-repair', - repairKind: 'final_artifact_gate', - }), - }), - ); - }); - - it('returns failed when the repair agent edits no allowed file', async () => { - const { workdir, trace } = await makeHarness(); - const result = await repairFinalGateFailure({ - agentRunner: { runLoop: vi.fn(async () => ({ stopReason: 'natural' as const })) }, - workdir, - gateError: 'final artifact gates failed:\naccount-segments: unknown semantic-layer entity', - allowedPaths: ['wiki/global/account-segments.md'], - trace, - repairKind: 'final_artifact_gate', - maxAttempts: 1, - stepBudget: 8, - }); - - expect(result).toEqual({ - status: 'failed', - attempts: 1, - reason: 'gate repair completed without editing an allowed path', - }); - await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('gate_repair_failed'); - }); -}); -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/final-gate-repair.test.ts -``` - -Expected: FAIL because `./final-gate-repair.js` does not exist. - -- [ ] **Step 3: Commit the failing tests** - -Run: - -```bash -git add packages/context/src/ingest/final-gate-repair.test.ts -git commit -m "test(ingest): cover isolated diff gate repair" -``` - -### Task 2: Implement the final gate repair module - -**Files:** -- Create: `packages/context/src/ingest/final-gate-repair.ts` -- Test: `packages/context/src/ingest/final-gate-repair.test.ts` - -- [ ] **Step 1: Add the repair module** - -Create `packages/context/src/ingest/final-gate-repair.ts`: - -```ts -import { mkdir, readFile, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { z } from 'zod'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; -import type { IngestTraceWriter } from './ingest-trace.js'; -import { traceTimed } from './ingest-trace.js'; - -export type FinalGateRepairKind = 'patch_semantic_gate' | 'final_artifact_gate'; - -export type FinalGateRepairResult = - | { status: 'repaired'; attempts: number; changedPaths: string[] } - | { status: 'failed'; attempts: number; reason: string }; - -export interface RepairFinalGateFailureInput { - agentRunner: AgentRunnerPort; - workdir: string; - gateError: string; - allowedPaths: string[]; - trace: IngestTraceWriter; - repairKind: FinalGateRepairKind; - maxAttempts?: number; - stepBudget?: number; -} - -const readRepairFileSchema = z.object({ - path: z.string().min(1), -}); - -const writeRepairFileSchema = z.object({ - path: z.string().min(1), - content: z.string(), -}); - -function normalizeRepoPath(path: string): string { - const normalized = path.replace(/\\/g, '/').replace(/^\/+/, ''); - const parts = normalized.split('/').filter((part) => part.length > 0); - if (parts.length === 0 || parts.some((part) => part === '.' || part === '..')) { - throw new Error(`gate repair path must be a repository-relative path: ${path}`); - } - return parts.join('/'); -} - -function assertAllowedPath(path: string, allowedPaths: ReadonlySet): string { - const normalized = normalizeRepoPath(path); - if (!allowedPaths.has(normalized)) { - throw new Error(`gate repair path not allowed: ${normalized}`); - } - return normalized; -} - -async function readOptionalFile(path: string): Promise<{ exists: boolean; content: string }> { - try { - return { exists: true, content: await readFile(path, 'utf-8') }; - } catch (error) { - if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') { - return { exists: false, content: '' }; - } - throw error; - } -} - -function buildGateRepairSystemPrompt(): string { - return ` -You repair one KTX isolated-diff artifact gate failure inside the integration worktree. - - - -- Use read_gate_error first. -- Read only files exposed by read_repair_file. -- Edit only paths exposed by write_repair_file. -- Prefer the smallest text edit that makes the gate pass. -- Preserve accepted work-unit, reconciliation, and deterministic projection content. -- Do not invent warehouse facts, business definitions, or semantic-layer entities. -- If the gate error requires choosing between conflicting facts without evidence, stop without editing. -`; -} - -function buildGateRepairUserPrompt(input: { - gateError: string; - allowedPaths: string[]; - repairKind: FinalGateRepairKind; - attempt: number; - maxAttempts: number; -}): string { - return `Repair isolated-diff artifact gates. - -Repair kind: ${input.repairKind} -Attempt: ${input.attempt} of ${input.maxAttempts} - -Allowed files: -${input.allowedPaths.map((path) => `- ${path}`).join('\n')} - -Gate error: -${input.gateError} - -Use read_gate_error first. Then inspect only the allowed files, write the -minimal repaired content, and stop.`; -} - -function buildToolSet(input: { - workdir: string; - gateError: string; - allowedPaths: ReadonlySet; - editedPaths: Set; -}): KtxRuntimeToolSet { - return { - read_gate_error: { - name: 'read_gate_error', - description: 'Read the artifact gate failure that must be repaired.', - inputSchema: z.object({}), - execute: async () => ({ - markdown: input.gateError, - structured: { gateError: input.gateError }, - }), - }, - read_repair_file: { - name: 'read_repair_file', - description: 'Read one allowed file from the integration worktree.', - inputSchema: readRepairFileSchema, - execute: async ({ path }: z.infer) => { - const normalized = assertAllowedPath(path, input.allowedPaths); - const file = await readOptionalFile(join(input.workdir, normalized)); - return { - markdown: file.exists ? file.content : `(missing file: ${normalized})`, - structured: { path: normalized, exists: file.exists }, - }; - }, - }, - write_repair_file: { - name: 'write_repair_file', - description: 'Replace one allowed integration worktree file with repaired text content.', - inputSchema: writeRepairFileSchema, - execute: async ({ path, content }: z.infer) => { - const normalized = assertAllowedPath(path, input.allowedPaths); - const fullPath = join(input.workdir, normalized); - await mkdir(dirname(fullPath), { recursive: true }); - await writeFile(fullPath, content, 'utf-8'); - input.editedPaths.add(normalized); - return { - markdown: `Wrote ${normalized}`, - structured: { path: normalized, bytes: Buffer.byteLength(content) }, - }; - }, - }, - }; -} - -export function finalGateRepairPaths(input: { - changedWikiPageKeys: string[]; - touchedSlSources: TouchedSlSource[]; -}): string[] { - return [ - ...new Set([ - ...input.touchedSlSources.map((source) => `semantic-layer/${source.connectionId}/${source.sourceName}.yaml`), - ...input.changedWikiPageKeys.map((pageKey) => `wiki/global/${pageKey}.md`), - ]), - ].sort(); -} - -export async function repairFinalGateFailure( - input: RepairFinalGateFailureInput, -): Promise { - const allowedPaths = new Set(input.allowedPaths.map(normalizeRepoPath)); - const maxAttempts = input.maxAttempts ?? 1; - const stepBudget = input.stepBudget ?? 16; - let lastFailure = 'gate repair did not run'; - - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const editedPaths = new Set(); - const sortedAllowedPaths = [...allowedPaths].sort(); - const traceData = { - repairKind: input.repairKind, - attempt, - maxAttempts, - allowedPaths: sortedAllowedPaths, - gateError: input.gateError, - }; - const result = await traceTimed(input.trace, 'gate_repair', 'gate_repair', traceData, async () => - input.agentRunner.runLoop({ - modelRole: 'repair', - systemPrompt: buildGateRepairSystemPrompt(), - userPrompt: buildGateRepairUserPrompt({ - gateError: input.gateError, - allowedPaths: sortedAllowedPaths, - repairKind: input.repairKind, - attempt, - maxAttempts, - }), - toolSet: buildToolSet({ - workdir: input.workdir, - gateError: input.gateError, - allowedPaths, - editedPaths, - }), - stepBudget, - telemetryTags: { - operationName: 'ingest-isolated-diff-gate-repair', - source: input.trace.context.sourceKey, - jobId: input.trace.context.jobId, - repairKind: input.repairKind, - }, - }), - ); - - if (result.stopReason === 'error') { - lastFailure = result.error?.message ?? 'gate repair agent loop errored'; - await input.trace.event('error', 'gate_repair', 'gate_repair_failed', traceData, result.error); - continue; - } - - const changedPaths = [...editedPaths].sort(); - if (changedPaths.length === 0) { - lastFailure = 'gate repair completed without editing an allowed path'; - await input.trace.event('error', 'gate_repair', 'gate_repair_failed', { - ...traceData, - reason: lastFailure, - }); - continue; - } - - await input.trace.event('debug', 'gate_repair', 'gate_repair_repaired', { - ...traceData, - changedPaths, - }); - return { status: 'repaired', attempts: attempt, changedPaths }; - } - - return { status: 'failed', attempts: maxAttempts, reason: lastFailure }; -} -``` - -- [ ] **Step 2: Run the repair module tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/final-gate-repair.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Commit the repair module** - -Run: - -```bash -git add packages/context/src/ingest/final-gate-repair.ts packages/context/src/ingest/final-gate-repair.test.ts -git commit -m "feat(ingest): add isolated diff gate repair agent" -``` - -### Task 3: Repair patch-level semantic gate failures - -**Files:** -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- Test: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` - -- [ ] **Step 1: Add patch integrator repair regressions** - -Append these tests inside -`describe('integrateWorkUnitPatch', ...)` in -`packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`: - -```ts - it('repairs semantic gate failures after a patch applies cleanly', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - const childDir = join(homeDir, 'child-semantic-repair'); - await git.addWorktree(childDir, 'child-semantic-repair', baseSha); - const childGit = git.forWorktree(childDir); - await writeFile(join(childDir, 'wiki/global/a.md'), 'bad semantic ref\n'); - await childGit.commitFiles(['wiki/global/a.md'], 'bad semantic edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/semantic-repair.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-semantic-repair/trace.jsonl'), - jobId: 'job-semantic-repair', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - const validateAppliedTree = vi - .fn() - .mockRejectedValueOnce(new Error('final artifact gates failed:\na: unknown semantic-layer entity')) - .mockResolvedValueOnce(undefined); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-repairable', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree, - slDisallowed: false, - allowedTargetConnectionIds: new Set(['c1']), - repairGateFailure: vi.fn(async (context) => { - expect(context).toMatchObject({ - unitKey: 'wu-repairable', - patchPath, - touchedPaths: ['wiki/global/a.md'], - }); - await writeFile(join(configDir, 'wiki/global/a.md'), 'repaired semantic ref\n', 'utf-8'); - return { - status: 'repaired' as const, - attempts: 1, - changedPaths: ['wiki/global/a.md'], - }; - }), - }); - - expect(result).toMatchObject({ - status: 'accepted', - touchedPaths: ['wiki/global/a.md'], - gateRepair: { - status: 'repaired', - attempts: 1, - changedPaths: ['wiki/global/a.md'], - }, - }); - expect(validateAppliedTree).toHaveBeenCalledTimes(2); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('repaired semantic ref\n'); - await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('patch_accepted_after_gate_repair'); - }); - - it('keeps the pre-apply tree when semantic gate repair fails', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - const childDir = join(homeDir, 'child-semantic-repair-fails'); - await git.addWorktree(childDir, 'child-semantic-repair-fails', baseSha); - const childGit = git.forWorktree(childDir); - await writeFile(join(childDir, 'wiki/global/a.md'), 'bad semantic ref\n'); - await childGit.commitFiles(['wiki/global/a.md'], 'bad semantic edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'patches/semantic-repair-fails.patch'); - await childGit.writeBinaryNoRenamePatch(baseSha, 'HEAD', patchPath); - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-semantic-repair-fails/trace.jsonl'), - jobId: 'job-semantic-repair-fails', - connectionId: 'c1', - sourceKey: 'fake', - level: 'trace', - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-not-repaired', - patchPath, - integrationGit: git, - trace, - author: { name: 'KTX Test', email: 'system@ktx.local' }, - validateAppliedTree: vi.fn().mockRejectedValue(new Error('final artifact gates failed')), - slDisallowed: false, - allowedTargetConnectionIds: new Set(['c1']), - repairGateFailure: vi.fn(async () => ({ - status: 'failed' as const, - attempts: 1, - reason: 'gate repair completed without editing an allowed path', - })), - }); - - expect(result).toMatchObject({ - status: 'semantic_conflict', - gateRepair: { - status: 'failed', - attempts: 1, - reason: 'gate repair completed without editing an allowed path', - }, - }); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('old\n'); - }); -``` - -- [ ] **Step 2: Run the patch integrator tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: FAIL because `integrateWorkUnitPatch()` does not accept -`repairGateFailure` and does not return `gateRepair`. - -- [ ] **Step 3: Add gate repair metadata to the patch integrator** - -Modify `packages/context/src/ingest/isolated-diff/patch-integrator.ts`: - -```ts -import type { FinalGateRepairResult } from '../final-gate-repair.js'; -``` - -Replace the `PatchIntegrationResult` type with: - -```ts -export type PatchIntegrationResult = - | { - status: 'accepted'; - commitSha: string; - touchedPaths: string[]; - textualResolution?: PatchIntegrationTextualResolution; - gateRepair?: FinalGateRepairResult; - } - | { - status: 'textual_conflict'; - reason: string; - touchedPaths: string[]; - textualResolution?: PatchIntegrationTextualResolution; - gateRepair?: FinalGateRepairResult; - } - | { - status: 'semantic_conflict'; - reason: string; - touchedPaths: string[]; - textualResolution?: PatchIntegrationTextualResolution; - gateRepair?: FinalGateRepairResult; - }; -``` - -Add this optional callback to `IntegrateWorkUnitPatchInput`: - -```ts - repairGateFailure?(input: { - unitKey: string; - patchPath: string; - touchedPaths: string[]; - reason: string; - }): Promise; -``` - -Replace the current `catch` block for the non-textual -`semantic_gate` section with this block: - -```ts - } catch (error) { - const reason = errorMessage(error); - await input.trace.event('error', 'integration', 'patch_semantic_conflict', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason, - }); - - if (input.repairGateFailure) { - const gateRepair = await input.repairGateFailure({ - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason, - }); - - if (gateRepair.status === 'failed') { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - return { - status: 'semantic_conflict', - reason: gateRepair.reason, - touchedPaths, - gateRepair, - }; - } - - try { - await traceTimed( - input.trace, - 'integration', - 'semantic_gate_after_gate_repair', - { unitKey: input.unitKey, touchedPaths: gateRepair.changedPaths }, - async () => { - await input.validateAppliedTree(gateRepair.changedPaths); - }, - ); - } catch (repairValidationError) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - return { - status: 'semantic_conflict', - reason: errorMessage(repairValidationError), - touchedPaths: gateRepair.changedPaths, - gateRepair, - }; - } - - const commit = await input.integrationGit.commitFiles( - gateRepair.changedPaths, - `ingest: repair WorkUnit ${input.unitKey} gates`, - input.author.name, - input.author.email, - ); - if (!commit.created) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - return { - status: 'semantic_conflict', - reason: 'gate repair produced no committable changes', - touchedPaths: gateRepair.changedPaths, - gateRepair, - }; - } - - await input.trace.event('debug', 'integration', 'patch_accepted_after_gate_repair', { - unitKey: input.unitKey, - commitSha: commit.commitHash, - touchedPaths: gateRepair.changedPaths, - attempts: gateRepair.attempts, - }); - return { - status: 'accepted', - commitSha: commit.commitHash, - touchedPaths: gateRepair.changedPaths, - gateRepair, - }; - } - - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - return { - status: 'semantic_conflict', - reason, - touchedPaths, - }; - } -``` - -- [ ] **Step 4: Run the patch integrator tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit patch-level gate repair** - -Run: - -```bash -git add packages/context/src/ingest/isolated-diff/patch-integrator.ts packages/context/src/ingest/isolated-diff/patch-integrator.test.ts -git commit -m "feat(ingest): repair isolated diff semantic gate failures" -``` - -### Task 4: Wire final gate repair into the runner - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/reports.ts` -- Modify: `packages/context/src/ingest/report-snapshot.ts` -- Modify: `packages/context/src/ingest/report-snapshot.test.ts` - -- [ ] **Step 1: Add report fields and parser coverage** - -In `packages/context/src/ingest/reports.ts`, extend -`IngestReportBody.isolatedDiff`: - -```ts - gateRepairAttempts?: number; - gateRepairs?: number; - gateRepairFailures?: number; -``` - -In `packages/context/src/ingest/report-snapshot.ts`, extend the -`isolatedDiff` schema: - -```ts - gateRepairAttempts: z.number().int().min(0).default(0), - gateRepairs: z.number().int().min(0).default(0), - gateRepairFailures: z.number().int().min(0).default(0), -``` - -Append this test to `packages/context/src/ingest/report-snapshot.test.ts`: - -```ts - it('parses isolated-diff gate repair counters', () => { - const snapshot = parseIngestReportSnapshot({ - id: 'report-1', - runId: 'run-1', - jobId: 'job-1', - connectionId: 'warehouse', - sourceKey: 'metabase', - createdAt: '2026-05-18T00:00:00.000Z', - body: { - status: 'completed', - syncId: 'sync-1', - diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 }, - commitSha: 'abc123', - isolatedDiff: { - enabled: true, - acceptedPatches: 1, - textualConflicts: 0, - semanticConflicts: 1, - gateRepairAttempts: 1, - gateRepairs: 1, - gateRepairFailures: 0, - }, - workUnits: [], - failedWorkUnits: [], - reconciliationSkipped: true, - conflictsResolved: [], - evictionsApplied: [], - unmappedFallbacks: [], - evictionInputs: [], - unresolvedCards: [], - supersededBy: null, - overrideOf: null, - provenanceRows: [], - toolTranscripts: [], - }, - }); - - expect(snapshot.body.isolatedDiff).toMatchObject({ - gateRepairAttempts: 1, - gateRepairs: 1, - gateRepairFailures: 0, - }); - }); -``` - -- [ ] **Step 2: Run report snapshot tests to verify they pass** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Import the gate repair module in the runner** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add: - -```ts -import { finalGateRepairPaths, repairFinalGateFailure } from './final-gate-repair.js'; -``` - -- [ ] **Step 4: Add gate repair counters to the isolated summary** - -In the `isolatedDiffSummary` object, add: - -```ts - gateRepairAttempts: 0, - gateRepairs: 0, - gateRepairFailures: 0, -``` - -- [ ] **Step 5: Pass patch-level gate repair to `integrateWorkUnitPatch()`** - -In the `integrateWorkUnitPatch()` call, add this callback next to -`resolveTextualConflict`: - -```ts - repairGateFailure: (context) => - repairFinalGateFailure({ - agentRunner: this.deps.agentRunner, - workdir: sessionWorktree.workdir, - gateError: context.reason, - allowedPaths: context.touchedPaths, - trace: runTrace, - repairKind: 'patch_semantic_gate', - maxAttempts: 1, - stepBudget: 16, - }), -``` - -After the existing `integration.textualResolution` counter block, add: - -```ts - if (integration.gateRepair) { - isolatedDiffSummary.gateRepairAttempts += integration.gateRepair.attempts; - if (integration.gateRepair.status === 'repaired') { - isolatedDiffSummary.semanticConflicts += 1; - isolatedDiffSummary.gateRepairs += 1; - } else { - isolatedDiffSummary.gateRepairFailures += 1; - } - } -``` - -- [ ] **Step 6: Replace final artifact gate throw-through with bounded repair** - -Replace the current `await traceTimed(... 'final_artifact_gates' ...)` block in -`packages/context/src/ingest/ingest-bundle.runner.ts` with: - -```ts - try { - await traceTimed( - runTrace, - 'final_gates', - 'final_artifact_gates', - finalArtifactGateTraceData, - async () => { - await validateFinalIngestArtifacts({ - connectionIds: repairConnectionIds, - changedWikiPageKeys: finalChangedWikiPageKeys, - touchedSlSources: finalTouchedSlSources, - wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - validateTouchedSources: (touched) => - validateWuTouchedSources( - { - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - connections: this.deps.connections, - configService: sessionWorktree.config, - gitService: sessionWorktree.git, - slSourcesRepository: this.deps.slSourcesRepository, - probeRowCount: this.deps.settings.probeRowCount, - slValidator: this.deps.slValidator, - }, - touched, - ), - tableExists: (connectionId, tableRef) => - this.tableRefExistsInSemanticLayer( - this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - [connectionId], - tableRef, - ), - }); - }, - ); - } catch (error) { - const gateError = this.errorMessage(error); - const repairPaths = finalGateRepairPaths({ - changedWikiPageKeys: finalChangedWikiPageKeys, - touchedSlSources: finalTouchedSlSources, - }); - const gateRepair = await repairFinalGateFailure({ - agentRunner: this.deps.agentRunner, - workdir: sessionWorktree.workdir, - gateError, - allowedPaths: repairPaths, - trace: runTrace, - repairKind: 'final_artifact_gate', - maxAttempts: 1, - stepBudget: 16, - }); - - isolatedDiffSummary.gateRepairAttempts += gateRepair.attempts; - if (gateRepair.status === 'failed') { - isolatedDiffSummary.gateRepairFailures += 1; - activeFailureDetails = { - ...finalArtifactGateTraceData, - gateRepair, - gateError, - }; - throw new Error(`${gateError}\ngate repair failed: ${gateRepair.reason}`); - } - - isolatedDiffSummary.gateRepairs += 1; - await traceTimed( - runTrace, - 'final_gates', - 'final_artifact_gates_after_gate_repair', - { - ...finalArtifactGateTraceData, - repairedPaths: gateRepair.changedPaths, - }, - async () => { - await validateFinalIngestArtifacts({ - connectionIds: repairConnectionIds, - changedWikiPageKeys: finalChangedWikiPageKeys, - touchedSlSources: finalTouchedSlSources, - wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir), - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - validateTouchedSources: (touched) => - validateWuTouchedSources( - { - semanticLayerService: this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - connections: this.deps.connections, - configService: sessionWorktree.config, - gitService: sessionWorktree.git, - slSourcesRepository: this.deps.slSourcesRepository, - probeRowCount: this.deps.settings.probeRowCount, - slValidator: this.deps.slValidator, - }, - touched, - ), - tableExists: (connectionId, tableRef) => - this.tableRefExistsInSemanticLayer( - this.deps.semanticLayerService.forWorktree(sessionWorktree.workdir), - [connectionId], - tableRef, - ), - }); - }, - ); - - const repairCommit = await sessionWorktree.git.commitFiles( - gateRepair.changedPaths, - `ingest(${job.sourceKey}): repair final gates syncId=${syncId}`, - this.deps.storage.systemGitAuthor.name, - this.deps.storage.systemGitAuthor.email, - ); - if (!repairCommit.created) { - isolatedDiffSummary.gateRepairFailures += 1; - throw new Error('final gate repair produced no committable changes'); - } - await runTrace.event('debug', 'final_gates', 'final_gate_repair_committed', { - commitSha: repairCommit.commitHash, - repairedPaths: gateRepair.changedPaths, - }); - } -``` - -- [ ] **Step 7: Run the runner and report tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit runner wiring and report fields** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/reports.ts packages/context/src/ingest/report-snapshot.ts packages/context/src/ingest/report-snapshot.test.ts -git commit -m "feat(ingest): wire isolated diff gate repair" -``` - -### Task 5: Add isolated runner gate repair regressions - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Test: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Add a final gate repair success regression** - -Append this test inside -`describe('IngestBundleRunner isolated diff path', ...)`: - -```ts - it('repairs final wiki body refs before squash when the repair agent edits the scoped page', async () => { - const runtime = await makeRealGitRuntime(); - try { - await mkdir(join(runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n', - ); - await writeFile( - join(runtime.configDir, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\n---\n\nExisting ARR uses `mart_account_segments.total_contract_arr_cents`.\n', - ); - await runtime.git.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], - 'seed stale wiki body ref', - 'KTX Test', - 'system@ktx.local', - ); - - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'source-only', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName === 'ingest-isolated-diff-gate-repair') { - const gateError = await params.toolSet.read_gate_error.execute({}); - expect(gateError.markdown).toContain('total_contract_arr_cents'); - const page = await params.toolSet.read_repair_file.execute({ - path: 'wiki/global/account-segments.md', - }); - await params.toolSet.write_repair_file.execute({ - path: 'wiki/global/account-segments.md', - content: page.markdown.replace('total_contract_arr_cents', 'total_contract_arr'), - }); - return { stopReason: 'natural' as const }; - } - if (params.modelRole === 'reconcile') { - return { stopReason: 'natural' as const }; - } - - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ - target: 'sl', - type: 'updated', - key: 'mart_account_segments', - detail: 'Rename ARR measure', - targetConnectionId: 'warehouse', - rawPaths: ['cards/source.json'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml'], - 'wu source rename', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' as const }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]); - - const result = await runner.run({ - jobId: 'job-final-gate-repair', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }); - - expect(result.commitSha).toBeTruthy(); - await expect(readFile(join(runtime.configDir, 'wiki/global/account-segments.md'), 'utf-8')).resolves.toContain( - 'mart_account_segments.total_contract_arr', - ); - await expect(readFile(join(runtime.configDir, 'wiki/global/account-segments.md'), 'utf-8')).resolves.not.toContain( - 'total_contract_arr_cents', - ); - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0] as any; - expect(reportCreate.body.isolatedDiff).toMatchObject({ - gateRepairAttempts: 1, - gateRepairs: 1, - gateRepairFailures: 0, - }); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-final-gate-repair/trace.jsonl'), 'utf-8'); - expect(trace).toContain('gate_repair_repaired'); - expect(trace).toContain('final_artifact_gates_after_gate_repair_finished'); - expect(trace).toContain('final_gate_repair_committed'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Add a final gate repair no-edit failure regression** - -Append this test inside the same `describe(...)` block: - -```ts - it('fails before squash when final gate repair makes no edit', async () => { - const runtime = await makeRealGitRuntime(); - try { - await mkdir(join(runtime.configDir, 'semantic-layer/warehouse'), { recursive: true }); - await mkdir(join(runtime.configDir, 'wiki/global'), { recursive: true }); - await writeFile( - join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr_cents\n expr: sum(contract_arr)\n', - ); - await writeFile( - join(runtime.configDir, 'wiki/global/account-segments.md'), - '---\nsummary: Account segments\nusage_mode: auto\n---\n\nExisting ARR uses `mart_account_segments.total_contract_arr_cents`.\n', - ); - await runtime.git.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml', 'wiki/global/account-segments.md'], - 'seed stale wiki body ref', - 'KTX Test', - 'system@ktx.local', - ); - const preRunHead = await runtime.git.revParseHead(); - - const { deps, adapter } = makeDeps(runtime); - adapter.chunk.mockResolvedValue({ - workUnits: [{ unitKey: 'source-only', rawFiles: ['cards/source.json'], peerFileIndex: [], dependencyPaths: [] }], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName === 'ingest-isolated-diff-gate-repair') { - return { stopReason: 'natural' as const }; - } - if (params.modelRole === 'reconcile') { - return { stopReason: 'natural' as const }; - } - - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\ngrain: [account_id]\ncolumns: [{name: account_id, type: string}]\njoins: []\nmeasures:\n - name: total_contract_arr\n expr: sum(contract_arr)\n', - ); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'mart_account_segments'); - currentSession.actions.push({ - target: 'sl', - type: 'updated', - key: 'mart_account_segments', - detail: 'Rename ARR measure', - targetConnectionId: 'warehouse', - rawPaths: ['cards/source.json'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/mart_account_segments.yaml'], - 'wu source rename', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' as const }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['cards/source.json', 'h1']]); - - await expect( - runner.run({ - jobId: 'job-final-gate-repair-fails', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).rejects.toThrow(/gate repair completed without editing an allowed path/); - - expect(await runtime.git.revParseHead()).toBe(preRunHead); - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0] as any; - expect(reportCreate.body.status).toBe('failed'); - expect(reportCreate.body.isolatedDiff).toMatchObject({ - gateRepairAttempts: 1, - gateRepairs: 0, - gateRepairFailures: 1, - }); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-final-gate-repair-fails/trace.jsonl'), 'utf-8'); - expect(trace).toContain('gate_repair_failed'); - expect(trace).not.toContain('squash_finished'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 3: Run the isolated runner tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. - -- [ ] **Step 4: Commit runner regressions** - -Run: - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "test(ingest): verify isolated diff final gate repair" -``` - -### Task 6: Final verification - -**Files:** -- Verify: `packages/context/src/ingest/final-gate-repair.ts` -- Verify: `packages/context/src/ingest/final-gate-repair.test.ts` -- Verify: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` -- Verify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Verify: `packages/context/src/ingest/reports.ts` -- Verify: `packages/context/src/ingest/report-snapshot.ts` -- Verify: `packages/context/src/ingest/report-snapshot.test.ts` - -- [ ] **Step 1: Run focused tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/final-gate-repair.test.ts src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/ingest-bundle.runner.isolated-diff.test.ts src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the existing isolated-diff safety suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-trace.test.ts src/ingest/wiki-body-refs.test.ts src/ingest/artifact-gates.test.ts src/ingest/semantic-layer-target-policy.test.ts src/ingest/isolated-diff/git-patch.test.ts src/ingest/isolated-diff/work-unit-executor.test.ts src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/isolated-diff/textual-conflict-resolver.test.ts src/ingest/ingest-bundle.runner.isolated-diff.test.ts src/ingest/report-snapshot.test.ts src/sl/tools/sl-write-source.tool.test.ts src/sl/tools/sl-edit-source.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run package type checks** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS or only pre-existing findings unrelated to these files. - -- [ ] **Step 5: Run formatting and diff checks** - -Run: - -```bash -pnpm exec prettier --check packages/context/src/ingest/final-gate-repair.ts packages/context/src/ingest/final-gate-repair.test.ts packages/context/src/ingest/isolated-diff/patch-integrator.ts packages/context/src/ingest/isolated-diff/patch-integrator.test.ts packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts packages/context/src/ingest/reports.ts packages/context/src/ingest/report-snapshot.ts packages/context/src/ingest/report-snapshot.test.ts docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair.md -git diff --check -``` - -Expected: PASS. - -- [ ] **Step 6: Commit final verification adjustments** - -If verification required formatting or type-only adjustments, run: - -```bash -git add packages/context/src/ingest/final-gate-repair.ts packages/context/src/ingest/final-gate-repair.test.ts packages/context/src/ingest/isolated-diff/patch-integrator.ts packages/context/src/ingest/isolated-diff/patch-integrator.test.ts packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts packages/context/src/ingest/reports.ts packages/context/src/ingest/report-snapshot.ts packages/context/src/ingest/report-snapshot.test.ts docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-gate-repair.md -git commit -m "chore(ingest): verify isolated diff gate repair" -``` - -Expected: commit is created only when Step 1 through Step 5 produced tracked -source changes after the previous task commits. - -## Self-review - -Spec coverage: - -- The plan implements the remaining gate repair stage from the spec. -- Patch-level semantic gate failures get one bounded repair attempt after the - patch applies cleanly. -- Final composed-tree artifact gate failures get one bounded repair attempt - before provenance validation and squash. -- Repair tools are scoped to touched wiki and semantic-layer files. -- Target-policy, patch-policy, textual conflict, provenance, and squash - failures remain non-repairable in this plan. -- Connector rollout, default promotion, old-path removal, and deterministic - semantic merge helpers remain non-v1 follow-up work. - -Placeholder scan: - -- No deferred implementation markers remain. -- Every code-changing step includes concrete code or exact insertion snippets. - -Type consistency: - -- The report field names are `gateRepairAttempts`, `gateRepairs`, and - `gateRepairFailures` in `reports.ts`, `report-snapshot.ts`, runner code, and - tests. -- The repair result type is `FinalGateRepairResult`. -- The repair function is `repairFinalGateFailure()`. -- The path helper is `finalGateRepairPaths()`. diff --git a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-shared-worktree-removal.md b/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-shared-worktree-removal.md deleted file mode 100644 index 980ef8f3..00000000 --- a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-shared-worktree-removal.md +++ /dev/null @@ -1,980 +0,0 @@ -# Isolated Diff Ingestion V1 Shared Worktree Removal Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the old shared-worktree WorkUnit execution path so every -non-override bundle ingest uses isolated WorkUnit diffs. - -**Architecture:** Keep `IngestBundleRunner` with one non-override execution -path: raw snapshot, optional deterministic projection, child WorkUnit -worktrees, patch integration, reconciliation, final gates, provenance -validation, and squash. Delete the private fallback routing setting and all -legacy tests, traces, and agent instructions that existed only for shared -WorkUnit state. - -**Tech Stack:** TypeScript, Vitest, pnpm, KTX ingest runner, Git worktrees. - ---- - -## Audit summary - -This audit read the original design in -`docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md`, every -implemented plan matching -`docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-*.md` and -`docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-*.md`, and the -current implementation under `packages/context/src/ingest/`, -`packages/context/prompts/`, and `packages/context/skills/`. - -Implemented v1 rollout coverage: - -- Rollout steps 1 and 2 exist in code: isolated child worktrees, binary - no-rename patch collection, and `git apply --3way --index` patch integration. -- Rollout step 3 exists in code: - `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts` is - wired through the patch integrator and runner. -- Rollout steps 4, 5, and 6 exist in code: final wiki and semantic-layer gates, - provenance validation before squash, target policy checks, bounded gate - repair, failed reports, and trace counters. -- Rollout step 7 exists in code: the Metabase stale body-reference regression - is covered in `ingest-bundle.runner.isolated-diff.test.ts`. -- Rollout step 8 is committed: Notion, LookML, Looker, dbt, and MetricFlow - route through isolated child worktrees, and MetricFlow projection runs before - WorkUnits. -- Rollout step 10 is committed: non-override ingests default to isolated diffs, - and the old branch is reachable only through the private - `sharedWorktreeSourceKeys` fallback setting. - -## Remaining gaps - -The remaining v1-blocking gaps are all part of rollout step 11: - -- `packages/context/src/ingest/ports.ts` still exposes the private - `sharedWorktreeSourceKeys?: string[]` setting. -- `packages/context/src/ingest/isolated-diff/source-routing.ts` and its test - exist only to support the fallback setting. -- `packages/context/src/ingest/local-bundle-runtime.ts` still installs - `sharedWorktreeSourceKeys: []`. -- `packages/context/src/ingest/ingest-bundle.runner.ts` still checks - `isSharedWorktreeFallbackEnabled()` and contains the - `shared_worktree_path_enabled` branch that runs WorkUnits against the mutable - integration worktree. -- `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - still has a regression proving the shared-worktree fallback is reachable. -- `packages/context/src/ingest/ingest-bundle.runner.test.ts` keeps broad runner - tests on the legacy path through `sharedWorktreeSourceKeys`; those tests must - either use the isolated mock harness or move coverage into the real-git - isolated suite. -- `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md` and - `packages/context/skills/ingest_triage/SKILL.md` still tell WorkUnit agents - that prior WorkUnit writes in the same job are visible in the current working - branch. That instruction is false after isolated diffs and must be removed - with the shared path. - -Non-blocking gaps after this plan: - -- Rollout step 9 deterministic semantic merge helpers remain intentionally - deferred until resolver metrics show frequent mechanical repairs. -- Semantic-layer dependency expansion remains direct declared joins only; the - spec explicitly defers transitive SQL-projection closure. -- Provenance remains in the ingest provenance store and report body; moving it - to worktree files is a separate schema migration. -- Resolver context can later include richer transcript excerpts and explicit - overlap summaries for every previously applied patch. -- Failures before an ingest run row exists still have deterministic trace files - but no stored ingest report. - -## File structure - -- Modify `packages/context/src/ingest/ports.ts`. Remove the private fallback - setting from `IngestSettingsPort`. -- Modify `packages/context/src/ingest/local-bundle-runtime.ts`. Stop importing - and installing default shared-worktree fallback settings. -- Delete `packages/context/src/ingest/isolated-diff/source-routing.ts`. This - helper has no responsibility once fallback routing is removed. -- Delete `packages/context/src/ingest/isolated-diff/source-routing.test.ts`. - Its assertions exist only for the fallback helper. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. Delete - `isSharedWorktreeFallbackEnabled()`, the old shared-worktree WorkUnit branch, - and helper methods that only served that branch. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Remove fallback reachability coverage and add a stale-setting regression that - proves a runtime object cannot opt out of isolated diffs. -- Modify `packages/context/src/ingest/ingest-bundle.runner.test.ts`. Remove - the fallback setting from the broad test harness and make its mocked Git - session support no-op isolated patch collection. -- Modify `packages/context/src/ingest/local-bundle-runtime.test.ts`. Assert - local runtime settings do not contain the fallback key. -- Modify `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md`. - Replace shared-branch WorkUnit visibility instructions with isolated-diff - instructions. -- Modify `packages/context/skills/ingest_triage/SKILL.md`. Remove Stage 3 - prior-WorkUnit visibility language and keep cross-WorkUnit sweep guidance in - Stage 4 reconciliation. - ---- - -### Task 1: Add removal-contract regressions - -**Files:** -- Modify: `packages/context/src/ingest/local-bundle-runtime.test.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Update the local runtime settings type** - -In `packages/context/src/ingest/local-bundle-runtime.test.ts`, replace -`RuntimeWithSettingsDeps` with: - -```ts -type RuntimeWithSettingsDeps = { - deps: { - settings: Record; - }; -}; -``` - -- [ ] **Step 2: Replace the local runtime fallback-setting assertion** - -In `packages/context/src/ingest/local-bundle-runtime.test.ts`, replace the test -named `defaults local bundle ingest to isolated diffs without an allowlist` with: - -```ts - it('defaults local bundle ingest to isolated diffs without a shared-worktree fallback setting', () => { - const runtime = createLocalBundleIngestRuntime({ - project, - adapters: [new FakeSourceAdapter()], - agentRunner: testAgentRunner(), - }); - - const settings = (runtime.runner as unknown as RuntimeWithSettingsDeps).deps.settings; - - expect(settings).not.toHaveProperty('sharedWorktreeSourceKeys'); - expect(Object.keys(settings).sort()).toEqual([ - 'ingestTraceLevel', - 'memoryIngestionModel', - 'probeRowCount', - 'workUnitFailureMode', - 'workUnitMaxConcurrency', - 'workUnitStepBudget', - ]); - }); -``` - -- [ ] **Step 3: Remove the source-routing import from the isolated runner test** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -delete this import: - -```ts -import { defaultSharedWorktreeSourceKeys } from './isolated-diff/source-routing.js'; -``` - -Then remove the `sharedWorktreeSourceKeys` line from the `settings` object in -`makeDeps()`: - -```ts - settings: { - memoryIngestionModel: 'test', - probeRowCount: 1, - ingestTraceLevel: 'trace', - ...settings, - }, -``` - -- [ ] **Step 4: Replace the shared fallback reachability test** - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -replace the test named -`keeps the shared-worktree path reachable through explicit private fallback settings` -with this stale-setting regression: - -```ts - it('does not support shared-worktree fallback settings', async () => { - const runtime = await makeRealGitRuntime(); - try { - const sourceKey = 'legacy-source'; - const staleSettings = { - sharedWorktreeSourceKeys: ['legacy-source'], - } as Partial & Record; - const { deps, adapter } = makeDeps(runtime, sourceKey, staleSettings); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { - unitKey: 'legacy-wiki', - rawFiles: ['legacy/page.json'], - peerFileIndex: [], - dependencyPaths: [], - }, - ], - }); - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName !== 'ingest-bundle-wu') { - return { stopReason: 'natural' }; - } - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'wiki/global'), { recursive: true }); - await writeFile( - join(root, 'wiki/global/legacy-isolated.md'), - '---\nsummary: Legacy isolated write\nusage_mode: auto\n---\n\nLegacy isolated write.\n', - 'utf-8', - ); - currentSession.actions.push({ - target: 'wiki', - type: 'created', - key: 'legacy-isolated', - detail: 'Legacy isolated write', - rawPaths: ['legacy/page.json'], - }); - await currentSession.gitService.commitFiles( - ['wiki/global/legacy-isolated.md'], - 'legacy isolated wiki', - 'KTX Test', - 'system@ktx.local', - ); - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [['legacy/page.json', 'h1']], sourceKey); - - await expect( - runner.run({ - jobId: 'job-legacy-isolated', - connectionId: 'warehouse', - sourceKey, - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }), - ).resolves.toMatchObject({ - jobId: 'job-legacy-isolated', - failedWorkUnits: [], - workUnitCount: 1, - }); - - const trace = await readFile( - join(runtime.configDir, '.ktx/ingest-traces/job-legacy-isolated/trace.jsonl'), - 'utf-8', - ); - expect(trace).toContain('isolated_diff_enabled'); - expect(trace).toContain('work_unit_child_created'); - expect(trace).not.toContain('shared_worktree_path_enabled'); - - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0]; - const reportBody = reportCreate?.body as { isolatedDiff?: unknown } | undefined; - expect(reportBody?.isolatedDiff).toMatchObject({ - enabled: true, - acceptedPatches: 1, - }); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 5: Run the removal regressions and confirm they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/local-bundle-runtime.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - -t "shared-worktree fallback|stale|defaults local bundle ingest|unlisted direct-writing source" -``` - -Expected: FAIL. The local runtime still exposes `sharedWorktreeSourceKeys`, and -the stale-setting runner test still reaches `shared_worktree_path_enabled`. - ---- - -### Task 2: Remove the fallback setting and routing module - -**Files:** -- Modify: `packages/context/src/ingest/ports.ts` -- Modify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Delete: `packages/context/src/ingest/isolated-diff/source-routing.ts` -- Delete: `packages/context/src/ingest/isolated-diff/source-routing.test.ts` - -- [ ] **Step 1: Remove the fallback setting from the runner settings port** - -In `packages/context/src/ingest/ports.ts`, replace `IngestSettingsPort` with: - -```ts -export interface IngestSettingsPort { - memoryIngestionModel: string; - probeRowCount: number; - workUnitMaxConcurrency?: number; - workUnitStepBudget?: number; - workUnitFailureMode?: 'abort' | 'continue'; - ingestTraceLevel?: IngestTraceLevel; -} -``` - -- [ ] **Step 2: Remove the local runtime source-routing import** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, delete this import: - -```ts -import { defaultSharedWorktreeSourceKeys } from './isolated-diff/source-routing.js'; -``` - -- [ ] **Step 3: Remove the local runtime fallback setting** - -In `packages/context/src/ingest/local-bundle-runtime.ts`, replace the settings -object with: - -```ts - settings: { - memoryIngestionModel: options.project.config.llm.models.default ?? 'local-ingest-model', - probeRowCount: 0, - workUnitMaxConcurrency: options.project.config.ingest.workUnits.maxConcurrency, - workUnitStepBudget: options.project.config.ingest.workUnits.stepBudget, - workUnitFailureMode: options.project.config.ingest.workUnits.failureMode, - ingestTraceLevel: ingestTraceLevelFromEnv(), - }, -``` - -- [ ] **Step 4: Delete the fallback routing helper files** - -Delete: - -```bash -git rm packages/context/src/ingest/isolated-diff/source-routing.ts -git rm packages/context/src/ingest/isolated-diff/source-routing.test.ts -``` - -- [ ] **Step 5: Confirm no fallback helper imports remain** - -Run: - -```bash -rg -n "defaultSharedWorktreeSourceKeys|isSharedWorktreeFallbackSourceKey|source-routing" packages/context/src -``` - -Expected: FAIL with no matches. `rg` exits with status 1 when the cleanup is -complete. - ---- - -### Task 3: Delete the shared-worktree runner branch - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` - -- [ ] **Step 1: Remove helper methods used only by the shared branch** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, delete these private -methods: - -```ts - private buildFailedWorkUnitOutcome(wu: WorkUnit, error: unknown): WorkUnitOutcome { - return { - unitKey: wu.unitKey, - status: 'failed', - reason: error instanceof Error ? error.message : String(error), - preSha: '', - postSha: '', - actions: [], - touchedSlSources: [], - slDisallowed: wu.slDisallowed, - slDisallowedReason: wu.slDisallowedReason, - }; - } - - private formatWorkUnitFailure(outcome: WorkUnitOutcome): string { - return `WorkUnit ${outcome.unitKey} failed: ${outcome.reason ?? 'unknown failure'}`; - } - - private isSharedWorktreeFallbackEnabled(sourceKey: string): boolean { - return (this.deps.settings.sharedWorktreeSourceKeys ?? []).includes(sourceKey); - } -``` - -- [ ] **Step 2: Make non-override isolated routing unconditional** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, replace: - -```ts - const isolatedDiffEnabled = !overrideReport && !this.isSharedWorktreeFallbackEnabled(job.sourceKey); -``` - -with: - -```ts - const isolatedDiffEnabled = !overrideReport; -``` - -Then replace: - -```ts - if (!overrideReport && isolatedDiffEnabled) { -``` - -with: - -```ts - if (!overrideReport) { -``` - -- [ ] **Step 3: Delete the old shared-worktree branch** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, delete the whole -branch that starts with: - -```ts - } else if (!overrideReport) { - await runTrace.event('info', 'routing', 'shared_worktree_path_enabled', { - sourceKey: job.sourceKey, - reason: 'explicit_private_fallback', - }); -``` - -and ends with: - -```ts - latestReportWorkUnits = this.toReportWorkUnits(stageIndex); - } -``` - -After the deletion, the surrounding code must read: - -```ts - } - - } - const carryForwardResult = - contextReport && this.deps.contextCandidateCarryforward - ? await this.deps.contextCandidateCarryforward.carryForward({ - runId: runRow.id, - connectionId: job.connectionId, - sourceKey: job.sourceKey, - }) - : null; -``` - -- [ ] **Step 4: Confirm the branch trace event is gone** - -Run: - -```bash -rg -n "shared_worktree_path_enabled|explicit_private_fallback|isSharedWorktreeFallbackEnabled|sharedWorktreeSourceKeys" packages/context/src/ingest/ingest-bundle.runner.ts -``` - -Expected: FAIL with no matches. - ---- - -### Task 4: Update runner tests for isolated-only execution - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.test.ts` -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Remove the fallback setting from the broad runner test harness** - -In `packages/context/src/ingest/ingest-bundle.runner.test.ts`, replace the -`settings` block in `buildRunner()` with: - -```ts - settings: { - probeRowCount: 1, - memoryIngestionModel: 'test-model', - }, -``` - -- [ ] **Step 2: Add no-op isolated patch support to the broad mock Git** - -In `packages/context/src/ingest/ingest-bundle.runner.test.ts`, replace the -`scopedGit` object in `makeDeps()` with: - -```ts - const scopedGit = { - revParseHead: vi.fn().mockResolvedValue('h'), - commitFiles: vi.fn().mockResolvedValue({ created: true, commitHash: 'h' }), - commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'h' }), - resetHardTo: vi.fn(), - assertWorktreeClean: vi.fn().mockResolvedValue(undefined), - writeBinaryNoRenamePatch: vi.fn(async (_base: string, _head: string, patchPath: string) => { - await writeFile(patchPath, '', 'utf-8'); - }), - applyPatchFile3WayIndex: vi.fn(), - diffNameStatus: vi.fn().mockResolvedValue([]), - }; -``` - -- [ ] **Step 3: Update the custom sequencer test Git mock** - -In the test named -`refuses to squash-merge when the session worktree has an in-progress sequencer op`, -replace the `sessionGit` object with: - -```ts - const sessionGit = { - revParseHead: vi.fn().mockResolvedValue('h'), - commitFiles: vi.fn().mockResolvedValue({ created: true, commitHash: 'h' }), - commitStaged: vi.fn().mockResolvedValue({ created: false, commitHash: 'h' }), - resetHardTo: vi.fn(), - assertWorktreeClean: vi.fn().mockRejectedValue(assertError), - writeBinaryNoRenamePatch: vi.fn(async (_base: string, _head: string, patchPath: string) => { - await writeFile(patchPath, '', 'utf-8'); - }), - applyPatchFile3WayIndex: vi.fn(), - diffNameStatus: vi.fn().mockResolvedValue([]), - }; -``` - -- [ ] **Step 4: Move the failed-WorkUnit integration regression to the isolated suite** - -In `packages/context/src/ingest/ingest-bundle.runner.test.ts`, delete the test -named `squash-merges only successful WUs into main when one WU fails sl_validate`. - -In `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`, -add this test near the other real-git isolated runner regressions: - -```ts - it('does not integrate failed isolated WorkUnit patches', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps, adapter } = makeDeps(runtime, 'fake'); - adapter.chunk.mockResolvedValue({ - workUnits: [ - { unitKey: 'wu-good', rawFiles: ['good.raw'], peerFileIndex: [], dependencyPaths: [] }, - { unitKey: 'wu-bad', rawFiles: ['bad.raw'], peerFileIndex: [], dependencyPaths: [] }, - ], - }); - deps.diffSetService.compute = vi.fn().mockResolvedValue({ - added: ['good.raw', 'bad.raw'], - modified: [], - deleted: [], - unchanged: [], - }); - deps.slValidator.validateSingleSource = vi.fn( - async (_validationDeps: unknown, _connectionId: string, sourceName: string) => ({ - errors: sourceName === 'bad' ? [{ message: 'bad source rejected' }] : [], - warnings: [], - }), - ) as never; - - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName !== 'ingest-bundle-wu') { - return { stopReason: 'natural' }; - } - const unitKey = params.telemetryTags.unitKey; - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true }); - if (unitKey === 'wu-good') { - await writeFile(join(root, 'semantic-layer/warehouse/good.yaml'), 'name: good\n', 'utf-8'); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'good'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'good', - detail: 'good source', - targetConnectionId: 'warehouse', - rawPaths: ['good.raw'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/good.yaml'], - 'test: add good source', - 'KTX Test', - 'system@ktx.local', - ); - } - if (unitKey === 'wu-bad') { - await writeFile(join(root, 'semantic-layer/warehouse/bad.yaml'), 'name: bad\n', 'utf-8'); - addTouchedSlSource(currentSession.touchedSlSources, 'warehouse', 'bad'); - currentSession.actions.push({ - target: 'sl', - type: 'created', - key: 'bad', - detail: 'bad source', - targetConnectionId: 'warehouse', - rawPaths: ['bad.raw'], - }); - await currentSession.gitService.commitFiles( - ['semantic-layer/warehouse/bad.yaml'], - 'test: add bad source', - 'KTX Test', - 'system@ktx.local', - ); - } - return { stopReason: 'natural' }; - }) as never; - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles( - runner, - runtime, - [ - ['good.raw', 'good-hash'], - ['bad.raw', 'bad-hash'], - ], - 'fake', - ); - - const result = await runner.run({ - jobId: 'job-failed-wu-isolated', - connectionId: 'warehouse', - sourceKey: 'fake', - trigger: 'upload', - bundleRef: { kind: 'upload', uploadId: 'upload' }, - }); - - expect(result.failedWorkUnits).toEqual(['wu-bad']); - await expect(readFile(join(runtime.configDir, 'semantic-layer/warehouse/good.yaml'), 'utf-8')).resolves.toContain( - 'good', - ); - await expect(readFile(join(runtime.configDir, 'semantic-layer/warehouse/bad.yaml'), 'utf-8')).rejects.toThrow(); - - const reportCreate = vi.mocked(deps.reports.create).mock.calls.at(-1)?.[0]; - const reportBody = reportCreate?.body as { isolatedDiff?: { acceptedPatches?: number }; failedWorkUnits?: string[] }; - expect(reportBody.failedWorkUnits).toEqual(['wu-bad']); - expect(reportBody.isolatedDiff).toMatchObject({ enabled: true, acceptedPatches: 1 }); - - const trace = await readFile( - join(runtime.configDir, '.ktx/ingest-traces/job-failed-wu-isolated/trace.jsonl'), - 'utf-8', - ); - expect(trace).toContain('work_unit_failed_before_patch'); - expect(trace).toContain('patch_accepted'); - expect(trace).not.toContain('shared_worktree_path_enabled'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 5: Run the updated focused runner tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - src/ingest/local-bundle-runtime.test.ts \ - -t "does not support shared-worktree|does not integrate failed isolated|defaults local bundle ingest|unlisted direct-writing source" -``` - -Expected: PASS. The traces contain `isolated_diff_enabled`, child worktree -events, and no `shared_worktree_path_enabled`. - -- [ ] **Step 6: Run the broad runner suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.test.ts -``` - -Expected: PASS. Broad runner coverage no longer depends on -`sharedWorktreeSourceKeys`. - -- [ ] **Step 7: Commit the runner removal** - -Run: - -```bash -git add \ - packages/context/src/ingest/ports.ts \ - packages/context/src/ingest/local-bundle-runtime.ts \ - packages/context/src/ingest/local-bundle-runtime.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.ts \ - packages/context/src/ingest/ingest-bundle.runner.test.ts \ - packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts \ - packages/context/src/ingest/isolated-diff/source-routing.ts \ - packages/context/src/ingest/isolated-diff/source-routing.test.ts -git commit -m "refactor(ingest): remove shared worktree WorkUnit path" -``` - -Expected: commit succeeds. The deleted routing files are included as deletions. - ---- - -### Task 5: Remove shared-branch agent instructions - -**Files:** -- Modify: `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md` -- Modify: `packages/context/skills/ingest_triage/SKILL.md` -- Test: `packages/context/src/ingest/ingest-prompts.test.ts` -- Test: `packages/context/src/ingest/ingest-runtime-assets.test.ts` - -- [ ] **Step 1: Update the WorkUnit role text** - -In `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md`, replace -the `` block with: - -```md - -You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit -gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, -Metabase card JSONs, Notion pages, or similar) and you must translate that -slice into KTX semantic-layer sources and/or knowledge wiki pages, in one pass. -You run in an isolated WorkUnit worktree. Deterministic projection output, -existing project memory, and listed dependency paths are visible; sibling -WorkUnit edits from this same job are not visible until the runner integrates -accepted patches. - -``` - -- [ ] **Step 2: Update the WorkUnit workflow text** - -In the same prompt, replace workflow steps 2 and 4 with: - -```md -2. Load the per-source review skill first (for example `lookml_ingest`, - `metricflow_ingest`, or `dbt_ingest`), then `sl_capture` and - `wiki_capture`, and `ingest_triage` last. The triage skill tells you how to - react when existing project memory, deterministic projection output, or - prior provenance overlaps with what this WorkUnit is about to write. -4. For each raw file: call `read_raw_file` (or `read_raw_span` for slicing large - files) to load content. Before writing a new SL source or wiki page, call - `discover_data` for each candidate source, table, metric, or topic name to - find existing wiki pages, SL sources, deterministic projection output, prior - sync artifacts, and raw warehouse matches; apply `ingest_triage` when you hit - one, and apply any matching canonical pin before deciding whether to edit, - rename, or skip. -``` - -- [ ] **Step 3: Update the WorkUnit do-not rule** - -In the same prompt, replace: - -```md -- Do not silently accept a name collision with a prior WU's write when the formula differs. Trigger `ingest_triage`. -``` - -with: - -```md -- Do not silently accept a name collision with visible existing memory, - deterministic projection output, or prior provenance when the formula differs. - Trigger `ingest_triage`. -``` - -- [ ] **Step 4: Update ingest triage caller guidance** - -In `packages/context/skills/ingest_triage/SKILL.md`, replace: - -```md -This skill is loaded in two contexts: -- By a Stage 3 WorkUnit agent when `sl_discover` reveals that a prior WU (or a prior sync) already wrote something that overlaps with what the current WU is about to write. -- By the Stage 4 reconciliation agent for cross-WU sweeps and for eviction decisions. -``` - -with: - -```md -This skill is loaded in two contexts: -- By a Stage 3 WorkUnit agent when `sl_discover`, deterministic projection - output, existing project memory, or prior provenance overlaps with what the - current WorkUnit is about to write. -- By the Stage 4 reconciliation agent for cross-WorkUnit sweeps, accepted patch - overlap, and eviction decisions. -``` - -- [ ] **Step 5: Update same-ingest wording in ingest triage** - -In `packages/context/skills/ingest_triage/SKILL.md`, replace: - -```md -4. **If there's no prior-sync row (both are from THIS job), check for same-ingest contradictions:** -``` - -with: - -```md -4. **If reconciliation sees accepted patches from this same job with no -prior-sync row, check for same-ingest contradictions:** -``` - -- [ ] **Step 6: Search for stale shared-state prompt language** - -Run: - -```bash -rg -n "prior WU|prior-WU|Prior WorkUnits|same job may have already written|visible on the working branch|shared_worktree_path_enabled|shared-worktree path reachable" packages/context/prompts packages/context/skills packages/context/src/ingest -``` - -Expected: FAIL with no matches. - -- [ ] **Step 7: Run prompt asset tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-prompts.test.ts \ - src/ingest/ingest-runtime-assets.test.ts -``` - -Expected: PASS. Prompt assets still load from packaged KTX assets. - -- [ ] **Step 8: Commit the prompt cleanup** - -Run: - -```bash -git add \ - packages/context/prompts/memory_agent_bundle_ingest_work_unit.md \ - packages/context/skills/ingest_triage/SKILL.md -git commit -m "docs(ingest): align WorkUnit prompts with isolated diffs" -``` - -Expected: commit succeeds. - ---- - -### Task 6: Final verification - -**Files:** -- Verify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Verify: `packages/context/src/ingest/ports.ts` -- Verify: `packages/context/src/ingest/local-bundle-runtime.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.test.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Verify: `packages/context/prompts/memory_agent_bundle_ingest_work_unit.md` -- Verify: `packages/context/skills/ingest_triage/SKILL.md` - -- [ ] **Step 1: Run the isolated-diff focused suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run \ - src/ingest/ingest-trace.test.ts \ - src/ingest/wiki-body-refs.test.ts \ - src/ingest/artifact-gates.test.ts \ - src/ingest/semantic-layer-target-policy.test.ts \ - src/ingest/isolated-diff/git-patch.test.ts \ - src/ingest/isolated-diff/work-unit-executor.test.ts \ - src/ingest/isolated-diff/patch-integrator.test.ts \ - src/ingest/isolated-diff/textual-conflict-resolver.test.ts \ - src/ingest/final-gate-repair.test.ts \ - src/ingest/report-snapshot.test.ts \ - src/ingest/ingest-bundle.runner.isolated-diff.test.ts -``` - -Expected: PASS. The output includes the isolated-diff runner tests and no -`source-routing.test.ts`. - -- [ ] **Step 2: Run the full context test suite** - -Run: - -```bash -pnpm --filter @ktx/context run test -``` - -Expected: PASS. - -- [ ] **Step 3: Run context type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. There are no `sharedWorktreeSourceKeys` type errors because the -setting no longer exists. - -- [ ] **Step 4: Run dead-code checks** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS. Knip does not report deleted source-routing exports, and Biome -does not report stale imports. - -- [ ] **Step 5: Search for removed legacy path names** - -Run: - -```bash -rg -n "sharedWorktreeSourceKeys|defaultSharedWorktreeSourceKeys|isSharedWorktreeFallbackSourceKey|shared_worktree_path_enabled|explicit_private_fallback|source-routing" packages docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-shared-worktree-removal.md -``` - -Expected: matches only in this plan file. There must be no matches under -`packages/`. - -- [ ] **Step 6: Confirm docs-site does not need an update** - -Run: - -```bash -rg -n "sharedWorktree|isolatedDiffSourceKeys|sharedWorktreeSourceKeys|executionMode|planningStrategy|conflictPolicy" docs-site README.md packages/*/README.md -``` - -Expected: either no matches or matches unrelated to a public user-facing knob. -This change removes an internal runner fallback and does not add, remove, or -rename public CLI behavior, configuration, or docs-site content. - -- [ ] **Step 7: Commit final verification notes if files changed** - -Run: - -```bash -git status --short -``` - -Expected: clean after the two implementation commits. If this command reports -new changes, stop and inspect them before finishing; final verification should -not create extra source changes. - -## Self-review - -Spec coverage: - -- Rollout step 11 is covered by Tasks 1 through 4: the private fallback setting, - helper module, old runner branch, trace event, and fallback tests are deleted. -- The isolated-diff WorkUnit flow remains covered by existing real-git tests and - the new failed-WorkUnit regression in Task 4. -- Agent-facing instructions are aligned with the spec's worktree invariant in - Task 5: sibling WorkUnit edits are not visible inside a child worktree. -- Override ingestion remains outside the WorkUnit execution branch and still - uses prior report materialization plus serial reconciliation. - -Placeholder scan: - -- This plan contains exact file paths, test names, replacement snippets, - commands, and expected results. -- There are no deferred implementation markers or unspecified edge-case - instructions. - -Type consistency: - -- `IngestSettingsPort` no longer includes `sharedWorktreeSourceKeys`. -- `isolatedDiffEnabled` remains the runner's internal summary flag and is - equivalent to `!overrideReport`. -- The removed trace event is `shared_worktree_path_enabled`; retained isolated - events include `isolated_diff_enabled`, `work_unit_child_created`, and - `work_unit_patch_collected`. - -Execution handoff: - -Plan complete and saved to -`docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-shared-worktree-removal.md`. - -Two execution options: - -1. **Subagent-Driven (recommended)** - Dispatch a fresh subagent per task, - review between tasks, and keep iteration fast. -2. **Inline Execution** - Execute tasks in this session using - `superpowers:executing-plans`, with batch execution and checkpoints. diff --git a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-textual-conflict-resolver.md b/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-textual-conflict-resolver.md deleted file mode 100644 index 43ad036b..00000000 --- a/docs/superpowers/plans/2026-05-18-isolated-diff-ingestion-v1-textual-conflict-resolver.md +++ /dev/null @@ -1,1241 +0,0 @@ -# Isolated Diff Ingestion V1 Textual Conflict Resolver Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add bounded resolver-agent handling for textual isolated-diff patch -conflicts so overlapping WorkUnit edits can be repaired, globally gated, and -committed before the runner fails the ingest. - -**Architecture:** Keep patch policy failures and semantic gate failures -fail-fast. When an allowed patch fails `git apply --3way --index`, the -integration worktree resets to the pre-apply `HEAD`, one repair agent runs with -tools limited to the failed patch's touched paths, the existing artifact gates -validate the repaired files, and the runner records resolver attempts, repairs, -and failures in traces and reports. Gate repair for cleanly applied but -semantically invalid trees remains a separate plan. - -**Tech Stack:** TypeScript ESM/NodeNext, Vitest, zod, Node `fs/promises`, -existing `AgentRunnerPort`, `GitService`, `IngestTraceWriter`, -`integrateWorkUnitPatch`, and `IngestBundleRunner`. - ---- - -## Audit Summary - -The source spec is -`docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md`. - -Plans already based on this spec: - -| Plan | Implementation status | Evidence | -| --- | --- | --- | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-core.md` | Implemented | `packages/context/src/ingest/isolated-diff/*`, `ingest-trace.ts`, `wiki-body-refs.ts`, `artifact-gates.ts`, and `ingest-bundle.runner.isolated-diff.test.ts` exist. Git history includes `cae5c4b feat: add isolated diff ingestion v1 core`, `1013bb6 test: cover isolated diff ingestion regressions`, and `c481f1c feat: route selected ingest sources through isolated diffs`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-gates-and-trace-closure.md` | Implemented | Final gates run after reconciliation, traces and failed reports are stored, and child worktree cleanup is covered. Git history includes `656e584 test(ingest): verify isolated diff postmortem coverage` and `87f1193 chore(ingest): verify isolated diff gate closure`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-provenance-gate-closure.md` | Implemented | `validateProvenanceRawPaths()` runs before squash, and the isolated runner has a pre-squash provenance regression. Git history includes `977a610 fix(ingest): gate provenance before isolated diff squash`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-reference-and-target-gate-closure.md` | Implemented | `semantic-layer-target-policy.ts`, SL write/edit target checks, patch target checks, and final wiki ref checks exist. Git history includes `5ec6396 fix(ingest): gate final wiki references` and `c61c50b test(ingest): cover isolated diff reference and target gates`. | -| `docs/superpowers/plans/2026-05-17-isolated-diff-ingestion-v1-global-wiki-reference-gate-closure.md` | Implemented | `wikiPageKeysForFinalGates()` expands to all global pages when semantic-layer sources change or wiki pages are removed. Git history includes `ba534fb fix(ingest): gate global wiki references`. | - -Focused verification passed before writing this plan: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-trace.test.ts src/ingest/wiki-body-refs.test.ts src/ingest/artifact-gates.test.ts src/ingest/semantic-layer-target-policy.test.ts src/ingest/isolated-diff/git-patch.test.ts src/ingest/isolated-diff/work-unit-executor.test.ts src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/ingest-bundle.runner.isolated-diff.test.ts src/sl/tools/sl-write-source.tool.test.ts src/sl/tools/sl-edit-source.tool.test.ts -``` - -Current result: `10 passed`, `61 passed`. - -The next spec gap is bounded textual conflict resolution. Today -`packages/context/src/ingest/isolated-diff/patch-integrator.ts` rolls back and -returns `textual_conflict` as soon as `git apply --3way --index` fails. The -spec requires expected cross-WorkUnit overlap to get one bounded repair attempt -before the run fails. - -## Scope - -This plan implements only textual conflict repair for allowed patches that fail -Git application. It does not repair: - -- patch policy failures such as `slDisallowed`, unauthorized target connection - paths, executable modes, or binary changes under text artifact roots; -- semantic conflicts where the patch applies but artifact gates fail; -- final gate failures after reconciliation or post-processing; -- broad connector rollout beyond the existing runner-owned Metabase allowlist; -- isolated-diff default promotion; or -- removal of the shared-worktree fallback path. - -## File Structure - -- Create `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts`. - Owns the bounded repair-agent loop and its read/write/delete tools. -- Create `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts`. - Covers allowed-path scoping, failed-patch visibility, successful repair, and - no-edit failure. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.ts`. - Calls the resolver after Git textual conflicts, validates repaired files, and - commits the repair. -- Modify `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`. - Covers repair success and repair failure while preserving pre-apply state. -- Modify `packages/context/src/ingest/ingest-bundle.runner.ts`. - Wires the resolver into the isolated-diff integration loop and increments - resolver counters. -- Modify `packages/context/src/ingest/report-snapshot.ts`. - Parses resolver counters from stored report bodies. -- Modify `packages/context/src/ingest/reports.ts`. - Adds resolver counters to the `isolatedDiff` report body type. -- Modify `packages/context/src/ingest/report-snapshot.test.ts`. - Covers the new report fields. -- Modify `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`. - Adds an end-to-end same-source conflict regression. - ---- - -### Task 1: Add Resolver Unit Tests - -**Files:** -- Create: `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts` - -- [ ] **Step 1: Write the failing resolver tests** - -Create `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts`: - -```ts -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import { FileIngestTraceWriter } from '../ingest-trace.js'; -import { resolveTextualConflict } from './textual-conflict-resolver.js'; - -async function makeHarness() { - const root = await mkdtemp(join(tmpdir(), 'ktx-textual-resolver-')); - const workdir = join(root, 'workdir'); - const patchPath = join(root, 'failed.patch'); - const trace = new FileIngestTraceWriter({ - tracePath: join(root, 'trace.jsonl'), - jobId: 'job-1', - connectionId: 'warehouse', - sourceKey: 'metabase', - runId: 'run-1', - syncId: 'sync-1', - level: 'trace', - }); - await mkdir(join(workdir, 'wiki/global'), { recursive: true }); - await writeFile(join(workdir, 'wiki/global/account.md'), 'accepted line\n', 'utf-8'); - await writeFile( - patchPath, - [ - 'diff --git a/wiki/global/account.md b/wiki/global/account.md', - 'index 8877391..6f63f4d 100644', - '--- a/wiki/global/account.md', - '+++ b/wiki/global/account.md', - '@@ -1 +1 @@', - '-base line', - '+proposal line', - '', - ].join('\n'), - 'utf-8', - ); - return { root, workdir, patchPath, trace }; -} - -describe('resolveTextualConflict', () => { - it('lets the repair agent read the failed patch and write only touched paths', async () => { - const { workdir, patchPath, trace } = await makeHarness(); - const agentRunner = { - runLoop: vi.fn(async (params: any) => { - const current = await params.toolSet.read_integration_file.execute({ path: 'wiki/global/account.md' }); - expect(current.structured).toEqual({ path: 'wiki/global/account.md', exists: true }); - expect(current.markdown).toContain('accepted line'); - - const patch = await params.toolSet.read_failed_patch.execute({}); - expect(patch.markdown).toContain('proposal line'); - - await expect( - params.toolSet.write_integration_file.execute({ - path: 'wiki/global/not-allowed.md', - content: 'bad\n', - }), - ).rejects.toThrow(/resolver path not allowed/); - - await params.toolSet.write_integration_file.execute({ - path: 'wiki/global/account.md', - content: 'accepted line\nproposal line\n', - }); - return { stopReason: 'natural' }; - }), - }; - - const result = await resolveTextualConflict({ - agentRunner, - workdir, - unitKey: 'wu-a', - patchPath, - touchedPaths: ['wiki/global/account.md'], - trace, - reason: 'patch failed: wiki/global/account.md', - maxAttempts: 1, - stepBudget: 8, - }); - - expect(result).toEqual({ - status: 'repaired', - attempts: 1, - changedPaths: ['wiki/global/account.md'], - }); - await expect(readFile(join(workdir, 'wiki/global/account.md'), 'utf-8')).resolves.toBe( - 'accepted line\nproposal line\n', - ); - expect(agentRunner.runLoop).toHaveBeenCalledWith( - expect.objectContaining({ - modelRole: 'repair', - stepBudget: 8, - telemetryTags: expect.objectContaining({ - operationName: 'ingest-isolated-diff-textual-resolver', - jobId: 'job-1', - unitKey: 'wu-a', - }), - }), - ); - }); - - it('fails when the repair agent completes without editing any touched path', async () => { - const { workdir, patchPath, trace } = await makeHarness(); - const result = await resolveTextualConflict({ - agentRunner: { runLoop: vi.fn(async () => ({ stopReason: 'natural' })) }, - workdir, - unitKey: 'wu-a', - patchPath, - touchedPaths: ['wiki/global/account.md'], - trace, - reason: 'patch failed: wiki/global/account.md', - maxAttempts: 1, - stepBudget: 8, - }); - - expect(result).toEqual({ - status: 'failed', - attempts: 1, - reason: 'resolver completed without editing an allowed path', - }); - }); -}); -``` - -- [ ] **Step 2: Run the resolver tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/textual-conflict-resolver.test.ts -``` - -Expected: FAIL with a module resolution error for -`./textual-conflict-resolver.js`. - -- [ ] **Step 3: Commit the failing tests** - -```bash -git add packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts -git commit -m "test(ingest): cover isolated diff textual conflict resolver" -``` - ---- - -### Task 2: Add Patch Integrator Resolver Contract Tests - -**Files:** -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` - -- [ ] **Step 1: Add resolver contract regressions** - -Append these tests inside `describe('integrateWorkUnitPatch', ...)` in -`packages/context/src/ingest/isolated-diff/patch-integrator.test.ts`: - -```ts - it('repairs a textual conflict through the bounded resolver and commits repaired files', async () => { - const { homeDir, configDir, git, baseSha } = await makeRepo(); - await mkdir(join(configDir, 'wiki/global'), { recursive: true }); - await writeFile(join(configDir, 'wiki/global/a.md'), 'base\n', 'utf-8'); - await git.commitFiles(['wiki/global/a.md'], 'base page', 'System User', 'system@example.com'); - const conflictBase = await git.revParseHead(); - - await writeFile(join(configDir, 'wiki/global/a.md'), 'accepted\n', 'utf-8'); - await git.commitFiles(['wiki/global/a.md'], 'accepted edit', 'System User', 'system@example.com'); - - const childDir = join(homeDir, 'child-conflict'); - await git.addWorktree(childDir, 'child-conflict', conflictBase); - const childGit = git.forWorktree(childDir); - await writeFile(join(childDir, 'wiki/global/a.md'), 'proposal\n', 'utf-8'); - await childGit.commitFiles(['wiki/global/a.md'], 'proposal edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'proposal.patch'); - await childGit.writeBinaryNoRenamePatch(conflictBase, 'HEAD', patchPath); - - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-resolver/trace.jsonl'), - jobId: 'job-resolver', - connectionId: 'warehouse', - sourceKey: 'metabase', - level: 'trace', - }); - - const validateAppliedTree = vi.fn(async (paths: string[]) => { - expect(paths).toEqual(['wiki/global/a.md']); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe( - 'accepted\nproposal\n', - ); - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-conflict', - patchPath, - integrationGit: git, - trace, - author: { name: 'System User', email: 'system@example.com' }, - slDisallowed: false, - allowedTargetConnectionIds: new Set(['warehouse']), - validateAppliedTree, - resolveTextualConflict: vi.fn(async (context) => { - expect(context).toMatchObject({ - unitKey: 'wu-conflict', - patchPath, - touchedPaths: ['wiki/global/a.md'], - }); - await writeFile(join(configDir, 'wiki/global/a.md'), 'accepted\nproposal\n', 'utf-8'); - return { - status: 'repaired', - attempts: 1, - changedPaths: ['wiki/global/a.md'], - }; - }), - }); - - expect(result).toMatchObject({ - status: 'accepted', - touchedPaths: ['wiki/global/a.md'], - textualResolution: { - status: 'repaired', - attempts: 1, - changedPaths: ['wiki/global/a.md'], - }, - }); - expect(validateAppliedTree).toHaveBeenCalledOnce(); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe( - 'accepted\nproposal\n', - ); - await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('patch_accepted_after_textual_resolution'); - expect(await git.revParseHead()).not.toBe(baseSha); - }); - - it('keeps the pre-apply integration tree when the resolver cannot repair a textual conflict', async () => { - const { homeDir, configDir, git } = await makeRepo(); - await mkdir(join(configDir, 'wiki/global'), { recursive: true }); - await writeFile(join(configDir, 'wiki/global/a.md'), 'base\n', 'utf-8'); - await git.commitFiles(['wiki/global/a.md'], 'base page', 'System User', 'system@example.com'); - const conflictBase = await git.revParseHead(); - - await writeFile(join(configDir, 'wiki/global/a.md'), 'accepted\n', 'utf-8'); - await git.commitFiles(['wiki/global/a.md'], 'accepted edit', 'System User', 'system@example.com'); - const acceptedHead = await git.revParseHead(); - - const childDir = join(homeDir, 'child-conflict-fails'); - await git.addWorktree(childDir, 'child-conflict-fails', conflictBase); - const childGit = git.forWorktree(childDir); - await writeFile(join(childDir, 'wiki/global/a.md'), 'proposal\n', 'utf-8'); - await childGit.commitFiles(['wiki/global/a.md'], 'proposal edit', 'System User', 'system@example.com'); - const patchPath = join(homeDir, 'proposal-fails.patch'); - await childGit.writeBinaryNoRenamePatch(conflictBase, 'HEAD', patchPath); - - const trace = new FileIngestTraceWriter({ - tracePath: join(homeDir, '.ktx/ingest-traces/job-resolver-fails/trace.jsonl'), - jobId: 'job-resolver-fails', - connectionId: 'warehouse', - sourceKey: 'metabase', - level: 'trace', - }); - - const result = await integrateWorkUnitPatch({ - unitKey: 'wu-conflict', - patchPath, - integrationGit: git, - trace, - author: { name: 'System User', email: 'system@example.com' }, - slDisallowed: false, - allowedTargetConnectionIds: new Set(['warehouse']), - validateAppliedTree: vi.fn(async () => {}), - resolveTextualConflict: vi.fn(async () => ({ - status: 'failed', - attempts: 1, - reason: 'resolver completed without editing an allowed path', - })), - }); - - expect(result).toMatchObject({ - status: 'textual_conflict', - textualResolution: { - status: 'failed', - attempts: 1, - reason: 'resolver completed without editing an allowed path', - }, - }); - expect(await git.revParseHead()).toBe(acceptedHead); - await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('accepted\n'); - }); -``` - -- [ ] **Step 2: Run the patch integrator tests to verify they fail** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: FAIL because `integrateWorkUnitPatch()` does not accept -`resolveTextualConflict` and does not return `textualResolution`. - -- [ ] **Step 3: Commit the failing integrator tests** - -```bash -git add packages/context/src/ingest/isolated-diff/patch-integrator.test.ts -git commit -m "test(ingest): cover isolated diff resolver integration" -``` - ---- - -### Task 3: Implement the Textual Conflict Resolver - -**Files:** -- Create: `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts` -- Modify: `packages/context/src/ingest/isolated-diff/patch-integrator.ts` - -- [ ] **Step 1: Add the resolver module** - -Create `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts`: - -```ts -import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { z } from 'zod'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js'; -import type { IngestTraceWriter } from '../ingest-trace.js'; -import { traceTimed } from '../ingest-trace.js'; - -export type TextualConflictResolutionResult = - | { status: 'repaired'; attempts: number; changedPaths: string[] } - | { status: 'failed'; attempts: number; reason: string }; - -export interface ResolveTextualConflictInput { - agentRunner: AgentRunnerPort; - workdir: string; - unitKey: string; - patchPath: string; - touchedPaths: string[]; - trace: IngestTraceWriter; - reason: string; - maxAttempts?: number; - stepBudget?: number; -} - -const readIntegrationFileSchema = z.object({ - path: z.string().min(1), -}); - -const writeIntegrationFileSchema = z.object({ - path: z.string().min(1), - content: z.string(), -}); - -const deleteIntegrationFileSchema = z.object({ - path: z.string().min(1), -}); - -function normalizeRepoPath(path: string): string { - const normalized = path.replace(/\\/g, '/').replace(/^\/+/, ''); - const parts = normalized.split('/').filter((part) => part.length > 0); - if (parts.length === 0 || parts.some((part) => part === '.' || part === '..')) { - throw new Error(`resolver path must be a repository-relative path: ${path}`); - } - return parts.join('/'); -} - -function assertAllowedPath(path: string, allowedPaths: ReadonlySet): string { - const normalized = normalizeRepoPath(path); - if (!allowedPaths.has(normalized)) { - throw new Error(`resolver path not allowed: ${normalized}`); - } - return normalized; -} - -async function readOptionalFile(path: string): Promise<{ exists: boolean; content: string }> { - try { - return { exists: true, content: await readFile(path, 'utf-8') }; - } catch (error) { - if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') { - return { exists: false, content: '' }; - } - throw error; - } -} - -function buildResolverSystemPrompt(): string { - return ` -You repair one failed KTX isolated-diff patch inside the integration worktree. - - - -- Preserve accepted integration content that is unrelated to the failed patch. -- Incorporate the failed patch only when the patch evidence is compatible with the current file. -- Edit only paths exposed by the resolver tools. -- Prefer the smallest text edit that makes the composed artifact coherent. -- Do not create new facts that are absent from the current file or failed patch. -- Stop after writing the repaired file content. -`; -} - -function buildResolverUserPrompt(input: { - unitKey: string; - patchPath: string; - touchedPaths: string[]; - reason: string; - attempt: number; - maxAttempts: number; -}): string { - return `Repair isolated-diff textual conflict. - -WorkUnit: ${input.unitKey} -Attempt: ${input.attempt} of ${input.maxAttempts} -Patch path: ${input.patchPath} -Touched paths: -${input.touchedPaths.map((path) => `- ${path}`).join('\n')} - -Git apply failure: -${input.reason} - -Use read_failed_patch first. Then read the touched integration files, write the -repaired content, and stop.`; -} - -function buildToolSet(input: { - workdir: string; - patchPath: string; - allowedPaths: ReadonlySet; - editedPaths: Set; -}): KtxRuntimeToolSet { - return { - read_failed_patch: { - name: 'read_failed_patch', - description: 'Read the failed Git patch that could not be applied to the integration worktree.', - inputSchema: z.object({}), - execute: async () => { - const patch = await readFile(input.patchPath, 'utf-8'); - return { - markdown: patch, - structured: { patchPath: input.patchPath, bytes: Buffer.byteLength(patch) }, - }; - }, - }, - read_integration_file: { - name: 'read_integration_file', - description: 'Read one allowed file from the current integration worktree.', - inputSchema: readIntegrationFileSchema, - execute: async ({ path }: z.infer) => { - const normalized = assertAllowedPath(path, input.allowedPaths); - const file = await readOptionalFile(join(input.workdir, normalized)); - return { - markdown: file.exists ? file.content : `(missing file: ${normalized})`, - structured: { path: normalized, exists: file.exists }, - }; - }, - }, - write_integration_file: { - name: 'write_integration_file', - description: 'Replace one allowed integration worktree file with repaired text content.', - inputSchema: writeIntegrationFileSchema, - execute: async ({ path, content }: z.infer) => { - const normalized = assertAllowedPath(path, input.allowedPaths); - const fullPath = join(input.workdir, normalized); - await mkdir(dirname(fullPath), { recursive: true }); - await writeFile(fullPath, content, 'utf-8'); - input.editedPaths.add(normalized); - return { - markdown: `Wrote ${normalized}`, - structured: { path: normalized, bytes: Buffer.byteLength(content) }, - }; - }, - }, - delete_integration_file: { - name: 'delete_integration_file', - description: 'Delete one allowed integration worktree file when the failed patch proves the deletion is correct.', - inputSchema: deleteIntegrationFileSchema, - execute: async ({ path }: z.infer) => { - const normalized = assertAllowedPath(path, input.allowedPaths); - await rm(join(input.workdir, normalized), { force: true }); - input.editedPaths.add(normalized); - return { - markdown: `Deleted ${normalized}`, - structured: { path: normalized }, - }; - }, - }, - }; -} - -export async function resolveTextualConflict( - input: ResolveTextualConflictInput, -): Promise { - const allowedPaths = new Set(input.touchedPaths.map(normalizeRepoPath)); - const maxAttempts = input.maxAttempts ?? 1; - const stepBudget = input.stepBudget ?? 12; - let lastFailure = 'resolver did not run'; - - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const editedPaths = new Set(); - const traceData = { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths: [...allowedPaths].sort(), - attempt, - maxAttempts, - reason: input.reason, - }; - const result = await traceTimed(input.trace, 'resolver', 'textual_conflict_resolver', traceData, async () => - input.agentRunner.runLoop({ - modelRole: 'repair', - systemPrompt: buildResolverSystemPrompt(), - userPrompt: buildResolverUserPrompt({ - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths: [...allowedPaths].sort(), - reason: input.reason, - attempt, - maxAttempts, - }), - toolSet: buildToolSet({ - workdir: input.workdir, - patchPath: input.patchPath, - allowedPaths, - editedPaths, - }), - stepBudget, - telemetryTags: { - operationName: 'ingest-isolated-diff-textual-resolver', - source: input.trace.context.sourceKey, - jobId: input.trace.context.jobId, - unitKey: input.unitKey, - }, - }), - ); - - if (result.stopReason === 'error') { - lastFailure = result.error?.message ?? 'resolver agent loop errored'; - await input.trace.event('error', 'resolver', 'textual_conflict_resolver_failed', traceData, result.error); - continue; - } - - const changedPaths = [...editedPaths].sort(); - if (changedPaths.length === 0) { - lastFailure = 'resolver completed without editing an allowed path'; - await input.trace.event('error', 'resolver', 'textual_conflict_resolver_failed', { - ...traceData, - reason: lastFailure, - }); - continue; - } - - await input.trace.event('debug', 'resolver', 'textual_conflict_resolver_repaired', { - ...traceData, - changedPaths, - }); - return { status: 'repaired', attempts: attempt, changedPaths }; - } - - return { status: 'failed', attempts: maxAttempts, reason: lastFailure }; -} -``` - -- [ ] **Step 2: Run resolver tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/textual-conflict-resolver.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Update the patch integrator types and conflict path** - -In `packages/context/src/ingest/isolated-diff/patch-integrator.ts`, add the -import: - -```ts -import type { TextualConflictResolutionResult } from './textual-conflict-resolver.js'; -``` - -Replace the result type and input interface with: - -```ts -export type PatchIntegrationTextualResolution = - | { status: 'repaired'; attempts: number; changedPaths: string[] } - | { status: 'failed'; attempts: number; reason: string }; - -export type PatchIntegrationResult = - | { status: 'accepted'; commitSha: string; touchedPaths: string[]; textualResolution?: PatchIntegrationTextualResolution } - | { status: 'textual_conflict'; reason: string; touchedPaths: string[]; textualResolution?: PatchIntegrationTextualResolution } - | { status: 'semantic_conflict'; reason: string; touchedPaths: string[]; textualResolution?: PatchIntegrationTextualResolution }; - -export interface IntegrateWorkUnitPatchInput { - unitKey: string; - patchPath: string; - integrationGit: GitService; - trace: IngestTraceWriter; - author: { name: string; email: string }; - slDisallowed: boolean; - allowedTargetConnectionIds: ReadonlySet; - validateAppliedTree(touchedPaths: string[]): Promise; - resolveTextualConflict?(input: { - unitKey: string; - patchPath: string; - touchedPaths: string[]; - reason: string; - }): Promise; -} -``` - -Inside the `catch` block that currently handles `patch_apply` errors, replace -the existing return with: - -```ts - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - const reason = errorMessage(error); - await input.trace.event('error', 'integration', 'patch_textual_conflict', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason, - }); - - if (!input.resolveTextualConflict) { - return { - status: 'textual_conflict', - reason, - touchedPaths, - }; - } - - const textualResolution = await input.resolveTextualConflict({ - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths, - reason, - }); - - if (textualResolution.status === 'failed') { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - return { - status: 'textual_conflict', - reason: textualResolution.reason, - touchedPaths, - textualResolution, - }; - } - - try { - await traceTimed( - input.trace, - 'integration', - 'semantic_gate_after_textual_resolution', - { unitKey: input.unitKey, touchedPaths: textualResolution.changedPaths }, - async () => { - await input.validateAppliedTree(textualResolution.changedPaths); - }, - ); - } catch (semanticError) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths: textualResolution.changedPaths, - reason: errorMessage(semanticError), - }); - return { - status: 'semantic_conflict', - reason: errorMessage(semanticError), - touchedPaths: textualResolution.changedPaths, - textualResolution, - }; - } - - const commit = await input.integrationGit.commitFiles( - textualResolution.changedPaths, - `ingest: resolve WorkUnit ${input.unitKey} conflict`, - input.author.name, - input.author.email, - ); - if (!commit.created) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } - const noChangeReason = 'textual resolver produced no committable changes'; - await input.trace.event('error', 'integration', 'textual_conflict_resolver_noop', { - unitKey: input.unitKey, - patchPath: input.patchPath, - touchedPaths: textualResolution.changedPaths, - }); - return { - status: 'textual_conflict', - reason: noChangeReason, - touchedPaths: textualResolution.changedPaths, - textualResolution, - }; - } - - await input.trace.event('debug', 'integration', 'patch_accepted_after_textual_resolution', { - unitKey: input.unitKey, - commitSha: commit.commitHash, - touchedPaths: textualResolution.changedPaths, - attempts: textualResolution.attempts, - }); - return { - status: 'accepted', - commitSha: commit.commitHash, - touchedPaths: textualResolution.changedPaths, - textualResolution, - }; -``` - -Leave the earlier patch policy rejection branch unchanged so policy failures -cannot invoke the resolver. - -- [ ] **Step 4: Run patch integrator tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/patch-integrator.test.ts -``` - -Expected: PASS. - -- [ ] **Step 5: Commit resolver implementation** - -```bash -git add packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts packages/context/src/ingest/isolated-diff/patch-integrator.ts -git commit -m "feat(ingest): repair isolated diff textual conflicts" -``` - ---- - -### Task 4: Wire the Resolver into the Runner and Reports - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.ts` -- Modify: `packages/context/src/ingest/reports.ts` -- Modify: `packages/context/src/ingest/report-snapshot.ts` -- Modify: `packages/context/src/ingest/report-snapshot.test.ts` - -- [ ] **Step 1: Import the resolver in the runner** - -In `packages/context/src/ingest/ingest-bundle.runner.ts`, add: - -```ts -import { resolveTextualConflict } from './isolated-diff/textual-conflict-resolver.js'; -``` - -- [ ] **Step 2: Add resolver counters to the isolated-diff summary** - -In the `isolatedDiffSummary` initializer in -`packages/context/src/ingest/ingest-bundle.runner.ts`, add: - -```ts - resolverAttempts: 0, - resolverRepairs: 0, - resolverFailures: 0, -``` - -- [ ] **Step 3: Pass the resolver callback to `integrateWorkUnitPatch()`** - -Inside the isolated-diff integration loop, add this property to the -`integrateWorkUnitPatch({ ... })` call: - -```ts - resolveTextualConflict: (context) => - resolveTextualConflict({ - agentRunner: this.deps.agentRunner, - workdir: sessionWorktree.workdir, - unitKey: context.unitKey, - patchPath: context.patchPath, - touchedPaths: context.touchedPaths, - trace: runTrace, - reason: context.reason, - maxAttempts: 1, - stepBudget: 12, - }), -``` - -- [ ] **Step 4: Record resolver outcomes after each integration attempt** - -Immediately after `const integration = await integrateWorkUnitPatch({ ... });`, -add: - -```ts - if (integration.textualResolution) { - isolatedDiffSummary.resolverAttempts += integration.textualResolution.attempts; - if (integration.textualResolution.status === 'repaired') { - isolatedDiffSummary.textualConflicts += 1; - isolatedDiffSummary.resolverRepairs += 1; - } else { - isolatedDiffSummary.resolverFailures += 1; - } - } -``` - -Keep the existing textual-conflict and semantic-conflict branches after this -counter update. - -- [ ] **Step 5: Add report body fields** - -In `packages/context/src/ingest/reports.ts`, extend -`IngestReportBody['isolatedDiff']` with: - -```ts - resolverAttempts?: number; - resolverRepairs?: number; - resolverFailures?: number; -``` - -In `packages/context/src/ingest/report-snapshot.ts`, extend the -`isolatedDiff` object schema with: - -```ts - resolverAttempts: z.number().int().min(0).default(0), - resolverRepairs: z.number().int().min(0).default(0), - resolverFailures: z.number().int().min(0).default(0), -``` - -- [ ] **Step 6: Add the report parser regression** - -Append this test to `packages/context/src/ingest/report-snapshot.test.ts`: - -```ts - it('parses isolated-diff textual resolver counters', () => { - const snapshot = parseIngestReportSnapshot({ - id: 'report-1', - runId: 'run-1', - jobId: 'job-1', - connectionId: 'warehouse', - sourceKey: 'metabase', - createdAt: '2026-05-18T00:00:00.000Z', - body: { - status: 'completed', - syncId: 'sync-1', - diffSummary: { added: 0, modified: 1, deleted: 0, unchanged: 0 }, - commitSha: 'abc123', - isolatedDiff: { - enabled: true, - acceptedPatches: 2, - textualConflicts: 1, - semanticConflicts: 0, - resolverAttempts: 1, - resolverRepairs: 1, - resolverFailures: 0, - }, - workUnits: [], - failedWorkUnits: [], - reconciliationSkipped: true, - conflictsResolved: [], - evictionsApplied: [], - unmappedFallbacks: [], - artifactResolutions: [], - evictionInputs: [], - unresolvedCards: [], - supersededBy: null, - overrideOf: null, - provenanceRows: [], - toolTranscripts: [], - }, - }); - - expect(snapshot.body.isolatedDiff).toMatchObject({ - resolverAttempts: 1, - resolverRepairs: 1, - resolverFailures: 0, - }); - }); -``` - -- [ ] **Step 7: Run report tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 8: Commit runner and report wiring** - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.ts packages/context/src/ingest/reports.ts packages/context/src/ingest/report-snapshot.ts packages/context/src/ingest/report-snapshot.test.ts -git commit -m "feat(ingest): report isolated diff resolver outcomes" -``` - ---- - -### Task 5: Add End-to-End Resolver Regression - -**Files:** -- Modify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` - -- [ ] **Step 1: Add the end-to-end test** - -Append this test inside `describe('IngestBundleRunner isolated diff path', ...)` -in `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts`: - -```ts - it('repairs additive same-source textual conflicts before final gates and squash', async () => { - const runtime = await makeRealGitRuntime(); - try { - const { deps } = makeDeps(runtime); - let currentSession: any = null; - deps.toolsetFactory.createIngestWuToolset = vi.fn((toolSession: any) => { - currentSession = toolSession; - return { toRuntimeTools: vi.fn(() => ({})) }; - }); - deps.agentRunner.runLoop = vi.fn(async (params: any) => { - if (params.telemetryTags.operationName === 'ingest-isolated-diff-textual-resolver') { - const current = await params.toolSet.read_integration_file.execute({ - path: 'semantic-layer/warehouse/mart_account_segments.yaml', - }); - expect(current.markdown).toContain('total_contract_arr_cents'); - const patch = await params.toolSet.read_failed_patch.execute({}); - expect(patch.markdown).toContain('account_count'); - await params.toolSet.write_integration_file.execute({ - path: 'semantic-layer/warehouse/mart_account_segments.yaml', - content: - 'name: mart_account_segments\n' + - 'grain: [account_id]\n' + - 'columns: [{name: account_id, type: string}]\n' + - 'joins: []\n' + - 'measures:\n' + - ' - name: total_contract_arr_cents\n' + - ' expr: sum(contract_arr)\n' + - ' - name: account_count\n' + - ' expr: count_distinct(account_id)\n', - }); - return { stopReason: 'natural' }; - } - - const root = rootOfConfig(currentSession.configService, runtime.configDir); - await mkdir(join(root, 'semantic-layer/warehouse'), { recursive: true }); - if (params.telemetryTags.unitKey === 'card-wiki') { - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\n' + - 'grain: [account_id]\n' + - 'columns: [{name: account_id, type: string}]\n' + - 'joins: []\n' + - 'measures:\n' + - ' - name: total_contract_arr_cents\n' + - ' expr: sum(contract_arr)\n', - ); - } else if (params.telemetryTags.unitKey === 'card-source') { - await writeFile( - join(root, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'name: mart_account_segments\n' + - 'grain: [account_id]\n' + - 'columns: [{name: account_id, type: string}]\n' + - 'joins: []\n' + - 'measures:\n' + - ' - name: account_count\n' + - ' expr: count_distinct(account_id)\n', - ); - } - addTouchedSlSource(currentSession.touchedSlSources, { - connectionId: 'warehouse', - sourceName: 'mart_account_segments', - }); - return { stopReason: 'natural' }; - }); - - const runner = new IngestBundleRunner(deps); - await mockStageRawFiles(runner, runtime, [ - ['cards/wiki.json', 'hash-a'], - ['cards/source.json', 'hash-b'], - ]); - - const result = await runner.run({ - jobId: 'job-resolver-e2e', - connectionId: 'warehouse', - sourceKey: 'metabase', - trigger: 'manual_resync', - bundleRef: { kind: 'upload', uploadId: 'upload-1' }, - }); - - expect(result.commitSha).toBeTruthy(); - const source = await readFile( - join(runtime.configDir, 'semantic-layer/warehouse/mart_account_segments.yaml'), - 'utf-8', - ); - expect(source).toContain('total_contract_arr_cents'); - expect(source).toContain('account_count'); - expect(deps.agentRunner.runLoop).toHaveBeenCalledWith( - expect.objectContaining({ - modelRole: 'repair', - telemetryTags: expect.objectContaining({ - operationName: 'ingest-isolated-diff-textual-resolver', - unitKey: 'card-source', - }), - }), - ); - const successReport = (deps.reports.create as any).mock.calls.at(-1)?.[0]?.body; - expect(successReport.isolatedDiff).toMatchObject({ - acceptedPatches: 2, - textualConflicts: 1, - semanticConflicts: 0, - resolverAttempts: 1, - resolverRepairs: 1, - resolverFailures: 0, - }); - const trace = await readFile(join(runtime.configDir, '.ktx/ingest-traces/job-resolver-e2e/trace.jsonl'), 'utf-8'); - expect(trace).toContain('textual_conflict_resolver_repaired'); - expect(trace).toContain('patch_accepted_after_textual_resolution'); - } finally { - await rm(runtime.homeDir, { recursive: true, force: true }); - } - }); -``` - -- [ ] **Step 2: Run the isolated-diff runner regression** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-bundle.runner.isolated-diff.test.ts -t "repairs additive same-source textual conflicts" -``` - -Expected: PASS. - -- [ ] **Step 3: Commit the end-to-end regression** - -```bash -git add packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts -git commit -m "test(ingest): verify isolated diff textual conflict repair" -``` - ---- - -### Task 6: Final Verification - -**Files:** -- Verify: `packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts` -- Verify: `packages/context/src/ingest/isolated-diff/patch-integrator.test.ts` -- Verify: `packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts` -- Verify: `packages/context/src/ingest/report-snapshot.test.ts` - -- [ ] **Step 1: Run the focused resolver and isolated-diff tests** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/isolated-diff/textual-conflict-resolver.test.ts src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/ingest-bundle.runner.isolated-diff.test.ts src/ingest/report-snapshot.test.ts -``` - -Expected: PASS. - -- [ ] **Step 2: Run the existing isolated-diff safety suite** - -Run: - -```bash -pnpm --filter @ktx/context exec vitest run src/ingest/ingest-trace.test.ts src/ingest/wiki-body-refs.test.ts src/ingest/artifact-gates.test.ts src/ingest/semantic-layer-target-policy.test.ts src/ingest/isolated-diff/git-patch.test.ts src/ingest/isolated-diff/work-unit-executor.test.ts src/ingest/isolated-diff/patch-integrator.test.ts src/ingest/ingest-bundle.runner.isolated-diff.test.ts src/sl/tools/sl-write-source.tool.test.ts src/sl/tools/sl-edit-source.tool.test.ts -``` - -Expected: PASS. - -- [ ] **Step 3: Run package type-check** - -Run: - -```bash -pnpm --filter @ktx/context run type-check -``` - -Expected: PASS. - -- [ ] **Step 4: Run dead-code analysis** - -Run: - -```bash -pnpm run dead-code -``` - -Expected: PASS, or only pre-existing findings unrelated to the files changed -by this plan. - -- [ ] **Step 5: Decide docs-site impact** - -No `docs-site/content/docs/` update is required for this plan because the -change is an internal ingest correctness behavior and report diagnostics -extension. If execution changes public CLI output while implementing this -plan, add a follow-up docs-site plan for the affected CLI/status page. - -- [ ] **Step 6: Commit verification notes only if files changed** - -If verification updates snapshots or checked-in fixtures, commit only those -intended files: - -```bash -git add packages/context/src/ingest -git commit -m "chore(ingest): verify isolated diff textual conflict repair" -``` - -If no files changed during verification, do not create an empty commit. - ---- - -## Self-Review - -Spec coverage: - -- Bounded resolver-agent handling for textual conflicts is covered by Tasks 1 - through 5. -- The resolver receives the failed patch, current integration files, touched - path scope, and trace context. -- Patch policy failures remain non-repairable, preserving the existing - `slDisallowed`, target-connection, binary, and executable-mode gates. -- Repaired files run through the existing artifact gates before commit and - before squash. -- Resolver attempts, repaired files, failures, and trace events are reported. - -Remaining spec gaps after this plan: - -- Gate repair for cleanly applied trees that fail final gates. -- Resolver context that includes work-unit transcript excerpts and all - previously applied overlapping patches. -- Broader connector rollout for Notion, LookML, Looker, dbt, and MetricFlow. -- Isolated-diff default promotion after at least one non-Metabase connector - passes. -- Shared-worktree WorkUnit path removal. - -Placeholder scan: - -- The plan contains exact file paths, commands, expected outcomes, and concrete - code blocks for every code-changing step. -- The plan does not contain deferred implementation markers. - -Type consistency: - -- `TextualConflictResolutionResult`, `PatchIntegrationTextualResolution`, and - `textualResolution` use the same `status`, `attempts`, `changedPaths`, and - `reason` fields across resolver, integrator, runner, and tests. -- Report fields use `resolverAttempts`, `resolverRepairs`, and - `resolverFailures` consistently in `reports.ts`, `report-snapshot.ts`, and - runner report bodies. diff --git a/docs/superpowers/specs/2026-05-11-agent-friendly-docs-site-design.md b/docs/superpowers/specs/2026-05-11-agent-friendly-docs-site-design.md deleted file mode 100644 index 69784f5e..00000000 --- a/docs/superpowers/specs/2026-05-11-agent-friendly-docs-site-design.md +++ /dev/null @@ -1,171 +0,0 @@ -# Agent-Friendly Docs Site Design - -## Goal - -Make `docs-site` easier for coding agents and LLM readers to discover, ingest, -and use. The work applies the Vercel Academy agent-friendly docs patterns to the -KTX documentation site while preserving the current Fumadocs + Next.js -architecture. - -Success means agents can: - -- Discover the documentation from well-known root files. -- Fetch all documentation in one plain-text response. -- Fetch any docs page as markdown without parsing the HTML UI. -- Follow CLI, MCP, setup, integration, and semantic-layer workflows from - structured examples. -- Recover from common setup and command failures using explicit troubleshooting - notes. - -## Current State - -`docs-site` is a Next 15 app using Fumadocs. Source pages live under -`docs-site/content/docs`, and rendered docs are served under `/docs`. - -The site currently has good human-facing MDX pages, but it does not expose: - -- `/llms.txt` -- `/llms-full.txt` -- raw markdown routes such as `/docs/getting-started/quickstart.md` -- markdown content negotiation - -Many docs pages already use tables and code blocks, but the structure is not -consistently optimized for literal agent parsing. CLI and agent-facing pages are -the highest-priority content because agents are most likely to copy commands and -JSON examples directly. - -## Design - -### Machine-readable access - -Add a small LLM docs utility layer inside `docs-site`: - -- `docs-site/lib/llm-docs.ts` - - Converts Fumadocs pages to raw or LLM-readable markdown. - - Builds a stable ordered list of docs pages from `source.getPages()`. - - Produces the `llms.txt` index content. - - Produces the `llms-full.txt` bundled content. - -Add routes: - -- `docs-site/app/llms.txt/route.ts` - - Returns `text/plain; charset=utf-8`. - - Includes `# KTX`, a blockquote summary, a short description, and sections - linking to key docs, markdown docs, CLI reference pages, integration pages, - and `/llms-full.txt`. - -- `docs-site/app/llms-full.txt/route.ts` - - Returns `text/plain; charset=utf-8`. - - Concatenates all docs pages in source order. - - Prefixes each page with a stable heading and canonical `/docs/...` URL. - -- `docs-site/app/llms.mdx/docs/[[...slug]]/route.ts` - - Returns one docs page as `text/markdown; charset=utf-8`. - - Uses the same slug shape as `/docs/[[...slug]]`. - - Returns 404 for unknown pages. - -Add a Next rewrite in `docs-site/next.config.mjs`: - -- `/docs/:path*.md` rewrites to `/llms.mdx/docs/:path*` - -Add a markdown negotiation proxy for `/docs/...` requests: - -- Requests whose `Accept` header prefers markdown are rewritten to the matching - LLM markdown route. -- Normal browser requests continue to render the existing Fumadocs UI. -- The proxy must leave `/llms.txt`, `/llms-full.txt`, assets, and non-docs - routes unchanged. - -### Content rewrite pass - -Rewrite the existing MDX content in a bounded, high-impact pass. The intent is -not to expand every page; it is to make every page more literal and consistent -for agents. - -Apply these patterns across docs: - -- Put command signatures in fenced code blocks. -- Use tables for flags, options, inputs, outputs, supported values, and - environment variables. -- Use realistic values in copy-paste examples. -- Show complete expected command output when output shape matters. -- Add explicit "Common errors" or "Recovery" sections for workflows where a - command can fail for predictable reasons. -- Add workflow sections that chain commands in the order an agent should use - them. -- Avoid placeholders that an agent could copy literally, unless the placeholder - is clearly marked as a value to replace. - -Priority pages: - -1. `getting-started/quickstart.mdx` - - Add a compact workflow summary. - - Make prerequisites and generated files explicit. - - Add troubleshooting for missing API keys, failed connection tests, daemon - startup, and unbuilt context. - -2. `guides/serving-agents.mdx` - - Treat MCP tools and `ktx agent` commands as agent-facing API references. - - Add tool/command input tables, output expectations, safety constraints, and - workflows for answering analytics questions. - -3. `guides/writing-context.mdx` - - Add semantic-source schema tables. - - Add workflows for listing, reading, editing, validating, querying, and - writing wiki knowledge. - -4. `cli-reference/*.mdx` - - Normalize every command page to: command signature, subcommands table, - option tables, examples, output modes, common errors, and related workflows - where useful. - -5. `integrations/agent-clients.mdx`, `integrations/primary-sources.mdx`, and - `integrations/context-sources.mdx` - - Normalize integration setup sections into structured config tables, - copy-paste examples, authentication requirements, and recovery notes. - -6. Concept and benchmark pages - - Keep narrative content, but add compact "Agent usage notes" where it helps - agents decide when to read or cite the page. - -### Documentation boundaries - -The first pass should not introduce a separate public docs tree or a generated -API reference system. It should work with the existing MDX source files and -Fumadocs loader. - -Do not add stale compatibility aliases or rename KTX concepts. Keep examples -aligned with commands and files that exist in the standalone KTX repository. - -### Testing - -Verification commands: - -- `pnpm --filter ktx-docs build` -- `pnpm --filter ktx-docs exec tsc --noEmit` after generated Fumadocs source - files exist. -- Route checks against a local docs server: - - `GET /llms.txt` returns 200 and `text/plain`. - - `GET /llms-full.txt` returns 200 and `text/plain`. - - `GET /docs/getting-started/quickstart.md` returns 200 and - `text/markdown`. - - unknown markdown docs paths return 404. - -For content checks, inspect the generated markdown responses to confirm they -contain: - -- realistic command examples, -- tables, -- full output examples where documented, -- workflow sections, -- recovery/error sections. - -## Acceptance Criteria - -- `/llms.txt` gives agents a concise index with links to key KTX docs and - `/llms-full.txt`. -- `/llms-full.txt` returns all docs content in source order as plain text. -- Every Fumadocs page can be fetched through a `.md` URL. -- High-priority docs pages use consistent agent-friendly structure. -- The docs site builds successfully. -- Verification results and any skipped checks are reported clearly. diff --git a/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md b/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md deleted file mode 100644 index a2f12a75..00000000 --- a/docs/superpowers/specs/2026-05-11-demo-guided-tour-design.md +++ /dev/null @@ -1,252 +0,0 @@ -# Demo Guided Tour - Design Spec - -## Problem - -The "Try KTX with packaged demo data" option in `ktx setup` is completely -disconnected from the real setup wizard. It bypasses all wizard steps, plays -an animated replay in a temp directory, and exits with no bridge to actually -using KTX. Users don't learn the real setup flow and hit a dead end. - -## Solution - -Redesign the demo option as a **guided tour** that walks the user through the -same setup wizard steps with pre-filled, read-only selections. The tour ends -with a real interactive agents step so the user can immediately use the demo -project with their coding agent. - -## Design Decisions - -| Decision | Choice | Rationale | -|----------|--------|-----------| -| Implementation strategy | Demo mode flag on existing wizard steps | Maximum code reuse; wizard changes automatically apply to demo | -| LLM/embeddings steps | Skipped | Not relevant to pre-packaged demo data | -| Database selection | PostgreSQL (read-only card) | Pre-filled, matches demo dataset | -| Context sources | dbt, Metabase, Notion (read-only card) | Pre-filled, matches demo dataset | -| Context build | Replay through real progress visualization | Same spinners, progress bars, status icons as real build | -| Agents step | Real interactive step | User actually connects their agent | -| Project location | Temp directory (`/tmp/ktx-demo-{hex}`) | Frictionless, no directory prompt | -| Navigation | Enter to advance, Escape to go back | Consistent with rest of wizard | - -## Flow - -``` -Entry menu: "Try KTX with packaged demo data" - │ - ▼ -Create demo project in /tmp/ktx-demo-{hex} -Copy pre-packaged assets (demo DB, replay, context artifacts) - │ - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ Demo banner (persistent, shown on every step) │ -│ │ -│ Demo mode - data has been pre-processed and KTX context is │ -│ already built. This walkthrough illustrates the setup steps. │ -│ Selections are pre-filled and read-only. │ -└────────────────────────────────────────────────────────────────┘ - │ - ▼ -Read-only card: Database connection - ▸ PostgreSQL (demo warehouse) - [Enter → next, Escape → back to entry menu] - │ - ▼ -Read-only card: Context sources - ▸ dbt - ▸ Metabase - ▸ Notion - [Enter → next, Escape → back to database card] - │ - ▼ -Context build replay - Same renderContextBuildView() / repainter as real wizard - Sources: demo-warehouse, dbt, metabase, notion - Replay at slightly faster-than-real pace - Completion summary: business areas, query definitions, knowledge pages - [Enter → next, Escape → back to sources card] - │ - ▼ -Transition message: - "Demo project is ready - let's connect your agent" - │ - ▼ -Interactive agents step (real runKtxSetupAgentsStep()) - User selects agent target, scope, install mode - [Normal interactive navigation; Escape goes back to replay summary] - │ - ▼ -Final summary: - ★ KTX demo is ready - Agent connected, project path shown - ⚠ Temp directory warning - Pointer to `ktx setup` for real data -``` - -## Step Details - -### Demo Banner - -Shown at the top of every read-only step. Uses clack box-drawing style: - -``` -┌ Demo mode - data has been pre-processed and KTX context is already built. -│ This walkthrough illustrates the setup steps. Selections are pre-filled and read-only. -``` - -### Read-Only Step Cards - -Rendered by a shared `renderDemoCard()` helper: - -```typescript -async function renderDemoCard( - title: string, - selections: string[], - io: KtxCliIo, -): Promise<'forward' | 'back'> -``` - -- Renders a clack-style box with title, bullet list of pre-filled selections, - and navigation hint ("Press Enter to continue, Escape to go back") -- Listens for raw keypresses: Enter → `'forward'`, Escape → `'back'` -- Uses same box-drawing characters and colors as `@clack/prompts` - -Card format: - -``` -┌ {title} -│ -│ ▸ {selection 1} -│ ▸ {selection 2} -│ ... -│ -│ Press Enter to continue, Escape to go back -└ -``` - -### Demo Step Sequence - -The demo reuses the main wizard's step loop with these steps: - -```typescript -const demoSteps = ['databases', 'sources', 'context', 'agents']; -``` - -Steps `databases` and `sources` dispatch to `renderDemoCard()` instead of -their real interactive functions when demo mode is active. Step `context` -dispatches to the replay visualization. Step `agents` runs the real -`runKtxSetupAgentsStep()`. - -Back navigation reuses `previousNavigableStepIndex()`. Escaping from the -first step (databases) returns to the entry menu. - -### Context Build Replay - -Uses the same rendering pipeline as the real context build: - -- `renderContextBuildView()` for the progress display -- `createRepainter()` for terminal repainting -- Same spinner frames, progress bars (`████░░░░`), status icons (`✓`, `⠹`, `○`) -- Same source grouping (Primary sources / Context sources) - -Sources shown: - -``` -Primary sources: - ✓ demo-warehouse completed · Xs - -Context sources: - ✓ dbt completed · Xs - ✓ metabase completed · Xs - ✓ notion completed · Xs -``` - -Replay timing: events from the pre-packaged replay file are played back at -a slightly faster pace than real-time (compressed to feel brisk but not -instant). - -Completion summary uses the existing format: - -``` -★ KTX finished ingesting your data - - ✓ Analyzed X business areas - ✓ Reconciled - 0 conflicts - - KTX created: - 📊 X query definitions - 📝 X knowledge pages - - Press Enter to continue, Escape to go back -``` - -The exact counts and artifact names come from the pre-packaged demo results -(to be provided by the user as improved demo data). - -### Agents Step Transition - -A brief message bridges from the read-only tour to the interactive step: - -``` -┌ Demo project is ready - let's connect your agent -│ -│ Your KTX context has been built with demo data. -│ Select an agent to start using it. -└ -``` - -Then `runKtxSetupAgentsStep()` runs with the demo project directory, -normal interactive prompts enabled. - -### Final Summary - -``` -★ KTX demo is ready - - Your agent is connected to a demo KTX project. - - ⚠ This project is in a temporary directory and will be - cleaned up by your system. To set up KTX with your own - data, run: ktx setup - - Project: /tmp/ktx-demo-a1b2c3 -``` - -If the user skips the agents step, replace the first line with manual -agent connection instructions (`ktx setup --agents --project-dir /tmp/...`). - -## Implementation Approach - -Thread a `demoMode` flag through the main setup loop in `setup.ts`. When -active: - -1. Skip `models` and `embeddings` steps entirely -2. Replace `databases` and `sources` step dispatch with `renderDemoCard()` -3. Replace `context` step dispatch with replay visualization -4. Run `agents` step normally -5. Show demo-specific completion summary instead of ready menu - -The `renderDemoCard()` helper is a new function in a new file -(e.g. `setup-demo-cards.ts`) that handles read-only card rendering and -keypress listening. - -The context build replay reuses existing `renderContextBuildView()` and -`createRepainter()` from `context-build-view.ts`, fed with events from -the pre-packaged replay file at an accelerated playback rate. - -## Files Changed - -| File | Change | -|------|--------| -| `packages/cli/src/setup.ts` | Add `demoMode` flag to setup loop; skip models/embeddings; dispatch to demo cards for databases/sources; show demo banner; demo completion summary | -| `packages/cli/src/setup-demo-cards.ts` | New file: `renderDemoCard()` helper, demo banner renderer, demo step definitions | -| `packages/cli/src/setup-context.ts` | Support replay mode for demo: feed pre-packaged events at accelerated pace through existing progress view | -| `packages/cli/src/demo.ts` | Remove or simplify `runKtxSetupDemoFromEntryMenu()` - now dispatches to the main setup loop with `demoMode: true` | -| `packages/cli/src/demo-assets.ts` | Update asset list if new demo data is provided; ensure demo project setup writes valid `ktx.yaml` for agent use | - -## Open Items - -- **Demo data**: User will provide improved pre-packaged results (Postgres, - dbt, Metabase, Notion). Current demo assets may need updating. -- **Replay speed**: Exact acceleration factor TBD - should feel brisk but - give users time to read source names and status transitions. Start with - ~2x real-time and adjust. diff --git a/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md b/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md deleted file mode 100644 index d27b6ea8..00000000 --- a/docs/superpowers/specs/2026-05-11-historic-sql-redesign-design.md +++ /dev/null @@ -1,677 +0,0 @@ -# Historic SQL Ingestion - Redesign - -**Status:** draft -**Date:** 2026-05-11 -**Owner:** Andrey Avtomonov - -## 1. Motivation - -The current historic-SQL ingestion adapter (`packages/context/src/ingest/adapters/historic-sql/`) is slow, complex, and structurally cannot answer the questions a research/BI agent actually asks. - -Concrete pain points observed: - -- A full run takes **30+ minutes against a tiny demo Postgres database**. The hot loop calls `SqlAnalysisPort.analyzeForFingerprint()` once per query via HTTP to the Python daemon, so thousands of RPC round-trips dominate runtime. -- **Two completely different code paths** for Postgres (baseline-diff against `pg_stat_statements`) versus BigQuery/Snowflake (timestamp cursor over `INFORMATION_SCHEMA.JOBS` / `QUERY_HISTORY`). Postgres further cannot produce the same outputs as the others (no per-execution samples, no literal-slot bindings, error rate stuck at zero). -- The output is **fingerprint-fragmented**: the pipeline emits one document per fingerprint, expands categorical literal slots into per-value sub-clusters, and ranks templates with a recency-decayed score. The result is many near-duplicate documents per fingerprint and gratuitous churn across runs. -- The output is **rigid and shallow**: deterministic slot classification (constant / categorical / runtime) and triage-signal buckets do not produce narrative an agent can use. The current downstream skills (`historic_sql_ingest`, `historic_sql_curator`) try to recover narrative from these templates but at high cost. -- Lots of moving parts (baseline files, reset detection, atomic per-connection commit, slot heuristics, ranking formula) for what is fundamentally "find interesting queries and tell agents about them." - -The end goal - per the user - is for ingested content to be **searchable by `ktx wiki search` and `ktx sl search` to help consumer research agents do data analysis and agentic BI**. - -## 2. Design principles - -1. **LLMs are the right tool for narrative and clustering.** Deterministic heuristics (slot classification, ranking formulas, categorical expansion) get replaced by LLM judgement applied to aggregated, bucketed inputs. -2. **The adapter stays LLM-free.** The existing convention - adapters are deterministic, skills do LLM work - is preserved. -3. **One pipeline across dialects.** A single reader interface, a single staging shape, a single set of skills. Dialect-specific behavior lives only in the snapshot query. -4. **No work where no signal changed.** Daily reruns should LLM only the things that actually changed. -5. **Lean context for caller agents.** Each retrieval tier (search hit → source read → pattern read) carries only what the agent needs to make the next decision. The principle lives in prompt instructions, not in defensive schema constraints. -6. **Simplification over backward compatibility.** Hard cutover, delete the old code path, no parallel implementations. - -## 3. Architecture - -``` - ┌────────────────────────── LLM-free, deterministic ─────────────────────────┐ -Reader (unified) ─▶ Aggregated snapshot ─▶ Batch SQL parse ─▶ Bucket by table - │ - ▼ - Staged dir: - manifest.json - tables/{schema}.{name}.json (one per touched table) - patterns-input.json - │ - ▼ - chunk() → WorkUnits - │ - ┌───────────────────────────────────────────────┴────────────────────────────┐ - ▼ ▼ - ┌────── LLM via skill ──────┐ ┌────── LLM via skill ──────┐ - │ historic_sql_table_digest │ (N WorkUnits, parallel) │ historic_sql_patterns │ - │ produces TableUsage │ │ produces Pattern[] │ - │ evidence per table │ │ evidence │ - └───────────────────────────┘ └───────────────────────────┘ - │ │ - └──────────────────────────┬───────────────────────────────────────────────────┘ - ▼ - onPullSucceeded() projection (no LLM): - Pass A - merge `usage` into _schema/{shard}.yaml (per-shard atomic, scan-managed keys) - Pass B - write/update pattern wiki pages (slug stability + stale handling) - Pass C - trigger SL search re-index for changed sources -``` - -## 4. Hot path (LLM-free) - -### 4.1 Unified reader interface - -```typescript -interface HistoricSqlReader { - probe(client: HistoricSqlQueryClient): Promise; - fetchAggregated( - client: HistoricSqlQueryClient, - window: { start: Date; end: Date }, - ): AsyncIterable; -} -``` - -`AggregatedTemplate` is one record per template, already aggregated by the warehouse. Schema in §9. - -**Trailing-window only.** No cursor, no baseline file. Every run reads "what was hot in the last N days." Idempotency comes from per-WorkUnit content hashing via the framework's `DiffSetComputerPort`. - -### 4.2 Snapshot queries (one per dialect) - -**Postgres** - `pg_stat_statements` collapsed to `queryid`: - -```sql -SELECT queryid::text AS template_id, - query AS canonical_sql, - SUM(calls)::bigint AS executions, - COUNT(DISTINCT userid) AS distinct_users, - SUM(total_exec_time) / NULLIF(SUM(calls), 0) AS mean_ms, - SUM(total_rows)::bigint AS rows_produced -FROM pg_stat_statements -WHERE toplevel = true -GROUP BY queryid, query -HAVING SUM(calls) >= @min_executions -``` - -`firstSeen` derives from `pg_stat_statements_info.stats_reset`; `lastSeen` is `now()`. `p50RuntimeMs` / `p95RuntimeMs` collapse to `mean_ms`. `errorRate = 0` (PG doesn't track failures in PGSS). - -**BigQuery** - warehouse-side aggregation over `INFORMATION_SCHEMA.JOBS_BY_PROJECT`: - -```sql -SELECT query_hash AS template_id, - MIN(query) AS canonical_sql, - COUNT(*) AS executions, - COUNT(DISTINCT user_email) AS distinct_users, - MIN(creation_time) AS first_seen, - MAX(creation_time) AS last_seen, - APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(50)] AS p50_ms, - APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(95)] AS p95_ms, - SAFE_DIVIDE(COUNTIF(error_result IS NOT NULL), COUNT(*)) AS error_rate -FROM `{project}.region-{region}.INFORMATION_SCHEMA.JOBS_BY_PROJECT` -WHERE job_type = 'QUERY' - AND statement_type IN ('SELECT', 'MERGE') - AND creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @window_days DAY) -GROUP BY query_hash -HAVING COUNT(*) >= @min_executions -``` - -**Snowflake** - analogous, over `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY`: - -```sql -SELECT query_hash AS template_id, - MIN(query_text) AS canonical_sql, - COUNT(*) AS executions, - COUNT(DISTINCT user_name) AS distinct_users, - MIN(start_time) AS first_seen, - MAX(start_time) AS last_seen, - APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms, - APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms, - DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate, - SUM(rows_produced) AS rows_produced -FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY -WHERE query_text IS NOT NULL - AND query_type IN ('SELECT', 'MERGE') - AND start_time >= DATEADD(day, -@window_days, CURRENT_TIMESTAMP()) -GROUP BY query_hash -HAVING COUNT(*) >= @min_executions -``` - -### 4.3 Batch SQL parse - -After collecting all `AggregatedTemplate` rows, **one** call to a new daemon endpoint: - -```typescript -const parsed = await sqlAnalysis.analyzeBatch( - templates.map(t => ({ id: t.templateId, sql: t.canonicalSql })), - dialect, -); -// → Map, error?: string }> -``` - -The endpoint is implemented in `python/ktx-daemon` and uses `sqlglot` internally with `ProcessPoolExecutor` parallelism over the batch. Replaces the per-query HTTP roundtrip pattern that dominates today's runtime. - -Per-row parse failures are non-fatal: the template loses table grounding (excluded from per-table bucketing and from patterns) but the failure is logged to `manifest.warnings` as `parse_failed:`. - -### 4.4 Filtering (three layers) - -**Layer A - Warehouse-side (in the SQL above):** - -- Noise prefixes (`SHOW`, `DESCRIBE`, `EXPLAIN`, `USE`, `SET`). -- System catalogs (`INFORMATION_SCHEMA`, `SNOWFLAKE.ACCOUNT_USAGE`, `pg_*`, `system.*`). -- DDL / non-analytical statement types via `statement_type` / `query_type` columns (PG falls back to prefix regex). -- Trivial probes (`SELECT 1`, `SELECT NOW()`, `SELECT VERSION()`) - configurable. -- Minimum executions threshold (`@min_executions`, default 5). -- Trailing window (`@window_days`, default 90) - BQ/SF only. - -**Layer B - Post-fetch, in-memory:** - -- Service-account exclusion/inclusion via configurable regex patterns; three modes (`exclude` default, `include`, `mark-only`). -- Orchestrator boilerplate (dbt/Looker/Metabase markers) - default `mark-only` (do not drop; dbt-generated queries are often the actual business logic). -- Failed-query filter (BQ/SF only): templates with `errorRate > 0.9 AND executions < 10`. - -**Layer C - Post-parse:** - -- Zero-table templates (parsed cleanly but touch no real tables) are dropped from per-table bucketization and from patterns. - -### 4.5 Bucketize by table - -In-memory pass: a single template touching N tables ends up in N table buckets. - -### 4.6 Staged artifacts - -``` -{stagedDir}/ - manifest.json - tables/ - {schema}.{name}.json # one per touched table - patterns-input.json -``` - -`manifest.json` is small (summary, window, counts, warnings - schema in §9). - -`tables/{schema}.{name}.json` contains **bucketed** content so that DiffSet content hashes are stable when nothing material changed: - -```jsonc -{ - "table": "public.orders", - "stats": { - "executionsBucket": "1k-5k", - "distinctUsersBucket": "5-10", - "errorRateBucket": "low", - "p95RuntimeBucket": "100ms-1s", - "recencyBucket": "current" - }, - "columnsByClause": { - "select": [["amount","high"], ["status","high"]], - "where": [["status","high"], ["created_at","mid"]], - "join": [["customer_id","high"]], - "groupBy": [["status","low"]] - }, - "observedJoins": [ - { "withTable": "public.customers", "on": ["customer_id"], "freq": "high" }, - { "withTable": "public.line_items", "on": ["order_id"], "freq": "high" } - ], - "topTemplates": [ - { "id": "...", "canonicalSql": "...", "topUsers": [...] } - ] -} -``` - -`patterns-input.json` contains every template in compact form (`id`, `canonicalSql`, `tablesTouched`, `executionsBucket`, `distinctUsersBucket`, `dialect`). Pulls double duty as the patterns skill input and as the audit log; no separate `templates.jsonl`. - -Bucket bands are defined deterministically in code (e.g. `executionsBucket`: `<10`, `10-100`, `100-1k`, `1k-5k`, `5k-50k`, `>50k`). Exact thresholds set during implementation; the principle is that small fluctuations don't change the bucket. - -### 4.7 `chunk()` (trivial, convention-following) - -One `WorkUnit` per `tables/*.json` file (handled by `historic_sql_table_digest`) + one `WorkUnit` referencing `patterns-input.json` (handled by `historic_sql_patterns`). No custom diff logic - the framework's `DiffSetComputerPort` already filters to changed files. - -## 5. Cold path (LLM, via skills) - -Both skills produce **evidence**; the adapter's `onPullSucceeded()` projects evidence to its final homes. This avoids write contention between parallel skill invocations on the same shard file. - -### 5.1 `historic_sql_table_digest` - -One invocation per changed table's `WorkUnit`. Input: the table's staged JSON plus dependency reference to the existing `_schema` entry (so the LLM sees the actual column list and doesn't hallucinate). - -**Prompt cache split** (`cacheControl: { type: 'ephemeral', ttl: '5m' }`, auto-bump to `'1h'` when the run is expected to exceed ~4 minutes wall clock): - -- **Cached prefix:** role, output JSON schema generated from `tableUsageOutputSchema` via Zod 4's `z.toJSONSchema()`, extraction rules, 1–2 few-shot examples. -- **Variable suffix:** table name, existing columns list, existing AI description, staged usage input. - -**Output schema** (zod, in `historic-sql/skill-schemas.ts`): - -```typescript -export const tableUsageOutputSchema = z.object({ - narrative: z.string(), - frequencyTier: z.enum(['high', 'mid', 'low', 'unused']), - commonFilters: z.array(z.string()), - commonGroupBys: z.array(z.string()).optional(), - commonJoins: z.array(z.object({ - table: z.string(), - on: z.array(z.string()), - })), - staleSince: z.iso.datetime().nullable().optional(), -}); -``` - -No hard length/cap constraints in the schema. Concision is a behavioral instruction in the prompt prefix. - -**Concurrency:** `runWithConcurrency()` from `packages/context/src/scan/description-generation.ts:147` (the same utility scan-description uses). Default 12, configurable in `ktx.yaml`. - -**Idempotency:** when `tables/{name}.json`'s content hash hasn't changed (bucketed stats stable), DiffSet marks the file `unchanged`, no WorkUnit is emitted, no LLM call happens. Steady-state daily runs LLM only the meaningfully changed tables. - -### 5.2 `historic_sql_patterns` - -One invocation per run (or a small handful if `patterns-input.json` exceeds a context budget - split deterministically by `tablesTouched` cardinality stratification). - -**Prompt:** identifies recurring analytical intents that span ≥2 tables with ≥mid executionsBucket and ≥2-5 distinct users. Output is a list of `PatternOutput`. - -**Output schema:** - -```typescript -export const patternOutputSchema = z.object({ - slug: z.string(), - title: z.string(), - narrative: z.string(), - definitionSql: z.string(), - tablesInvolved: z.array(z.string()), - slRefs: z.array(z.string()), - constituentTemplateIds: z.array(z.string()), -}); -``` - -**Cache control:** skip. Single call per run; cache write premium doesn't amortize. - -**Slug stability across runs:** the projection step (§5.3) does a deterministic similarity check against existing pattern pages. For each new pattern, find an existing slug whose `tablesInvolved` ∪ `constituentTemplateIds` overlap ≥60% with the new one and reuse it; else mint a new slug. Pure post-process, no LLM call. - -### 5.3 Projection inside `onPullSucceeded()` - -After all skills complete and evidence is committed, run two passes. Both are pure data transformations, no LLM calls. - -**Pass A - `_schema` shard reconciliation:** - -1. Collect all `historic_sql_table_usage` evidence written this run. -2. Group by `shardKey` (`catalog.schema`). -3. For each shard: - - Load existing `_schema/{shardKey}.yaml`. - - For each table entry: if new evidence exists, merge under `usage` via `mergeUsagePreservingExternal()` (only `historicSql`-managed keys touched; user-added keys preserved - same pattern as `mergeDescriptionsPreservingExternal` at `local-enrichment-artifacts.ts:237-242`). - - For tables previously present with `historicSql`-managed `usage` but absent from this run's snapshot: set `usage.staleSince = lastSnapshotSeenAt`, clear other historicSql-managed fields. - - Atomic write to `_schema/{shardKey}.yaml`. -4. Trigger SL search re-index for changed sources via the existing flow (`sl-search.service.ts:91-99` detects search-text drift). - -**Pass B - wiki pattern pages:** - -1. Collect all `historic_sql_pattern` evidence written this run. -2. Load existing wiki pages with tags `['historic-sql', 'pattern']` for this connection. -3. Run slug-stability matching. -4. For each pattern (existing or new): - - Build `LocalKnowledgePage` with `key: historic-sql/{slug}`, `scope: GLOBAL`, `tags: ['historic-sql', 'pattern']`, `slRefs` to relevant SL sources, `refs` to other historic-sql pages. - - `writeLocalKnowledgePage(...)`. -5. For existing patterns not seen this run: append frontmatter `stale_since: {today}` and add `tag: stale`. Don't delete; preserve for historical lookups. -6. After `staleArchiveAfterDays` threshold (default 90 days, configurable): move the page key under `historic-sql/_archived/` and add `tag: archived`. - -## 6. Search-surface plumbing - -### 6.1 `ktx wiki search` - no plumbing required - -Pattern pages are written to `knowledge/global/historic-sql/{slug}.md` and are discovered by the existing `searchLocalKnowledgePages()` walk. Tags `['historic-sql', 'pattern']` enable faceted search. - -### 6.2 `ktx sl search` - small extension - -**6.2.1 - `SemanticLayerSource.usage` field** - -Add an optional `usage` field to `SemanticLayerSource` in `packages/context/src/sl/schemas.ts`, reusing the same `tableUsageOutputSchema` from `skill-schemas.ts`. Single source of truth end-to-end. - -**6.2.2 - `_schema` → `SemanticLayerSource` projection carries `usage`** - -The existing projection step in `local-sl.ts` (or wherever the manifest reader builds `SemanticLayerSource` objects) needs one new field copy: `entry.usage → source.usage`. - -**6.2.3 - `buildSemanticLayerSourceSearchText()` extension** - -Extend the function at `sl-search.service.ts:8-74` to include usage content in the FTS5/embedding text: - -```typescript -if (source.usage) { - const u = source.usage; - parts.push(`usage: ${u.narrative}`); - parts.push(`frequency: ${u.frequencyTier}`); - if (u.commonFilters?.length) parts.push(`commonly filtered by: ${u.commonFilters.join(', ')}`); - if (u.commonGroupBys?.length) parts.push(`commonly grouped by: ${u.commonGroupBys.join(', ')}`); - for (const j of u.commonJoins ?? []) { - parts.push(`commonly joined to ${j.table} on ${j.on.join(',')}`); - } - if (u.staleSince) parts.push(`stale since ${u.staleSince}`); -} -``` - -**6.2.4 - Re-index trigger** - -Already wired. Per-source content-hash detection at `sl-search.service.ts:91-99` ensures only sources whose `usage` changed re-embed. - -**6.2.5 - Query-mode result enrichment** - -Extend the search result shape returned by `agent sl list --query` to include `score` and an FTS5 `snippet()` per hit. Implementation: small SQL change in `sqlite-sl-sources-index.ts` to select `snippet(local_sl_sources_fts, ...)` alongside the source row. - -Result shape becomes: - -```jsonc -{ - "connectionId": "warehouse", - "name": "public.orders", - "table": "orders", - "columnCount": 12, - "measureCount": 3, - "joinCount": 2, - "description": "...", - "score": 0.81, - "frequencyTier": "high", - "snippet": "commonly filtered by status, joined to customers" -} -``` - -The full `usage` block lives in the `SemanticLayerSource` returned by `agent sl read `. - -## 7. Three-tier retrieval model - -| Tier | Surface | What an agent gets | -|---|---|---| -| Search hit | `agent sl list --query "..."` | name, table, counts, description, score, frequencyTier, snippet | -| Source read | `agent sl read ` | full SemanticLayerSource YAML including columns, measures, joins, and `usage` block | -| Pattern read | `agent wiki read historic-sql/{slug}` | title, narrative, canonical SQL, tables involved, slRefs | - -Agents pull deeper only when they need to. The bytes per tier are governed by prompt-side concision instructions, not by schema constraints. - -## 8. Configuration - -Per-connection block in `ktx.yaml`: - -```yaml -connections: - warehouse: - driver: postgres - connectionUrl: postgres://... - historicSql: - enabled: true - # everything below is optional; defaults from the zod schema - windowDays: 90 - minExecutions: 5 - concurrency: 12 - filters: - serviceAccounts: - patterns: ['^etl-', '@bot\.'] - mode: exclude # exclude | include | mark-only - orchestrators: - mode: mark-only # include | exclude | mark-only - dropTrivialProbes: true - dropFailedBelow: - errorRate: 0.9 - executions: 10 - redactionPatterns: ['password', 'api_key'] - staleArchiveAfterDays: 90 -``` - -CLI setup wizard (`ktx setup`) flags map onto this block. `--historic-sql-min-calls` is renamed `--historic-sql-min-executions` (cross-dialect clarity); both names accepted for one release. - -Doctor command (`ktx dev doctor`) retains PG-specific validation: version ≥ 14, extension installed, `pg_read_all_stats` grant, `pg_stat_statements.track != 'none'`. The `pg_stat_statements.max ≥ 5000` check is downgraded from a warning to an informational note (deallocation churn no longer threatens delta-tracking integrity, because there is no delta tracking). - -## 9. Schemas (zod) - -Lives in `packages/context/src/ingest/adapters/historic-sql/types.ts` unless noted. - -```typescript -export const historicSqlPullConfigSchema = z.object({ - dialect: z.enum(['postgres', 'bigquery', 'snowflake']), - windowDays: z.number().int().positive().default(90), - minExecutions: z.number().int().nonnegative().default(5), - concurrency: z.number().int().positive().default(12), - filters: z.object({ - serviceAccounts: z.object({ - patterns: z.array(z.string()).default([]), - mode: z.enum(['exclude', 'include', 'mark-only']).default('exclude'), - }).optional(), - orchestrators: z.object({ - mode: z.enum(['exclude', 'include', 'mark-only']).default('mark-only'), - }).optional(), - dropTrivialProbes: z.boolean().default(true), - dropFailedBelow: z.object({ - errorRate: z.number(), - executions: z.number().int(), - }).optional(), - }).optional(), - redactionPatterns: z.array(z.string()).default([]), - staleArchiveAfterDays: z.number().int().positive().default(90), -}); - -export const aggregatedTemplateSchema = z.object({ - templateId: z.string(), - canonicalSql: z.string(), - dialect: z.enum(['postgres', 'bigquery', 'snowflake']), - stats: z.object({ - executions: z.number().int(), - distinctUsers: z.number().int(), - firstSeen: z.iso.datetime(), - lastSeen: z.iso.datetime(), - p50RuntimeMs: z.number().nullable(), - p95RuntimeMs: z.number().nullable(), - errorRate: z.number(), - rowsProduced: z.number().int().nullable(), - }), - topUsers: z.array(z.object({ - user: z.string().nullable(), - executions: z.number().int(), - })), -}); - -export const stagedTableInputSchema = z.object({ - table: z.string(), - stats: z.object({ - executionsBucket: z.string(), - distinctUsersBucket: z.string(), - errorRateBucket: z.string(), - p95RuntimeBucket: z.string(), - recencyBucket: z.string(), - }), - columnsByClause: z.record(z.string(), z.array(z.tuple([z.string(), z.string()]))), - observedJoins: z.array(z.object({ - withTable: z.string(), - on: z.array(z.string()), - freq: z.string(), - })), - topTemplates: z.array(z.object({ - id: z.string(), - canonicalSql: z.string(), - topUsers: z.array(z.object({ user: z.string().nullable() })), - })), -}); - -export const stagedPatternsInputSchema = z.object({ - templates: z.array(z.object({ - id: z.string(), - canonicalSql: z.string(), - tablesTouched: z.array(z.string()), - executionsBucket: z.string(), - distinctUsersBucket: z.string(), - dialect: z.enum(['postgres', 'bigquery', 'snowflake']), - })), -}); - -export const stagedManifestSchema = z.object({ - source: z.literal('historic-sql'), - connectionId: z.string(), - dialect: z.enum(['postgres', 'bigquery', 'snowflake']), - fetchedAt: z.iso.datetime(), - windowStart: z.iso.datetime(), - windowEnd: z.iso.datetime(), - snapshotRowCount: z.number().int(), - touchedTableCount: z.number().int(), - parseFailures: z.number().int(), - warnings: z.array(z.string()), - probeWarnings: z.array(z.string()), -}); -``` - -In `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` - the **single source of truth for LLM I/O shapes**, imported by the prompt builder, the evidence parser, the projection step, the `SemanticLayerSource` type, and the `_schema` manifest entry type: - -```typescript -export const tableUsageOutputSchema = z.object({ - narrative: z.string(), - frequencyTier: z.enum(['high', 'mid', 'low', 'unused']), - commonFilters: z.array(z.string()), - commonGroupBys: z.array(z.string()).optional(), - commonJoins: z.array(z.object({ - table: z.string(), - on: z.array(z.string()), - })), - staleSince: z.iso.datetime().nullable().optional(), -}); -export type TableUsageOutput = z.infer; - -export const patternOutputSchema = z.object({ - slug: z.string(), - title: z.string(), - narrative: z.string(), - definitionSql: z.string(), - tablesInvolved: z.array(z.string()), - slRefs: z.array(z.string()), - constituentTemplateIds: z.array(z.string()), -}); -export const patternsArraySchema = z.array(patternOutputSchema); -export type PatternOutput = z.infer; -``` - -**Extensions to existing types:** - -- `packages/context/src/sl/schemas.ts` - `SemanticLayerSource.usage: tableUsageOutputSchema.optional()`. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` - `LiveDatabaseManifestTableEntry.usage?: TableUsageOutput`. - -The `_schema/{shard}.yaml` manifest version need not bump - `usage` is an additive, optional field. Validators must allow unknown future keys (audit during step 1 of §10). - -## 10. Cutover plan - -Hard cutover. No parallel codepaths. Single coordinated PR (or PR train). - -### 10.1 Code that gets deleted - -Within `packages/context/src/ingest/adapters/historic-sql/`: - -- `stage.ts` - rewritten -- `stage-pgss.ts` - **deleted** (no baseline tracking) -- `stage-pgss.test.ts`, `stage-pgss-golden.test.ts` - **deleted** -- `historic-sql.adapter.ts` - rewritten -- `historic-sql.adapter.test.ts` - rewritten -- `chunk.ts` / `chunk.test.ts` - rewritten (becomes trivial) -- `detect.ts` / `detect.test.ts` - trivial update -- `postgres-pgss-query-history-reader.ts` - rewritten as `postgres-pgss-reader.ts`; baseline-tracking code removed -- `bigquery-query-history-reader.ts` / `snowflake-query-history-reader.ts` - rewritten; cursor logic removed; warehouse-side GROUP BY -- `types.ts` - rewritten -- **new** `skill-schemas.ts` -- `errors.ts` - keep (probe errors); prune unused - -Old skills `historic_sql_ingest` and `historic_sql_curator` - audit; if only consumed by historic-sql, delete. - -`expandCategoricalTemplates`, `classifySlot`, `rankTemplate`, slot-related types - gone. - -### 10.2 Existing artifacts - -| Artifact | Where | Decision | -|---|---|---| -| Old per-template wiki pages | `knowledge/global/...` (legacy `historic-sql-template` tag or matching key prefix) | **One-time cleanup** in `onPullSucceeded()` on first run after upgrade. Idempotent: subsequent runs no-op. | -| PG baseline files | `.ktx/cache/historic-sql/{connectionId}/pgss-baseline.json` | **Delete on first run.** Cache; no signal lost. | -| Old `raw-sources/{connectionId}/historic-sql/{syncId}/` snapshots | `raw-sources/...` | **Leave alone.** Per-sync audit; framework handles retention. | - -### 10.3 Ordering - -1. **Foundations** (independent, no behavioral change): - - Daemon `analyze-batch` endpoint + `SqlAnalysisPort.analyzeBatch()` (old method still in place, unused). - - `SemanticLayerSource.usage` field (no producer yet). - - `LiveDatabaseManifestTableEntry.usage` field (no producer yet). - - `mergeUsagePreservingExternal()` utility + tests. -2. **Search enrichment** (independent, ships an unrelated win): - - `buildSemanticLayerSourceSearchText()` extension. - - FTS5 `snippet()` + score in query-mode results. -3. **New adapter** (replaces old in a single commit per dialect): - - PG path first (smallest surface, has the doctor command for validation). - - BQ + SF together (share aggregation pattern). -4. **Skills + projection:** - - `historic_sql_table_digest` + `historic_sql_patterns`. - - `onPullSucceeded` projection passes. - - One-time legacy cleanup. -5. **Delete the old codepath** - same PR as step 3, ideally. -6. **Docs + setup wizard** updates. - -### 10.4 Verification before merging - -- **Demo DB end-to-end:** `examples/postgres-historic/` ingest completes in **under 60 seconds** (current 30-minute baseline becomes the regression bar). -- **Cross-dialect smoke:** at least one run against each of PG / BQ / SF ends with non-empty `_schema/{shard}.yaml` `usage` blocks and ≥0 pattern pages. -- **Idempotency:** a second run immediately after the first produces zero `historic_sql_table_digest` LLM calls. -- **Drift:** a run where one table disappears from the snapshot sets `usage.staleSince` on that table's `_schema` entry; reappearance clears it. -- **Search retrieval:** `agent sl list --query` returns hits with non-empty snippets; `agent wiki search ""` returns the pattern page directly. -- **No old code paths:** `git grep -E "stagePgStatStatementsTemplates|expandCategoricalTemplates|classifySlot|pgss-baseline"` returns zero results. -- **Doctor still passes** on a properly configured PG with the new adapter. - -### 10.5 Out of scope - -- Embedding-based pattern clustering (rejected in favor of LLM-driven intent detection). -- Wiki shard pages (rejected - patterns are sparse; per-page is correct). -- Incremental dialect-by-dialect rollout behind a flag. -- A `ktx historic-sql migrate` command - cleanup runs automatically once. -- Framework-level `raw-sources/` retention policy (separate concern; not introduced here). -- Per-table wiki pages (the very problem `_schema` shards exist to avoid - see §11). - -### 10.6 Risks - -| Risk | Mitigation | -|---|---| -| Daemon `analyze-batch` slower than hoped on huge templates | `ProcessPoolExecutor` parallelism; configurable batch size cap | -| `_schema` shard concurrent writes (scan + historic-sql) | Atomic per-shard write + scan-managed-keys merge (`mergeUsagePreservingExternal`); new test covers concurrent invocation | -| Pattern slug churn between runs | Slug-stability matcher in projection; ≥60% overlap reuses existing slug; falls back to new mint if no match | -| Existing manifest validators reject `usage` field | Audit validators in step 1 of §10.3; extend allowed-fields list | -| User-edited `usage` fields clobbered | `mergeUsagePreservingExternal` follows the same scan-managed-keys discipline as descriptions; covered by tests | - -## 11. Rejected alternatives - -Documented so future readers don't relitigate. - -**Per-table wiki pages** - one `.md` per table under `knowledge/global/historic-sql/`. Rejected: reintroduces the per-table-file proliferation problem (`writeLocalKnowledgePage` writes one file per page) that `_schema` shards exist to avoid. ~800 markdown files for a 1000-table warehouse, ~100 churning daily. - -**Single-file all-usage page** - one giant page containing every table. Rejected: ~700 KB blob; FTS5 snippets all come from the same source; `wiki read` returns an unusable mass. - -**One file per table in a new `_usage/` directory** - same file-count problem as per-table wiki, plus needs new search plumbing. - -**New parallel `_usage/{shard}.yaml` shards** - same sharding benefit as merging into `_schema` but without riding SL search. Plumbing required without offsetting win. - -**One wiki page per `catalog.schema`** - workable, but pages get large (200 tables per page) and only rides wiki search, not SL search. The chosen design rides both. - -**Single staged `snapshot.json`** - to reduce `raw-sources/` accumulation. Rejected: required custom diff logic in `chunk()`, broke framework convention, saved bounded disk for a framework-level concern (sync retention). Per-table staged files with bucketed content is cleaner. - -**Embedding-based pattern clustering** - using sentence-transformer embeddings to cluster templates into themes before naming via LLM. Rejected: reintroduces clustering hyperparameters and determinism the redesign aims to avoid. The LLM does the grouping in one call from the full template list, no embedding step. - -**Skip pattern pages entirely** - ship only `_schema` enrichment for a leaner v1. Rejected: leaves `ktx wiki search` empty of historic-sql content (loses one of two stated consumption surfaces) and forces agents to synthesize cross-cutting intents from fragmented per-table mentions. - -**TypeScript-native SQL parser** instead of sqlglot via daemon - `node-sql-parser`, `pgsql-parser` (WASM), etc. Rejected: materially worse dialect coverage on Snowflake/BigQuery edge cases; duplicates parser logic when KTX already uses sqlglot elsewhere (`python/ktx-daemon/src/ktx_daemon/lookml.py`); AGENTS.md explicitly mandates sqlglot. Batch endpoint on the existing daemon achieves the perf win. - -**Hard length/count caps in zod output schemas** (e.g. `narrative.max(250)`, `commonFilters.max(5)`). Rejected: arbitrary thresholds, brittle retry-on-violation paths, defensive coding for a soft concern. Concision belongs in prompt instructions; the schema validates shape. - -## 12. Cost / scale envelope - -For a representative mid-size warehouse (~200 touched tables): - -| Phase | Calls | Cost @ Sonnet | -|---|---|---| -| Hot path (deterministic) | 0 | $0 | -| First-run table digest (uncached + cached mix) | ~200 | ~$5–7 | -| First-run patterns | 1 | ~$0.05 | -| Embeddings (changed tables) | ~200 | ~$0.02 | -| **First run total** | | **~$5–7** | -| Daily steady-state (hash-skipped) | ~10–30 changed | ~$0.10–$0.25 | - -Wall-clock: first run ~1–3 min on mid; demo DB <60s end-to-end. - -For a large warehouse (~800 touched tables): first-run ~$20–30, daily ~$0.20–$1.00. - -## 13. Open questions - -- Exact bucket thresholds for `executionsBucket`, `distinctUsersBucket`, etc. - to be chosen during implementation based on what produces stable hashes in practice. -- Final naming of the daemon endpoint (`/sql/analyze-batch` vs alternatives). -- Whether `historic_sql_ingest` / `historic_sql_curator` skills are consumed elsewhere - audit during step 1. -- Whether to delete legacy wiki pages automatically or behind a confirmation flag - design assumes automatic. diff --git a/docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md b/docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md deleted file mode 100644 index bc089765..00000000 --- a/docs/superpowers/specs/2026-05-11-npm-managed-python-runtime-design.md +++ /dev/null @@ -1,234 +0,0 @@ -# npm-managed Python runtime design - -This spec defines how KTX ships as one visible npm package while still using -Python for sqlglot, semantic-layer planning, database-agent compute, and local -embeddings. The goal is a user experience where users install or run only -`@kaelio/ktx`, and KTX manages its Python runtime automatically when a command -needs it. - -## Goals - -KTX must be usable through the npm package `@kaelio/ktx` with a `ktx` binary. -Users can run KTX without learning about the Python packages that power parts of -the system. - -The first release must support these invocation modes: - -- `npx @kaelio/ktx setup demo` -- `npx @kaelio/ktx sl query ...` -- `npm install @kaelio/ktx`, followed by `npx ktx ...` -- `npm install -g @kaelio/ktx`, followed by `ktx ...` - -KTX-owned Python code must ship inside the npm package as a bundled wheel. KTX -doesn't need to publish its own Python code to PyPI for this release. - -## Non-goals - -This release does not need to provide a public TypeScript SDK split across -multiple npm packages. The internal workspace package layout can remain useful -for development, but the public npm surface is a single package. - -This release does not need a fully offline install. KTX's own Python wheel is -bundled, but third-party Python dependencies can come from PyPI through `uv`. - -This release does not install local embedding dependencies by default. Local -embeddings remain lazy because `sentence-transformers`, `torch`, and model -downloads are large. - -## Package model - -KTX publishes one public npm package: - -```text -@kaelio/ktx -``` - -That package exposes one binary: - -```json -{ - "bin": { - "ktx": "./dist/bin.js" - } -} -``` - -The npm package includes these assets: - -- Bundled JavaScript CLI output. -- Packaged demo assets. -- One KTX-owned Python wheel, for example - `python/kaelio_ktx-0.1.0-py3-none-any.whl`. -- A wheel checksum or runtime manifest that lets the CLI verify the bundled - Python payload before installation. - -The Python wheel contains the current `semantic_layer` and `ktx_daemon` -modules. It exposes at least the `ktx-daemon` console script. - -## Runtime installation - -KTX creates a managed Python runtime only when a command needs Python-backed -behavior. The runtime lives outside the npm cache so it survives `npx` runs. - -The runtime root is platform-specific: - -- macOS: `~/Library/Application Support/kaelio/ktx/runtime` -- Linux: `${XDG_DATA_HOME:-~/.local/share}/kaelio/ktx/runtime` -- Windows: `%LOCALAPPDATA%/Kaelio/KTX/runtime` - -The runtime is versioned by the npm package version. A versioned runtime avoids -mixing JavaScript and Python code from incompatible releases. - -The installer performs these steps: - -1. Locate `uv`. -2. Create a virtual environment under the versioned runtime directory. -3. Install the bundled KTX wheel into that environment. -4. Write a runtime manifest with the CLI version, wheel checksum, Python - executable, daemon executable, and installed feature set. - -For lightweight Python support, the install command uses the bundled wheel's -default dependency set. For local embeddings, the installer adds the embeddings -extra only when selected: - -```bash -uv pip install "/path/to/kaelio_ktx-0.1.0-py3-none-any.whl" -uv pip install "/path/to/kaelio_ktx-0.1.0-py3-none-any.whl[local-embeddings]" -``` - -## Feature installation levels - -KTX manages Python runtime features in levels so first use stays fast. - -`core` includes: - -- `sqlglot` -- `pydantic` -- `pyyaml` -- `fastapi` -- `uvicorn` -- lightweight daemon dependencies - -`local-embeddings` adds: - -- `sentence-transformers` -- `torch` -- model download support for `all-MiniLM-L6-v2` - -Commands that only need semantic-layer SQL generation require `core`. -Commands that need local embeddings require `local-embeddings`. - -## Command behavior - -Pure TypeScript commands run without the managed Python runtime. - -Python-backed one-shot operations use the managed `ktx-daemon` executable -directly. Examples include semantic query compilation, semantic validation, -semantic source generation, and sqlglot-backed table identifier parsing. - -Repeated or expensive operations use a managed HTTP daemon. Local embeddings use -the daemon because loading the model for every one-shot process is too slow. - -KTX provides runtime management commands: - -```bash -ktx runtime install -ktx runtime status -ktx runtime start -ktx runtime stop -ktx runtime doctor -ktx runtime prune -``` - -Normal commands can install the runtime lazily. Runtime commands make that -behavior inspectable and debuggable. - -## Daemon lifecycle - -The daemon binds to `127.0.0.1` on an available random port. KTX writes daemon -state to the runtime manifest or an adjacent state file: - -```json -{ - "pid": 12345, - "port": 58731, - "version": "0.1.0", - "features": ["core", "local-embeddings"], - "startedAt": "2026-05-11T00:00:00Z" -} -``` - -Before reusing a daemon, KTX checks that the process is alive, the port responds -to `/health`, and the daemon version matches the CLI version. If any check -fails, KTX treats the daemon as stale and starts a new one. - -KTX uses one-shot Python for short operations by default. It starts the daemon -only when a command benefits from process reuse. - -## Interactive and CI behavior - -In an interactive terminal, KTX prompts before installing the managed runtime -for the first time. The prompt states that Python dependencies will be -downloaded. - -With `--yes`, KTX installs the required runtime features without prompting. - -With `--no-input`, KTX fails if a required runtime feature is missing and no -explicit auto-install flag is present. The error prints the exact command to -prepare the runtime. - -For local embeddings, KTX prompts separately because the dependency and model -downloads are larger than the core runtime. - -## Error handling - -If `uv` is missing, KTX prints a focused error that explains how to install it -and how to retry. A later release can add a bundled or downloaded `uv` strategy. - -If Python runtime installation fails, KTX preserves install logs in the runtime -directory and prints the log path. - -If the daemon fails to start, KTX prints the captured daemon stdout and stderr -path. It falls back to one-shot mode only when the requested operation supports -one-shot execution. - -If JavaScript and Python versions don't match, KTX reinstalls the managed -runtime for the current npm package version. - -## Release flow - -The release builds the Python wheel before packing npm artifacts. The npm pack -step includes the wheel as an asset. - -Release checks must cover: - -1. Clean install of the packed npm package. -2. `npx` execution of the packed package. -3. First-run managed runtime install from the bundled wheel. -4. One-shot semantic-layer query through the managed runtime. -5. Runtime status and doctor output. -6. Daemon start, health check, reuse, and stop. -7. Optional local embeddings smoke in a separate job or opt-in check. - -## Open decisions - -KTX still needs a final decision on whether `uv` is a hard prerequisite or a -bootstrap dependency that KTX downloads automatically. - -KTX also needs the final Python distribution name. This spec uses -`kaelio-ktx` as the distribution name and `kaelio_ktx` in wheel filenames. - -## Success criteria - -Users can run `npx @kaelio/ktx ...` and complete Python-backed KTX operations -without manually installing a KTX Python package. - -Users who install `@kaelio/ktx` locally can run `npx ktx ...` through the local -project's npm binary resolution. - -The first Python-backed command installs only the core runtime. Local embedding -dependencies install only after the user selects local embeddings or explicitly -requests the `local-embeddings` runtime feature. - -KTX can diagnose and repair stale or mismatched managed runtimes without asking -users to delete directories manually. diff --git a/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md b/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md deleted file mode 100644 index 84764501..00000000 --- a/docs/superpowers/specs/2026-05-12-notion-ingestion-warehouse-verification-design.md +++ /dev/null @@ -1,331 +0,0 @@ -# Warehouse Verification Tools for Ingestion Synthesis - -**Date:** 2026-05-12 -**Author:** Andrey Avtomonov -**Status:** Design - pending implementation plan - -## Background and motivation - -KTX's ingest pipeline synthesises wiki pages and semantic-layer (SL) sources from third-party content (Notion, LookML, Looker, Metabase, dbt, MetricFlow, historic SQL, live-database scans, and chat). The synthesis stage is an LLM call that runs once per WorkUnit, governed by a skill prompt (e.g. `notion_synthesize`) and a set of allowed tools. - -A real-world inspection (project `/tmp/ktx-proj-1`) surfaced two failure modes the synthesis stage produces: - -1. **Fictional identifiers laundered into wiki output.** A Notion page mentioned `orbit_analytics.customer` as a legacy "customer source" table with a `plan_tier in {free, pro, enterprise}` column. Neither the table, the column, nor those values exist in the configured warehouse. The synthesis LLM faithfully copied them into `knowledge/global/orbit/customers-source.md` as a "Conflict Note", giving the fabricated names full wiki frontmatter, a `Source:` citation, and apparent authority. -2. **Column attribution drift.** The same wiki page documents columns under `orbit_raw.accounts` but states the `paying_account_count` measure filters on `normalized_plan_code` and `contract_status`. Those columns live on `orbit_analytics.mart_account_segments`, not on `accounts`. A reader (or a downstream agent) following the page will write `accounts.normalized_plan_code` and get a `column does not exist` error. - -Root cause analysis (`packages/context/skills/notion_synthesize/SKILL.md`, `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts`, `packages/context/src/wiki/tools/wiki-write.tool.ts`) showed three contributing factors: - -- The synthesis LLM has no verification primitive that distinguishes a real warehouse identifier from a fabricated one. `sl_discover` only finds objects already promoted into the semantic layer; raw warehouse scans (which already exist on disk under `raw-sources//live-database//`) are not surfaced to the LLM at all. -- `wiki_write` performs no body-text validation - anything the LLM emits is written. -- The skill prompt itself uses `orbit_analytics.customer` as a canonical example string (`SKILL.md:70`), reinforcing the same fictional name the LLM ends up emitting. - -Kaelio's server-side ingest WU agent (`/Users/andrey/conductor/workspaces/kaelio-main2/douala/server/src/tools/toolset-factory.service.ts`) had four verification tools that KTX dropped during the open-source extraction: `discover_data`, `entity_details`, `dictionary_search`, and `sql_execution`. The underlying connector infrastructure (`KtxScanConnector`, dialect classes, `assertReadOnlySql`, `SemanticLayerService.executeQuery`) is present in KTX, so the gap is at the tool layer, not the platform layer. - -## Goal - -Give every ingest adapter's synthesis-time LLM call the tools and skill-prompt instructions needed to verify warehouse identifiers (`schema.table`, `schema.table.column`) and sample values before emitting them into wiki pages, SL sources, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback` records. - -## Non-goals - -- Not changing `wiki_write` itself. A complementary spec covers hard write-time validation; this spec focuses on giving the LLM the tools to self-validate. -- Not modifying any Notion fetch/chunk/cluster behaviour. -- Not changing the `_schema/*.yaml` format. -- Not introducing a UUID layer for tables or columns; KTX keeps `(connection, catalog, db, name)` as the canonical table identity. -- Not adding `semantic_query` to the synthesis toolset. `semantic_query` is a future tool for the research/chat-time agent; synthesis creates SL sources rather than queries them, so the wrong shape. -- Not adding `dictionary_search`. `entity_details` already returns per-column `sampleValues` from the relationship-profile, and `sql_execution` covers the rarer "where does this literal live?" case more accurately than a sampled-JSON full-text scan. - -## What already exists in KTX - -The dialect/driver/connection architecture is fully ported from Kaelio. The new tools sit on top of three already-shipping primitives: - -| Primitive | Location | -|---|---| -| `KtxTableRef = { catalog: string\|null, db: string\|null, name: string }` | `packages/context/src/scan/types.ts:168` | -| `SemanticLayerService.executeQuery(connectionId, sql)` | `packages/context/src/sl/semantic-layer.service.ts:1004`, used today by `sl_validate` | -| `assertReadOnlySql` / `limitSqlForExecution` | `packages/context/src/connections/read-only-sql.ts` | -| 7 connectors with parallel layout (postgres, mysql, sqlserver, snowflake, bigquery, clickhouse, sqlite), each exporting a dialect class | `packages/connector-*` | -| Raw scan artefacts: `tables/...json` and `enrichment/relationship-profile.json` (with `nativeType`, `nullable`, `primaryKey`, `foreignKeys`, `rowCount`, `nullCount`, `distinctCount`, `sampleValues`, descriptions) | `raw-sources//live-database//` | -| `wiki_search`, `sl_discover`, `sl_read_source`, `sl_validate`, `emit_unmapped_fallback` | already wired into synthesis stages | - -The only meaningfully new code is `WarehouseCatalogService`, a small `getDialectForDriver` dispatch, the three tool files, and the wiring in `ingest-bundle.runner.ts`. - -## Architecture - -### Module layout - -``` -packages/context/src/ingest/tools/warehouse-verification/ - discover-data.tool.ts - entity-details.tool.ts - sql-execution.tool.ts - warehouse-catalog.service.ts - index.ts # exports createWarehouseVerificationTools() -packages/context/src/connections/ - dialects.ts # adds getDialectForDriver() -packages/context/skills/_shared/ - identifier-verification.md # the protocol snippet referenced from every synthesis skill -``` - -### Canonical table identity - -Every tool that names a warehouse object uses the tuple `(connectionName, catalog, db, name[, column])`. `connectionName` is the slug from `ktx.yaml` (e.g., `"warehouse"`), validated against `^[a-zA-Z0-9][a-zA-Z0-9_-]*$`. There is no UUID layer. - -`display` strings the LLM picks up from source pages (e.g., `"orbit_raw.accounts"` for Postgres or `"project.dataset.table"` for BigQuery) are parsed by `WarehouseCatalogService.resolveDisplay`, which knows the connection's driver via `getDialectForDriver`. Ambiguous parses (e.g., a 2-part display on BigQuery) return a candidates list instead of guessing. - -Dialect mapping: - -| Driver | catalog | db | name | Display | -|---|---|---|---|---| -| postgres | `null` | schema | table | `schema.table` | -| mysql | `null` | schema | table | `schema.table` | -| sqlserver | catalog | schema | table | `catalog.schema.table` | -| snowflake | database | schema | table | `db.schema.table` | -| bigquery | project | dataset | table | `project.dataset.table` | -| clickhouse | `null` | database | table | `database.table` | -| sqlite | `null` | `null` | table | `table` | - -### `WarehouseCatalogService` - -Stateless except for a per-WorkUnit cache. Reads raw scan files under `raw-sources//live-database//`. - -```ts -class WarehouseCatalogService { - getTable(ref: { connectionName: string } & KtxTableRef): Promise; - listTables(connectionName: string): Promise; - resolveDisplay(connectionName: string, display: string): Promise<{ - resolved: KtxTableRef | null; - candidates: KtxTableRef[]; // ranked by edit distance when resolved is null - dialect: string; - }>; - searchByName(connectionName: string, query: string, limit: number): Promise>; - getLatestSyncId(connectionName: string): Promise; -} -``` - -`getTable` merges the raw schema file (native types, PK, FK, nullable) with the enrichment profile (row counts, null rates, distinct counts, sample values, AI-generated descriptions). When no scan exists for the connection, every read returns `null`; tools surface this as a distinct "no scan available" state rather than as "identifier not found", so the LLM doesn't conclude a real table is fictional just because a scan hasn't run yet. - -### `getDialectForDriver` - -```ts -// packages/context/src/connections/dialects.ts -export type SupportedDriver = 'postgres'|'postgresql'|'mysql'|'sqlserver'|'snowflake'|'bigquery'|'clickhouse'|'sqlite'|'sqlite3'; -export function getDialectForDriver(driver: SupportedDriver): KtxDialect; -``` - -Sync dispatch. The connectors' existing dialect classes already expose the same shape - `formatTableName(KtxTableRef)`, `quoteIdentifier(string)`, `mapToDimensionType(nativeType)`. The implementation plan introduces a minimal `KtxDialect` interface that these classes already satisfy structurally; no connector-internal changes required. Used by tools only for display-string parsing and error-message formatting; tools never construct executable SQL. - -## Tool contracts - -### `entity_details` - -```ts -input = { - connectionName: string, - targets: Array< // 1..50, mixed shapes allowed - | { display: string } // "orbit_raw.accounts" or "orbit_raw.accounts.account_id" - | { catalog: string|null, db: string, name: string, column?: string } - >, -} -``` - -Output (markdown, per target): - -``` -### orbit_raw.accounts -Type: table | Native columns: 11 | PK: account_id | FKs: parent_account_id → orbit_raw.accounts.account_id -Description: One row per customer account… - -Columns: -- account_id (text, nullable=false, PK) - sample: ["acct_001","acct_002",…] -- parent_account_id (text, nullable=true, FK → orbit_raw.accounts.account_id) -- account_name (text, nullable=false) -- … - -Profile: rowCount=4321 distinctCount(account_id)=4321 nullRate(parent_account_id)=0.62 -``` - -When `column` is provided in a target, output is scoped to that one column. When a target doesn't resolve, output is `Not found in scan. Closest matches: …` with up to 5 candidates from `searchByName`. When the connection has no `live-database` scan, output is `No live-database scan available for connection ""; run \`ktx scan\` first.` - distinct from the "not found" state. - -Structured output: `{ resolved: TableDetail[], missing: Array<{target, candidates}>, scanAvailable: boolean }`. - -Refuses `connectionName` values not in the WU-stage's `allowedConnectionNames` set. - -### `sql_execution` - -```ts -input = { - connectionName: string, - sql: string, // single SELECT or WITH only - rowLimit?: number, // default 100, hard cap 1000 -} -``` - -Pipeline: - -1. `assertReadOnlySql(sql)` - regex rejects anything starting with `insert|update|delete|merge|alter|drop|create|truncate|grant|revoke|copy|call|do|vacuum|analyze|refresh`. -2. `limitSqlForExecution(sql, rowLimit)` - wraps as `select * from () as ktx_query_result limit N`. -3. `SemanticLayerService.executeQuery(connectionName, wrappedSql)`. -4. Format as markdown table; first ~20 rows inline; if truncated, append `… +N more rows`. - -Structured output: `{ headers, rows, rowCount, truncated, sql, wrappedSql }`. - -Connector errors surface verbatim (e.g., Postgres `relation "orbit_analytics.customer" does not exist`). That error message is the most valuable verification signal - it tells the LLM the identifier is fictional. - -Refuses `connectionName` not in `allowedConnectionNames`. Each connector's driver-level read-only enforcement (Postgres read-only transaction, BigQuery query-only jobs) is a second defence under the regex gate. - -### `discover_data` - -```ts -input = { - query: string, - connectionName?: string, // omit to search all configured warehouse connections - limit?: number, // default 10 per section - sourceName?: string, // SL source detail mode (delegates to sl_discover) -} -``` - -Composes three searches and groups output into three sections, omitting empty sections: - -1. **Wiki Pages** - `wiki_search({query, limit})`. Routing hint: *use `wiki_read(blockKey)` for full content*. -2. **Semantic Layer Sources** - `sl_discover({query, connectionName})`. Routing hint: *use `sl_read_source(sourceName)` for the YAML, or `entity_details` for warehouse-shape details*. -3. **Raw Warehouse Schema** - `WarehouseCatalogService.searchByName(connectionName, query, limit)`. Routing hint: *use `entity_details({connectionName, targets: [{display}]})` for full DDL + sample values*. - -When `sourceName` is set, delegates entirely to `sl_discover` inspect mode and skips other sections. When all three sections are empty, output is `No matches for "" across wiki, semantic layer, or raw warehouse schema. Try broader terms; this concept may not exist yet.` - -Structured output: `{ wiki: WikiSearchStructured|null, sl: SlDiscoverStructured|null, raw: RawSchemaHits|null }`. - -## Wiring - -`packages/context/src/ingest/ingest-bundle.runner.ts` already plumbs `emit_unmapped_fallback` into both the WorkUnit stage (`createEmitUnmappedFallbackTool` around line 726) and the reconcile stage (around line 962), with merging done via `packages/context/src/ingest/stages/build-wu-context.ts` and `build-reconcile-context.ts`. - -Add a parallel factory next to those existing calls: - -```ts -const warehouseTools = createWarehouseVerificationTools({ - semanticLayerService: scopedSemanticLayerService, - warehouseCatalog: new WarehouseCatalogService({ fileStore, projectDir }), - dialects: getDialectForDriver, - allowedConnectionNames: slConnectionIds, // reuse existing scoping - sqlExecutionRowLimit: 100, -}); -// Merge `entity_details`, `sql_execution`, `discover_data` into both stage tool maps -// alongside emit_unmapped_fallback. -``` - -`createWarehouseVerificationTools` returns `Record` with three keys. The set is wired into every adapter's synthesis stage - no per-adapter opt-in. - -## Skill-prompt updates - -### Shared protocol - -`packages/context/skills/_shared/identifier-verification.md`: - -```md -## Identifier Verification Protocol - -Before writing a wiki page or SL source on any topic: -1. `discover_data({query: ""})` - see what wikis, SL sources, and raw tables - already exist. Prefer updating existing pages over creating new ones. - -Before emitting any `schema.table` or `schema.table.column` into a wiki body, -SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`: -2. `entity_details({connectionName, targets: [{display: ""}]})` - - confirm the identifier resolves; inspect native types, FK/PK, and sampleValues. -3. For literal values from the source (status codes, plan tiers): check whether - they appear in `entity_details`' `sampleValues` for the relevant column. - If `sampleValues` is short or you suspect the sample missed real values, run - a `sql_execution` probe: `SELECT DISTINCT
    FROM LIMIT 50`. -4. If the candidate identifier still doesn't resolve, do one of: - (a) Use `sql_execution` with `SELECT 1 FROM LIMIT 0`. If it errors, - the identifier is fictional. - (b) Wrap the identifier in `[unverified - from ]` in the wiki body, - citing the exact raw path that mentioned it. - (c) When recording `emit_unmapped_fallback` with `no_physical_table`, - include the failing probe error in `clarification`. -5. Never copy `.
    ` placeholder strings from these instructions - into output. -``` - -Each affected skill inlines this block verbatim (skill files are independent prompts; KTX has no cross-skill include mechanism today). - -### Per-skill diffs - -Two skills are deliberately excluded from updates: `ingest_triage` (read-only triage; produces no wiki or SL output) and `sl` (umbrella reference doc; cross-links to the protocol but doesn't need its own copy). - -| Skill | Changes | -|---|---| -| `notion_synthesize` | Inline protocol; append `discover_data`, `entity_details`, `sql_execution` to `Allowed:` (line 74); replace `orbit_analytics.customer` example on line 70 with `.
    ` | -| `dbt_ingest` | Inline protocol; line 24: replace `wiki_sl_search` → `discover_data` and `sl_describe_table` → `entity_details`; strengthen the "not permission to invent physical columns" paragraph by naming `entity_details` as the verification call | -| `lookml_ingest` | Inline protocol; add: "Verify each `sql_table_name` from the LookML view with `entity_details` before mapping to an SL source" | -| `looker_ingest` | Inline protocol; add: "For every Looker field reference, call `entity_details` on the underlying `(schema, table, column)` before promoting to `sl_refs` or quoting in wiki body" | -| `metabase_ingest` | Inline protocol; add: "Before writing a wiki page derived from a Metabase question's SQL, verify each `schema.table.column` mentioned with `entity_details`" | -| `metricflow_ingest` | Inline protocol; add: "Verify each MetricFlow model's source table with `entity_details` before producing the corresponding `sl_write_source`" | -| `live_database_ingest` | Inline protocol; add: "Sample values come from the scan record; do not invent values not present in `relationship-profile.json`" | -| `historic_sql_table_digest` | Shortened protocol focused on column attribution: "Only mention columns visible in the table's scan record. Use `entity_details({display})` if uncertain" | -| `historic_sql_patterns` | Inline protocol; add: "Every join column mentioned in pattern descriptions must be verified via `entity_details` for both sides of the join" | -| `knowledge_capture` | Inline protocol; update line 44: "First call `discover_data` to find existing wiki pages, SL sources, and raw tables on the topic" | -| `sl_capture` | Inline protocol; add: "Before `sl_write_source`, call `entity_details` on the target table to confirm column names and types match the YAML being written" | - -### Cleanups beyond the four-tool addition - -- `notion_synthesize/SKILL.md:70` - remove `orbit_analytics.customer` (placeholder). -- `packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts:67` - same example string in the Zod `.describe()` - replace with `.
    `. -- `dbt_ingest/SKILL.md:24` - fix `wiki_sl_search` and `sl_describe_table` (neither tool exists in KTX). -- `packages/context/src/sl/tools/sl-warehouse-validation.ts:93` - inline error message references the non-existent `sl_describe_table`. Replace with `sl_read_source`. - -## Testing strategy - -### Unit tests - -| Component | Tests | -|---|---| -| `getDialectForDriver` | Every supported driver returns a dialect; unknown driver throws with a clear list of supported drivers | -| `WarehouseCatalogService.getTable` | Reads and merges `tables/.json` and `relationship-profile.json`; returns `null` when no sync exists; returns `null` for unknown `(catalog, db, name)` | -| `WarehouseCatalogService.resolveDisplay` | Postgres 2-part display → `{catalog: null, db, name}`; BigQuery 3-part display → `{catalog, db, name}`; ambiguous 2-part on BigQuery returns candidates list; unknown displays produce closest-match candidates ordered by edit distance | -| `WarehouseCatalogService.searchByName` | Substring and token match; tiers (exact-name → token-match) ordered correctly; cache hit on second call within same instance | -| `entity_details` | Resolves `{display}` and structured inputs; reports "Not found" with candidates for unknown ref; reports "no scan available" distinctly when scan dir missing; truncates above 50 targets | -| `discover_data` | Three sections present when all three have hits; sections omitted when empty; `sourceName` inspect mode delegates to `sl_discover` and skips other sections; `allowedConnectionNames` scope honoured | -| `sql_execution` | `assertReadOnlySql` rejects each mutating verb; row-limit wrap visible in `wrappedSql`; connector errors surface verbatim with the failing SQL; rejects `connectionName` not in `allowedConnectionNames` | - -### Integration tests - -- Extend `packages/context/src/ingest/ingest-bundle.runner.test.ts` to verify the three new tools are present in both WU-stage and reconcile-stage tool maps and refuse out-of-scope `connectionName` values. -- New fixture-based test: stage a small `raw-sources//live-database//` directory with 2 tables + 1 enrichment profile, then call each tool through the runner's tool map and assert the markdown contains the expected fields. Uses the same fake-LLM harness as `notion.adapter.test.ts`. -- One end-to-end regression test reproducing the `orbit_analytics.customer` hallucination: a fake Notion page mentioning the fictional table is fed to the synthesis stage; the run produces a wiki page where the fictional name is wrapped in `[unverified - …]` or omitted, not promoted to `tables:` frontmatter. - -### Prompt-bundling tests - -Extend `packages/context/src/memory/memory-runtime-assets.test.ts`: - -- Every skill in the synthesis-writers list embeds the verification-protocol block (assert by stable header text). -- Every such skill lists the three new tools when it has a `## Tools / Allowed` section, or mentions them inline in a workflow step otherwise. -- No skill file contains any of the banned strings: `orbit_analytics.customer`, `wiki_sl_search`, `sl_describe_table`. - -### Performance guards - -`WarehouseCatalogService` caches the per-connection table list per stage (one WorkUnit's lifetime). Tests assert second call is a cache hit. No DB index for `searchByName` in this iteration - linear scan over scan artefacts is acceptable up to ~50K columns. If volume warrants it later, a follow-up PR adds a SQLite FTS index. - -## Rollout - -Four mergeable PRs: - -| PR | Lands | -|---|---| -| 1 | `getDialectForDriver` + `WarehouseCatalogService` + `entity_details` tool + wiring in `ingest-bundle.runner.ts` + unit/integration tests | -| 2 | `sql_execution` tool + tests + the `orbit_analytics.customer` regression test (which exercises protocol steps 4a/4c) | -| 3 | `discover_data` tool + tests | -| 4 | All 11 skill prompts updated with the verification protocol + the three cleanups + extended `memory-runtime-assets.test.ts` | - -Skill prompts land last so they can reference the tools that already exist. - -## Out of scope - -- **Hard write-time validation in `wiki_write` / `emit_unmapped_fallback`.** A complementary spec covers regex-based identifier validation at the write boundary. Defence-in-depth - separate concern. -- **SQLite FTS index for `searchByName`.** Deferred until the linear scan benchmark fails. -- **`raw_schema_search` as a standalone tool.** `discover_data`'s raw section covers the concept-search case. -- **`semantic_query` in the synthesis toolset.** `semantic_query` will exist in KTX for the research/chat-time agent; it is deliberately excluded from synthesis because synthesis creates SL sources rather than queries them. -- **`dictionary_search`.** `entity_details` already returns per-column `sampleValues`; for the rarer "where does this literal live?" case, `sql_execution` is more accurate than a sampled-JSON scan. -- **UUID layer for tables/columns.** KTX deliberately stays string-keyed on `(connection, catalog, db, name)`. diff --git a/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md b/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md deleted file mode 100644 index f6e4835e..00000000 --- a/docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md +++ /dev/null @@ -1,593 +0,0 @@ -# Unified Ingest UX Design - -**Date:** 2026-05-13 -**Author:** Andrey Avtomonov -**Status:** Design - pending implementation plan - -## Background - -KTX currently exposes multiple user-facing ideas for one product action: -building context from configured connections. Database connections use -`ktx scan `, source connections use -`ktx ingest run --connection-id --adapter `, and setup uses a -context-build wrapper that plans database scans before source ingestion. - -The implementation already points toward one concept. `ktx scan` runs a -stage-only ingest with the `live-database` adapter, then writes scan-specific -reports, schema manifests, and enrichment artifacts. `ktx setup` already -builds context from all configured connections by routing database connections -to scan internals and source connections to source-ingest internals. - -The user-facing model must become simpler: - -- Setup configures KTX. -- Ingest builds or refreshes context. -- Status explains readiness. - -`scan`, `live-database`, and adapter selection are implementation details. - -## Goals - -The redesign makes `ktx ingest` the single public context-building command and -keeps the foreground experience rich, clear, and robust. - -- Remove `ktx scan` as a normal external verb. -- Remove `live-database` from user-facing CLI help, output, docs, and - `ktx.yaml`. -- Treat database schema ingest as mandatory baseline behavior for database - connections. -- Keep slow AI-heavy database behavior explicit with `--deep`; keep fast, - deterministic behavior explicit with `--fast`. -- Fold query-history ingestion into database connection ingest as an optional - facet. -- Keep `ktx setup` guided. It stores defaults in `ktx.yaml` and uses the same - foreground context-build engine as `ktx ingest`. -- Remove detach, attach, watch, resume, stop, and background context-build - flows. -- Preserve a polished foreground progress view for TTY users and scriptable - output for non-TTY and JSON users. - -## Non-goals - -This spec does not redesign the semantic-layer YAML format, the ingest bundle -agent loop, or warehouse verification tools. - -- Do not remove the internal scan implementation if it remains the cleanest - module boundary. -- Do not remove internal adapter/source keys in one large rename. User-facing - terminology changes first; internal cleanup can follow where it reduces - complexity. -- Do not make query-history ingestion mandatory. -- Do not make AI enrichment mandatory for database connections. -- Do not add `--fast` or `--deep` to top-level `ktx setup`. -- Do not preserve compatibility shims for old public `scan` or - `ingest run --adapter live-database` usage unless an implementation plan - explicitly chooses a short deprecation window. - -## Public command model - -`ktx ingest` becomes the direct command for building context from one -connection or all configured connections. - -```bash -ktx ingest warehouse -ktx ingest warehouse --fast -ktx ingest warehouse --deep -ktx ingest warehouse --deep --query-history -ktx ingest warehouse --no-query-history -ktx ingest notion -ktx ingest --all -ktx ingest --all --deep -``` - -The command dispatches by connection driver: - -- Database drivers run database ingest. -- Source drivers run source ingest. -- `--all` runs database ingest targets first, then source ingest targets. - -The old `ktx ingest run --connection-id --adapter ` command is -removed from the public interface. Normal users configure and ingest -connections, not adapters. - -`ktx scan` is no longer a documented public command. Database schema scanning -continues as an internal phase of database ingest. - -Stored report inspection is separate from live context-build control. The -public `ktx ingest` namespace has no subcommands, so `run`, `status`, `watch`, -and `replay` are ordinary connection IDs: - -```bash -ktx ingest run -ktx ingest status -ktx ingest watch -ktx ingest replay -``` - -No setup or config validation rejects those names. Old adapter-backed command -shapes such as `ktx ingest run --connection-id warehouse --adapter -live-database` fail through normal option parsing because `--connection-id` and -`--adapter` are not public `ktx ingest` options. - -## Database ingest depth - -Database ingest always includes a schema baseline. The depth controls how much -extra work KTX may perform. - -Depth is the public abstraction over the current scan engine: - -- `fast` maps to `KtxScanMode: structural` with `detectRelationships: false`. -- `deep` maps to `KtxScanMode: enriched` and requests relationship detection. -- The internal `relationships` scan mode remains an advanced implementation - detail. It is not a separate public depth in this v1. - -Deep mode includes relationship discovery when the project's -`scan.relationships.enabled` setting is true. Relationship validation thresholds -and budgets remain governed by the existing internal `scan.relationships` -configuration; users do not get a separate public relationship flag in this -surface. If `scan.relationships.enabled` is false, `--deep` still runs enriched -database ingest but relationship discovery remains disabled. - -### Fast - -`--fast` means KTX builds deterministic schema context quickly. - -- No LLM calls. -- No embeddings. -- No AI-generated descriptions. -- No expensive relationship discovery that depends on sampling, read-only SQL, - or model calls. -- Introspect tables, columns, native types, comments, declared primary keys, - and declared foreign keys when the connector can read them. -- Write or update database schema context that agents can use as grounding. -- Do not run query-history synthesis, because the current query-history path - uses ingest work units and model-backed synthesis. - -This is the safe default for new database connections, CI, smoke tests, and -large unknown warehouses. - -### Deep - -`--deep` means KTX builds richer database context through the enriched scan path -and uses slower capabilities. - -- Requires LLM, embedding, and scan-enrichment readiness before work starts. -- Generates table and column descriptions. -- Generates embeddings. -- May sample or query data through read-only connector capabilities. -- Discovers and validates relationships when relationship discovery is enabled. -- May process query history into usage patterns when query history is enabled. - -Deep mode is the best agent-readiness mode, but it can take longer and can -require model, embedding, and database permissions. - -KTX must not silently downgrade an explicit or stored `deep` request to `fast`. -For a single database target, if the project is missing the model, embedding, or -scan-enrichment configuration required for deep ingest, KTX errors before -starting the run and tells the user to run `ktx setup` or rerun with `--fast`. -For `--all`, deep-readiness failures follow the per-target rule in -**Error handling and warnings**. - -### Flag rules - -`--fast` and `--deep` are mutually exclusive. Passing both is an error. - -When neither flag is passed, `ktx ingest` uses the stored connection default. -If no default exists, database connections use `fast`. - -If a depth flag is passed for a non-database source, KTX prints a warning and -continues: - -```text ---deep affects database ingest only; ignoring it for notion. -``` - -For `--all`, KTX aggregates warnings instead of repeating noisy lines: - -```text ---deep ignored for 2 non-database sources. -``` - -## Query history - -Historic SQL becomes the database connection's query-history facet. The term -`historic-sql` remains an internal source key unless a later cleanup renames -it. - -Query history is optional because it can require extra grants and can expose -sensitive SQL text. Setup asks about it only for database drivers that support -it. - -```bash -ktx ingest warehouse --query-history -ktx ingest warehouse --no-query-history -ktx ingest warehouse --query-history-window-days 30 -``` - -Query-history flags apply only to database connections that support the feature. -In v1, supported query-history drivers are `postgres` or `postgresql`, -`bigquery`, and `snowflake`. They map to the existing historic-SQL dialects -`postgres`, `bigquery`, and `snowflake`. `sqlite`, `mysql`, `clickhouse`, and -`sqlserver` are database ingest targets but do not support query history in v1. - -Non-applicable query-history flags produce warnings and continue when the target -can otherwise be ingested. For a single unsupported database target, -`--query-history` or `--query-history-window-days` runs schema ingest, skips the -query-history facet, and prints a warning. For `--all`, KTX aggregates those -warnings and continues other eligible targets. Stored -`connections..context.queryHistory.enabled: true` on an unsupported driver -is a config warning and is skipped for that driver; it must not abort schema -ingest for that target. - -Query history uses schema context as grounding. KTX must run the database -schema facet before query-history processing in the same ingest run. If a user -explicitly enables query history for a run, the output states that schema -ingest runs first. - -Because query-history synthesis is model-backed in the current architecture, -`--query-history` upgrades the effective database depth to deep for that run. -KTX prints a warning when a user combines `--fast` with `--query-history`: - -```text ---query-history requires deep ingest; running warehouse with --deep. -``` - -Stored `connections..context.queryHistory.enabled: true` has the same -depth requirement. When no explicit depth flag is passed, stored query-history -enablement upgrades the effective database depth to `deep` for that run. When a -user explicitly passes `--fast` and does not pass `--query-history`, KTX honors -the explicit fast request, skips stored query-history processing for that run, -does not modify `ktx.yaml`, and prints a warning: - -```text -warehouse has query history enabled in ktx.yaml, but --fast skips query-history processing. -``` - -`--query-history-window-days ` overrides -`connections..context.queryHistory.windowDays` only for the current run. It -must not rewrite `ktx.yaml`. The effective value flows into the same -`historicSqlUnifiedPullConfigSchema.windowDays` field used by the current -historic-SQL pull path. - -## Configuration model - -User-authored `ktx.yaml` becomes connection-centric. Database schema ingest is -implied by the database connection and no longer appears as an ingest adapter. - -```yaml -connections: - warehouse: - driver: postgres - readonly: true - context: - depth: fast - queryHistory: - enabled: false - - notion: - driver: notion - context: - enabled: true -``` - -Deep database defaults and query history use the same connection-local shape: - -```yaml -connections: - warehouse: - driver: postgres - readonly: true - context: - depth: deep - queryHistory: - enabled: true - windowDays: 90 - minExecutions: 5 - filters: - dropTrivialProbes: true - serviceAccounts: - mode: exclude - patterns: - - "^svc_" - redactionPatterns: [] -``` - -`context.queryHistory` is the canonical user-facing shape. Runtime code maps it -to the existing historic-SQL pull config as follows: - -- `dialect` is derived from the database driver (`postgres` or `postgresql`, - `bigquery`, or `snowflake`) and is not normally user-authored. -- `windowDays`, `minExecutions`, and `redactionPatterns` copy through directly. -- `filters.dropTrivialProbes` defaults to `true`. -- `filters.serviceAccounts.patterns` and `filters.serviceAccounts.mode` map to - the existing service-account filter fields. The default mode is `exclude`. -- `concurrency`, `staleArchiveAfterDays`, - `filters.orchestrators.mode`, and `filters.dropFailedBelow` are advanced - query-history fields. When present, they map directly to the same fields in - `historicSqlUnifiedPullConfigSchema`. When absent, KTX uses the existing - historic-SQL schema defaults and omitted-field behavior. - -Existing `connection.historicSql` blocks are legacy cutover input. Setup or the -explicit config rewrite path must migrate them into -`connection.context.queryHistory` while preserving all mapped query-history -fields, including the advanced fields listed above. `ktx ingest` must not -rewrite `ktx.yaml`; it may read legacy `historicSql` blocks for the current run -and emit a cleanup warning. If both `context.queryHistory` and `historicSql` are -present, `context.queryHistory` wins and KTX emits a config-cleanup warning -instead of running both. - -Config migration must be idempotent. A setup or explicit rewrite pass that -migrates a connection removes the legacy `connection.historicSql` block after -copying preserved fields, does not regenerate normal `ingest.adapters` entries, -and produces the same `ktx.yaml` on repeated runs. If `ktx ingest` sees a legacy -block before cleanup, the warning may repeat because ingest is config-read-only. - -`ingest.adapters` is no longer normal user config. Existing `ingest.adapters` -entries load as advanced/internal overrides during the transition, but -public `ktx ingest ` must not fail solely because the -driver-to-adapter mapping chooses an adapter missing from that list. The rule -applies to database internals (`live-database` and `historic-sql`) and to all -source adapters selected from configured drivers, including `notion`, `dbt`, -`metabase`, `looker`, `metricflow`, and `lookml`. - -The implementation can satisfy this by bypassing the adapter allow-list for -connection-centric public ingest, or by synthesizing the adapters required by -configured connections before dispatch. The old adapter-backed advanced command -may continue to honor `ingest.adapters` while it exists. Normal generated -`ktx.yaml` must not include `live-database`, `historic-sql`, or source adapter -entries just to make public `ktx ingest ` work. - -## Setup flow - -`ktx setup` remains a guided configuration flow. It does not expose -`ktx setup --fast` or `ktx setup --deep`. - -During interactive setup, KTX asks for database context depth when a database -connection is configured or when setup reaches the context-build step: - -```text -How much database context should KTX build? - -Fast: schema only, no AI, quickest -Deep: AI descriptions, embeddings, relationships, slower -``` - -The recommended selection depends on readiness: - -- Recommend Fast when model, embedding, or scan-enrichment configuration is - missing. -- Recommend Deep when model, embedding, and scan-enrichment configuration are - ready. - -The recommendation is based on the final configuration produced by the current -setup run, not on an earlier intermediate state. Setup must either ask the depth -question after the model, embedding, and scan-enrichment setup paths complete, -or defer or repeat the depth prompt before the foreground context build starts -when those capabilities are configured later in the same setup run. - -Setup stores the chosen default in `connections..context.depth`. The -foreground context build uses that stored default. Setup can still expose a -non-prominent automation flag later, such as `--context-depth fast`, if -headless setup needs it, but the main product surface is guided. - -Setup readiness is depth-aware: - -- For `fast`, a database context is ready when the latest non-dry-run - structural scan for the connection completed and wrote schema manifest shards. - Model, embedding, description-enrichment, and scan-enrichment checks are - skipped for fast contexts. -- For `deep`, a database context is ready only when the enriched scan completed - table descriptions, column descriptions, embeddings, and schema manifest - shards. When relationship discovery is enabled, readiness requires the - relationship stage to have completed for the latest enriched scan. A - completed relationship stage with zero accepted, review, rejected, or skipped - relationships still counts as ready; readiness must not require non-empty - relationship artifacts or accepted relationships. If relationship discovery is - disabled, the relationship stage is not part of the readiness gate. - -The missing-input gate uses the same rule. Missing model, embedding, or -scan-enrichment configuration must not block a user who selected `fast`. The -same missing inputs must block `deep` before the foreground build starts, with a -message that offers `fast` as the no-AI path. - -## Foreground progress UX - -KTX keeps a rich foreground progress view. It removes detach and background -execution. - -The shared build view groups work by user-facing source type: - -```text -Building KTX context (2/4 · 1m 12s) -─────────────────────────────────── - -Databases - ✓ warehouse 42 tables · 6 changed · relationships found - ⠹ billing reading schema · 18/64 tables - -Context sources - ✓ dbt 18 models · 42 metrics - ○ notion queued - -Warnings - --deep ignored for notion; it only applies to database connections. -``` - -The view must not show `scan` or `live-database` in normal mode. It uses: - -- `Databases` instead of `Primary sources`. -- `Context sources` for docs, BI, metrics, and modeling sources. -- `reading schema` or `building schema context` instead of `scanning`. -- `query history` or `usage patterns` instead of `historic-sql`. - -Non-TTY output remains append-only and scriptable. `--json` returns structured -results. Routine artifact paths and internal adapter names appear only in -`--debug` or JSON output. - -## Removing detach and watch - -The context build is foreground only. - -- `Ctrl+C` stops the current run. -- KTX records interrupted or failed state where useful for status reporting. -- Rerunning `ktx setup` or `ktx ingest` starts a fresh foreground build or - reuses existing completed artifacts when safe. - -Remove these user-facing concepts from context build: - -- detach -- attach -- watch -- resume -- stop -- background context-build subprocesses -- prompts that offer "Watch progress" -- hints such as `d to detach` - -Existing `running` or `detached` state from older versions must be treated as -stale or interrupted with a clear rerun instruction. - -`.ktx/setup/context-build.json` remains only as a foreground status cache, not a -background control plane. New writes may use `not_started`, `running`, -`completed`, `failed`, `interrupted`, or `stale`. `running` means the current -foreground process is active; a later setup process that finds a leftover -`running` record from an older process must mark it `stale` or `interrupted` -before offering a fresh run. `detached` and `paused` are legacy-only statuses -and must be normalized to `stale` or `interrupted` on read or on the next setup -write. - -The state file must not keep user-facing `watch`, `resume`, or `stop` command -affordances after this redesign. It may retain run ids, report ids, artifact -paths, source progress, failure details, and a retry/build command when those -help status reporting. - -## Internal naming and migration - -User-facing surfaces must stop saying `live-database`. - -This includes: - -- CLI help. -- Normal command output. -- Setup prompts. -- Generated `ktx.yaml`. -- README quickstart and examples. -- Friendly errors and warnings. - -Internal paths and source keys can keep `live-database` during the first -implementation if renaming them would add risk. Debug output and JSON may -include internal names when they are necessary for troubleshooting. - -The implementation plan must also update stale command suggestions. For -example, setup source recovery must no longer tell users to run -`ktx ingest run --connection-id ... --adapter `. It must suggest the -new connection-centric command: - -```bash -ktx ingest -``` - -## Error handling and warnings - -Warnings are non-fatal when KTX can still perform the requested ingest. - -- Ignored depth flag on a non-database source: warn and continue. -- Ignored query-history flag on an unsupported database: warn and continue if - schema ingest can run. -- Both `--fast` and `--deep`: error before any work starts. -- Explicit or stored `deep` without required model, embedding, or - scan-enrichment readiness: error before any work starts for that target. -- `--query-history` without required model, embedding, or scan-enrichment - readiness: error before any work starts for that target because query history - upgrades the run to `deep`. -- Query-history requested without required grants: fail that query-history - facet and keep schema results when schema ingest succeeded. -- Database schema ingest failure: fail that database target. - -`--all` isolates target failures. It runs all database targets first, then all -source targets, even when one or more database targets fail. Source targets may -therefore run against previously completed database context if the current -database refresh failed. The final exit code is non-zero when any target or -required facet fails, and the summary identifies partial failures by -connection. - -For `--all`, readiness is evaluated per target after resolving each target's -effective depth and query-history settings. A database target whose effective -run requires deep readiness but lacks model, embedding, or scan-enrichment -configuration fails before work starts for that target; eligible database and -source targets still run. Command-level errors that make target planning -impossible, such as mutually exclusive flags, an unreadable project config, or -no eligible targets, still abort before any target work starts. - -Failure messages focus on the connection and user action: - -```text -warehouse failed: connection refused. -Retry: ktx ingest warehouse --deep -``` - -They do not mention internal adapter names unless debug output is enabled. - -## Acceptance criteria - -The implementation is complete when these conditions hold: - -- `ktx ingest ` works for database and source connections. -- `ktx ingest --all` runs database targets before source targets. -- `ktx ingest ` does not require `ingest.adapters` entries for - any adapter chosen from the configured connection driver. -- Connection ids that collide with surviving `ktx ingest` subcommands are - rejected during setup or config validation. -- `--fast` and `--deep` control database depth and are mutually exclusive. -- `--fast` maps to structural database ingest without relationship detection. -- `--deep` maps to enriched database ingest with relationship detection when - `scan.relationships.enabled` is true. -- `--deep` and `--query-history` fail before work starts when required model, - embedding, or scan-enrichment configuration is missing. -- `ktx ingest --all` continues independent targets after partial failures and - exits non-zero when any target or required facet fails. -- `ktx ingest --all` treats deep-readiness failures as per-target failures - after target planning, rather than aborting eligible independent targets. -- `ktx setup` stores a database context depth without exposing top-level - `--fast` or `--deep`. -- `ktx setup` bases the recommended/default database context depth on the final - model, embedding, and scan-enrichment readiness reached by the setup run. -- `ktx setup` treats fast database context as ready after completed structural - schema ingest and does not require AI descriptions or embeddings for fast. -- Generated `ktx.yaml` does not include `live-database` for normal projects. -- Generated `ktx.yaml` uses `connections..context.queryHistory`, not - `connections..historicSql`, for query-history configuration. -- Normal CLI help and output do not mention `live-database`. -- Normal CLI help and output do not present `scan` as a public verb. -- Normal CLI help and output do not present `ktx ingest watch` as live context - build control. -- Query history is optional, connection-local, and overridable per ingest run. -- Query history is supported only for `postgres` or `postgresql`, `bigquery`, - and `snowflake` in v1; unsupported database drivers warn and skip the - query-history facet without blocking schema ingest. -- Stored query-history enablement upgrades default database ingest to deep, but - explicit `--fast` skips stored query history for that run with a warning. -- `--query-history-window-days` overrides the effective historic-SQL - `windowDays` pull config for the current run only and does not rewrite - `ktx.yaml`. -- Legacy `connection.historicSql` migration is idempotent, preserves all mapped - query-history fields, and is performed by setup or an explicit config rewrite, - not by `ktx ingest`. -- Context build has no detach, attach, watch, resume, stop, or background - execution path. -- `.ktx/setup/context-build.json` is retained only as foreground status cache - state; legacy `detached` or `paused` records do not trigger background - recovery branches. -- Existing setup context progress UX is consolidated with `ktx ingest` rather - than duplicated. -- Non-TTY and JSON output remain suitable for scripts. - -## Open implementation questions - -The implementation plan must decide these lower-level details: - -- Whether old `ktx scan` exits with an error, is hidden, or remains as a - temporary undocumented debug command. -- Whether internal artifact paths keep `raw-sources//live-database` - for the first implementation. -- Whether setup needs a headless `--context-depth fast|deep` flag for CI. diff --git a/docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md b/docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md deleted file mode 100644 index b65c77b6..00000000 --- a/docs/superpowers/specs/2026-05-14-research-agent-mcp-tools-design.md +++ /dev/null @@ -1,933 +0,0 @@ -# Research Agent MCP Tools Design - -**Date:** 2026-05-14 -**Author:** Andrey Avtomonov -**Status:** Design — pending implementation plan - -## Background - -KTX positions itself as a standalone context layer for database agents. -External agents — Claude Code, Cursor, Codex, opencode — should be able to -connect to a local KTX instance via MCP and perform research against -configured data connections. - -The existing MCP surface (`packages/context/src/mcp/context-tools.ts`) already -exposes strong **context** primitives: wiki search/read/write, semantic-layer -list/read/write/validate/query, ingest and scan run management, memory -capture. What it is missing is the **active investigation** primitives a -research agent needs: - -- The agent cannot run raw SQL against a connection. `sl_query` only covers - semantic-layer-defined queries. -- The agent cannot inspect raw table or column metadata for tables that are - not yet modeled in the semantic layer. -- The agent cannot find which column holds a literal value mentioned by the - user (e.g., "Acme Corp"). -- The agent must call multiple separate search tools (`wiki_search`, - `sl_list_sources`) and reconcile results manually instead of getting a - unified ranked discovery view. - -The Kaelio research agent (reference implementation at -`/Users/andrey/conductor/workspaces/kaelio-main2/douala/server/src/cores/research-execution.core.ts`) -addresses these gaps with tools named `sql_execution`, `entity_details`, -`dictionary_search`, and `discover_data`, used in a discovery → inspection → -query loop. The corresponding KTX infrastructure already exists in pieces: - -- `KtxScanConnector.executeReadOnly` on every connector - (`packages/connector-postgres/src/connector.ts:447` and siblings) — read-only - SQL execution with `assertReadOnlySql` and `limitSqlForExecution`. -- `KtxSchemaSnapshot` from scan reports — full table/column/FK metadata. -- `SlDictionaryEntry` extraction over relationship-profiling artifacts - (`packages/context/src/sl/sl-dictionary-profile.ts`). -- Hybrid search core with Reciprocal Rank Fusion - (`packages/context/src/search/{hybrid-search-core,rrf}.ts`). - -This design exposes those primitives as four new MCP tools, adds a research -skill to guide external agents, and introduces an HTTP-only `ktx mcp` daemon -to host the MCP server. - -## Goals - -- Expose four new MCP tools that turn KTX into a research-capable context - layer for any MCP-compatible client: `discover_data`, `entity_details`, - `dictionary_search`, `sql_execution`. -- Ship a `ktx-research` skill installable via `ktx setup-agents`, describing - the discover → inspect → query → capture workflow for external agents. -- Provide a `ktx mcp` CLI subtree that runs the MCP server over HTTP on - localhost, with the same lifecycle pattern as the existing managed Python - daemon (`packages/cli/src/managed-python-daemon.ts`). -- Make `ktx setup-agents` install MCP client configuration for the configured - targets pointing at the local HTTP endpoint. v1 splits this by client: for - claude-code and cursor (JSON config), `setup-agents` writes the entry - directly; for codex (TOML) and opencode (different JSON wrapper), - `setup-agents` prints a copy-pasteable snippet rather than writing the file. - See the client matrix below for full per-target behavior. -- Reuse existing infrastructure (connector `executeReadOnly`, schema - snapshots, dictionary profile, hybrid search + RRF) rather than building - parallel implementations. - -## Non-goals - -- This spec does not build an agent loop inside KTX. The system prompt, step - budget, tool dispatch, and methodology tracking remain in the external - client. KTX is a context provider, not an agent runner. -- This spec does not expose Python code execution. The `ktx-daemon` - `/code/execute` endpoint exists but is not surfaced via MCP. That is a - separate design with its own sandboxing and security considerations. -- This spec does not ship widget rendering, chart creation, or scheduled - report execution. Those are presentation concerns the external client owns. -- This spec does not implement stdio MCP transport. HTTP-only. -- This spec does not implement OS-level auto-start (launchd, systemd user - units). `ktx mcp start` must be run explicitly. -- This spec does not implement remote network exposure beyond loopback. Token - auth and non-`127.0.0.1` binding are supported but TLS, audit logging, and - multi-tenant isolation are out of scope for v1. - -## Tool inventory - -Four new MCP tools, registered in `packages/context/src/mcp/context-tools.ts` -alongside the existing tools. - -### Relationship to existing warehouse-verification tools - -KTX already ships ingest-side implementations of `sql_execution`, -`entity_details`, and `discover_data` at -`packages/context/src/ingest/tools/warehouse-verification/{sql-execution,entity-details,discover-data}.tool.ts`, -backed by `warehouse-catalog.service.ts`. Their contracts differ from the -MCP shapes proposed below in three concrete ways: - -- They currently take `connectionName` (slug-shaped); this spec renames - them to `connectionId` in the same change (see below). -- They take `targets` (a discriminated `display` vs. `{catalog,db,name}` - union) and `rowLimit`, not `entities` / `maxRows`. -- They return `{ markdown, structured }` with scan availability, candidate - matches, and ingest-session-allowed-connection scoping, not the - MCP-shaped pure-structured outputs in this spec. - -To avoid two divergent contracts for the same primitives, the MCP tools -**must be implemented by extracting the shared logic out of -`warehouse-verification/*` and into reusable services** -(e.g., `WarehouseCatalogService` as the source of truth for table/column -resolution and discovery, plus a shared read-only SQL executor that wraps -`assertReadOnlySql`/`limitSqlForExecution`). The ingest tools and the new -MCP tools then become thin adapters around those services with their own -input/output shapes appropriate to each surface. - -KTX has no public users yet, so the same change that introduces the MCP -tools renames the ingest-side `connectionName` parameter to `connectionId` -across `warehouse-verification/*.tool.ts`, `warehouse-catalog.service.ts`, -and any callers. `connectionId` matches the rest of the in-process MCP -surface (`sl_query`, `sl_list_sources`, `scan_trigger`, etc.) and the new -MCP tool inputs. The ingest tools and the new MCP tools then share both -the service layer and the parameter name; only their input/output shapes -differ (markdown+structured for the ingest surface, pure structured for -the MCP surface). - -### discover_data - -Unified ranked search across wiki, semantic-layer sources/measures/dimensions, -and raw schema tables/columns. Returns refs only with a uniform shape; the -agent dereferences top hits using the existing `wiki_read`, `sl_read_source`, -or `entity_details` tools. - -**Input schema:** - -```typescript -{ - query: z.string().min(1), - connectionId: z.string().optional(), // omit → all connections - kinds: z.array(z.enum([ - 'wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column', - ])).optional(), // omit → all kinds - limit: z.number().int().min(1).max(50).default(15).optional(), -} -``` - -**Output:** `{ refs: Ref[] }` — the MCP protocol requires `structuredContent` -to be a JSON object, so the array of matches is wrapped under `refs`. Each -ref is shaped: - -```typescript -{ - kind: 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column', - id: string, // stable id: wiki key, source name, or driver-qualified table/column display string - score: number, // RRF fused score, 0-1 range - summary: string | null, // one-line description; null when no source field is populated - snippet: string | null, // short context snippet, ≤200 chars; null when nothing meaningful to show - matchedOn: // why this result matched (powers the snippet for non-description kinds) - | 'name' | 'display' | 'description' | 'comment' | 'expr' | 'sample_value' | 'body', - connectionId?: string, // present for non-wiki kinds - tableRef?: { // present for kind 'table' and 'column' - catalog: string | null, - db: string | null, - name: string, - }, - columnName?: string, // present for kind 'column' -} -``` - -The structured `tableRef` mirrors the live `KtxSchemaTable` identity -(`packages/context/src/scan/types.ts:74-83`) so callers can pass refs into -`entity_details` without losing `catalog`/`db` qualification on drivers -that need it (BigQuery `project.dataset.table`, Snowflake/SQL Server -`database.schema.table`). - -#### `summary` and `snippet` provenance per kind - -Both fields are derived from existing source data, never invented or -LLM-generated. The resolver is pure and deterministic per kind. When no -source field exists for a given kind, the field is `null`; agents must -not assume a missing snippet means "no context" — they should dereference -the ref via `wiki_read`, `sl_read_source`, or `entity_details` to get -authoritative content. - -| Kind | `summary` source | `snippet` source | -|---|---|---| -| `wiki` | `WikiFrontmatter.summary` (`packages/context/src/wiki/types.ts:15`) — populated at write time | Up to 200 chars from the wiki body around the match position; falls back to first 200 chars of body when `matchedOn === 'name'`/`'display'` | -| `sl_source` | `resolveDescription(source.descriptions, priority)` (`packages/context/src/sl/descriptions.ts:16-34`) over the `user|ai|dbt|db` priority chain (`packages/context/src/sl/types.ts:5`) | When `matchedOn === 'description'`/`'body'`: a window of the resolved description; otherwise the source's `name` + first 1–2 measure or dimension names as context | -| `sl_measure` | `measure.description` (`packages/context/src/sl/types.ts:37`) | `measure.expr` truncated to 200 chars — the calculation is the most informative one-line context for a measure | -| `sl_dimension` | `resolveDescription(column.descriptions, priority)` (same precedence as `sl_source`); when empty, fall back to `null` | `${column.name} (${column.type})` formatted exactly like the existing inline rendering in `sl-search.service.ts:29-41` | -| `table` | `firstDescription(table.descriptions)` then `table.comment` (precedence already used by `warehouse-catalog.service.ts:286-287`); `null` when both are empty | When `matchedOn === 'description'`/`'comment'`: a window of that string; when `matchedOn === 'name'`/`'display'`: a comma-joined list of up to 5 of the table's column names | -| `column` | `resolveDescription(column.descriptions)` then `column.comment` (`warehouse-catalog.service.ts:228-245`); `null` when both are empty | When `matchedOn === 'description'`/`'comment'`: that text; when `matchedOn === 'sample_value'`: `${column.nativeType} · samples: ` formatted from `column.sampleValues` (`warehouse-catalog.service.ts:18-23`); otherwise `${column.nativeType}` | - -The `matchedOn` field is the same concept as the existing -`RawSchemaHit.matchedOn` in `warehouse-catalog.service.ts:40-54`, -extended to the wiki and SL kinds. Snippets always come from a single -already-stored field; the resolver never concatenates across sources or -invents bridging text. Length cap is enforced at the producer side (≤200 -chars after a single-pass slice; no ellipsis appended — clients render -one if they want). - -**Implementation:** new module `packages/context/src/search/discover.ts`. -Composes three sub-searches in parallel: - -1. Wiki search via the existing wiki search backend. -2. SL search over sources/measures/dimensions using existing - `sl-sources-index` (or a new lightweight index if needed for measure - granularity). -3. Raw schema search over tables and columns from `KtxSchemaSnapshot`, - indexed at scan time and stored alongside other scan artifacts. - -Results from each sub-search are fused with `packages/context/src/search/rrf.ts` -using equal weights. The `kinds` filter constrains which sub-searches run. - -### entity_details - -Read structured metadata for one or more raw tables (and optionally specific -columns) from the latest scan snapshot. The raw-data equivalent of -`sl_read_source`. - -**Input schema:** - -```typescript -{ - connectionId: z.string().min(1), - entities: z.array(z.object({ - // table accepts either a driver-display string ("project.dataset.table", - // "schema.name", "db.schema.name") or a structured ref. The resolver - // returns a structured error when the input is ambiguous across multiple - // schemas/catalogs. - table: z.union([ - z.string().min(1), - z.object({ - catalog: z.string().nullable(), - db: z.string().nullable(), - name: z.string().min(1), - }), - ]), - columns: z.array(z.string()).optional(), // omit → all columns - })).min(1).max(20), -} -``` - -**Output:** for each entity, a structured record: - -```typescript -{ - connectionId: string, - tableRef: { // structured identity, lossless on every driver - catalog: string | null, // BigQuery project, Snowflake/SQL Server database - db: string | null, // schema/dataset - name: string, - }, - display: string, // driver-formatted display string - // (e.g. "project.dataset.table", "schema.name") - kind: 'table' | 'view' | 'external' | 'event_stream', // matches KtxSchemaTableKind - comment: string | null, - estimatedRows: number | null, - columns: Array<{ - name: string, - nativeType: string, - normalizedType: string, - dimensionType: 'time' | 'string' | 'number' | 'boolean', - nullable: boolean, - primaryKey: boolean, - comment: string | null, - }>, - foreignKeys: Array<{ - fromColumn: string, - toCatalog: string | null, // qualified FK target, preserves cross-db FKs - toDb: string | null, - toTable: string, - toColumn: string, - constraintName: string | null, - }>, - snapshot: { // freshness metadata, present on every response - syncId: string, // latest scan/sync identifier - extractedAt: string, // ISO-8601 UTC of the snapshot - scanRunId: string | null, // scan run id if available - }, -} -``` - -Output fields mirror `KtxSchemaTable` / `KtxSchemaColumn` / -`KtxSchemaForeignKey` from `packages/context/src/scan/types.ts:51-82`. The -full `KtxSchemaTableKind` set is preserved so BigQuery `external` tables -and warehouses with event-stream sources are not silently coerced. FK -target qualification (`toCatalog`/`toDb`) carries through so agents can -write valid SQL for cross-schema or cross-database references without -re-resolving. - -If `columns` is provided, only the requested columns appear in the `columns` -array (PKs and FKs still report on the full table). - -**Implementation:** new module `packages/context/src/scan/entity-details.ts`. -Reads `KtxSchemaSnapshot` from the same store the existing `scan_*` tools -read. No new infrastructure. If the requested table is not in the latest -snapshot, the tool returns a structured error with a suggestion to run -`ktx ingest `. - -**Cache freshness.** Today `WarehouseCatalogService` caches `ConnectionCatalog` -per connection name with no invalidation -(`packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts:248-249`, -`:404-411`). For an ingest tool that runs inside a single short-lived ingest -session that is acceptable, but the MCP daemon is long-lived and serves -clients across multiple `scan_trigger` / `ktx ingest` runs. The MCP adapter -**must** key its cache on the latest scan artifact identity (the `syncId` -derived from the artifact path, or the artifact file mtime) and re-read when -that identity advances. The same rule applies to the shared services backing -`discover_data` and `dictionary_search`. The implementation plan must -either: - -1. Extend `WarehouseCatalogService` (and equivalent dictionary/discover - services) to invalidate cached entries when the underlying artifact - identity advances, or -2. Wrap those services in an MCP-adapter cache layer that performs the - identity check before returning cached values. - -### dictionary_search - -Find which connection, source, and column **profile-sampled** a given literal -value (or substring) such as "Acme Corp" or "shipped". Backed by the existing -`SlDictionaryEntry` extraction over relationship-profiling artifacts. - -**Authoritativeness.** The dictionary index is built from *sampled* values -captured during relationship profiling — by default 5 values per column, -drawn from a sample of up to 10,000 rows -(`packages/context/src/scan/relationship-profiling.ts:409-410`, -`packages/context/src/sl/sl-dictionary-profile.ts:70`). A hit confirms a -column contains the value; a miss is **not** proof that the value is absent -from the column or warehouse — the value may simply have been outside the -profile sample. The tool must surface this distinction in its output and the -research skill must teach agents not to treat a miss as exhaustive. - -**Input schema:** - -```typescript -{ - values: z.array(z.string().min(1)).min(1).max(20), - connectionId: z.string().optional(), // omit → all connections -} -``` - -**Output:** for each input value, the list of matching entries plus -per-connection provenance. Coverage and miss reasons are connection-scoped -because `loadLatestSlDictionaryEntries` iterates each connection's profile -artifact independently -(`packages/context/src/sl/sl-dictionary-profile.ts:96-112`); a single -all-connections call can mix `no_profile_artifact` (one connection never -ran an enriched scan), `value_not_in_sample` (another connection ran but -the literal was outside the sample), and matches in the same response. - -```typescript -{ - // The set of connections actually searched on this call. When the input - // omits connectionId this is every configured connection; otherwise it - // contains the single requested connection. - searched: Array<{ - connectionId: string, - coverage: { - sampledRows: number | null, // profileSampleRows used at profile time - valuesPerColumn: number | null, // sampleValuesPerColumn used at profile time - profiledColumns: number, // count of columns in the dictionary index for this connection - syncId: string | null, // identifier of the profile artifact (null when missing) - profiledAt: string | null, // ISO-8601 UTC of the profile artifact (null when missing) - }, - // Per-connection status, independent of any specific input value: - // ready — profile present with profiled columns - // no_profile_artifact — enriched scan never ran for this connection - // no_candidate_columns — profile present but no columns profile-eligible - status: 'ready' | 'no_profile_artifact' | 'no_candidate_columns', - }>, - results: Array<{ - value: string, // input value - matches: Array<{ - connectionId: string, - sourceName: string, - columnName: string, - matchedValue: string, // actual value found (may differ in case) - cardinality: number | null, // column cardinality if known - }>, - // Per-connection miss reasons for this value, present when that - // connection produced no match. Connections that matched do not appear - // in `misses`. For ready connections with no match, the reason is - // 'value_not_in_sample' (non-authoritative miss). For unready - // connections, the reason mirrors their `status` above. - misses: Array<{ - connectionId: string, - reason: - | 'no_profile_artifact' - | 'no_candidate_columns' - | 'value_not_in_sample', - }>, - }>, -} -``` - -**Matching semantics:** case-insensitive substring match against the -profile-sampled values. Misses are never authoritative — they only state -that the value was not in the captured sample for the listed connection. -`misses[].reason` distinguishes "no enriched scan has run on this -connection" (`no_profile_artifact`), "enriched scan ran but no columns -were profile-eligible" (`no_candidate_columns`), and "scan ran but value -was not in the sample" (`value_not_in_sample`). The research skill must -direct agents to follow up a `value_not_in_sample` miss with -`sql_execution` against the most plausible columns, not to conclude the -value is absent. - -**Cache freshness:** the dictionary index is keyed on the profile artifact -identity (the `syncId` derived from its path or the artifact mtime). When -that identity advances, the daemon re-reads the artifact on next call. See -the `entity_details` cache-freshness note above for the shared rule. - -**Implementation:** new module `packages/context/src/sl/dictionary-search.ts`. -Loads `SlDictionaryEntry` records via the existing extraction code path, -builds a per-connection in-memory index on first call, caches it for the -lifetime of the MCP daemon. Invalidated on next ingest run (the daemon -watches `.ktx/db.sqlite` for changes, or simply re-reads on each call when -the artifact mtime advances). - -### sql_execution - -Execute a read-only SQL query against a configured connection and return the -result. The fallback path for questions the semantic layer does not cover. - -**Input schema:** - -```typescript -{ - connectionId: z.string().min(1), - sql: z.string().min(1), - maxRows: z.number().int().min(1).max(10_000).default(1000).optional(), -} -``` - -**Output:** - -```typescript -{ - headers: string[], - headerTypes?: string[], // driver-mapped type names, one per header; optional - rows: Array>, - rowCount: number, -} -``` - -`headerTypes` is optional because not every connector exposes per-column -type metadata. The current contract makes it optional -(`KtxQueryResult.headerTypes` in `packages/context/src/scan/types.ts:272-277`), -and the SQLite connector currently omits it -(`packages/connector-sqlite/src/connector.ts:237-240`, `:301-308`). When a -connector returns header types, the MCP adapter passes them through -verbatim. When a connector does not, the MCP adapter omits the field rather -than fabricating values. - -**Implementation:** delegates to `KtxScanConnector.executeReadOnly` on the -matching connector. The connector calls `assertReadOnlySql` and -`limitSqlForExecution` (`packages/context/src/connections/read-only-sql.ts`). - -**Read-only enforcement is lexical, not parser-backed.** The current guard -inspects the first token with regex: it accepts queries whose first non-space -token is `SELECT` or `WITH`, and rejects queries whose first non-space token -matches a fixed mutating-verb list. Implications: - -- A CTE that nests a data-modifying statement (e.g., `WITH x AS (INSERT ... - RETURNING *) SELECT ...`, valid in Postgres) passes the first-token check - and would reach the connector. -- Dialect-specific read/write constructs and procedure calls that do not - start with a listed verb are not caught. - -Because `sql_execution` exposes this boundary to external MCP clients, the -tool **must not** be enabled until one of the following holds: - -1. The guard is upgraded to a sqlglot/AST-based read-only check that - inspects every statement and CTE node, with explicit tests for CTE-DML, - `CALL`, `DO`, vendor pragmas, and multi-statement payloads; or -2. Connector-side execution forces a read-only transaction / session (e.g., - `SET TRANSACTION READ ONLY` for Postgres, `READ ONLY` connection for - MySQL, equivalent for each connector), so the guard is defense-in-depth - rather than the sole boundary. - -The implementation plan that follows this spec is required to choose and -land one of those before registering `sql_execution` in the MCP surface. -Errors from `assertReadOnlySql` (whichever implementation) are returned as -structured tool errors so the agent can correct the query and retry. - -## Tool naming convention - -Match the existing KTX MCP convention (no prefix): `discover_data`, -`entity_details`, `dictionary_search`, `sql_execution`. The existing tools -(`wiki_search`, `sl_list_sources`, `scan_trigger`, `memory_capture`) all use -unprefixed snake_case; the new tools follow suit. - -## Connection model - -- `sql_execution` and `entity_details` require `connectionId` — these tools - cannot operate without a target. -- `discover_data` and `dictionary_search` make `connectionId` optional. Omit - it to search across all configured connections; provide it to scope. This - matches the existing pattern for `sl_list_sources({ connectionId? })`. -- All tools are project-locked: the MCP daemon runs in one KTX project dir; - to operate on a different project, restart the daemon with a different - `--project-dir` or `cwd`. - -## MCP daemon: `ktx mcp` - -A new CLI subtree in `packages/cli/src/commands/mcp-commands.ts`, wired into -`cli-program.ts` alongside `setup`, `connection`, `ingest`, `wiki`, `sl`, -`status`, `dev`. - -### Commands - -```bash -ktx mcp start [--port ] [--host ] [--token ] [--foreground] \ - [--allowed-host ...] [--allowed-origin ...] -ktx mcp stop -ktx mcp status -ktx mcp logs [--follow] -``` - -`--allowed-host` and `--allowed-origin` are repeatable. They extend (not -replace) the defaults defined in the security model below. - -### `ktx mcp start` - -Starts a long-lived HTTP MCP server bound to the configured host and port, -serving every tool registered by `createKtxMcpServer`. The server stays alive -until `ktx mcp stop` is invoked or the process is terminated. - -- Default `--host` is `127.0.0.1`. Any value other than `127.0.0.1` or - `localhost` **requires** `--token` (or `KTX_MCP_TOKEN` in the environment); - the command refuses otherwise. -- Default `--port` is 7878. If the port is in use, the command exits with an - error explaining how to choose another. Allocated port is persisted to - `.ktx/mcp.json` for subsequent `status`, `stop`, `logs`, and - `setup-agents` calls. -- `--foreground` runs the server in the foreground and pipes all logs to - stdout, for debugging. Default is background. -- Background runs detach via the same pattern as the managed Python daemon - (`packages/cli/src/managed-python-daemon.ts`): spawn a detached child, - write `pid`, `port`, `startedAt` to `.ktx/mcp.json`, return immediately - with the URL the user should configure in their client. -- Logs go to `.ktx/logs/mcp.log` (matches existing log layout). - -### `ktx mcp stop` - -Reads `.ktx/mcp.json` for the daemon PID, sends SIGTERM, waits up to 10 -seconds for graceful exit, then SIGKILLs if still running. Removes the state -file on success. - -### `ktx mcp status` - -Reads `.ktx/mcp.json`, checks the process is alive, hits the server's -`/health` endpoint, and reports: - -- Running / stopped / stale (state file present but process not alive) -- Port, host, started-at, pid -- Whether token auth is enabled -- Configured project dir - -### `ktx mcp logs` - -Tails or follows `.ktx/logs/mcp.log`. Standard `--follow` flag. - -### Lifecycle - -Manual: the user runs `ktx mcp start` after each reboot or whenever they -want the server running. No auto-start on other `ktx` commands (matches the -explicit pattern established by the daemon model). - -### Transport - -HTTP-only via `StreamableHTTPServerTransport` from -`@modelcontextprotocol/sdk/server/streamableHttp.js`. - -The `/mcp` endpoint must implement the full Streamable HTTP contract, not -just `POST`: - -- `POST /mcp` — JSON-RPC requests (and the `initialize` handshake when no - session exists). On the first `initialize` post, the server allocates a - session id and returns it in the `Mcp-Session-Id` response header. -- `GET /mcp` — opens an SSE stream for server-initiated messages on an - existing session. Requires a valid `Mcp-Session-Id` header. -- `DELETE /mcp` — explicit session termination by the client. Requires a - valid `Mcp-Session-Id` header; the server must drop the session and any - associated SSE streams. - -**Session model.** v1 ships **stateful** sessions: the server generates a -session id with `randomUUID()` on `initialize`, stores the transport in an -in-memory map keyed by session id, reuses it on subsequent -`POST`/`GET`/`DELETE` calls that carry the same `Mcp-Session-Id`, and -removes it on `DELETE` or transport close. Requests that carry an unknown -session id are rejected with HTTP 404 so the client knows to re-initialize. - -Health: `GET /health` returns `{ status: 'ok', projectDir, port }` for -liveness checks. `/health` is separate from `/mcp` and is not subject to -session-id requirements (but is subject to host/origin validation; see -below). - -### Security model - -- `127.0.0.1` binding is the default and requires no token auth (loopback - only). Even on loopback, the server enforces **Host and Origin header - validation** on every `/mcp` and `/health` request to defend against - browser-driven DNS-rebinding attacks (the same defense the MCP SDK - exposes in `createMcpExpressApp` / `createMcpHonoApp`). -- **Host validation** compares the incoming `Host` header to the allowed-host - list after normalizing: lowercase, strip any port, strip surrounding - brackets from IPv6 literals (`[::1]:7878` → `::1`). Comparison is exact - on the normalized host string. The default allowed-host list is - `['localhost', '127.0.0.1', '::1']`. `--allowed-host` values are appended - after the same normalization. -- **Origin validation** compares the full browser `Origin` header (scheme + - host + port) to the allowed-origin list. The default allowed-origin list - is empty: any request that carries an `Origin` header is rejected unless - an explicit `--allowed-origin` entry matches. Non-browser clients that - do not send an `Origin` header (Claude Code, Cursor, Codex, opencode - HTTP transports) are accepted regardless of `Origin`. Each - `--allowed-origin` value must be a full origin string - (e.g., `http://localhost:7878`); KTX validates the format at startup. -- Non-loopback binding requires `--token ` or `KTX_MCP_TOKEN`. The - server checks `Authorization: Bearer ` on **every** `/mcp` method — - `POST`, `GET` (SSE), and `DELETE` — and rejects with HTTP 401 otherwise. - Token enforcement is independent of the session check; both must pass. - When `--host` is non-loopback, the allowed-host list expands to include - the normalized bound host plus any user-supplied `--allowed-host` - values. -- TLS is out of scope. For remote access, document running KTX behind a - reverse proxy (Caddy, nginx) that terminates TLS. - -## Client config installation via `ktx setup-agents` - -`ktx setup-agents` extends its existing per-target file installation -(`plannedKtxAgentFiles` in `packages/cli/src/setup-agents.ts:64`) to also -write MCP server entries. - -The per-client config matrix is **not uniform**. Each client has its own -file location, scope semantics, and entry shape; `setup-agents` must -produce the correct shape per target rather than emit one JSON blob. - -| Target | Scope | MCP config path | Writer behavior | -|---|---|---|---| -| claude-code | user (global) | `~/.claude.json` → root `mcpServers.ktx` | write JSON | -| claude-code | local (per-project, private) | `~/.claude.json` → `projects[].mcpServers.ktx` | write JSON | -| claude-code | project (shared, checked in) | `/.mcp.json` → `mcpServers.ktx` | write JSON | -| cursor | global | `~/.cursor/mcp.json` → `mcpServers.ktx` | write JSON | -| cursor | project | `/.cursor/mcp.json` → `mcpServers.ktx` | write JSON | -| codex | user (global) | `~/.codex/config.toml` → `[mcp_servers.ktx]` (TOML) | print instructions; do not auto-write in v1 | -| opencode | user (global) | `~/.config/opencode/opencode.json` → `mcp.ktx` | print instructions; do not auto-write in v1 | -| opencode | project | `/opencode.json` → `mcp.ktx` | print instructions; do not auto-write in v1 | - -The shared global `~/.claude.json` and per-project `~/.claude.json` → -`projects[...]` scope are both supported because Claude Code's "user" vs. -"local" scopes write to different sub-trees of the same file; `setup-agents` -must select the scope explicitly per invocation. - -Codex and opencode entries are **printed as copy-pasteable snippets** in v1 -because their config formats (TOML for codex, a different JSON wrapper for -opencode) diverge enough from the JSON writers above that mixing them into -the same writer codepath risks silently producing invalid files. This is a -deliberate v1 scoping decision, not a permanent limitation. - -#### Entry shapes by target - -Claude Code (HTTP): - -```jsonc -{ - "mcpServers": { - "ktx": { - "type": "http", - "url": "http://localhost:7878/mcp" - // when token auth is active, env-var expansion only: - // "headers": { "Authorization": "Bearer ${KTX_MCP_TOKEN}" } - } - } -} -``` - -Cursor (HTTP, project `.cursor/mcp.json` or global `~/.cursor/mcp.json`): - -```jsonc -{ - "mcpServers": { - "ktx": { - "url": "http://localhost:7878/mcp" - // when token auth is active, env-var expansion only: - // "headers": { "Authorization": "Bearer ${KTX_MCP_TOKEN}" } - } - } -} -``` - -Codex (printed snippet, `~/.codex/config.toml`): - -```toml -[mcp_servers.ktx] -url = "http://localhost:7878/mcp" -# Codex MCP config does not currently document a headers field; if token -# auth is active, instruct the user to either run KTX on loopback without a -# token or wait for codex header support before enabling. -``` - -opencode (printed snippet, `opencode.json`): - -```jsonc -{ - "mcp": { - "ktx": { - "type": "remote", - "url": "http://localhost:7878/mcp", - "enabled": true - // when token auth is active, env-var expansion only: - // "headers": { "Authorization": "Bearer ${KTX_MCP_TOKEN}" } - } - } -} -``` - -#### Token handling per client - -When `--token` / `KTX_MCP_TOKEN` is active, `setup-agents` writes the bearer -token **only via environment-variable reference** (`Bearer ${KTX_MCP_TOKEN}`), -never as a literal token value. Claude Code, Cursor, and opencode all -support environment-variable expansion inside `headers` values; the -written entry references `${KTX_MCP_TOKEN}` and the user is responsible -for exporting it in the shell that launches the MCP client. - -Rules: - -- **No literal-token writes, anywhere.** Even the user-scope (private) - Claude Code / Cursor config receives env-var references, not the raw - token. This keeps the same writer codepath for every scope and avoids a - branch that materializes secrets. -- **Project-scope (shared, checked-in) configs are gated.** When a token is - active and the user requests a shared scope — `/.mcp.json` - for Claude Code, `/.cursor/mcp.json` for Cursor — `setup-agents` - prints a warning and offers a choice: (a) write the entry with the - `${KTX_MCP_TOKEN}` reference (the file is safe to commit; readers must - export the variable locally), or (b) skip the shared entry and rely on a - user-scope entry instead. The default is (a). -- **Verify header support per client before writing.** The matrix below - reflects the current state of each client's MCP config docs: - - claude-code: supports `headers` with `${VAR}` expansion on HTTP entries. - - cursor: supports `headers` with `${VAR}` expansion on HTTP entries. - - opencode: supports `headers` with `${VAR}` expansion on remote MCP - entries. - - codex: **not currently supported** in published config docs. When a - token is active and the user selects codex, `setup-agents` prints a - warning and skips the codex entry rather than writing an entry that - codex will silently ignore. The recommended workaround is to bind KTX - to loopback without a token for codex users. -- **Implementation acceptance test.** Setup-agents writer tests must assert - that no rendered output contains the literal token string for any - scope/target combination — only the `${KTX_MCP_TOKEN}` reference. - -Port is read from `.ktx/mcp.json` if present, falling back to 7878. The -install manifest (`agentInstallManifestPath`, -`packages/cli/src/setup-agents.ts:60`) tracks each **written** entry so -`ktx setup-agents --remove` can roll back cleanly. The current manifest -entry kinds are `file` and `json-key` -(`packages/cli/src/setup-agents.ts:42-50`); the MCP client writers for -claude-code and cursor add `json-key` entries for their respective config -files. Printed-only snippets for codex and opencode are **not** tracked in -the manifest, and `--remove` does not attempt to mutate user-written -files for those targets; the printed instructions tell the user how to -remove the entry by hand. - -If the daemon is not running when `setup-agents` runs, the command prints a -follow-up hint: "Run `ktx mcp start` to enable the configured KTX MCP -server." It does **not** auto-start the daemon (matches the manual -lifecycle decision). - -## Research skill - -A new skill source file at `packages/cli/src/skills/research/SKILL.md`, -installed by `ktx setup-agents` to all configured targets. The skill is -separate from the existing setup skill (different triggers: "work in a KTX -project" vs. "answer a data question") and lives in its own per-target -folder so global vs. project scope and removal stay clean. - -`plannedKtxAgentFiles` in `packages/cli/src/setup-agents.ts:64` is extended -to return both the existing `ktx` entries and new `ktx-research` entries: - -| Target | Scope | Path | -|---|---|---| -| claude-code | global | `~/.claude/skills/ktx-research/SKILL.md` | -| claude-code | project | `.claude/skills/ktx-research/SKILL.md` | -| codex | global | `${CODEX_HOME}/skills/ktx-research/SKILL.md` | -| codex | project | `.agents/skills/ktx-research/SKILL.md` | -| cursor | project | `.cursor/rules/ktx-research.mdc` | -| opencode | project | `.opencode/commands/ktx-research.md` | -| universal | project | `.agents/skills/ktx-research/SKILL.md` | - -The skill body is identical across targets; only the wrapper format and -file path differ to match each target's convention. - -### Skill content - -```markdown ---- -name: ktx-research -description: Use when answering a question that needs data from a KTX-connected database — investigating, analyzing, "how many", "show me", "what's the breakdown of", finding records by value, exploring tables, comparing periods, or any data-investigation request. Triggers even when the user does not say "research"; if the answer requires querying a configured KTX connection, this skill applies. ---- - -# KTX Research Workflow - -You have access to KTX MCP tools for investigating data. Follow this workflow. - - -1. **Discover** — call `discover_data(query)` first to see what exists across wiki, semantic-layer sources, and raw tables. Returns refs only. -2. **Inspect top hits in parallel** — for each promising ref: - - `kind: 'wiki'` → `wiki_read(key)` - - `kind: 'sl_source'` / `'sl_measure'` / `'sl_dimension'` → `sl_read_source(connectionId, sourceName)` - - `kind: 'table'` / `'column'` → `entity_details(connectionId, entities)` -3. **Resolve literals** — if the user named a value (e.g., "Acme Corp", "status=shipped"), call `dictionary_search(values)` to find which column holds it. -4. **Query** — - - Prefer `sl_query` when the semantic layer covers the question (joins, measures pre-defined). - - Use `sql_execution` only for things the semantic layer doesn't cover. -5. **Capture learnings** — at the end of the turn, call `memory_capture(userMessage, assistantMessage)` so future turns benefit. Skip when the answer carries no durable knowledge (e.g., the user only asked for schema info). - - - -- Always run `discover_data` before writing SQL. Do not guess table names. -- Prefer the semantic layer over raw SQL when both can answer the question — measures are the source of truth. -- Read entity details before writing SQL against an unfamiliar table; do not assume column names. -- Treat `sql_execution` as read-only. Writes are rejected by the server. -- Validate value mentions with `dictionary_search` instead of guessing case/spelling — but treat a `dictionary_search` *miss* as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent. - - - -**Input:** "How many orders did Acme Corp place last month?" - -**Output workflow:** -1. `dictionary_search(["Acme Corp"])` → finds `customers.name` -2. `discover_data("orders customer monthly")` → finds `orders_facts` SL source -3. `sl_read_source("warehouse", "orders_facts")` → confirms measure `order_count`, dim `customer_name`, dim `ordered_at` -4. `sl_query({ measures: ["order_count"], filters: ["customer_name = 'Acme Corp'", "ordered_at >= date_trunc('month', now() - interval '1 month')"], dimensions: [{ field: "ordered_at", granularity: "month" }] })` -5. `memory_capture(userMessage, assistantMessage)` - ---- - -**Input:** "What columns does the events table have?" - -**Output workflow:** -1. `discover_data("events table")` → top hit `kind: 'table', id: 'analytics.events'` -2. `entity_details("warehouse", [{ table: "analytics.events" }])` → returns columns, types, FKs -3. Answer directly. (No query needed; no `memory_capture` since no durable learning.) - -``` - -## Files - -### New - -- `packages/context/src/scan/entity-details.ts` — derives entity-detail - records from `KtxSchemaSnapshot`, sharing resolution logic with - `warehouse-verification/warehouse-catalog.service.ts` (refactored or - imported, not duplicated). -- `packages/context/src/sl/dictionary-search.ts` — builds and queries the - dictionary index over relationship-profiling artifacts. -- `packages/context/src/search/discover.ts` — composes wiki, SL, and raw - schema searches; fuses results via `rrf.ts`. Reuses the same wiki/SL/raw - search building blocks as `warehouse-verification/discover-data.tool.ts`. -- `packages/cli/src/commands/mcp-commands.ts` — `ktx mcp start|stop|status|logs`. -- `packages/cli/src/managed-mcp-daemon.ts` — daemon lifecycle (spawn, - pidfile, log management), mirroring `managed-python-daemon.ts`. -- `packages/cli/src/skills/research/SKILL.md` — research workflow skill. -- Tests for each new module following existing patterns - (`*.test.ts` siblings), including coverage of the per-client config - writer/printer matrix. - -### Modified - -- `packages/context/src/mcp/context-tools.ts` — register the four new tools - with their Zod schemas. -- `packages/context/src/mcp/server.ts` — extend `KtxMcpContextPorts` with the - new ports (`sqlExecution`, `entityDetails`, `dictionarySearch`, `discover`). -- `packages/context/src/mcp/types.ts` — add the new port interface - definitions. -- `packages/cli/src/cli-program.ts` — register the `mcp` command subtree. -- `packages/cli/src/setup-agents.ts` — install the research skill and write - MCP client config entries to each configured target. - -## Testing strategy - -- Unit tests for each new module (`entity-details.ts`, - `dictionary-search.ts`, `discover.ts`) using existing fixture patterns. -- MCP-level integration test in `packages/context/src/mcp/server.test.ts` - that registers a fake server, invokes each tool, and asserts the - responses. -- CLI integration test for `ktx mcp start|stop|status` lifecycle following - the pattern in `managed-python-daemon.test.ts`. -- Setup-agents tests verifying behavior per target: claude-code and cursor - writers add the correct JSON entry and a corresponding `json-key` - manifest entry that `--remove` cleans up; codex and opencode targets - produce printed snippet output and do not mutate any user config file - or add manifest entries in v1. -- Verification commands per CLAUDE.md: `pnpm --filter @ktx/context run test` - and `pnpm --filter @ktx/cli run test` for the affected packages, plus - `pnpm run type-check`. - -## Out of scope / follow-ups - -- **Python code execution via MCP.** The daemon's `/code/execute` endpoint - exists; surfacing it via MCP is a separate design with sandbox/security - considerations. -- **Stdio MCP transport.** HTTP-only for now. Stdio can be added later as an - additional transport mode without changing the tool surface. -- **OS-level auto-start.** Manual `ktx mcp start` only. Adding launchd / - systemd unit installation is a UX polish for a later release. -- **TLS in the daemon itself.** Reverse proxy is the documented path. Native - TLS support if/when demand emerges. -- **Multi-project / project-switching MCP.** One daemon per project. A - cross-project model would require per-call `projectDir` arguments or a - `set_project_dir` tool and is deferred. -- **Audit logging, rate limiting, per-tool authorization.** Not in scope for - v1; the security boundary is loopback or bearer token. - -## Open trade-offs - -- **`dictionary_search` requires `--deep` (enriched) scan to have run.** The - relationship-profiling artifact that powers the dictionary index is only - produced by enriched scans. The tool reports this distinctly when missing, - but the dependency is real: without enriched scan, the tool returns - empty. -- **`entity_details` reads from the latest snapshot, not live.** If the - database schema changes after the last scan, the tool will reflect the - scan state, not reality. Surfacing this clearly in the tool's response - (snapshot timestamp) is part of the implementation. -- **No streaming for `sql_execution`.** Large results are capped at - `maxRows` (default 1000, max 10k). The tool returns the full result set - in one response. Streaming partial results is left for a later iteration - if real workloads demand it. diff --git a/docs/superpowers/specs/2026-05-15-claude-code-backend-design.md b/docs/superpowers/specs/2026-05-15-claude-code-backend-design.md deleted file mode 100644 index ed139296..00000000 --- a/docs/superpowers/specs/2026-05-15-claude-code-backend-design.md +++ /dev/null @@ -1,698 +0,0 @@ -# Brainstorm: `claude-code` backend with full KTX LLM parity - -Adds a `claude-code` backend that gives KTX full parity with the existing -`ANTHROPIC_API_KEY`-based `anthropic` backend for **all KTX LLM calls**. The -backend uses `@anthropic-ai/claude-agent-sdk` and reuses the user's existing -local Claude Code authentication. Users select it in `ktx.yaml`. - -This is not an implementation plan. It is the revised design after expanding -the requirement from "`ktx ingest` works with Claude Code" to "every KTX LLM -call works with Claude Code." The follow-up implementation plan should be -written separately. - -## Core decision - -`claude-code` is a first-class global LLM backend. Any code path that currently -works with `llm.provider.backend: anthropic` must work with -`llm.provider.backend: claude-code`, unless it is not an LLM call at all. - -This includes: - -- Agent loops implemented through `AgentRunnerService.runLoop(...)`. -- Text generation through `generateKtxText(...)`. -- Structured object generation through `generateKtxObject(...)`. -- Local ingest and MCP-triggered local ingest flows. -- Page triage and light extraction. -- Context-candidate curation and reconciliation. -- Memory capture. -- Scan/enrichment internals and relationship LLM proposals. -- Future KTX LLM call sites that use the shared runtime boundary. - -Commands that do not use LLMs do not need special Claude Code behavior. There -must be no silent fallback from `claude-code` to gateway, Anthropic API-key -execution, or deterministic output. - -## Goals - -- Let a KTX user run all KTX LLM-backed behavior through their existing local - Claude Code session without provisioning `ANTHROPIC_API_KEY`, Vertex - credentials, or an AI Gateway key. -- Preserve the existing user-facing CLI and MCP behavior. `claude-code` changes - how LLM calls execute, not which KTX workflows exist. -- Preserve role-based model selection. `llm.models.default`, `triage`, - `candidateExtraction`, `curator`, `reconcile`, and `repair` remain the source - of model selection for every LLM call. -- Preserve KTX's curated tool boundaries. Claude Code built-ins, - filesystem-discovered MCP servers, hooks, skills, plugins, agents, and slash - commands must not become invokable in KTX agent loops. The Agent SDK init - message may still report host-discovered slash commands, skills, and agents; - KTX treats that metadata as diagnostic only and restricts execution through - `tools: []`, exact KTX MCP `allowedTools`, `disallowedTools`, and - deny-by-default `canUseTool`. -- Keep embeddings independent. Claude does not provide embeddings; users keep - configuring `ingest.embeddings` and scan/enrichment embeddings as they do - today. -- Fail fast with a clear message if local Claude Code authentication is not - usable. - -## Non-goals - -- **Embedding parity.** Embeddings remain separate from LLM execution. -- **Tool-call repair parity in the first pass.** The AI SDK runner uses - `experimental_repairToolCall` (`packages/llm/src/repair.ts:35-88`). The Claude - Agent SDK has no transparent same-step repair hook. MVP behavior is next-turn - self-correction from schema errors or a normal tool-failure count. -- **OTEL telemetry parity in the first pass.** The AI SDK runner uses - `experimental_telemetry`. The Agent SDK exposes hooks such as - `PostToolUseFailure` and `SessionEnd`, but no drop-in OTEL switch. MVP ships - without telemetry parity on this backend. -- **Productizing Claude subscription limits.** Documentation must frame this as - "use your own local Claude Code session," not as a third-party Claude Max or - Claude.ai product feature. - -## Approaches considered - -### Recommended: global LLM runtime port - -Introduce a backend-neutral KTX LLM runtime port for operations, not just model -construction: - -```ts -interface KtxLlmRuntimePort { - generateText(input: KtxGenerateTextInput): Promise; - generateObject(input: KtxGenerateObjectInput): Promise; - runAgentLoop(params: RunLoopParams): Promise; -} -``` - -The existing `anthropic`, `vertex`, and `gateway` backends implement the runtime -through the AI SDK and existing `KtxLlmProvider`. The new `claude-code` backend -implements the same runtime through `@anthropic-ai/claude-agent-sdk`. - -This is the recommended approach because KTX call sites need operations: -"generate text," "generate a structured object," and "run an agent loop." They -do not inherently need direct access to an AI SDK `LanguageModel`. The Agent SDK -is a session/agent API, not an AI SDK model factory, so the runtime port avoids -pretending those APIs are the same. - -### Rejected: fake AI SDK `LanguageModel` for Claude Code - -Trying to make Claude Code look like an AI SDK `LanguageModel` would be brittle. -The Agent SDK owns session execution, permissions, MCP tools, structured output, -and result messages. Those semantics do not map cleanly onto a normal -`getModel(...)` return value. - -### Rejected: branch at every call site - -Adding `if backend === "claude-code"` around each LLM call would work briefly -but would duplicate prompt wrapping, structured output handling, debug logging, -tool conversion, auth checks, and error mapping. It would also make future LLM -call sites easy to miss. - -## Architecture - -```text -ktx.yaml - llm.provider.backend: anthropic | vertex | gateway | claude-code - llm.models.: model alias or model ID - -createLocalKtxLlmRuntimeFromConfig(project.config.llm) - -> AiSdkKtxLlmRuntime - - wraps existing KtxLlmProvider - - generateText / Output.object / AgentRunnerService - -> ClaudeCodeKtxLlmRuntime - - uses @anthropic-ai/claude-agent-sdk query() - - implements text, object, and agent-loop operations - -All KTX LLM call sites - -> KtxLlmRuntimePort -``` - -The runtime is selected at the same boundaries that currently construct an -`llmProvider` or `AgentRunnerService`: - -- `packages/context/src/llm/local-config.ts` -- `packages/context/src/ingest/local-bundle-runtime.ts` -- `packages/context/src/memory/local-memory.ts` -- `packages/context/src/scan/local-scan.ts` -- `packages/context/src/mcp/local-project-ports.ts` -- Any CLI setup/status/doctor code that validates LLM readiness - -After the change, services should not need to know whether the configured -backend is AI SDK based or Claude Code based. They call the runtime operation -they need. - -## LLM call-site migration - -The implementation plan must migrate every current KTX LLM call site to the -runtime port: - -- `packages/context/src/llm/generation.ts`: `generateKtxText` and - `generateKtxObject` become runtime-backed helpers or are folded into the - runtime. -- `packages/context/src/agent/agent-runner.service.ts`: the AI SDK agent loop - becomes the AI SDK implementation of `runAgentLoop`. -- `packages/context/src/ingest/page-triage/page-triage.service.ts`: page triage - and light extraction depend on `KtxLlmRuntimePort`, not raw `KtxLlmProvider`. -- `packages/context/src/scan/description-generation.ts`: AI descriptions use - the runtime text-generation operation. -- `packages/context/src/scan/relationship-llm-proposal.ts`: relationship - proposals use the runtime object-generation operation. -- `packages/context/src/ingest/stages/stage-3-work-units.ts`, - `packages/context/src/ingest/stages/stage-4-reconciliation.ts`, - `packages/context/src/ingest/context-candidates/curator-pagination.service.ts`, - and `packages/context/src/memory/memory-agent.service.ts`: agent loops use the - runtime agent-loop operation or a thin `AgentRunnerPort` backed by it. -- Test helpers and MCP local project ports that inject `llmProvider` or - `agentRunner` must either inject the runtime port or use compatibility test - adapters during the migration. - -The plan must include a grep-based audit so new or overlooked `getModel(...)`, -`generateKtxText(...)`, `generateKtxObject(...)`, `AgentRunnerService`, and -`llmProvider` usages are either migrated or explicitly proven non-runtime. - -## Config design - -The config should make `claude-code` a first-class backend: - -```yaml -llm: - provider: - backend: claude-code - models: - default: sonnet - triage: haiku - candidateExtraction: sonnet - curator: sonnet - reconcile: sonnet - repair: sonnet -``` - -Implementation implications: - -- Extend `KTX_LLM_BACKENDS` in `packages/context/src/project/config.ts` and - `KtxLlmBackend` in `packages/llm/src/types.ts`. -- Update setup, status, doctor, schema generation, examples, and docs so - `claude-code` is understood everywhere `anthropic` is understood. -- Update `createKtxLlmProvider` / `createModelFactory` so unsupported backend - values throw instead of falling through to gateway. -- Keep `llm.models` as the per-role binding source. The Claude Code runtime maps - each KTX role to the configured model string for the current call. -- Define accepted model aliases, such as `sonnet`, `opus`, and `haiku`, and full - model IDs supported by the pinned SDK version. - -## Claude Agent SDK runtime behavior - -Every Agent SDK call must be isolated enough for KTX execution. Use explicit -options even when SDK defaults currently match the desired value. - -For agent loops with tools: - -```ts -query({ - prompt, - options: { - cwd: project.projectDir, - systemPrompt, - model: resolveModel(modelRole), - maxTurns: stepBudget, - settingSources: [], - skills: [], - plugins: [], - mcpServers: { ktx: createSdkMcpServer({ name: "ktx", tools }) }, - tools: [], - allowedTools: [/* exact mcp__ktx__ ids generated from the tool map */], - canUseTool: ktxCanUseTool, - permissionMode: "dontAsk", - persistSession: false, - env: ktxClaudeCodeEnv - } -}); -``` - -`ktxClaudeCodeEnv` is the controlled environment described in -"Agent SDK environment and auth boundary" below; it must be passed on every -KTX `query()` call. - -For plain text generation: - -- Use the same `query()` runtime with `maxTurns: 1`. -- Pass `settingSources: []`, `skills: []`, `plugins: []`, `tools: []`, - `permissionMode: "dontAsk"`, `persistSession: false`, and - `env: ktxClaudeCodeEnv`. -- Do not expose MCP tools unless the KTX call explicitly passed tools. -- Return the final result message text. - -For structured object generation: - -- Use the same `query()` runtime with the Agent SDK structured output option - for JSON schema output, plus the same isolation tuple including - `env: ktxClaudeCodeEnv`. -- Convert KTX Zod schemas at the runtime boundary. -- Parse and validate the returned object with the original KTX schema before - returning it to the caller. - -The plan must confirm the exact option names against the pinned SDK version, but -the required outcome is fixed: - -- Filesystem settings are not loaded. The SDK's documented default for an - omitted `settingSources` is `["user", "project", "local"]` - (`@anthropic-ai/claude-agent-sdk@0.3.142` `sdk.d.ts:1686-1695`), - which would inherit the user's Claude Code filesystem settings. Every KTX - `query()` call site - agent loops, text generation, object generation, and - the auth probe - MUST pass `settingSources: []` explicitly, along with - `skills: []`, `plugins: []`, `tools: []`, `persistSession: false`, and no - `mcpServers` entries other than the KTX MCP server (omitted entirely when - the call site does not expose tools). The implementation MUST assert from - the SDK init message that the controlled execution surface matches KTX's - expectations: - - - `message.tools` equals the exact generated KTX MCP tool ids for the current - call. - - `message.mcp_servers` equals the expected KTX MCP server set: `[]` when the - call exposes no tools, or `["ktx"]` when it does. - - `message.plugins` is empty. - - The implementation MUST NOT reject a run solely because - `message.slash_commands`, `message.skills`, or `message.agents` contain - host-discovered names. In `@anthropic-ai/claude-agent-sdk@0.3.142`, those - fields can report host discovery even when KTX passes the isolation options. - They are not part of the KTX execution surface when `tools: []`, - `allowedTools`, `disallowedTools`, and deny-by-default `canUseTool` are set. -- `skills: []` is a context filter in the pinned SDK - (`sdk.d.ts:1697-1718`): unlisted skills are hidden from the model's skill - listing and rejected by the Skill tool, but discovered skill names may still - appear in init metadata. KTX must still pass `skills: []`. -- Plugins are disabled with `plugins: []`, and the runtime asserts that - `message.plugins` is empty in the init message. -- Built-in tools are disabled by setting `tools: []`. The pinned SDK type - (`@anthropic-ai/claude-agent-sdk@0.3.142`, `sdk.d.ts`) documents `tools` as - the base set of built-in tools, with `[]` meaning "disable all built-ins"; - `tools` does not accept MCP tool ids and cannot be used to restrict MCP - availability. -- MCP tool availability is granted by registering the KTX MCP server through - `mcpServers`. The SDK does not document a wildcard like `mcp__ktx__*` for - any tool field; KTX must enumerate exact generated MCP tool ids of the form - `mcp__ktx__` (derived from the tool map handed to - `createSdkMcpServer`) wherever a list of tool ids is required. -- Pre-approval under `permissionMode: "dontAsk"` is configured by listing those - same exact `mcp__ktx__` ids in `allowedTools` (documented as - auto-allow without prompting). Treat `allowedTools` as auto-approval, not - restriction. -- Defense-in-depth restriction uses `canUseTool`. The KTX runtime supplies a - `canUseTool` handler that allows only tool names in the current KTX MCP tool - map and denies everything else, so host-discovered slash commands, skills, - agents, future SDK defaults, or a misconfigured MCP server cannot expand the - execution surface. -- `disallowedTools` MUST additionally list the current built-in tool names - (`Agent`, `Task`, `AskUserQuestion`, `Bash`, `Read`, `Edit`, `Write`, `Glob`, - `Grep`, `WebFetch`, `WebSearch`, `TodoWrite`) as redundant insurance. -- `cwd` is `project.projectDir`, resolved at startup via `resolveKtxProjectDir`, - not `process.cwd()`. -- Sessions are not persisted unless the plan identifies a concrete debugging - feature that needs persistence. - -## Agent SDK environment and auth boundary - -The Agent SDK's `query()` option `env` (`@anthropic-ai/claude-agent-sdk@0.3.142` -`sdk.d.ts:1265-1279`) is the environment passed to the Claude Code child -process and defaults to `process.env`. Without an explicit `env`, the SDK -inherits the parent's environment, including any `ANTHROPIC_API_KEY`, -`ANTHROPIC_AUTH_TOKEN`, `ANTHROPIC_BASE_URL`, gateway/AI-Gateway tokens, -`GOOGLE_APPLICATION_CREDENTIALS` / `CLOUD_ML_REGION` (Vertex), and -`AWS_*` (Bedrock) credentials — any of which can switch the Claude Code CLI's -authentication source to API-key or another provider, bypassing the user's -local Claude Code session. That would silently violate the core requirement -that `claude-code` runs through the user's existing local Claude Code session -and that there is no silent fallback to gateway, Anthropic API-key, or other -provider execution. - -Every `claude-code` `query()` call site - agent loops, text generation, -object generation, and the auth probe - MUST pass an explicit `env` -(`ktxClaudeCodeEnv`) constructed from `process.env` with the following -denylist removed: - -- `ANTHROPIC_API_KEY` -- `ANTHROPIC_AUTH_TOKEN` -- `ANTHROPIC_BASE_URL` -- `ANTHROPIC_MODEL` (provider-routing override) -- `ANTHROPIC_VERTEX_PROJECT_ID`, `CLOUD_ML_REGION`, - `GOOGLE_APPLICATION_CREDENTIALS`, `GOOGLE_CLOUD_PROJECT` -- `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, - `AWS_REGION`, `AWS_PROFILE` -- `CLAUDE_CODE_USE_BEDROCK`, `CLAUDE_CODE_USE_VERTEX` -- Any future provider-routing variables the pinned SDK version documents - -The denylist is the source of truth and lives next to the runtime constructor -so adding a variable is a single-file change. - -Acceptance criteria: - -- The constructed `ktxClaudeCodeEnv` does not contain any denylisted key, and - this is verified by a unit test that seeds each denylisted key in a fake - `process.env`. -- The auth probe fails with the same "authenticate Claude Code locally" - message even when `ANTHROPIC_API_KEY` (or any other denylisted credential) - is present in `process.env` and no valid local Claude Code session exists. -- Every KTX-originated `query()` invocation is spied to assert that `env` - was passed and that it does not contain any denylisted key; the test fails - if any code path falls back to the SDK default `process.env`. -- The "no silent fallback" rule is preserved end-to-end: a machine with - `ANTHROPIC_API_KEY` set but no local Claude Code authentication still fails - setup/status/doctor on `claude-code`. - -## Tool boundary - -Agent-loop tools cannot remain only raw AI SDK `Record` values if -two backends must consume them. The plan must define a backend-neutral tool -descriptor for the final tool map handed to an agent loop: - -```ts -interface KtxRuntimeToolDescriptor { - name: string; - description: string; - inputSchema: z.ZodObject; - execute(input: TInput): Promise>; -} - -interface KtxRuntimeToolOutput { - // What the model sees as the tool_result content. Always a markdown string; - // never a raw JS object. This matches BaseTool's existing - // `toModelOutput` contract (`packages/context/src/tools/base-tool.ts:154-162`) - // which sends only markdown to the LLM. - markdown: string; - // Out-of-band payload preserved for tool callers (transcripts, debug, - // verification ledger, downstream KTX consumers). Not sent to the model. - structured?: TOutput; -} -``` - -Every composed tool entry must produce this descriptor shape, including: - -- `BaseTool` outputs from factory toolsets, which already return - `{ markdown, structured }`. -- Source-specific raw tools such as `emit_historic_sql_evidence` in - `packages/context/src/ingest/local-bundle-runtime.ts`. -- Stage-local tools in `buildWuToolSet` and `buildReconcileToolSet`. -- Inline `load_skill`, read/raw/span, stage/diff, eviction, and emit tools in - `packages/context/src/ingest/ingest-bundle.runner.ts`. -- Memory-agent `load_skill` in - `packages/context/src/memory/memory-agent.service.ts`. -- The `withVerificationLedger` wrapping layer, whose markdown/structured - guard outputs (`packages/context/src/ingest/tools/verification-ledger.tool.ts:40-97`) - already match the contract. - -### Tool output contract - -The runtime defines a single output contract for both backends so the model -sees the same content regardless of provider: - -- **Model-visible content**: the `markdown` field, mapped to the Agent SDK - tool handler return as `{ content: [{ type: "text", text: markdown }] }` for - `claude-code`, and surfaced through the existing `toModelOutput` markdown - path for AI SDK backends. The model never sees raw JS objects. -- **Structured payload**: the optional `structured` field, preserved on the - in-process tool-result envelope for transcript/debug capture, the - verification ledger, and any KTX caller that introspects results. The - Claude adapter does not put structured JSON into model-visible content - unless an individual call site explicitly opts in. -- **Normalization of existing raw tools**: tools that today return a bare - string (e.g. `load_skill` "Skill not available" responses in - `packages/context/src/ingest/ingest-bundle.runner.ts:697-721` and - `:924-936`, and `packages/context/src/memory/memory-agent.service.ts:128-152`) - must be wrapped at the descriptor boundary so `markdown` is the string and - `structured` is omitted. Tools that today return a plain object (e.g. - skill payload `{ name, content, skillDirectory }`) must be wrapped so - `markdown` is a deterministic human-readable rendering (e.g. the skill - body with a header) and the original object is preserved on `structured`. - No KTX tool may return a raw object as the model-visible payload on the - Claude Code backend, because the Agent SDK MCP handler will otherwise - stringify it and drop the structured fields. -- **AI SDK parity**: the AI SDK adapter MUST preserve BaseTool's existing - `toModelOutput` markdown-only behavior. Migrating BaseTool-derived tools - to the descriptor must not start sending structured JSON to the model. - -The AI SDK adapter converts descriptors to `tool(...)` with a `toModelOutput` -that emits `markdown` only. The Claude Code adapter converts descriptors to -Agent SDK `tool(name, description, schema.shape, handler)` entries inside -`createSdkMcpServer(...)` and returns `{ content: [{ type: "text", text: -markdown }] }`. - -Non-object schemas are unsupported for `claude-code` and must be rejected at -startup with a clear error. In practice KTX tool inputs are already `z.object`. - -## Stop reasons and failures - -The Claude runner maps the SDK's typed `SDKResultMessage` (union of -`SDKResultSuccess` and `SDKResultError` in -`@anthropic-ai/claude-agent-sdk@0.3.142`, `sdk.d.ts`) to -`RunLoopStopReason = "budget" | "natural" | "error"`. The mapping must consider -three typed signals in this precedence order, because each successive signal -may be present where the previous one is absent: - -1. `subtype`: `"error_max_turns"` -> `"budget"`; `"success"` -> `"natural"`; - other error subtypes (`"error_during_execution"`, - `"error_max_budget_usd"`, `"error_max_structured_output_retries"`) -> - `"error"`. -2. `terminal_reason` (optional `TerminalReason` field on both success and - error results): `"max_turns"` -> `"budget"`; `"completed"` -> `"natural"`; - any other terminal reason such as `"blocking_limit"`, - `"rapid_refill_breaker"`, `"prompt_too_long"`, `"image_error"`, - `"model_error"`, `"aborted_streaming"`, `"aborted_tools"`, - `"stop_hook_prevented"`, `"hook_stopped"`, or `"tool_deferred"` -> - `"error"`. -3. The assistant message `stop_reason`: `"max_turns"` -> `"budget"`; any - other non-null unsuccessful stop reason -> `"error"`. - -A `max_turns` signal arriving through any of the three sources must map to -`"budget"`; the runner MUST NOT classify a max-turn termination as -`"natural"` or as a generic `"error"` because it was reported via -`terminal_reason` instead of `subtype`. - -`Stop` hooks are not the authoritative stop-reason source because they do not -carry the terminal reason. They remain useful for lifecycle logging. Tool failure -counting should use `PostToolUseFailure` and feed the same mechanism that -`stage-3-work-units.ts` checks through `toolFailureCount?(wu.unitKey)`. - -For text and object generation, SDK authentication, billing, rate-limit, -permission, max-turn, structured-output, and execution errors must map to the -same error surfaces that KTX uses for the Anthropic API-key backend. - -## Agent-loop progress callbacks - -`RunLoopParams.onStepFinish` -(`packages/context/src/agent/agent-runner.service.ts:20`) is part of the -current agent-loop contract. The AI SDK runner increments `stepIndex` on each -`generateText` step and invokes the callback -(`agent-runner.service.ts:83-97`). KTX consumers depend on this: -`packages/context/src/ingest/ingest-bundle.runner.ts:782` emits -`work_unit_step` events from it, and `:1036` / `:1089` update reconciliation -progress for the user-visible "Reconciling results · step N" status. - -The `claude-code` runner MUST preserve `onStepFinish` semantics: - -- It MUST invoke `onStepFinish` exactly once per assistant turn (i.e. once per - step the SDK reports), incrementing `stepIndex` starting at 1. -- The plan MUST name the concrete SDK stream event used as the step boundary - (the implementation plan picks one of the documented assistant/result - message events from the pinned SDK version and justifies it). The chosen - event must produce the same `stepIndex` count as the AI SDK runner for an - equivalent run: N tool-using turns yield N callbacks. -- Callback errors MUST be caught and logged at `warn` level without aborting - the loop, matching `agent-runner.service.ts:90-96`. -- `stepBudget` passed to the callback MUST equal the `maxTurns` configured on - the SDK `query()` call. - -Acceptance criteria: - -- A `claude-code` agent loop run with `stepBudget: N` produces N - `work_unit_step` events when the loop runs to budget. -- A reconciliation run under `claude-code` produces the same - `updateProgress` calls (count and `stepIndex / stepBudget` ratio) as the - Anthropic API-key backend for an equivalent fixture. -- An `onStepFinish` callback that throws does not surface the error as the - loop result. - -## Prompt caching parity - -`packages/llm/src/types.ts:44, :61` exposes `llm.promptCaching` as a config -field, and the AI SDK message builder -(`packages/llm/src/message-builder.ts:62-114, :141-218`) applies -`anthropic.cacheControl: { type: "ephemeral", ttl }` markers to the system -message, the last history message, and sorted tools, with TTLs split into -`systemTtl`, `toolsTtl`, and `historyTtl`. `model-provider.test.ts:276` -verifies caching is enabled by default with those three TTLs. - -The Agent SDK does not expose KTX's marker-based contract. The closest -mechanism is `systemPrompt: string[]` with -`SYSTEM_PROMPT_DYNAMIC_BOUNDARY` (`sdk.d.ts:1746-1799`), which marks a static -prefix as cacheable but provides no per-tool, per-history, or per-TTL knobs. - -For the `claude-code` backend, the spec treats `llm.promptCaching` as -**partial parity**: - -- The Claude runtime MAY map a non-empty static system prefix to a cacheable - `systemPrompt` array using `SYSTEM_PROMPT_DYNAMIC_BOUNDARY` when - `cacheSystem` is enabled in the resolved `KtxPromptCachingConfig`. The - implementation plan decides whether to ship this mapping in the first pass - or defer it. -- `cacheTools`, `cacheHistory`, and the `systemTtl` / `toolsTtl` / - `historyTtl` fields have no Agent SDK equivalent. The runtime MUST NOT - silently drop them: when a user sets non-default values under - `llm.promptCaching` and the backend is `claude-code`, status/doctor and the - setup wizard MUST surface that these fields are ignored on this backend. -- Docs under `docs-site/content/docs/` MUST document this divergence in the - same pages that describe `claude-code` setup, so users do not assume the - TTL/tool/history knobs apply. - -Acceptance criteria: - -- A `claude-code` runtime constructed from a config with default - `promptCaching` does not throw and does not pass KTX `cacheControl` - markers to the Agent SDK (the AI-SDK-only markers stay on the AI SDK - path). -- A `claude-code` runtime constructed from a config with non-default - `promptCaching` values yields a warning surfaced through doctor/status - output identifying the ignored fields. - -## Auth and setup - -`ktx setup`, status, and doctor flows must validate that Claude Code SDK auth is -usable, not just that `~/.claude/` exists. Acceptable validation strategies: - -- A minimal SDK probe call with `settingSources: []`, `skills: []`, - `plugins: []`, `tools: []`, `persistSession: false`, no `mcpServers`, - `env: ktxClaudeCodeEnv`, and `maxTurns: 1`. The probe MUST NOT rely on - the SDK's documented default for any of these fields, because the default - for `settingSources` is `["user", "project", "local"]` (loads filesystem - settings) and the default for `env` is `process.env` (can route auth - through `ANTHROPIC_API_KEY` or other provider credentials and hide a - missing local Claude Code session). See "Agent SDK environment and auth - boundary" above for the `env` denylist. - The auth probe MUST tolerate init messages with non-empty `slash_commands`, - `skills`, and `agents` when `message.tools` is empty, `message.mcp_servers` - is empty, `message.plugins` is empty, and the query options contain the KTX - isolation tuple. Host discovery metadata is not an auth failure. -- An SDK-provided account/auth status method if the pinned version exposes one. -- A docs-endorsed file-presence check only if the official SDK docs explicitly - state that it proves auth usability. - -Failure copy should tell the user to authenticate Claude Code locally with the -Claude Code CLI, then rerun setup or the command they attempted. - -## Documentation impact - -Docs updates are required because this changes user-visible setup and LLM -provider behavior: - -- `docs-site/content/docs/getting-started/quickstart.mdx` -- `docs-site/content/docs/cli-reference/ktx-setup.mdx` -- `docs-site/content/docs/guides/building-context.mdx` -- Any config reference page that documents `llm.provider.backend` -- Any status or doctor docs that describe LLM readiness - -The docs must say that `claude-code` uses the user's own local Claude Code -session. Do not describe it as a way for KTX to resell, pool, or productize -Claude subscription limits. - -## Verified evidence - -- Current `KtxLlmProvider` returns AI SDK `LanguageModel` instances and only - supports `anthropic`, `vertex`, and `gateway` - (`packages/llm/src/types.ts`, `packages/llm/src/model-provider.ts`). -- Project config currently accepts `llm.provider.backend: none | anthropic | - vertex | gateway` (`packages/context/src/project/config.ts`). -- `generateKtxText` and `generateKtxObject` are shared non-agent generation - helpers (`packages/context/src/llm/generation.ts`). -- `AgentRunnerService` is the shared AI SDK agent-loop implementation - (`packages/context/src/agent/agent-runner.service.ts`). -- Page triage and light extraction currently use raw `KtxLlmProvider` - (`packages/context/src/ingest/page-triage/page-triage.service.ts`). -- Scan/enrichment internals currently use `createLocalKtxLlmProviderFromConfig`, - `generateKtxText`, and `generateKtxObject` - (`packages/context/src/scan/local-scan.ts`, - `packages/context/src/scan/description-generation.ts`, - `packages/context/src/scan/relationship-llm-proposal.ts`). -- Local ingest and MCP local project ports inject `llmProvider` and - `agentRunner` today (`packages/context/src/ingest/local-bundle-runtime.ts`, - `packages/context/src/mcp/local-project-ports.ts`). -- The Agent SDK TypeScript reference (`@anthropic-ai/claude-agent-sdk@0.3.142`, - `sdk.d.ts:1690-1697` and the `sdk.mjs` runtime default - `["user","project","local"]`) documents `settingSources` **defaulting to - loading user, project, and local filesystem settings** when omitted; passing - `[]` is the explicit opt-out ("SDK isolation mode"). The same reference - documents `allowedTools` as auto-approval rather than restriction, - `canUseTool` as the programmatic permission handler, - `permissionMode: "dontAsk"`, `tools` as the base built-in set with `[]` - meaning "disable all built-ins" and no MCP-id support, `disallowedTools`, - `maxTurns`, `mcpServers`, `cwd`, `persistSession`, and SDK result/hook - message shapes. -- `SDKResultMessage = SDKResultSuccess | SDKResultError` in - `@anthropic-ai/claude-agent-sdk@0.3.142` (`sdk.d.ts`); both variants expose - an optional `terminal_reason: TerminalReason`, where `TerminalReason` - includes `'max_turns' | 'completed'` alongside other terminal reasons. -- The Agent SDK MCP docs and SDK examples (e.g. Context7 - `/nothflare/claude-agent-sdk-docs` custom-tools guide) show registering MCP - servers in `query()` options and listing exact `mcp____` ids - in `allowedTools`; no SDK doc or type currently documents a wildcard form. -- BaseTool's `toModelOutput` already sends only `markdown` to the model while - preserving structured output for callers - (`packages/context/src/tools/base-tool.ts:154-162`); some raw AI SDK tools - in `packages/context/src/ingest/ingest-bundle.runner.ts:697-721, :924-936` - and `packages/context/src/memory/memory-agent.service.ts:128-152` currently - return bare strings or plain objects and must be normalized at the - descriptor boundary so both backends preserve the contract. -- The Agent SDK skills docs say the `skills` option is a context filter rather - than a sandbox. KTX must pass `skills: []`, but must not assert that - `message.skills` is empty in the SDK init message. -- `Options.env` in `@anthropic-ai/claude-agent-sdk@0.3.142` - (`sdk.d.ts:1265-1279`) is the environment passed to the Claude Code - process and defaults to `process.env`. Without an explicit `env`, the SDK - inherits the parent environment, including any provider-routing variables - (`ANTHROPIC_API_KEY`, Vertex/Bedrock credentials, gateway tokens) that - could change the active authentication source of the Claude Code CLI and - hide a missing local Claude Code session. - -## Open items for the implementation plan - -1. Confirm exact TypeScript option names and result-message discriminants - against the pinned `@anthropic-ai/claude-agent-sdk` version. -2. Define the final `KtxLlmRuntimePort` file location and package exports. -3. Define model alias validation for `sonnet`, `opus`, `haiku`, and full model - IDs. -4. Define the auth probe and make setup/status/doctor report actionable - messages. -5. Run a repo-wide audit for all LLM call sites and migrate each one to the - runtime boundary. -6. Write tests proving `claude-code` works for text generation, structured - object generation, and agent-loop execution. -7. Write tests proving page triage, scan/enrichment internals, memory capture, - MCP-triggered local ingest, and normal local ingest all use the - `claude-code` runtime when configured. -8. Write tests proving a raw built-in Claude Code tool request is denied, - host-discovered Skill/Agent/SlashCommand requests are denied by `canUseTool`, - and only exact `mcp__ktx__*` tools are allowed during KTX agent loops. -9. Write a test that asserts every KTX-originated `query()` invocation - (agent loop, text generation, object generation, auth probe) is called - with `settingSources: []`, `skills: []`, `plugins: []`, `tools: []`, and - `persistSession: false`, by spying on the SDK entry point. The test must - fail if any path falls back to SDK defaults for those fields. The test must - also prove that non-empty host-discovered `slash_commands`, `skills`, and - `agents` in the init message do not fail the auth probe or runtime when the - controlled tool, MCP server, and plugin surfaces match KTX expectations. -10. Write a test that asserts `onStepFinish` is invoked the expected number - of times for a fixed-budget `claude-code` agent loop, including the - work-unit and reconciliation progress paths. -11. Write a test that asserts every KTX-originated `query()` invocation - (agent loop, text generation, object generation, auth probe) is called - with an explicit `env` and that none of the denylisted provider-routing - variables (`ANTHROPIC_API_KEY`, `ANTHROPIC_AUTH_TOKEN`, - `ANTHROPIC_BASE_URL`, `ANTHROPIC_MODEL`, `ANTHROPIC_VERTEX_PROJECT_ID`, - `CLOUD_ML_REGION`, `GOOGLE_APPLICATION_CREDENTIALS`, - `GOOGLE_CLOUD_PROJECT`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, - `AWS_SESSION_TOKEN`, `AWS_REGION`, `AWS_PROFILE`, - `CLAUDE_CODE_USE_BEDROCK`, `CLAUDE_CODE_USE_VERTEX`) are present in - that env, by seeding each variable in a fake `process.env`. The test - must also assert that the auth probe still fails when - `ANTHROPIC_API_KEY` is set in `process.env` but no local Claude Code - session exists. diff --git a/docs/superpowers/specs/2026-05-15-semantic-layer-docs-design.md b/docs/superpowers/specs/2026-05-15-semantic-layer-docs-design.md deleted file mode 100644 index 34d7594c..00000000 --- a/docs/superpowers/specs/2026-05-15-semantic-layer-docs-design.md +++ /dev/null @@ -1,166 +0,0 @@ -# Semantic Layer Docs Design - -**Date:** 2026-05-15 -**Status:** Design - pending implementation plan - -## Goal - -Add a concise Concepts page that explains the semantic layer as the query -planning engine inside KTX's broader context layer. - -The page should make the technical depth visible to skeptical data users -without positioning KTX as only a semantic-layer product. Success means a reader -understands: - -- KTX is a context layer for agents. -- The semantic layer is one core subsystem inside that context layer. -- The join graph, grain declarations, and relationship metadata are what make - generated SQL safer than schema-only or markdown-only approaches. -- KTX maintains this semantic layer through ingest, validation, analyst edits, - and reviewable files. - -## Current State - -The docs currently explain semantic sources in two places: - -- `docs-site/content/docs/concepts/the-context-layer.mdx` describes semantic - sources as one pillar of KTX context. -- `docs-site/content/docs/guides/writing-context.mdx` documents the YAML fields - for sources, measures, joins, grain, validation, and common errors. - -That content is useful, but the differentiator is not visually obvious. The -semantic layer is embedded in longer narrative pages, so readers can miss the -hard parts: join graph construction, fan-out prevention, chasm traps, and query -planning. - -## Positioning - -Create a standalone Concepts page with a guarded title such as -`Semantic Layer Internals` or `The Semantic Engine Inside KTX`. - -The first screen must frame the product clearly: - -> KTX is a context layer. Its semantic layer is the query-planning core that -> turns reviewed context into safe SQL. - -The page should avoid a title like `Semantic Layer` by itself because that can -make KTX look like a narrow semantic-layer tool. The page should repeatedly show -the semantic layer between the broader context inputs and the agent workflows it -supports. - -Add a short cross-link from `the-context-layer.mdx` so the existing overview -keeps owning the product category. That section should say the semantic layer is -one critical pillar, then link to the internals page for readers who want the -mechanics. - -## Page Structure - -Add `docs-site/content/docs/concepts/semantic-layer-internals.mdx` and include -it in `docs-site/content/docs/concepts/meta.json` after `the-context-layer`. - -Recommended sections: - -1. `What this page explains` - - One short paragraph. - - A two-column `KTX is / KTX is not just` table. - -2. `Where the semantic layer fits` - - A visual block showing: - `context inputs -> semantic layer engine -> agent workflows`. - - Inputs include semantic YAML, wiki pages, scans, and provenance. - - Outputs include search, SQL generation, explanations, edits, and review. - -3. `The join graph` - - Explain nodes as semantic sources and edges as validated joins. - - Show a small graph with `orders`, `customers`, `order_items`, and - `refunds`. - - Keep text to one or two short paragraphs plus bullets. - -4. `How KTX builds it` - - Show a pipeline from database evidence and imported modeling tools to - reviewable YAML. - - Mention declared keys, inferred relationships, dbt/MetricFlow/LookML - imports, query history, validation, and analyst review. - -5. `How KTX maintains it` - - Show a feedback loop: - ingest evidence -> YAML diff -> validation -> analyst review -> agent use - -> corrections. - - Emphasize that files remain the source of truth. - -6. `Why grain and relationships matter` - - Use the fan-out problem as the central example. - - Compare a naive join against a safe semantic-layer plan. - - Explain many-to-one, one-to-many, many-to-many, chasm traps, and ambiguous - paths in compact bullets. - -7. `How the execution engine uses the graph` - - Explain path selection, unsafe path rejection, pre-aggregation into CTEs, - filter placement, and dialect transpilation. - - Include a small before/after SQL-shape diagram or table. - -8. `What this means for agents` - - Summarize why this is more than saving markdown: - agents can inspect, query, validate, edit, and review the same semantic - files. - - Link to `Writing Context` and `ktx sl`. - -## Scannability Rules - -The implementation should shorten long prose blocks across the touched pages. - -- Keep most text blocks to one or two paragraphs. -- Prefer bullets, tables, diagrams, and compact callout blocks between prose. -- Avoid four-paragraph narrative runs. -- Use diagrams before dense explanations when the concept is spatial. -- Keep examples concrete and copy-pasteable. - -## Visual Direction - -Use the existing docs-site MDX style rather than a new design system. The current -`the-context-layer.mdx` page already uses custom `not-prose` MDX diagrams with -Fumadocs color tokens; the new page should follow that pattern. - -The diagrams should feel like technical product documentation: - -- restrained, dense, and readable; -- high contrast for the semantic-layer engine box; -- visible arrows or adjacency that make flow obvious; -- tables for classification and comparison; -- no marketing hero, decorative gradients, or generic card-heavy layout. - -## Non-goals - -- Do not redesign the whole docs site. -- Do not rename KTX concepts, packages, commands, or directories. -- Do not claim KTX replaces every BI or semantic-layer system. -- Do not add implementation details that are not true in the current codebase. -- Do not expand the page into a long reference for every YAML field; keep that - in `Writing Context`. - -## Verification - -Because this is docs-only work, verification should focus on the docs site: - -- Run the docs build or the narrowest available docs-site type/build check. -- Run formatting or lint checks if the docs package exposes them. -- Preview the page locally and inspect desktop and mobile widths. -- Confirm the page is listed in Concepts navigation. -- Confirm the opening section clearly says KTX is a context layer, not just a - semantic-layer tool. - -If implementation changes only MDX and metadata, TypeScript workspace tests are -not required unless the page introduces shared components. - -## Acceptance Criteria - -- A standalone Concepts page explains the semantic-layer internals. -- The Context Layer page links to the new internals page without making the - overview longer. -- The new page includes diagrams for the system fit, join graph, maintenance - loop, and fan-out-safe execution path. -- Long prose is broken into scannable sections with bullets, tables, and visual - interruptions. -- The positioning consistently says KTX is a context layer with a semantic - execution core. -- Docs-site verification passes or any skipped check is reported with a reason. diff --git a/docs/superpowers/specs/2026-05-16-mcp-tool-polish-design.md b/docs/superpowers/specs/2026-05-16-mcp-tool-polish-design.md deleted file mode 100644 index 6dda60a9..00000000 --- a/docs/superpowers/specs/2026-05-16-mcp-tool-polish-design.md +++ /dev/null @@ -1,802 +0,0 @@ -# MCP Tool Polish: Slim Research Surface + Spec Compliance - -**Date:** 2026-05-16 -**Author:** Andrey Avtomonov -**Status:** Design — pending implementation plan - -## Background - -KTX currently exposes 25 MCP tools across context, semantic-layer, ingest, and -scan ports (`packages/context/src/mcp/context-tools.ts`). The -`ktx-analytics` SKILL.md (`packages/cli/src/skills/analytics/SKILL.md`) -is installed into every supported MCP client by `ktx setup --agents` and -already describes a Douala-equivalent research methodology (Discover → Inspect -→ Resolve → Plan → Query → Validate → Capture) that references nine of those -tools. - -A recent in-session audit surfaced three real bug classes: - -- **`structuredContent` shape** — `discover_data` returned a bare array; MCP - requires an object. Fixed. -- **Union-shape LLM drift** — `sl_query.order_by` accepted - `{ field, direction }` but Claude emitted Cube-style `{ id, desc }`. Fixed - via a `z.preprocess` that normalizes the alt-shape before strict validation. -- **Contract leak to the Python daemon** — `compileLocalSlQuery` skipped - `toResolvedWire` and sent TS-only authoring fields (`usage`, - `inherits_columns_from`) to a Pydantic model with `extra="forbid"`. Fixed. - -The audit also identified systemic gaps applicable across the surface: no -per-field `.describe()` outside `memory_capture*`, no `outputSchema` declared -anywhere, no MCP tool annotations, lingering union-drift risk on -`slQueryDimensionSchema` (`{ dimension, granularity }`) and -`entityDetailsTableRefSchema` (`{ schema, table }`), and inconsistent error -handling (only `sql_execution` wraps thrown errors in-band per MCP spec; the -other 24 let exceptions propagate as JSON-RPC errors). - -The current MCP spec (2025-11-25) provides several mechanisms KTX does not -use: `outputSchema` with `structuredContent` validation, tool annotations -(`readOnlyHint`/`destructiveHint`/`idempotentHint`/`openWorldHint`/`title`), -progress notifications via `_meta.progressToken`, and a clear in-band error -contract (`isError: true` with text content). - -The 25-tool surface is also wider than the agent needs. The `ktx-analytics` -SKILL only orchestrates nine of them; the rest are admin/setup/maintenance -operations that are better served by a `ktx`-CLI flow (and a future -`ktx-admin` SKILL — out of scope for this spec). - -## Goals - -- Reduce the MCP-registered surface to **11 tools** focused on the research - loop: `connection_list`, `discover_data`, `wiki_search`, `wiki_read`, - `entity_details`, `dictionary_search`, `sl_read_source`, `sl_query`, - `sql_execution`, `memory_ingest` (new), `memory_ingest_status` (new). The - remaining 14 tools become CLI-only by removing their MCP registration; their - implementations stay in `packages/context/src/` to back the CLI. -- Replace `memory_capture` / `memory_capture_status` with `memory_ingest` / - `memory_ingest_status`. The new tool takes free-form markdown `content` - plus optional `connectionId`; the memory agent triages into wiki and SL as - before. -- Apply the per-tool polish kit on every retained tool: MCP tool annotations, - `outputSchema`, per-field `.describe()`, rewritten tool descriptions, - standardized in-band error handling, union-drift normalization on the two - remaining at-risk schemas, and a type-narrowed `jsonToolResult`. -- Emit MCP progress notifications from `sql_execution` and `sl_query`. -- Update `ktx-analytics` SKILL.md to use the renamed tool, broaden the capture - step, and document multi-connection routing. - -## Non-Goals - -- Admin CLI skill (separate spec). -- Deleting the source code of the admin tools (deferred follow-up gated by - the admin CLI skill landing). -- MCP resources (subscribable wiki / SL). -- MCP prompts pushed by the server (the analytics SKILL is the equivalent). -- Elicitation, sampling, tool icons. -- A code-execution tool / Python sandbox (separate spec; the analytics - workflow does not require one for the goals above). -- Per-client schema-feature workarounds beyond what the audit findings - already cover. Codex's no-header-auth limitation is unrelated to tool - shape and is left to `setup-agents.ts` to document. -- Multi-tenancy, telemetry, rate limiting. - -## Design - -### 1. Surface change - -#### 1.1 Retained tools (11) - -| # | Tool | Port | -|---|---|---| -| 1 | `connection_list` | `KtxConnectionsMcpPort.list` | -| 2 | `discover_data` | `KtxDiscoverDataMcpPort.search` | -| 3 | `wiki_search` | `KtxKnowledgeMcpPort.search` | -| 4 | `wiki_read` | `KtxKnowledgeMcpPort.read` | -| 5 | `entity_details` | `KtxEntityDetailsMcpPort.read` | -| 6 | `dictionary_search` | `KtxDictionarySearchMcpPort.search` | -| 7 | `sl_read_source` | `KtxSemanticLayerMcpPort.readSource` | -| 8 | `sl_query` | `KtxSemanticLayerMcpPort.query` | -| 9 | `sql_execution` | `KtxSqlExecutionMcpPort.execute` | -| 10 | `memory_ingest` | New port `KtxMemoryIngestMcpPort.ingest` | -| 11 | `memory_ingest_status` | `KtxMemoryIngestMcpPort.status` | - -`connection_list` is retained because in multi-connection projects, the agent -needs a way to enumerate available connections before issuing a `sql_execution` -or `sl_query` against a specific one. - -**`connectionId` resolution per retained tool** (auto-resolution exists today -only on the local SL path; do not broaden it as part of this spec): - -| Tool | `connectionId` | Auto-resolves to single connection if omitted? | -|---|---|---| -| `connection_list` | n/a | n/a | -| `discover_data` | optional | no — search is run unscoped when omitted | -| `wiki_search` | n/a | n/a | -| `wiki_read` | n/a | n/a | -| `entity_details` | required | no | -| `dictionary_search` | optional | no — search is run unscoped when omitted | -| `sl_read_source` | required | no | -| `sl_query` | optional | yes — `resolveLocalConnectionId` (`packages/context/src/sl/local-query.ts`) auto-resolves when the project has exactly one connection | -| `sql_execution` | required | no | -| `memory_ingest` | optional | no — omitted means "global" knowledge (wiki only — see below) | -| `memory_ingest_status` | n/a | n/a | - -The skill update in §3 must reflect this matrix: when `connection_list` shows -multiple connections, the agent always passes `connectionId` for the required -tools and for `sl_query`/`discover_data`/`dictionary_search` whenever the user -intent pins a specific warehouse. - -**`memory_ingest` connectionId semantics — important constraint.** The -underlying `MemoryAgentService.ingest` derives `hasSL = !!input.connectionId` -(`packages/context/src/memory/memory-agent.service.ts:55`) and only wires the -SL-capable toolset when `connectionId` is supplied -(`packages/context/src/memory/memory-agent.service.ts:116-118`). Therefore -`memory_ingest` can update the semantic layer **only** when `connectionId` is -provided. Omit `connectionId` only for genuinely global wiki-only knowledge -(company-wide policies, vocabulary, user preferences); supply `connectionId` -for any knowledge that touches a specific warehouse — including measure -definitions, schema gotchas, and any wording like "in our warehouse" or "this -warehouse". The §3 SKILL update and the worked example must enforce this. - -#### 1.2 Removed from MCP registration - -`connection_test`, `wiki_write`, `sl_list_sources`, `sl_write_source`, -`sl_validate`, `ingest_trigger`, `ingest_status`, `ingest_report`, -`ingest_replay`, `scan_trigger`, `scan_status`, `scan_report`, -`scan_list_artifacts`, `scan_read_artifact`, -plus `memory_capture` and `memory_capture_status` (replaced). - -The conditional registration blocks in `registerKtxContextTools` for these -ports are removed. The underlying `KtxIngestMcpPort`, `KtxScanMcpPort`, etc. -implementations stay; the `ktx` CLI uses them directly. The `KtxMcpContextPorts` -type drops the removed `ingest?`, `scan?`, etc. fields. `MemoryCapturePort` is -renamed to `MemoryIngestPort`. - -#### 1.3 New tool — `memory_ingest` - -Replaces `memory_capture`. The change is a rename + a slightly relaxed input -contract; the underlying `MemoryCaptureService` -(`packages/context/src/memory/memory-runs.ts:81`) is reused as-is and renamed to -`MemoryIngestService`. No alias, no migration shim — per the standing -no-back-compat rule, the rename is atomic with the SKILL update. - -**Final internal API shape after the rename — no compatibility wrappers:** - -| Old name | New name | -|---|---| -| `MemoryCaptureService` (class) | `MemoryIngestService` | -| `MemoryCaptureService.capture(input)` (method) | `MemoryIngestService.ingest(input)` | -| `MemoryCaptureServiceDeps` | `MemoryIngestServiceDeps` | -| `MemoryCaptureStartResult` | `MemoryIngestStartResult` | -| `MemoryCaptureStatus` (return type) | `MemoryIngestStatus` | -| `MemoryCapturePort` (in `mcp/types.ts`) | `MemoryIngestPort` (with `.ingest()` and `.status()`) | -| `MemoryCapturePort.capture()` | `MemoryIngestPort.ingest()` | -| `TextMemoryCapturePort` (CLI, `text-ingest.ts`) | `TextMemoryIngestPort` (with `.ingest()`, `.waitForRun()`, `.status()`) | -| `createLocalProjectMemoryCapture` factory | `createLocalProjectMemoryIngest` | - -Every internal call site (`packages/context/src/mcp/server.ts`, -`packages/context/src/mcp/local-project-ports.ts`, -`packages/cli/src/mcp-server-factory.ts`, `packages/cli/src/text-ingest.ts`, -their tests, and the `packages/context/src/memory/index.ts` re-exports) is -updated in lockstep. The agent-facing `MemoryAgentService.ingest` method and -its `MemoryAgentInput` type are unchanged. - -**Mapping `memory_ingest` input → `MemoryAgentInput`** (defined in -`packages/context/src/memory/types.ts`): - -| `MemoryAgentInput` field | Value supplied by `memory_ingest` handler | -|---|---| -| `userId` | `userContext.userId` (existing pattern) | -| `chatId` | `mcp-${randomUUID()}` (existing pattern) | -| `userMessage` | synthetic framing string, e.g. `Ingest external knowledge into KTX memory.` | -| `assistantMessage` | input.`content` | -| `connectionId` | input.`connectionId` (when provided) | -| `sourceType` | `'external_ingest'` | - -The free-form markdown is routed into `assistantMessage` (not `userMessage`) -with a synthetic framing `userMessage`, mirroring the existing CLI text-ingest -path (`packages/cli/src/text-ingest.ts:295-302`). This mapping is required so -that `detectCaptureSignals` -(`packages/context/src/memory/capture-signals.ts:14`) can fire its -`assistantMessage`-keyed cues — SQL aggregates, LookML structure, and -definition tables — for artifact-like content. Routing content into -`userMessage` would lose those signals and silently degrade triage parity with -CLI ingest. The existing memory-agent prompt and tests already expect this -shape; no changes to the memory agent itself are required. - -Acceptance criterion: an MCP `memory_ingest` call with the same markdown -content as a CLI `ktx ingest text` invocation must produce identical -`CaptureSignals` (knowledge / sl / dialect / reasons) — covered by a parity -test that feeds the same fixture content through both ingest entry points and -asserts equal `detectCaptureSignals` output. - -**Input schema:** - -```typescript -const memoryIngestSchema = z.object({ - content: z - .string() - .min(1) - .describe( - 'Free-form markdown to ingest. Include the knowledge itself plus any ' + - 'context (source, the user\'s question, why this came up) that the ' + - 'memory agent should consider when triaging into wiki/SL.', - ), - connectionId: z - .string() - .min(1) - .optional() - .describe( - 'Scope this memory to a specific connection. REQUIRED when the knowledge ' + - 'is warehouse-specific (measure definitions, schema gotchas, anything ' + - 'tied to a particular warehouse) — without it the memory agent cannot ' + - 'update the semantic layer and the knowledge will land as wiki-only. ' + - 'Omit only for genuinely global wiki knowledge (company-wide policies, ' + - 'vocabulary, user preferences).', - ), -}); -``` - -**Tool description:** - -> Ingest free-form knowledge into KTX's durable memory so it is available to -> future turns. Call this whenever a research turn produces something worth -> remembering — business rules, metric definitions, gotchas, schema -> explanations, recurring findings — **or** whenever the user asks you to -> remember something. Pass everything in `content` as markdown: the finding, -> plus any source or context that helps the memory agent triage. KTX's memory -> agent decides whether the content belongs in the wiki, the semantic layer, -> or both. Each call is a feedback loop — better notes here mean smarter -> `discover_data` and `wiki_search` results for everyone next time. - -**Returns:** `{ runId: string }` — same shape as today's `memory_capture`. - -**`memory_ingest_status`** mirrors today's `memory_capture_status` exactly, -renamed only. - -### 2. Per-tool polish kit - -#### 2.0 Registration topology — memory tools must share the polish path - -Today `memory_capture` / `memory_capture_status` are registered in -`packages/context/src/mcp/server.ts` via direct `deps.server.registerTool` -calls (`server.ts:23,45`), **bypassing** `registerParsedTool` in -`context-tools.ts`. If left as-is, the polish kit below -(annotations, `outputSchema`, in-band error wrapping, per-field `.describe()`) -would not apply to `memory_ingest` / `memory_ingest_status`, contradicting the -"all 11 tools" acceptance criteria. - -Therefore, as part of the polish-kit PR (PR 2), one of the following must -happen — the implementation plan picks: - -1. **Preferred:** Move `memory_ingest` and `memory_ingest_status` registration - into `registerKtxContextTools` so they go through `registerParsedTool` like - every other tool. The `MemoryIngestPort` becomes a `contextTools.memoryIngest` - port and the standalone `registerMemoryCaptureTools` helper in `server.ts` - is deleted. -2. **Acceptable fallback:** Keep `registerMemoryIngestTools` in `server.ts` - but rewrite it to call `registerParsedTool` (exported from - `context-tools.ts`) so the same annotations / `outputSchema` / - error-wrapping plumbing is applied uniformly. - -Either way, every checklist item in §§2.1–2.4 must apply to the two memory -tools, and the §Verification annotations and `outputSchema` tests must cover -them. - -#### 2.1 Tool annotations - -Every tool gets annotations and a `title`: - -| Tool | title | readOnly | destructive | idempotent | openWorld | -|---|---|:--:|:--:|:--:|:--:| -| `connection_list` | Connection List | ✓ | — | ✓ | — | -| `discover_data` | Discover Data | ✓ | — | — | — | -| `wiki_search` | Wiki Search | ✓ | — | — | — | -| `wiki_read` | Wiki Read | ✓ | — | ✓ | — | -| `entity_details` | Entity Details | ✓ | — | ✓ | — | -| `dictionary_search` | Dictionary Search | ✓ | — | — | — | -| `sl_read_source` | Semantic Layer Read Source | ✓ | — | ✓ | — | -| `sl_query` | Semantic Layer Query | ✓ | — | — | — | -| `sql_execution` | SQL Execution | ✓ | — | — | — | -| `memory_ingest` | Memory Ingest | — | ✓ | — | — | -| `memory_ingest_status` | Memory Ingest Status | ✓ | — | omit | — | - -`openWorldHint: false` for every tool — even `sql_execution` targets a -configured, bounded warehouse, not the web. `sql_execution` is `readOnlyHint: -true` because the server-side parser enforces read-only (`assertReadOnlySql`). -`destructiveHint` is omitted (defaults to `false`) for read-only tools per the -MCP spec; explicit `false` is fine but redundant. - -`ToolAnnotations` are static optional booleans per the MCP 2025-11-25 schema -(`title?`, `readOnlyHint?`, `destructiveHint?`, `idempotentHint?`, -`openWorldHint?` — no state-dependent variants). `idempotentHint` describes -whether repeated calls have additional environmental effect and is most -meaningful when `readOnlyHint` is `false`. For `memory_ingest_status`, which is -a polling read whose response shape changes while a run is active, leave -`idempotentHint` unset — the tool is read-only but not statically idempotent. - -`registerTool` accepts annotations in the `config` object today; this is a -plumbing change in `registerParsedTool` to forward them. - -#### 2.2 `outputSchema` on all 11 tools - -Per the MCP 2025-11-25 spec, clients SHOULD validate `structuredContent` -against `outputSchema` when declared. Authoring is mechanical: each response -shape already typed in `packages/context/src/mcp/types.ts` gets a parallel -Zod schema and is passed as `outputSchema` to `registerTool`. - -`registerParsedTool` is extended to accept an optional `outputSchema` arg and -forward it to `server.registerTool`. The Zod schemas live alongside the -input schemas in `context-tools.ts` (or a sibling `tool-output-schemas.ts` if -the file grows too large). - -Example for `discover_data`: - -```typescript -const discoverDataOutputSchema = z.object({ - refs: z.array( - z.object({ - kind: discoverDataKindSchema, - id: z.string(), - score: z.number(), - summary: z.string().nullable(), - snippet: z.string().nullable(), - matchedOn: z.enum(['name', 'display', 'description', 'comment', 'expr', 'sample_value', 'body']), - connectionId: z.string().optional(), - tableRef: z.object({ catalog: z.string().nullable(), db: z.string().nullable(), name: z.string() }).optional(), - columnName: z.string().optional(), - }), - ), -}); -``` - -#### 2.3 Per-field `.describe()` on every input - -Anthropic's documented mechanism for fighting model drift, already used in -`memory_capture*`. Applied to every input field on every retained tool. -Highest leverage: `sl_query`, `entity_details`, `dictionary_search`, -`sql_execution`, `memory_ingest`. Tool-level `description` strings are -rewritten to be longer with one concrete example shape inlined (the technique -that fixed `order_by` model drift in this session). - -#### 2.4 In-band error wrapping in `registerParsedTool` - -Per MCP spec, tools return handler/runtime errors as `isError: true` + text -content, not JSON-RPC errors. Move the try/catch into the `registerParsedTool` -helper so every tool consistently surfaces handler exceptions as -`jsonErrorToolResult`. `sql_execution`'s local try/catch is removed (the -helper handles it). - -**Scope — what becomes in-band vs. what stays JSON-RPC.** The MCP SDK -pre-validates incoming arguments against the registered `inputSchema` before -the tool callback runs, and surfaces validation failures as -`McpError(InvalidParams)` / JSON-RPC errors -(`@modelcontextprotocol/sdk/dist/esm/server/mcp.js` `validateToolInput`, -~line 166). KTX cannot intercept those without forking the SDK and we will not. -Therefore: - -- Schema-validation failures on input → remain JSON-RPC `InvalidParams` errors, - emitted by the SDK before our handler runs. This is the documented MCP - behavior; clients already handle it. -- Handler exceptions, port/driver errors, and any post-validation runtime - errors thrown inside the tool body → wrapped in-band as - `{ isError: true, content: [{ type: 'text', ... }] }` by - `registerParsedTool`'s catch. -- The redundant `inputSchema.parse(input)` inside `registerParsedTool` may be - kept as defense-in-depth (e.g., for the rare path where the SDK was given a - raw shape and a downstream change loosens validation) or removed; either is - acceptable. If kept, parse failures here are wrapped in-band as well, but in - practice they are unreachable for valid SDK registrations because the SDK - has already parsed against the same schema. - -```typescript -function registerParsedTool( - server: KtxMcpServerLike, - name: string, - config: { title: string; description: string; inputSchema: unknown; outputSchema?: unknown; annotations: ToolAnnotations }, - inputSchema: TInput, - handler: (input: z.infer) => Promise, - outputSchema?: TOutput, -): void { - server.registerTool(name, { ...config, outputSchema: outputSchema ? outputSchema : undefined }, async (input) => { - try { - return await handler(inputSchema.parse(input)); - } catch (error) { - return jsonErrorToolResult(formatToolError(error)); - } - }); -} -``` - -A small `formatToolError` helper renders Zod errors with `path: message` lines -and falls through to `error.message` / `String(error)` for non-Zod cases. - -Acceptance tests in §Verification must therefore split the error path: - -- Bad input shape (rejected by SDK pre-validation) → expect a thrown - `McpError`/`InvalidParams` JSON-RPC error, not `isError: true`. -- Handler-thrown / port-thrown error (e.g., unknown `connectionId`, driver - failure) → expect `{ isError: true, content: [{ type: 'text', ... }] }`. - -#### 2.5 Union-drift normalization - -Apply the same `z.preprocess` pattern used for `order_by` to the two remaining -at-risk unions: - -**`slQueryDimensionSchema`** — accept `{ dimension, granularity }` (Cube -convention) as an alias for `{ field, granularity }`. Bare strings continue to -work unchanged. - -```typescript -const slQueryDimensionSchema = z.preprocess( - (value) => { - if (typeof value === 'string') return { field: value }; - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('field' in obj) && typeof obj.dimension === 'string') obj.field = obj.dimension; - return obj; - } - return value; - }, - z.object({ - field: z.string().min(1).describe('Dimension to group by, e.g. "orders.created_at" or a SL dimension key.'), - granularity: z.string().min(1).optional().describe('Time granularity for time dimensions: day, week, month, quarter, year.'), - }), -); -``` - -**`entityDetailsTableRefSchema`** — accept `{ schema, table }` (BigQuery / -SQL-style convention) as an alias for `{ db, name }`. Today's schema requires -`catalog`, `db`, and `name` (`packages/context/src/mcp/context-tools.ts:169`), -so the alias path must also default `catalog` to `null` to satisfy the -validator. Either of the two equivalent shapes below is acceptable; the -implementation plan picks one: - -1. Make `catalog` (and `db`) `.nullable().default(null)` so the alias path - doesn't have to set them, and bare `{ schema, table }` is accepted with - `catalog === null`, `db === schema`, `name === table`. -2. Have the preprocess unconditionally fill missing `catalog`/`db` with `null`. - -Acceptance criterion: a tool call with `{ table: { schema: "public", table: "orders" } }` parses successfully and resolves to `{ catalog: null, db: "public", name: "orders" }`. Existing callers passing `{ catalog, db, name }` continue to work unchanged. - -```typescript -const entityDetailsTableRefSchema = z.preprocess( - (value) => { - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('db' in obj) && typeof obj.schema === 'string') obj.db = obj.schema; - if (!('name' in obj) && typeof obj.table === 'string') obj.name = obj.table; - if (!('catalog' in obj)) obj.catalog = null; - return obj; - } - return value; - }, - z.object({ - catalog: z.string().nullable().describe('Catalog/project (BigQuery project, Snowflake database). null when not applicable.'), - db: z.string().nullable().describe('Schema/database for the table. null when not applicable.'), - name: z.string().min(1).describe('Table name.'), - }), -); -``` - -`slQueryMeasureSchema` is **not** changed — bare strings cover the common case -and `{ expr, name }` matches Cube's measure-with-alias convention; no observed -drift. - -#### 2.6 Type-narrow `jsonToolResult` - -The goal is to forbid arrays at compile time without breaking the existing -`interface`-typed response shapes (e.g. `KtxKnowledgeSearchResponse` in -`packages/context/src/mcp/types.ts`). `Record` is **not** -acceptable as the constraint because TypeScript interfaces without an index -signature are not assignable to it; using it would cause widespread compile -failures on valid object responses. - -Use a non-array object constraint instead, e.g.: - -```typescript -type NonArrayObject = object & { length?: never }; - -export function jsonToolResult( - structuredContent: T, -): KtxMcpToolResult { ... } -``` - -Acceptance criteria: - -- A bare array literal at a call site fails to type-check (catches the - `discover_data` bug class). -- Every existing `interface`-typed response (`KtxKnowledgeSearchResponse`, - `KtxSemanticLayerQueryResponse`, `KtxSqlExecutionResponse`, - `KtxEntityDetailsResponse`, `KtxDictionarySearchResponse`, - `KtxDiscoverDataResponse`, `KtxConnectionTestResponse`, - `KtxSemanticLayerReadResponse`, `KtxSemanticLayerListResponse`, - `KtxKnowledgePage`, memory capture/status response shapes) continues to - type-check at every existing `jsonToolResult` call site without modification. - -Implementation plan can substitute an equivalent narrowing if it is more -idiomatic; the contract is "no arrays, no breaking interface assignability." -Pure defensive type change; no runtime effect on current code. - -#### 2.7 Enforce the `toResolvedWire` invariant - -Add a doc comment on `KtxSemanticLayerComputePort.query` and -`.validateSources` stating that callers must pass `toResolvedWire`-sanitized -sources to prevent the daemon `usage`-leak bug from regressing if a new code -path bypasses `SemanticLayerService`. - -The doc comment alone is not sufficient — there is already an unsanitized -caller. `loadComputableSources` in -`packages/context/src/mcp/local-project-ports.ts` (~line 311) parses YAML and -pushes the raw record into `validateSources` -(`packages/context/src/mcp/local-project-ports.ts:586`). It must be brought into -conformance by sanitizing each record with `toResolvedWire` before handing it -to `validateSources`, mirroring the existing sanitization in -`packages/context/src/sl/local-query.ts:76`. Acceptance criterion: every -`KtxSemanticLayerComputePort.query` and `.validateSources` call site in the -repo passes `toResolvedWire`-sanitized records, verified by code review and -covered by the existing `local-query.test.ts` / `local-project-ports.test.ts` -tests for the relevant paths. Note that `sl_validate` is removed from MCP -registration (§1.2) but the underlying port keeps backing the CLI, so the -invariant must hold for the CLI path too. - -#### 2.8 Progress notifications — `sql_execution` + `sl_query` - -Per MCP spec, the caller may include `params._meta.progressToken` in the -request; the server emits `notifications/progress` with `{ progressToken, -progress, total?, message }` at stage transitions. - -`KtxSqlExecutionMcpPort.execute` and `KtxSemanticLayerMcpPort.query` are -extended with an optional `onProgress?: (event: { progress: number; total?: number; message: string }) => void` parameter. The MCP tool handlers wire -`onProgress` to the SDK's notification channel via the handler-context object -passed by `server.registerTool`. Non-progress-supporting clients ignore the -events. - -Emitted stages: - -- `sql_execution`: `"Validating SQL"` (progress 0.0) → `"Executing"` (0.3) → `"Fetched N rows"` (1.0). -- `sl_query`: `"Compiling query"` (0.0) → `"Generating SQL"` (0.3) → `"Executing"` (0.6) → `"Fetched N rows"` (1.0). - -Progress emission is best-effort; if the underlying port can't report a stage -boundary (e.g., a driver doesn't expose progress callbacks), the stage is -simply skipped. - -`memory_ingest` does not emit progress — `runId` + `memory_ingest_status` -polling is the documented async pattern for it. - -### 3. `ktx-analytics` SKILL.md refinements - -File: `packages/cli/src/skills/analytics/SKILL.md`. - -**Step 7 rewrite** (current vs. new): - -Current: - -> 7. **Capture durable learnings** - at the end of the turn, call `memory_capture` when the investigation produced reusable business context, metric definitions, or schema knowledge. - -New: - -> 7. **Capture durable learnings** - call `memory_ingest` whenever a turn produces something worth remembering (business rules, metric definitions, schema gotchas, recurring findings) **or** whenever the user asks you to remember something. Pass markdown in `content` including any source context the memory agent should weigh. Each call is a feedback loop — better notes today mean smarter `discover_data` and `wiki_search` results tomorrow. - -**Tool-name updates** throughout: every `memory_capture` reference becomes -`memory_ingest`. - -**Multi-connection rule** added under `` — phrased to match the §1.1 -connection matrix so the agent does not over-scope unscoped tools: - -> When `connection_list` shows multiple connections, pass an explicit -> `connectionId` to every tool that takes one **and where user intent pins a -> specific warehouse**. The matrix is: -> -> - **Required:** `entity_details`, `sl_read_source`, `sql_execution`. -> - **Required when user intent is warehouse-specific (including any wording -> like "in our warehouse" / "this warehouse"):** `memory_ingest` — without -> `connectionId`, the memory agent cannot update the semantic layer and the -> knowledge will land as wiki-only. -> - **Pass when intent pins a warehouse, otherwise omit for unscoped -> discovery:** `sl_query`, `discover_data`, `dictionary_search`. -> - **Never pass `connectionId` (the tool does not accept one):** -> `connection_list`, `wiki_search`, `wiki_read`, `memory_ingest_status`. -> -> If intent is ambiguous for a required-or-scoped tool, ask the user which -> warehouse before calling — do not guess. - -**One new worked example** demonstrating user-driven ingest: - -> **Input:** "Heads up — ARR is always reported in cents in our warehouse." -> -> **Workflow:** -> 1. If multiple connections, call `connection_list` and pick the warehouse the -> user means (asking if ambiguous). Pass its id as `connectionId` so the -> memory agent can update the semantic layer, not just the wiki. -> 2. `memory_ingest({ connectionId: "", content: "ARR is reported in cents (not dollars) in this warehouse. Multiply by 0.01 for dollar amounts. Source: user clarification." })` — no analysis turn; just remember. - -The existing Discover → Inspect → Resolve → Plan → Query → Validate → Capture -workflow stays. The existing two examples are updated only to reflect the -`memory_capture` → `memory_ingest` rename. - -## Migration / sequencing - -Three landings, each independently mergeable, in this order: - -### PR 1 — Surface change (atomic) - -- Remove the 14 admin tools from `registerKtxContextTools` (conditional - registration blocks deleted). -- Rename `memory_capture` → `memory_ingest`, `memory_capture_status` → - `memory_ingest_status`. Rename `MemoryCapturePort` → `MemoryIngestPort`, - `MemoryCaptureService` → `MemoryIngestService`. Update the new tool's input - contract per §1.3. -- Update `packages/context/src/mcp/local-project-ports.ts` and - `packages/context/src/mcp/server.ts` to reflect the renames and the dropped - ports. -- Update `packages/cli/src/skills/analytics/SKILL.md` per §3 in the same - diff. -- Update all tests for the removed/renamed tools. -- Update `docs-site/content/docs/integrations/agent-clients.mdx` to replace - the existing "memory capture" wording (currently at line ~90) with - "memory ingest". This update is unconditional — the file already names the - tool family. -- Update `packages/cli/src/mcp-server-factory.ts` and - `packages/cli/src/text-ingest.ts` (plus their tests) to reflect the - `MemoryCapture*` → `MemoryIngest*` rename. Both files import - `createLocalProjectMemoryCapture` and use a `memoryCapture` variable, so the - rename does cross the CLI boundary even though the tool surface used by the - CLI is unchanged. Re-export rename in `packages/context/src/memory/index.ts` - is part of this PR. - -The CLI's runtime use of the removed admin-tool implementations is unchanged — -only the memory rename touches CLI code. - -### PR 2 — Polish kit - -Touches all 11 retained tools. Can be one PR or split per family -(annotations + outputSchema + descriptions + error wrapping + union-drift -fixes + `jsonToolResult` type narrowing + `toResolvedWire` doc comment). - -### PR 3 — Progress notifications - -Extends `KtxSqlExecutionMcpPort` and `KtxSemanticLayerMcpPort` with optional -`onProgress`, wires the MCP handler context's notification channel, emits at -the stage boundaries listed in §2.8. - -Eventual **deletion** of the 14 admin tool implementations is a separate -follow-up spec gated on the `ktx-admin` SKILL landing. Until then they remain -in `packages/context/src/` and are used only by the CLI. - -## Verification - -### Unit tests per retained tool - -For each of the 11 retained tools: - -- Input schema accepts canonical input. -- Input schema accepts each documented normalized alt-shape (Cube - `{ dimension, granularity }`, `{ schema, table }`, bare-string `order_by`, - Cube-style `{ id, desc }` `order_by`). -- Output schema accepts the response shape returned by the underlying port. -- Error path returns `{ isError: true, content: [{ type: 'text', ... }] }` - (not a thrown exception). - -### Schema snapshot test - -A `tools/list` snapshot test in `packages/context/src/mcp/server.test.ts` -captures the exact JSON Schema each client receives for every tool. Re-runs -across PRs catch accidental schema drift (e.g., a Zod change silently -broadening the contract). - -### Annotations test - -Assert every tool's registered config carries the expected `readOnlyHint`, -`destructiveHint`, `idempotentHint`, `openWorldHint`, and `title` per §2.1. - -### Multi-client end-to-end smoke - -Stdio (Claude Desktop) and Streamable HTTP (Claude Code) are the two -transports; the other four clients (Codex, Cursor, OpenCode, universal) share -one of these transports and their config files are static. Spin up `ktx mcp -stdio` and `ktx mcp start`, call each retained tool through both transports, -verify response shape against `outputSchema`. - -### Required commands - -The named test files include slow tests that are excluded from the default -`@ktx/context` `test` script and live in `test:slow` -(`packages/context/package.json:127-128`). Implementation must run both: - -```bash -pnpm --filter @ktx/context run test -pnpm --filter @ktx/context run test:slow -pnpm --filter @ktx/context run type-check -pnpm --filter @ktx/cli run type-check -pnpm --filter @ktx/cli run test -pnpm run dead-code -``` - -CLI checks are required because PR 1 renames cross the CLI boundary -(`packages/cli/src/mcp-server-factory.ts`, `packages/cli/src/text-ingest.ts`, -the `MemoryCapture*` re-exports, plus their vitest specs). -`pnpm --filter @ktx/cli run test:slow` should also be added if PR 1 ends up -touching any of the slow-test files enumerated in `packages/cli/package.json` -(`scan.test.ts`, `setup*.test.ts`, etc.); the rename diff today does not, but -the implementation plan must re-check before merging. - -When `docs-site/content/docs/integrations/agent-clients.mdx` is touched, also -run the docs-site scripts declared in `docs-site/package.json` — there is no -`lint` script: - -```bash -pnpm --filter ktx-docs run build -pnpm --filter ktx-docs run test -``` - -### Red-green regression - -For the union-drift fixes (§2.5): revert the preprocess in `local-query.ts` or -`context-tools.ts`, run the alt-shape test → expect failure, restore → -expect pass. Same pattern as the `order_by` and `usage`-leak fixes in this -session. - -## Risks - -- **Removed tools surprise a user who depended on them via MCP.** Mitigated - by the no-back-compat rule (KTX is pre-public) and by the SKILL update - landing atomically with the surface change. Users on `ktx setup --agents` - flow get the updated SKILL the next time they re-run setup. -- **`outputSchema` validation breaks a client that doesn't tolerate - unrecognized JSON Schema keywords.** Mitigated by emitting `outputSchema` - via the same `server.registerTool` path that already produces `inputSchema`, - so both schemas are serialized by the MCP SDK as JSON Schema 2020-12 (the - dialect the SDK's tool-list types declare — - `@modelcontextprotocol/sdk/.../types.js` `inputSchema` / `outputSchema` - fields, and Appendix B). The spec uses SHOULD-validate semantics, not MUST. - The snapshot test catches drift; the multi-client smoke confirms - compatibility. -- **Progress notifications increase notification volume for clients that - poll.** Mitigated by stage-based emission (3-4 events per call max). Clients - that don't support progress simply ignore the events. -- **Renaming `MemoryCapturePort`/`MemoryCaptureService` cascades through - internal callers.** Cascade is bounded to `packages/context/src/memory/`, - `packages/context/src/mcp/`, and their tests; type-checker catches missed - call sites. - -## Open Questions - -None at design time. Open items for the implementation plan: - -- Final wording for the rewritten tool descriptions on `discover_data`, - `entity_details`, `dictionary_search`, `sl_read_source`, `sl_query`, - `sql_execution`. (Drafts can be authored during PR 2.) -- Whether `formatToolError` should redact path elements for security - (probably not — these are local-only MCP servers, and the existing error - shape doesn't redact). -- Whether to split PR 2 into per-family sub-PRs or keep it monolithic. Default - is monolithic since the polish-kit changes touch the same files and tests. - -## Appendix A — File map - -| Change | File | -|---|---| -| Tool registration removed (14 tools) | `packages/context/src/mcp/context-tools.ts` | -| Memory rename + route memory_ingest tools through the shared polish path (§2.0) | `packages/context/src/mcp/server.ts`, `packages/context/src/mcp/context-tools.ts`, `packages/context/src/memory/*` | -| Local ports update | `packages/context/src/mcp/local-project-ports.ts` | -| Port types update | `packages/context/src/mcp/types.ts` | -| Annotations, outputSchema, describe, error wrapping, union preprocess | `packages/context/src/mcp/context-tools.ts` | -| `jsonToolResult` type narrowing | `packages/context/src/mcp/context-tools.ts` | -| `toResolvedWire` invariant comment | `packages/context/src/daemon/semantic-layer-compute.ts` | -| Progress callback plumbing | `packages/context/src/daemon/semantic-layer-compute.ts`, `packages/context/src/sl/local-query.ts`, `packages/context/src/mcp/local-project-ports.ts` (`executeValidatedReadOnlySql`) | -| Tests | `packages/context/src/mcp/server.test.ts`, `packages/context/src/mcp/local-project-ports.test.ts`, `packages/context/src/sl/local-query.test.ts` | -| Skill | `packages/cli/src/skills/analytics/SKILL.md` | -| Docs | `docs-site/content/docs/integrations/agent-clients.mdx` | - -## Appendix B — MCP-spec cross-reference - -- Tool annotations (`readOnlyHint`, `destructiveHint`, `idempotentHint`, - `openWorldHint`, `title`): MCP 2025-11-25 spec `/schema` "ToolAnnotations". -- `outputSchema` with `structuredContent` SHOULD-validate: spec - `/server/tools` "Tool Result > Structured Content". -- In-band error contract (`isError: true` + text content, not JSON-RPC - error): spec `/server/tools` "Tool Execution Error Example". -- Progress notifications via `params._meta.progressToken` → - `notifications/progress`: spec `/basic/utilities/progress`. -- `inputSchema.type: "object"` requirement and JSON Schema 2020-12 default - dialect: spec `/server/tools` "Tool > inputSchema" and SEP 1613. diff --git a/docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md b/docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md deleted file mode 100644 index b2428ae5..00000000 --- a/docs/superpowers/specs/2026-05-17-isolated-diff-ingestion-design.md +++ /dev/null @@ -1,612 +0,0 @@ -# Isolated-diff ingestion design - -**Date:** 2026-05-17 -**Author:** Andrey Avtomonov -**Status:** Design - pending implementation plan - -## Background - -KTX ingests third-party context sources into durable project memory: raw source -snapshots, wiki pages, semantic-layer sources, evidence documents, candidates, -and fallback records. The current bundle runner stages raw source data in one -ingestion session worktree, then runs work units against that same mutable -worktree. - -A Metabase ingestion run exposed the failure mode this design addresses. One -work unit inferred and wrote the semantic-layer measure -`mart_account_segments.total_contract_arr_cents`, a later work unit overwrote -the same source with `total_contract_arr`, and the generated wiki page kept -referencing the stale non-existent measure. The local per-work-unit checks did -not catch the final cross-artifact inconsistency because durable writes were -accepted into shared state before final integration. - -The fix is not a Metabase-only validation patch. The same class of risk exists -any time LLM-authored work units mutate durable wiki or semantic-layer files: -Metabase cards, Notion pages and clusters, dbt YAML, MetricFlow YAML, Looker -dashboards and explores, and LookML models and views can all produce overlapping -or contested memory artifacts. KTX needs one ingestion execution model that -isolates agent-authored changes, integrates them deliberately, and validates -the final project state globally. - -## Goals - -This design creates one opinionated ingestion algorithm for all context sources. -Connector-specific code stays responsible for source-shaped work: fetching raw -data, normalizing raw files, planning work units, and optionally projecting -deterministic facts. The shared runner owns execution correctness. - -The design has these goals: - -- Run all agent-authored durable writes in isolated per-work-unit worktrees. -- Treat each work unit's git diff as its proposal artifact. -- Integrate accepted diffs through a shared artifact-aware merge path. -- Resolve expected cross-work-unit overlap with bounded agent repair before - failing the run. -- Run final global semantic gates before any changes reach the main project - worktree. -- Keep connector variance minimal and source-shaped, not pipeline-shaped. -- Avoid proposal manifests, typed candidates, and extra reporting entities for - the first implementation. -- Preserve deterministic projections for source systems with authoritative - structured metadata. - -## Non-goals - -This design does not change the wiki frontmatter schema, wiki page file layout, -the semantic-layer YAML format, or the raw source snapshot layouts. It does add -a narrow author-facing inline-code grammar for explicit wiki body references to -semantic-layer entities and raw tables, because body text is part of the -stale-reference failure class. It also does not remove source adapters' current -fetch and chunk logic in one large rewrite. - -This design does not introduce public connector knobs such as -`executionMode`, `planningStrategy`, or `conflictPolicy`. The core runner -becomes more opinionated instead. - -This design does not require all connectors to stop using candidates. Candidate -storage remains valid for flows that intentionally defer wiki curation. The -isolation model applies when a work unit writes durable project files. - -## Locked design direction - -The ingestion runner uses one flow for every source that can produce durable -changes. - -```text -fetch raw - -> optional deterministic project - -> adapter plans WorkUnit[] - -> isolated WU diffs - -> artifact-aware integration - -> global semantic gates - -> squash -``` - -The important invariant is that the core runner does not know why a work unit -exists. A dbt adapter may plan by model, Notion may plan by page or cluster, -MetricFlow may plan by graph component, and Looker may plan by dashboard or -explore. Those differences describe the source system. They are not ingestion -execution modes. - -## Architecture - -The design splits ingestion into two layers with explicit responsibility -boundaries. - -### Source adapter layer - -The adapter owns source semantics. It fetches raw evidence, normalizes that -evidence into staged files, and plans work units from the staged snapshot and -diff scope. - -The adapter may also provide deterministic projectors. A projector is code that -converts authoritative source facts into KTX artifacts without an agent. Good -examples are live database schema introspection and straightforward MetricFlow -semantic-model import. - -The isolation-relevant adapter surface remains small: - -```ts -interface SourceAdapter { - source: string; - skillNames: string[]; - - fetch?(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise; - chunk(stagedDir: string, diffSet?: DiffSet): Promise; - - project?(ctx: DeterministicProjectionContext): Promise; - resolveSlTargets?(ctx: SlTargetResolutionContext): Promise; -} -``` - -This is the subset the isolated-diff runner needs to understand source-shaped -planning and deterministic projection. It is not a proposal to delete existing -`SourceAdapter` fields. Existing lifecycle and source-support fields such as -`detect`, `readFetchReport`, `listTargetConnectionIds`, `clusterWorkUnits`, -`describeScope`, `onPullSucceeded`, `evidenceIndexing`, `triageSupported`, -`getTriageSignals`, and `reconcileSkillNames` stay part of the adapter contract -until a separate cleanup intentionally removes them with migration impact -called out. - -`chunk()` returns ordinary `WorkUnit[]`. The runner does not need a -`planningStrategy` enum because the source adapter can plan by any domain shape -that makes sense. - -### Ingestion execution layer - -The runner owns correctness, isolation, and integration. After `WorkUnit[]` -exists, all connectors follow the same execution path. - -The runner is responsible for: - -- creating the ingestion integration worktree from the project base commit; -- committing deterministic projection in the integration worktree before child - worktree creation; -- creating one child worktree per work unit from the post-projection ingestion - base commit; -- scoping tools to the work unit's raw files and allowed target connections; -- running the agent loop inside the work unit worktree; -- validating touched artifacts before accepting the work unit diff; -- collecting the work unit git diff; -- applying accepted diffs into the integration worktree; -- resolving textual and artifact-level conflicts; -- running final global gates; and -- squashing the integration worktree back to the project main worktree. - -## Worktree model - -The design uses three levels of git state. - -```text -project main worktree - ingest integration worktree - per-work-unit worktree(s) -``` - -The project main worktree is the durable KTX project state. The ingestion -integration worktree stages raw snapshots, deterministic projections, accepted -work-unit diffs, reconciliation changes, and final gate repairs before one -squash merge back to main. - -Deterministic projection runs first in the integration worktree, after the raw -snapshot is staged and before any per-work-unit worktree is created. The runner -commits those projector changes as a single projection commit. The integration -worktree's post-projection HEAD is the ingestion base commit referenced in this -design. If the adapter has no projector, the raw-snapshot commit is the -ingestion base commit. - -Each per-work-unit worktree starts from the same ingestion base commit. A work -unit never observes another concurrent work unit's transient edits. This makes -the work unit diff a clean proposal against a stable base. Work units observe -deterministic projection outputs, including through `dependencyPaths` context, -and do not re-derive authoritative projected facts. - -The integration worktree and each per-work-unit worktree must share one Git -object database, created through `git worktree add` from the same repository. -This is required so `git apply --3way` can resolve the base blobs recorded in -each work-unit patch during integration. - -The runner creates and runs child worktrees under the existing -`workUnitMaxConcurrency` setting. A run may have many planned work units, but no -more than that bound may be active or left on disk at once. The default remains -serial execution. Child worktrees must be cleaned up after the diff, transcript, -and outcome metadata are persisted, including failure paths. Adapters with -large fan-out, such as Notion, may use `clusterWorkUnits` before execution to -keep work-unit count tractable, but clustering remains source-shaped planning -rather than a separate execution mode. - -## Work-unit lifecycle - -Each work unit follows a fixed lifecycle. - -1. Create a child worktree at the ingestion base commit. -2. Build a scoped tool session for the child worktree. -3. Run the source skill and agent loop. -4. Run work-unit-local gates against touched artifacts. -5. If gates pass, record `git diff --binary` from base to child HEAD. -6. If gates fail, mark the work unit failed and discard the child worktree. -7. Clean up the child worktree after the diff and transcript are persisted. - -The work unit outcome stores the existing operational metadata KTX already -records: unit key, status, actions, touched semantic-layer sources, failure -reason, raw files, and transcript path. It does not add a proposal manifest. -The diff is the proposal. - -For `slDisallowed` work units, isolation is defense in depth. The scoped -work-unit tools must withhold semantic-layer write and edit tools, and the -integration layer must reject any otherwise accepted diff from that work unit -that touches `semantic-layer/**`. This catches buggy or bypassed tool behavior -before an invalid LookML connection-mismatch write can reach the integration -worktree. - -### Diff proposal contract - -The proposal artifact is a Git patch with binary-safe content, not the existing -hash-based raw-source `DiffSet`. - -The first implementation must use one pinned patch contract: - -- collect `git diff --binary --no-renames ..HEAD`; -- disable rename and copy detection so renames are represented as delete plus - create in version one; -- preserve mode changes from the patch metadata, but reject unexpected - executable-mode or binary changes under known text artifact roots such as - `wiki/**` and `semantic-layer/**`; -- apply each accepted patch to the integration worktree with - `git apply --3way --index`; -- do not use `git apply --reject`, because partial hunk application is not an - accepted integration state; and -- if patch application fails, leaves conflicts, or touches a path disallowed for - that work unit, roll back the integration worktree to its pre-apply HEAD and - classify the outcome as a textual conflict. - -Delete-versus-edit, recreate-versus-edit, and delete-versus-create races are -therefore textual conflicts when Git cannot apply the patch cleanly. If Git -applies the patch but known artifact validators reject the resulting tree, the -outcome is a semantic conflict. - -## Integration lifecycle - -The integration worktree applies accepted work-unit diffs after local gates -pass. The runner applies diffs in a deterministic order, using the original -work-unit index unless a future implementation introduces explicit dependency -ordering. - -Integration has three conflict classes: - -- Clean patch application: the diff applies without conflict. -- Textual conflict: git cannot apply the patch cleanly. -- Semantic conflict: the patch applies textually but creates an invalid or - inconsistent artifact. - -Textual conflicts are resolved before semantic gates run when a bounded -resolver agent can produce a valid result. Overlapping work-unit writes are -normal, especially for Metabase cards that target the same semantic-layer marts -from different collections. The runner must treat overlap as an integration -case, not as a reason to fail immediately. - -Version one is agent-first. If `git apply --3way --index` leaves conflicts, -the runner starts a resolver agent in the integration worktree. The resolver -receives only the failed patch, already-applied patches, conflicted files, -relevant work-unit transcripts, raw evidence paths, and the final-gate rules. -The resolver must preserve all non-conflicting accepted content, resolve -duplicate or competing artifact entries from evidence, and edit only files -touched by the failed patch or already-applied overlapping patches. - -The runner then reruns artifact gates for the changed files and continues with -the remaining patches if validation passes. Resolver attempts are capped to -avoid an unbounded repair loop. A run fails only after the bounded resolver -attempts cannot produce a valid integration tree. - -Deterministic semantic merge is a later optimization, not a version-one -requirement. After measuring resolver latency, cost, and failure modes, KTX can -add merge helpers for common semantic-layer YAML cases, such as additive -`measures`, `segments`, `columns`, `joins`, and `descriptions` updates keyed by -their stable logical identifiers. Those helpers can replace agent calls for -mechanical merges once the measured v1 behavior justifies the added complexity. - -The integration worktree is preserved on failure with conflict markers or -resolver edits, work-unit patches, transcripts, trace events, and the failure -report. The runner never squashes a failed or partially repaired integration -tree back to the project main worktree. - -### Gate repair stage - -The gate repair stage handles cases where patches apply cleanly but the -combined tree fails final semantic or wiki gates. This is distinct from textual -conflict resolution: the tree is textually valid, but the artifacts violate KTX -contracts. - -After each patch integration and after reconciliation, the runner runs final -artifact gates for the affected scope. If gates fail, the runner classifies the -errors before deciding whether to repair or fail. - -Repairable gate errors include: - -- stale wiki body references to renamed semantic-layer entities; -- invalid `sl_refs` entries that point to entities instead of sources; -- inline prose that accidentally uses explicit SL reference syntax; -- duplicate measures, segments, or joins with equivalent definitions; -- missing or stale wiki references created by accepted patches; and -- join or source references that can be corrected from the composed manifest - and work-unit evidence. - -High-risk gate errors fail without automatic repair unless a later -implementation adds a stronger evidence contract: - -- two work units define the same measure with different business meaning; -- a required warehouse table or column does not exist; -- a SQL source fails execution and no obvious localized rewrite exists; or -- the repair would require choosing between conflicting facts without evidence. - -For repairable errors, the runner starts a gate repair agent with the exact -gate errors, changed files, relevant work-unit transcripts, raw evidence paths, -and final-gate rules. The agent may edit only the files involved in the gate -failure. The runner reruns gates after each repair attempt and caps attempts to -one or two passes per integration stage. If the tree still fails, the run stops -with the final gate report and preserved integration worktree. - -### Reconciliation in the new flow - -Reconciliation remains a shared runner stage, but it runs as a serial -integration-stage pass instead of a parallel work unit. - -The runner applies all accepted work-unit diffs to the integration worktree, -resolves textual conflicts that can be resolved, and then runs reconciliation in -that integration worktree before final global gates and before squash. -Reconciliation must see the integrated state because its job is to resolve -cross-work-unit duplicates, evictions, fallbacks, and source-specific -reconcile guidance. - -Reconciliation runs exactly once per integration pass, serially against the -integration worktree, after all accepted work-unit diffs have been applied and -after textual conflicts are resolved. It never runs inside a child worktree and -never overlaps with work-unit execution. This is the safety carve-out from the -isolation goal: concurrent agent writes are the failure mode being avoided, and -reconciliation is non-concurrent by construction. - -Reconciliation is not allowed to mutate project main directly. Its changes are -captured as a reconciliation diff against the pre-reconciliation integration -HEAD and recorded in the existing stage/report metadata. Reconciliation gates -validate the artifacts touched by the reconciliation diff plus any wiki page or -semantic-layer source referenced by changed frontmatter or body references, -using the same artifact-class validators as work-unit gates. Reconciliation may -write only to target connections authorized by the adapter for the ingest run, -but it is not subject to any single work unit's `slDisallowed` scope. The final -global gates validate the combined tree after reconciliation. If reconciliation -introduces an invalid wiki or semantic-layer reference, touches an unauthorized -target, or records an unresolvable artifact conflict, the runner sends -repairable failures through the gate repair stage and stops before squash only -when bounded repair cannot produce a valid tree. - -## Artifact-aware integration - -KTX durable artifacts are structured enough that git-only merge is not a strong -correctness boundary. Artifact-aware integration must parse and validate known -file classes after diffs are applied. - -The first implementation must cover these worktree file classes: - -- semantic-layer source YAML; -- wiki markdown frontmatter; -- wiki body references to semantic-layer sources, measures, dimensions, and raw - warehouse tables. - -Unmapped fallback records are not worktree files in version one. They remain -typed stage-index and report records emitted by `emit_unmapped_fallback`; the -integration layer validates their raw paths and structured reason codes as -report metadata, not as mergeable artifacts. - -Provenance also stays out of the worktree in version one. The source of truth is -the ingest provenance store and report body. Before inserting provenance rows, -the global gate derives the planned rows from accepted work-unit actions, -reconciliation actions, artifact-resolution records, and skipped raw files, then -checks those rows against the integrated worktree and staged raw hashes. Moving -provenance to on-disk files would be a separate schema migration, not part of -this design. - -Artifact-resolution records are the existing merged or subsumed reconciliation -outputs emitted through `emit_artifact_resolution` as -`ArtifactResolutionRecord` stage-index records. They are in-memory stage -records, not worktree files, and they feed the provenance gate. - -Artifact-aware integration starts with validation plus bounded agent repair. -It does not need semantic-layer YAML merge helpers in version one. If two diffs -contest the same source YAML or wiki page and bounded agent repair cannot prove -correctness, the runner must stop rather than silently accepting stale -references. Deterministic semantic merge helpers can be added after v1 metrics -show which conflicts are frequent, mechanical, and worth optimizing. - -## Global semantic gates - -Final gates run after every accepted diff, deterministic projection, and -reconciliation change has landed in the integration worktree. These gates are -global because the final failure can emerge only after independent valid diffs -combine. - -The final gates must include: - -- semantic-layer validation for touched and dependency sources; -- wiki `wiki_refs` validation; -- wiki frontmatter `sl_refs` validation, including source-level and - measure-level references; -- wiki body validation for explicit semantic-layer source, measure, dimension, - and table references; and -- provenance validation for raw paths referenced by new or changed artifacts - before those rows are inserted into SQLite. - -For semantic-layer validation, touched sources are sources changed by accepted -work-unit diffs, deterministic projection, or reconciliation. Dependency sources -are their direct declared-join neighbors in the composed semantic-layer graph, -including sources they join to and sources that join to them. Version one runs -the existing whole-connection structural checks and source-scoped checks with -the touched-and-dependency source set; it does not expand dependency scope to a -transitive SQL-projection closure. - -The wiki body gate needs a narrow grammar so ordinary prose does not become a -semantic-layer reference. In version one, an explicit body reference is one of -these Markdown forms outside fenced code blocks: - -- an inline code token in the form `source.entity`, where both parts are plain - identifier tokens, `source` matches a visible semantic-layer source, and - `entity` must match one of that source's measures, dimensions, or segments; -- an inline code token in the form `connectionId/source.entity`, where - `source.entity` follows the same plain-identifier rule and validates against - that specific target connection; -- an inline code token in the form `source:source_name`, which validates a - source-level semantic-layer reference; or -- an inline code token in the form `table:qualified_table_name`, which validates - a raw warehouse table reference against the visible raw table/catalog sources. - -The parser ignores unformatted prose, fenced SQL examples, wildcard patterns -such as `mart_nrr_quarterly.*_arr_cents`, inline SQL predicates such as -`users.is_internal = false`, and unprefixed single-token inline code. Two-part -inline code that does not name a visible semantic-layer source is not treated -as an SL entity reference; use the `table:` prefix for raw warehouse table -references. - -The `total_contract_arr_cents` incident is the regression case for this gate: -the integrated tree must fail if a wiki page references -`mart_account_segments.total_contract_arr_cents` as an inline-code body token -while the final semantic-layer source defines only `total_contract_arr`. - -## Deterministic projection - -Some connectors have authoritative structured inputs that do not need an LLM to -write KTX artifacts. Those connectors can provide deterministic projectors that -run in the integration worktree. - -Projection is different from work-unit execution: - -- projectors are code, not agents; -- projectors run against the integration worktree; -- projectors produce ordinary durable file changes; and -- projector outputs still pass final global gates. - -The runner infers hybrid behavior from the adapter. If an adapter has both -projectors and work units, it is hybrid. If it has only projectors, it is -deterministic. If it has only work units, it uses isolated diffs. No public -`executionMode` knob is needed. - -## Connector migration notes - -Each connector keeps its source-shaped planning logic. The migration changes -where durable writes happen and how they are integrated. - -### Metabase - -Metabase must move first because it produced the observed stale-measure wiki -reference. Collection and card chunking can remain adapter-specific, but direct -wiki and semantic-layer writes must happen in per-work-unit worktrees. - -The regression test must reproduce two work units that touch -`mart_account_segments`: one writes a wiki reference to an inferred measure and -another leaves the final source with a different measure name. The final global -gate must reject the integrated tree. - -### dbt - -dbt uses source-shaped planning by model or schema file. Deterministic -projection is appropriate for straightforward model, source, column, and -description facts when dbt artifacts are authoritative. Agent work units remain -useful for business wiki synthesis, ambiguous relationship interpretation, and -enrichment that is not directly represented in dbt YAML. - -### MetricFlow - -MetricFlow uses source-shaped planning by graph component. Existing -deterministic semantic-model import code becomes a projector in the ingestion -flow. Agent work units handle unsupported constructs, cross-model explanations, -and wiki synthesis. - -### Looker - -Looker already defers some dashboard and look knowledge through candidates. -That can continue. Any direct semantic-layer writes from explores or query -translation must run through isolated work-unit diffs. - -Looker-specific API and file-adapter collisions remain connector domain logic, -but final correctness still belongs to the shared integration gates. - -### LookML - -LookML already has useful source-shaped ownership rules: models, views, orphan -views, dashboards, and connection-mismatch guards. Those rules stay in the -adapter. Direct semantic-layer writes move into isolated work-unit diffs. - -Connection-mismatch work units can keep their existing write restrictions. The -runner enforces those restrictions through scoped tools and target connection -resolution. - -### Notion - -Notion pages and clusters can create overlapping durable wiki knowledge and can -write semantic-layer overlays after warehouse verification. Notion therefore -uses the same isolated-diff execution model for direct durable writes. - -Large Notion workspaces still need source-shaped clustering to control context -size and cost. Clustering remains adapter logic; correctness comes from isolated -diffs and final global gates. - -## Minimal connector variance - -New connectors must not choose from a menu of ingestion architectures. They -must provide the small amount of source-specific behavior the shared runner -needs. - -Every connector answers these questions: - -- How does KTX fetch or receive raw evidence? -- How does KTX normalize that evidence into staged files? -- How does KTX split the staged evidence into `WorkUnit[]`? -- Are any source facts authoritative enough for deterministic projection? -- Which target semantic-layer connections can the connector write to? - -Everything else is shared runner behavior. - -## Regression tests - -The implementation plan must start with narrow tests that prove the new -execution model prevents the known failure class. - -The first test creates a fake or Metabase-like adapter with two work units -starting from the same base: - -1. Work unit A writes a wiki page that references - `mart_account_segments.total_contract_arr_cents` as an inline-code body - token. -2. Work unit B writes or overwrites the final semantic-layer source with only - `total_contract_arr`. -3. Both work units pass their local gates in isolation. -4. Integration applies both diffs. -5. The final global gate fails the run before squash. - -Additional tests cover: - -- two work units editing different wiki pages without conflict; -- two work units editing the same semantic-layer overlay with additive changes, - where the resolver agent preserves both changes and gates the repaired file; -- two work units editing the same semantic-layer overlay with incompatible - definitions, where the resolver agent receives the conflict context and the - run fails only after bounded repair attempts cannot prove a result; -- a textual conflict in a wiki page where the resolver agent preserves - non-conflicting accepted content and gates the repaired page before squash; -- a cleanly merged tree that fails final gates, where the gate repair agent - fixes a stale wiki reference and the run continues; -- an unrepairable final-gate failure, such as a missing warehouse column, where - the runner stops with a preserved integration worktree and report; -- a hybrid adapter case where deterministic projector outputs are visible in a - child worktree before work-unit wiki synthesis, and the final global gate - catches any stale reference to a non-existent projected semantic-layer entity; -- Notion-style direct wiki writes with invalid `sl_refs`; and -- LookML-style `slDisallowed` work units where write tools are unavailable and - integration rejects any diff that still touches `semantic-layer/**`. - -## Rollout - -The rollout must be incremental because the current runner is shared by all -adapters. - -The rollout switch is runner-owned. During migration it may be a private -per-source allowlist, or an internal `IngestSettingsPort` map keyed by -`sourceKey`, but it must not become a `SourceAdapter` field or public connector -configuration knob. - -1. Add the per-work-unit worktree executor behind that internal runner setting. -2. Add diff collection and deterministic integration in the existing runner. -3. Add bounded resolver-agent handling for textual conflicts. -4. Add final global wiki and semantic-layer reference gates, including the wiki - body reference parser defined above. -5. Add bounded gate-repair-agent handling for repairable final-gate failures. -6. Instrument resolver latency, attempts, repaired files, and failure classes. -7. Migrate Metabase to the new execution path first. -8. Migrate Notion, LookML, Looker, dbt, and MetricFlow. -9. Add deterministic semantic merge helpers only after v1 metrics show which - agent repairs are frequent and mechanical enough to justify optimization. -10. Promote the new path to the default after the Metabase regression test and - at least one non-Metabase connector pass. -11. Remove the old shared-worktree work-unit execution path. - -The rollout is complete when every connector that permits agent-authored durable -writes uses isolated diffs and all integrations pass the same final global -gates. diff --git a/knip.json b/knip.json index 2fd48187..51459946 100644 --- a/knip.json +++ b/knip.json @@ -7,6 +7,7 @@ "ignoreDependencies": [ "@semantic-release/commit-analyzer", "@semantic-release/github", + "@semantic-release/npm", "@semantic-release/release-notes-generator", "conventional-changelog-conventionalcommits" ] diff --git a/package.json b/package.json index ad773b28..71970b58 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,6 @@ "setup:dev": "node scripts/setup-dev.mjs", "release:published-smoke": "node scripts/published-package-smoke.mjs --require-config", "release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", - "release:npm-publish": "node scripts/publish-public-npm-package.mjs", "release:readiness": "node scripts/release-readiness.mjs", "release:update-version": "node scripts/update-public-release-version.mjs", "relationships:acquire-public-fixtures": "node scripts/acquire-public-benchmark-fixtures.mjs", @@ -43,7 +42,7 @@ "test": "node --test scripts/*.test.mjs && pnpm --filter './packages/*' run test", "test:coverage": "pnpm run test:coverage:ts && pnpm run test:coverage:py", "test:coverage:py": "uv run pytest --cov=python/ktx-sl/semantic_layer --cov=python/ktx-daemon/src/ktx_daemon --cov-report=xml:coverage/python.xml --cov-report=term", - "test:coverage:ts": "pnpm --filter './packages/*' run build && pnpm --filter './packages/*' run test --coverage --coverage.reporter=lcov --coverage.exclude='dist/**'", + "test:coverage:ts": "pnpm --filter './packages/*' run build && pnpm --filter './packages/*' run test --coverage --coverage.reporter=lcov --coverage.exclude='dist/**' && node scripts/normalize-lcov-paths.mjs packages/*/coverage/lcov.info", "test:slow": "pnpm --filter @ktx/context run test:slow && pnpm --filter @ktx/cli run test:slow", "type-check": "pnpm --filter './packages/*' run type-check" }, @@ -54,6 +53,7 @@ "@semantic-release/exec": "^7.1.0", "@semantic-release/git": "^10.0.1", "@semantic-release/github": "^12.0.8", + "@semantic-release/npm": "^13.1.5", "@semantic-release/release-notes-generator": "^14.1.1", "@types/node": "^25.7.0", "better-sqlite3": "^12.10.0", diff --git a/packages/cli/src/commands/mcp-commands.test.ts b/packages/cli/src/commands/mcp-commands.test.ts index 9a54f22a..29235bce 100644 --- a/packages/cli/src/commands/mcp-commands.test.ts +++ b/packages/cli/src/commands/mcp-commands.test.ts @@ -79,7 +79,43 @@ describe('registerMcpCommands', () => { expect(startDaemon).toHaveBeenCalledTimes(1); expect(context.io.stdout.write).toHaveBeenCalledWith( - 'KTX MCP daemon already running: http://127.0.0.1:7878/mcp\n', + [ + 'KTX MCP daemon already running: http://127.0.0.1:7878/mcp', + '', + 'KTX is ready for configured agents.', + 'Open your agent for this KTX project and ask a data question, for example:', + ' "Use KTX to show me the available tables and metrics."', + '', + ].join('\n'), + ); + }); + + it('prints a friendly next step after starting the daemon', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const startDaemon = vi.fn().mockResolvedValue({ + status: 'started', + url: 'http://127.0.0.1:7878/mcp', + state: { + schemaVersion: 1, + pid: 4242, + host: '127.0.0.1', + port: 7878, + tokenAuth: false, + projectDir: '/tmp/ktx-started', + startedAt: '2026-05-14T00:00:00.000Z', + logPath: '/tmp/ktx-started/.ktx/logs/mcp.log', + }, + }); + const context = makeContext({ deps: { mcp: { startDaemon } } }); + registerMcpCommands(program, context); + + await program.parseAsync(['--project-dir', '/tmp/ktx-started', 'mcp', 'start'], { from: 'user' }); + + expect(context.io.stdout.write).toHaveBeenCalledWith( + expect.stringContaining('KTX MCP daemon started: http://127.0.0.1:7878/mcp\n\nKTX is ready for configured agents.'), + ); + expect(context.io.stdout.write).toHaveBeenCalledWith( + expect.stringContaining('"Use KTX to show me the available tables and metrics."'), ); }); diff --git a/packages/cli/src/commands/mcp-commands.ts b/packages/cli/src/commands/mcp-commands.ts index 9e583ce4..be7044a7 100644 --- a/packages/cli/src/commands/mcp-commands.ts +++ b/packages/cli/src/commands/mcp-commands.ts @@ -25,6 +25,17 @@ function binPath(): string { return fileURLToPath(new URL('../bin.js', import.meta.url)); } +function formatMcpStartResultMessage(input: { status: 'started' | 'already-running'; url: string }): string { + return [ + input.status === 'started' ? `KTX MCP daemon started: ${input.url}` : `KTX MCP daemon already running: ${input.url}`, + '', + 'KTX is ready for configured agents.', + 'Open your agent for this KTX project and ask a data question, for example:', + ' "Use KTX to show me the available tables and metrics."', + '', + ].join('\n'); +} + export function registerMcpCommands(program: Command, context: KtxCliCommandContext): void { const mcp = program.command('mcp').description('Run the KTX MCP HTTP server'); @@ -82,11 +93,7 @@ export function registerMcpCommands(program: Command, context: KtxCliCommandCont allowedOrigins: options.allowedOrigin, binPath: binPath(), }); - context.io.stdout.write( - result.status === 'started' - ? `KTX MCP daemon started: ${result.url}\n` - : `KTX MCP daemon already running: ${result.url}\n`, - ); + context.io.stdout.write(formatMcpStartResultMessage({ status: result.status, url: result.url })); }); mcp diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index fbf366c9..54628346 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -85,8 +85,6 @@ function optionWasSpecified(command: Command, optionName: string): boolean { function shouldShowSetupEntryMenu( options: { - new?: boolean; - existing?: boolean; agents?: boolean; target?: string; global?: boolean; @@ -98,7 +96,6 @@ function shouldShowSetupEntryMenu( anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; llmModel?: string; - anthropicModel?: string; vertexProject?: string; vertexLocation?: string; skipLlm?: boolean; @@ -108,7 +105,6 @@ function shouldShowSetupEntryMenu( skipEmbeddings?: boolean; database?: KtxSetupDatabaseDriver[]; databaseConnectionId?: string[]; - newDatabaseConnectionId?: string; databaseUrl?: string; databaseSchema?: string[]; enableQueryHistory?: boolean; @@ -160,8 +156,6 @@ function shouldShowSetupEntryMenu( } return ![ - 'new', - 'existing', 'agents', 'target', 'global', @@ -173,7 +167,6 @@ function shouldShowSetupEntryMenu( 'anthropicApiKeyEnv', 'anthropicApiKeyFile', 'llmModel', - 'anthropicModel', 'vertexProject', 'vertexLocation', 'skipLlm', @@ -181,7 +174,6 @@ function shouldShowSetupEntryMenu( 'embeddingApiKeyEnv', 'embeddingApiKeyFile', 'skipEmbeddings', - 'newDatabaseConnectionId', 'databaseUrl', 'enableQueryHistory', 'disableQueryHistory', @@ -214,8 +206,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo .command('setup') .description('Set up or resume a local KTX project') .addOption(new Option('--project-dir ', 'KTX project directory').hideHelp()) - .addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false)) - .addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false)) .option('--agents', 'Install agent integration only', false) .addOption( new Option('--target ', 'Agent target').choices([ @@ -230,7 +220,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo .option('--global', 'Install agent integration into the global target scope', false) .option('--local', 'Install Claude Code MCP config into the private per-project ~/.claude.json scope', false) .addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false)) - .option('--yes', 'Accept safe defaults in non-interactive setup', false) + .option('--yes', 'Accept project creation and runtime install defaults where setup confirms', false) .option('--no-input', 'Disable interactive terminal input') .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend).hideHelp()) .addOption( @@ -240,7 +230,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(), ) .addOption(new Option('--llm-model ', 'LLM model ID or backend model alias').hideHelp()) - .addOption(new Option('--anthropic-model ', 'Anthropic model ID to validate and save').hideHelp()) .addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp()) .addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp()) .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) @@ -269,16 +258,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo .default([] as string[]) .hideHelp(), ) - .addOption( - new Option('--new-database-connection-id ', 'Connection id for one new database connection') - .argParser((value) => { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - return value; - }) - .hideHelp(), - ) .addOption( new Option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(), ) @@ -365,11 +344,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo context.setExitCode(1); return; } - if (options.llmModel && options.anthropicModel) { - context.io.stderr.write('Choose only one LLM model flag: --llm-model or --anthropic-model.\n'); - context.setExitCode(1); - return; - } if ( options.llmBackend && options.llmBackend !== 'anthropic' && @@ -419,12 +393,18 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo return; } - const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto'; + const creatingDatabaseConnection = options.database.length > 0 || options.databaseUrl !== undefined; + if (creatingDatabaseConnection && options.databaseConnectionId.length > 1) { + context.io.stderr.write('Choose only one new database connection id when configuring a database.\n'); + context.setExitCode(1); + return; + } + const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project'; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), - mode, + mode: 'auto', agents: options.agents === true, ...(options.target ? { target: options.target } : {}), agentScope: resolvedAgentScope, @@ -436,7 +416,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo ...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}), ...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}), ...(options.llmModel ? { llmModel: options.llmModel } : {}), - ...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}), ...(options.vertexProject ? { vertexProject: options.vertexProject } : {}), ...(options.vertexLocation ? { vertexLocation: options.vertexLocation } : {}), skipLlm: options.skipLlm === true, @@ -445,8 +424,12 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo ...(options.embeddingApiKeyFile ? { embeddingApiKeyFile: options.embeddingApiKeyFile } : {}), skipEmbeddings: options.skipEmbeddings === true, ...(options.database.length > 0 ? { databaseDrivers: options.database } : {}), - ...(options.databaseConnectionId.length > 0 ? { databaseConnectionIds: options.databaseConnectionId } : {}), - ...(options.newDatabaseConnectionId ? { databaseConnectionId: options.newDatabaseConnectionId } : {}), + ...(options.databaseConnectionId.length > 0 && creatingDatabaseConnection + ? { databaseConnectionId: options.databaseConnectionId[0] } + : {}), + ...(options.databaseConnectionId.length > 0 && !creatingDatabaseConnection + ? { databaseConnectionIds: options.databaseConnectionId } + : {}), ...(options.databaseUrl ? { databaseUrl: options.databaseUrl } : {}), databaseSchemas: options.databaseSchema, ...(options.enableQueryHistory ? { enableQueryHistory: true } : {}), diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index f08e9d2d..2a397653 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -676,8 +676,7 @@ describe('runKtxDoctor', () => { ' adapters:', ' - live-database', ' embeddings:', - ' backend: deterministic', - ' model: deterministic', + ' backend: none', ' dimensions: 8', '', ].join('\n'), @@ -694,8 +693,8 @@ describe('runKtxDoctor', () => { ).resolves.toBe(0); expect(testIo.stdout()).toContain('Embeddings'); - expect(testIo.stdout()).toContain('deterministic'); - expect(testIo.stdout()).toContain('semantic search degraded'); + expect(testIo.stdout()).toContain('none'); + expect(testIo.stdout()).toContain('semantic search will be skipped'); delete process.env.ANTHROPIC_API_KEY; }); diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index a48da8d5..3c12e583 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -470,8 +470,6 @@ describe('runKtxCli', () => { expect(stdout).not.toContain('setup context'); for (const hiddenFlag of [ - '--new', - '--existing', '--agent-scope', '--skip-agents', '--llm-backend', @@ -480,7 +478,6 @@ describe('runKtxCli', () => { '--embedding-backend', '--database ', '--database-connection-id', - '--new-database-connection-id', '--enable-historic-sql', '--historic-sql-min-executions', '--enable-query-history', @@ -842,8 +839,12 @@ describe('runKtxCli', () => { it('rejects removed setup options', async () => { const setup = vi.fn(async () => 0); const cases = [ + ['setup', '--new'], + ['setup', '--existing'], ['setup', '--project'], ['setup', '--agent-scope', 'global'], + ['setup', '--anthropic-model', 'claude-sonnet-4-6'], + ['setup', '--new-database-connection-id', 'warehouse'], ['setup', '--skip-initial-source-ingest'], ]; @@ -1065,7 +1066,7 @@ describe('runKtxCli', () => { '--no-input', '--anthropic-api-key-env', 'ANTHROPIC_API_KEY', - '--anthropic-model', + '--llm-model', 'claude-sonnet-4-6', ], setupIo.io, @@ -1080,7 +1081,7 @@ describe('runKtxCli', () => { inputMode: 'disabled', cliVersion: '0.1.0-rc.1', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }), setupIo.io, @@ -1104,7 +1105,7 @@ describe('runKtxCli', () => { 'local-gcp-project', '--vertex-location', 'us-east5', - '--anthropic-model', + '--llm-model', 'claude-sonnet-4-6', ], setupIo.io, @@ -1121,7 +1122,7 @@ describe('runKtxCli', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }), setupIo.io, @@ -1239,7 +1240,7 @@ describe('runKtxCli', () => { '--skip-embeddings', '--database', 'postgres', - '--new-database-connection-id', + '--database-connection-id', 'warehouse', '--database-url', 'env:DATABASE_URL', @@ -1283,18 +1284,41 @@ describe('runKtxCli', () => { const setup = vi.fn(async () => 0); await expect( - runKtxCli(['setup', '--new-database-connection-id', 'status', '--no-input'], testIo.io, { setup }), + runKtxCli(['setup', '--database-connection-id', 'status', '--no-input'], testIo.io, { setup }), ).resolves.toBe(0); expect(setup).toHaveBeenCalledWith( expect.objectContaining({ command: 'run', - databaseConnectionId: 'status', + databaseConnectionIds: ['status'], }), testIo.io, ); }); + it('dispatches non-TTY agents setup with target without requiring --no-input or --yes', async () => { + const testIo = makeIo({ stdoutIsTty: false }); + const setup = vi.fn(async () => 0); + + await expect( + runKtxCli(['--project-dir', tempDir, 'setup', '--agents', '--target', 'claude-code'], testIo.io, { setup }), + ).resolves.toBe(0); + + expect(setup).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'run', + projectDir: tempDir, + agents: true, + target: 'claude-code', + agentScope: 'project', + inputMode: 'auto', + yes: false, + }), + testIo.io, + ); + expect(testIo.stderr()).toBe(''); + }); + it('dispatches setup source flags', async () => { const setup = vi.fn(async () => 0); const testIo = makeIo(); diff --git a/packages/cli/src/ingest-depth.ts b/packages/cli/src/ingest-depth.ts index f5706d8d..f8e5d06e 100644 --- a/packages/cli/src/ingest-depth.ts +++ b/packages/cli/src/ingest-depth.ts @@ -62,7 +62,6 @@ export function deepReadinessGaps(config: KtxProjectConfig): string[] { if ( !embeddings || embeddings.backend === 'none' || - embeddings.backend === 'deterministic' || !embeddings.model || embeddings.dimensions <= 0 ) { diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index 8b18716f..d64dd7e1 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -133,7 +133,7 @@ export async function writeMetabaseConfig(projectDir: string): Promise { ' adapters:', ' - metabase', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', @@ -502,7 +502,7 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync ' adapters:', ' - metabase', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index f4229dc9..aef06f41 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -262,7 +262,7 @@ describe('runKtxIngest', () => { { command: 'run', projectDir, - mode: 'new', + mode: 'auto', agents: false, agentScope: 'project', skipAgents: true, @@ -322,7 +322,7 @@ describe('runKtxIngest', () => { expect(runIo.stderr()).toContain('Configure a local Claude Code session or API-backed LLM, then rerun ingest:'); expect(runIo.stderr()).toContain(`ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`); expect(runIo.stderr()).toContain( - `ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`, + `ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, ); }); @@ -777,7 +777,7 @@ describe('runKtxIngest', () => { ' adapters:', ' - metabase', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', @@ -1869,7 +1869,7 @@ describe('runKtxIngest', () => { ' adapters:', ' - looker', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/managed-local-embeddings.test.ts b/packages/cli/src/managed-local-embeddings.test.ts index cbb9b5f1..85fa00c9 100644 --- a/packages/cli/src/managed-local-embeddings.test.ts +++ b/packages/cli/src/managed-local-embeddings.test.ts @@ -150,6 +150,8 @@ describe('ensureManagedLocalEmbeddingsDaemon', () => { }), ).resolves.toEqual({ baseUrl: 'http://127.0.0.1:61234', + stdoutLog: '/work/proj/.ktx/runtime/daemon.stdout.log', + stderrLog: '/work/proj/.ktx/runtime/daemon.stderr.log', env: { [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234', }, diff --git a/packages/cli/src/managed-local-embeddings.ts b/packages/cli/src/managed-local-embeddings.ts index 8c383ef5..f485a942 100644 --- a/packages/cli/src/managed-local-embeddings.ts +++ b/packages/cli/src/managed-local-embeddings.ts @@ -14,6 +14,8 @@ import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from '. export interface ManagedLocalEmbeddingsDaemon { baseUrl: string; + stdoutLog: string; + stderrLog: string; env: Record; } @@ -91,6 +93,8 @@ export async function ensureManagedLocalEmbeddingsDaemon( return { baseUrl: daemon.baseUrl, + stdoutLog: daemon.state.stdoutLog, + stderrLog: daemon.state.stderrLog, env: { [MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: daemon.baseUrl, }, diff --git a/packages/cli/src/managed-mcp-daemon.test.ts b/packages/cli/src/managed-mcp-daemon.test.ts index e28c4a3e..d72bb6a4 100644 --- a/packages/cli/src/managed-mcp-daemon.test.ts +++ b/packages/cli/src/managed-mcp-daemon.test.ts @@ -11,6 +11,8 @@ import { type KtxMcpDaemonState, } from './managed-mcp-daemon.js'; +type KtxMcpDaemonStartOptions = Parameters[0]; + function child(pid = 4242): KtxMcpDaemonChild { return { pid, unref: vi.fn() }; } @@ -40,6 +42,7 @@ describe('managed MCP daemon lifecycle', () => { }); afterEach(async () => { + vi.unstubAllEnvs(); await rm(tempDir, { recursive: true, force: true }); }); @@ -94,6 +97,33 @@ describe('managed MCP daemon lifecycle', () => { ); }); + it('sanitizes IPv6 CIDR entries from child NO_PROXY env', async () => { + vi.stubEnv('NO_PROXY', 'localhost,fd07:b51a:cc66:f0::/64'); + vi.stubEnv('no_proxy', '::1,fd00::/8,*.orb.local'); + const spawnDaemon = vi.fn>(() => child(5555)); + + await startKtxMcpDaemon({ + projectDir, + cliVersion: '0.0.0-test', + host: '127.0.0.1', + port: 7879, + allowedHosts: [], + allowedOrigins: [], + binPath: '/repo/packages/cli/dist/bin.js', + spawnDaemon, + processAlive: vi.fn(() => false), + portAvailable: vi.fn(async () => true), + now: () => new Date('2026-05-14T00:00:00.000Z'), + }); + + const env = spawnDaemon.mock.calls[0]?.[2].env; + if (!env) { + throw new Error('Expected MCP daemon spawn env'); + } + expect(env.NO_PROXY).toBe('localhost,::1,*.orb.local'); + expect(env.no_proxy).toBe(env.NO_PROXY); + }); + it('returns already-running without spawning when the daemon is alive at the same host/port', async () => { await mkdir(join(projectDir, '.ktx'), { recursive: true }); await writeFile(join(projectDir, '.ktx/mcp.json'), `${JSON.stringify(state(projectDir), null, 2)}\n`); diff --git a/packages/cli/src/managed-mcp-daemon.ts b/packages/cli/src/managed-mcp-daemon.ts index ef3df2a9..dd3fb821 100644 --- a/packages/cli/src/managed-mcp-daemon.ts +++ b/packages/cli/src/managed-mcp-daemon.ts @@ -4,6 +4,7 @@ import { createServer } from 'node:net'; import { dirname, join } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { z } from 'zod'; +import { sanitizeChildProxyEnv } from './proxy-env.js'; export interface KtxMcpDaemonState { schemaVersion: 1; @@ -166,11 +167,11 @@ export async function startKtxMcpDaemon(options: { const child = (options.spawnDaemon ?? defaultSpawnDaemon)(process.execPath, args, { detached: true, stdio: ['ignore', log.fd, log.fd], - env: { + env: sanitizeChildProxyEnv({ ...process.env, KTX_CLI_VERSION: options.cliVersion, ...(options.token ? { KTX_MCP_TOKEN: options.token } : {}), - }, + }), }); if (!child.pid) { throw new Error('Failed to start KTX MCP daemon: child process pid was not available.'); diff --git a/packages/cli/src/managed-python-command.test.ts b/packages/cli/src/managed-python-command.test.ts index a63f162e..767d8dd1 100644 --- a/packages/cli/src/managed-python-command.test.ts +++ b/packages/cli/src/managed-python-command.test.ts @@ -99,6 +99,7 @@ function installResult(features: KtxRuntimeFeature[] = ['core']): ManagedPythonR asset: { manifest: installedManifest.asset, wheelPath: '/assets/python/kaelio_ktx-0.2.0-py3-none-any.whl', + requiresPython: { specifier: '>=3.13', minimumVersion: '3.13' }, }, manifest: installedManifest, }; diff --git a/packages/cli/src/managed-python-daemon.test.ts b/packages/cli/src/managed-python-daemon.test.ts index c7009548..8797fb8f 100644 --- a/packages/cli/src/managed-python-daemon.test.ts +++ b/packages/cli/src/managed-python-daemon.test.ts @@ -4,7 +4,6 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { readManagedPythonDaemonStatus, - sanitizeProxyEnv, startManagedPythonDaemon, stopAllManagedPythonDaemons, stopManagedPythonDaemon, @@ -80,6 +79,7 @@ function installResult(root: string, features: Array<'core' | 'local-embeddings' asset: { manifest: manifest(root, features).asset, wheelPath: join(root, 'assets', 'python', 'kaelio_ktx-0.2.0-py3-none-any.whl'), + requiresPython: { specifier: '>=3.13', minimumVersion: '3.13' }, }, manifest: manifest(root, features), }; @@ -133,6 +133,7 @@ describe('managed Python daemon lifecycle', () => { }); afterEach(async () => { + vi.unstubAllEnvs(); await rm(tempDir, { recursive: true, force: true }); }); @@ -188,6 +189,27 @@ describe('managed Python daemon lifecycle', () => { }); }); + it('sanitizes IPv6 CIDR entries from child NO_PROXY env', async () => { + vi.stubEnv('NO_PROXY', 'localhost,fd07:b51a:cc66:f0::/64,127.0.0.0/8'); + vi.stubEnv('no_proxy', '::1,fd00::/8,*.orb.local'); + const spawnDaemon = makeSpawn(5555); + + await startManagedPythonDaemon({ + ...daemonOptionsBase(tempDir), + features: ['local-embeddings'], + installRuntime: vi.fn(async () => installResult(tempDir, ['core', 'local-embeddings'])), + spawnDaemon, + fetch: makeFetch(), + allocatePort: vi.fn(async () => 61234), + now: () => new Date('2026-05-11T00:00:00.000Z'), + pollIntervalMs: 1, + }); + + const env = vi.mocked(spawnDaemon).mock.calls[0]?.[2].env; + expect(env?.NO_PROXY).toBe('localhost,127.0.0.0/8,::1,*.orb.local'); + expect(env?.no_proxy).toBe(env?.NO_PROXY); + }); + it('makes a final health probe before reporting startup failure', async () => { const spawnDaemon = makeSpawn(5556); const installRuntime = vi.fn(async () => installResult(tempDir)); @@ -405,38 +427,3 @@ describe('managed Python daemon lifecycle', () => { expect(await readFile(layout(tempDir).daemonStatePath, 'utf8')).toContain('"pid": 4242'); }); }); - -describe('sanitizeProxyEnv', () => { - it('removes IPv6 CIDR entries from NO_PROXY that crash httpx', () => { - const cleaned = sanitizeProxyEnv({ - NO_PROXY: 'localhost,127.0.0.1,fd07:b51a:cc66:f0::/64,*.orb.internal', - no_proxy: 'localhost,127.0.0.1,fd07:b51a:cc66:f0::/64,*.orb.internal', - }); - expect(cleaned.NO_PROXY).toBe('localhost,127.0.0.1,*.orb.internal'); - expect(cleaned.no_proxy).toBe('localhost,127.0.0.1,*.orb.internal'); - }); - - it('deletes NO_PROXY entirely when every entry is unparseable', () => { - const cleaned = sanitizeProxyEnv({ NO_PROXY: 'fd07::/64,::1' }); - expect(cleaned.NO_PROXY).toBeUndefined(); - }); - - it('preserves IPv4 addresses, IPv4 CIDRs, hostnames, and wildcards', () => { - const cleaned = sanitizeProxyEnv({ - NO_PROXY: '127.0.0.0/8,10.0.0.1,localhost,*.example.com', - }); - expect(cleaned.NO_PROXY).toBe('127.0.0.0/8,10.0.0.1,localhost,*.example.com'); - }); - - it('leaves other env vars untouched', () => { - const cleaned = sanitizeProxyEnv({ PATH: '/usr/bin', NO_PROXY: '::1', FOO: 'bar' }); - expect(cleaned.PATH).toBe('/usr/bin'); - expect(cleaned.FOO).toBe('bar'); - expect(cleaned.NO_PROXY).toBeUndefined(); - }); - - it('does nothing when NO_PROXY is not set', () => { - const cleaned = sanitizeProxyEnv({ PATH: '/usr/bin' }); - expect(cleaned).toEqual({ PATH: '/usr/bin' }); - }); -}); diff --git a/packages/cli/src/managed-python-daemon.ts b/packages/cli/src/managed-python-daemon.ts index 402a1e14..bcf7b446 100644 --- a/packages/cli/src/managed-python-daemon.ts +++ b/packages/cli/src/managed-python-daemon.ts @@ -14,6 +14,7 @@ import { type ManagedPythonRuntimeInstallOptions, type ManagedPythonRuntimeInstallResult, } from './managed-python-runtime.js'; +import { sanitizeChildProxyEnv } from './proxy-env.js'; export interface ManagedPythonDaemonState { schemaVersion: 1; @@ -696,10 +697,10 @@ export async function startManagedPythonDaemon( { detached: true, stdio: ['ignore', stdout.fd, stderr.fd], - env: { - ...sanitizeProxyEnv(process.env), + env: sanitizeChildProxyEnv({ + ...process.env, KTX_DAEMON_VERSION: options.cliVersion, - }, + }), }, ); child.unref(); @@ -807,32 +808,3 @@ export async function stopAllManagedPythonDaemons( scanErrors: discovery.scanErrors, }; } - -/** - * Filter NO_PROXY/no_proxy values to remove entries httpx cannot parse. - * - * httpx (used by the Python daemon via huggingface_hub / sentence-transformers) - * treats each comma-separated NO_PROXY entry as a URL pattern. Raw IPv6 CIDR - * blocks like `fd07:b51a:cc66:f0::/64` raise `InvalidURL` and crash the daemon. - * OrbStack and similar Docker setups inject such entries by default. - * - * We drop any entry containing `::` (the unambiguous IPv6 marker) but keep - * IPv4 addresses, IPv4 CIDRs, hostnames, and wildcard hosts intact. - */ -export function sanitizeProxyEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - const result: NodeJS.ProcessEnv = { ...env }; - for (const key of ['NO_PROXY', 'no_proxy']) { - const value = result[key]; - if (typeof value !== 'string' || value.length === 0) continue; - const kept = value - .split(',') - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0 && !entry.includes('::')); - if (kept.length === 0) { - delete result[key]; - } else { - result[key] = kept.join(','); - } - } - return result; -} diff --git a/packages/cli/src/managed-python-runtime.test.ts b/packages/cli/src/managed-python-runtime.test.ts index e083368e..13b97a45 100644 --- a/packages/cli/src/managed-python-runtime.test.ts +++ b/packages/cli/src/managed-python-runtime.test.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { strToU8, zipSync } from 'fflate'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { MISSING_UV_RUNTIME_INSTALL_MESSAGE, @@ -14,10 +15,33 @@ import { type ManagedPythonRuntimeExec, } from './managed-python-runtime.js'; -async function writeAsset(root: string, contents = 'wheel-bytes') { +function runtimeWheelContents(input: { label?: string; requiresPython?: string | null } = {}): Buffer { + const label = input.label ?? 'runtime-wheel'; + const requiresPython = input.requiresPython === null ? [] : [`Requires-Python: ${input.requiresPython ?? '>=3.13'}`]; + return Buffer.from( + zipSync({ + 'kaelio_ktx-0.1.0.dist-info/METADATA': strToU8( + [ + 'Metadata-Version: 2.4', + 'Name: kaelio-ktx', + 'Version: 0.1.0', + ...requiresPython, + `Summary: ${label}`, + '', + ].join('\n'), + ), + }), + ); +} + +async function writeAsset( + root: string, + options: { label?: string; requiresPython?: string | null; contents?: Buffer } = {}, +) { const assetDir = join(root, 'assets', 'python'); await mkdir(assetDir, { recursive: true }); const wheelPath = join(assetDir, 'kaelio_ktx-0.1.0-py3-none-any.whl'); + const contents = options.contents ?? runtimeWheelContents(options); await writeFile(wheelPath, contents); await writeFile( join(assetDir, 'manifest.json'), @@ -30,7 +54,7 @@ async function writeAsset(root: string, contents = 'wheel-bytes') { wheel: { file: 'kaelio_ktx-0.1.0-py3-none-any.whl', sha256: createHash('sha256').update(contents).digest('hex'), - bytes: Buffer.byteLength(contents), + bytes: contents.byteLength, }, }, null, @@ -145,17 +169,18 @@ describe('verifyRuntimeAsset', () => { }); it('reads the manifest and verifies the wheel checksum', async () => { - const { assetDir, wheelPath } = await writeAsset(tempDir, 'valid-wheel'); + const { assetDir, wheelPath } = await writeAsset(tempDir, { label: 'valid-wheel' }); const asset = await verifyRuntimeAsset({ assetDir }); expect(asset.manifest.distributionName).toBe('kaelio-ktx'); expect(asset.manifest.normalizedName).toBe('kaelio_ktx'); expect(asset.wheelPath).toBe(wheelPath); + expect(asset.requiresPython).toEqual({ specifier: '>=3.13', minimumVersion: '3.13' }); }); it('rejects a wheel whose checksum does not match the manifest', async () => { - const { assetDir, wheelPath } = await writeAsset(tempDir, 'original'); + const { assetDir, wheelPath } = await writeAsset(tempDir, { label: 'original' }); await writeFile(wheelPath, 'tampered'); await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow( @@ -164,7 +189,7 @@ describe('verifyRuntimeAsset', () => { }); it('rejects an unsafe wheel filename in the manifest', async () => { - const { assetDir } = await writeAsset(tempDir, 'valid-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'valid-wheel' }); await writeFile( join(assetDir, 'manifest.json'), `${JSON.stringify({ @@ -190,6 +215,22 @@ describe('verifyRuntimeAsset', () => { /Missing bundled Python runtime manifest.*pnpm run artifacts:build/s, ); }); + + it('rejects a bundled wheel without Requires-Python metadata', async () => { + const { assetDir } = await writeAsset(tempDir, { requiresPython: null }); + + await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow( + /Bundled Python runtime wheel metadata is missing Requires-Python/, + ); + }); + + it('rejects a bundled wheel without a supported minimum Python version', async () => { + const { assetDir } = await writeAsset(tempDir, { requiresPython: '<4' }); + + await expect(verifyRuntimeAsset({ assetDir })).rejects.toThrow( + /Unsupported bundled Python runtime Requires-Python: <4/, + ); + }); }); describe('installManagedPythonRuntime', () => { @@ -204,7 +245,7 @@ describe('installManagedPythonRuntime', () => { }); it('creates a venv, installs the core wheel, and writes a manifest', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const commands: Array<{ command: string; args: string[] }> = []; const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { commands.push({ command, args }); @@ -222,7 +263,8 @@ describe('installManagedPythonRuntime', () => { expect(result.status).toBe('installed'); expect(commands).toEqual([ { command: 'uv', args: ['--version'] }, - { command: 'uv', args: ['venv', result.layout.venvDir, '--python', '3.13'] }, + { command: 'uv', args: ['python', 'install', '3.13'] }, + { command: 'uv', args: ['venv', '--python', '3.13', result.layout.venvDir] }, { command: 'uv', args: ['pip', 'install', '--python', result.layout.pythonPath, result.asset.wheelPath], @@ -240,7 +282,7 @@ describe('installManagedPythonRuntime', () => { }); it('disables repo uv config for managed runtime uv commands', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const commands: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = []; const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args, options) => { commands.push({ command, args, env: options?.env }); @@ -258,13 +300,14 @@ describe('installManagedPythonRuntime', () => { expect(commands.map((call) => [call.command, call.args[0], call.env?.UV_NO_CONFIG, call.env?.PATH])).toEqual([ ['uv', '--version', '1', '/opt/homebrew/bin'], + ['uv', 'python', '1', '/opt/homebrew/bin'], ['uv', 'venv', '1', '/opt/homebrew/bin'], ['uv', 'pip', '1', '/opt/homebrew/bin'], ]); }); it('installs the local-embeddings extra when requested', async () => { - const { assetDir } = await writeAsset(tempDir, 'embedding-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'embedding-wheel' }); const commands: Array<{ command: string; args: string[] }> = []; const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { commands.push({ command, args }); @@ -288,7 +331,7 @@ describe('installManagedPythonRuntime', () => { }); it('fails with the hard-prerequisite message when uv is missing', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const commands: Array<{ command: string; args: string[] }> = []; const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { commands.push({ command, args }); @@ -309,7 +352,7 @@ describe('installManagedPythonRuntime', () => { }); it('reuses an existing compatible runtime when force is false', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '', @@ -335,14 +378,17 @@ describe('installManagedPythonRuntime', () => { }); expect(second.status).toBe('ready'); - expect(exec).toHaveBeenCalledTimes(3); + expect(exec).toHaveBeenCalledTimes(4); }); it('keeps failed install logs in the versioned runtime directory', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => { if (command === 'uv' && args[0] === 'venv') { - throw Object.assign(new Error('uv venv failed'), { stdout: 'creating\n', stderr: 'bad python\n' }); + throw Object.assign(new Error('uv venv failed'), { + stdout: 'creating\n', + stderr: '× No solution found\n╰─▶ current Python version (3.12.3) does not satisfy Python>=3.13\n', + }); } return { stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '' }; }); @@ -355,11 +401,11 @@ describe('installManagedPythonRuntime', () => { features: ['core'], exec, }), - ).rejects.toThrow(/Python runtime install failed/); + ).rejects.toThrow(/current Python version \(3\.12\.3\) does not satisfy Python>=3\.13/); const log = await readFile(join(tempDir, 'runtime', '0.2.0', 'install.log'), 'utf8'); - expect(log).toContain('$ uv venv'); - expect(log).toContain('bad python'); + expect(log).toContain('$ uv venv --python 3.13'); + expect(log).toContain('current Python version (3.12.3) does not satisfy Python>=3.13'); }); }); @@ -386,7 +432,7 @@ describe('readManagedPythonRuntimeStatus', () => { }); it('reports ready when manifest and executables exist', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '', @@ -413,7 +459,7 @@ describe('readManagedPythonRuntimeStatus', () => { }); it('reports broken when an executable is missing', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '', @@ -449,7 +495,7 @@ describe('doctorManagedPythonRuntime', () => { }); it('checks uv, bundled assets, and installed runtime status', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const exec: ManagedPythonRuntimeExec = vi.fn(async (command, args) => ({ stdout: command === 'uv' && args[0] === '--version' ? 'uv 0.9.5\n' : '', stderr: '', @@ -471,7 +517,7 @@ describe('doctorManagedPythonRuntime', () => { }); it('reports uv as a hard prerequisite when uv is missing', async () => { - const { assetDir } = await writeAsset(tempDir, 'core-wheel'); + const { assetDir } = await writeAsset(tempDir, { label: 'core-wheel' }); const exec: ManagedPythonRuntimeExec = vi.fn(async () => { throw new Error('spawn uv ENOENT'); }); diff --git a/packages/cli/src/managed-python-runtime.ts b/packages/cli/src/managed-python-runtime.ts index 611f21ab..88b0fa2b 100644 --- a/packages/cli/src/managed-python-runtime.ts +++ b/packages/cli/src/managed-python-runtime.ts @@ -5,6 +5,7 @@ import { homedir } from 'node:os'; import { basename, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; +import { strFromU8, unzipSync } from 'fflate'; import { z } from 'zod'; const execFileAsync = promisify(execFile); @@ -12,16 +13,6 @@ const execFileAsync = promisify(execFile); export const runtimeFeatureSchema = z.enum(['core', 'local-embeddings']); export type KtxRuntimeFeature = z.infer; -/** - * Python version the managed runtime venv must be built with. KTX's bundled - * wheels declare `requires-python = ">=3.13"`; without an explicit `--python` - * flag, `uv venv` may pick a too-old system Python (Ubuntu 24.04 ships 3.12) - * and the subsequent `uv pip install` fails late with a confusing "package - * requires Python >=3.13" error. Pinning here pushes uv to auto-download the - * right interpreter via its python-management feature. - */ -const MANAGED_RUNTIME_PYTHON_VERSION = '3.13'; - const runtimeAssetManifestSchema = z.object({ schemaVersion: z.literal(1), distributionName: z.literal('kaelio-ktx'), @@ -88,6 +79,10 @@ export interface ManagedPythonDaemonLayout extends ManagedPythonRuntimeLayout { export interface ManagedRuntimeAsset { manifest: KtxRuntimeAssetManifest; wheelPath: string; + requiresPython: { + specifier: string; + minimumVersion: string; + }; } export type ManagedPythonRuntimeExec = ( @@ -206,6 +201,40 @@ function isErrnoException(error: unknown, code: string): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === code; } +function parseRequiresPythonFromWheel(input: { wheelPath: string; contents: Buffer }): ManagedRuntimeAsset['requiresPython'] { + let files: Record; + try { + files = unzipSync(new Uint8Array(input.contents)); + } catch (error) { + throw new Error( + `Unable to read bundled Python runtime wheel metadata: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const metadataEntry = Object.entries(files).find(([path]) => path.endsWith('.dist-info/METADATA')); + if (!metadataEntry) { + throw new Error(`Bundled Python runtime wheel metadata is missing: ${input.wheelPath}`); + } + + const metadata = strFromU8(metadataEntry[1]); + const requiresPython = metadata + .split(/\r?\n/) + .map((line) => line.match(/^Requires-Python:\s*(.+)\s*$/i)?.[1]?.trim()) + .find((value): value is string => typeof value === 'string' && value.length > 0); + if (!requiresPython) { + throw new Error('Bundled Python runtime wheel metadata is missing Requires-Python'); + } + + const minimumMatch = requiresPython.match(/(?:^|[,\s])>=\s*([0-9]+)\.([0-9]+)(?:\.[0-9]+)?\b/); + if (!minimumMatch) { + throw new Error(`Unsupported bundled Python runtime Requires-Python: ${requiresPython}`); + } + + return { + specifier: requiresPython, + minimumVersion: `${minimumMatch[1]}.${minimumMatch[2]}`, + }; +} + export async function verifyRuntimeAsset(input: { assetDir: string }): Promise { const manifestPath = join(input.assetDir, 'manifest.json'); let manifestData: unknown; @@ -231,7 +260,7 @@ export async function verifyRuntimeAsset(input: { assetDir: string }): Promise part.length > 0).join('\n'); + if (!output) { + return `Python runtime install failed. Install log: ${input.logPath}`; + } + return `Python runtime install failed.\n${output}\nInstall log: ${input.logPath}`; +} + async function runLogged(input: { exec: ManagedPythonRuntimeExec; logPath: string; @@ -298,7 +335,7 @@ async function runLogged(input: { if (output.stderr) { await appendFile(input.logPath, output.stderr.endsWith('\n') ? output.stderr : `${output.stderr}\n`); } - throw new Error(`Python runtime install failed. Install log: ${input.logPath}`); + throw new Error(installFailureMessage({ logPath: input.logPath, stdout: output.stdout, stderr: output.stderr })); } } @@ -344,7 +381,14 @@ export async function installManagedPythonRuntime( exec, logPath: layout.installLogPath, command: 'uv', - args: ['venv', layout.venvDir, '--python', MANAGED_RUNTIME_PYTHON_VERSION], + args: ['python', 'install', asset.requiresPython.minimumVersion], + env: uvEnv, + }); + await runLogged({ + exec, + logPath: layout.installLogPath, + command: 'uv', + args: ['venv', '--python', asset.requiresPython.minimumVersion, layout.venvDir], env: uvEnv, }); const wheelSpec = features.includes('local-embeddings') ? `${asset.wheelPath}[local-embeddings]` : asset.wheelPath; diff --git a/packages/cli/src/next-steps.test.ts b/packages/cli/src/next-steps.test.ts index d7904555..c2b15530 100644 --- a/packages/cli/src/next-steps.test.ts +++ b/packages/cli/src/next-steps.test.ts @@ -49,8 +49,10 @@ describe('KTX demo next steps', () => { const rendered = formatNextStepLines().join('\n'); expect(rendered).toContain('KTX context is ready for agents.'); + expect(rendered).toContain('KTX project directory'); expect(rendered).toContain('ask a data question'); expect(rendered).toContain('Verify with:'); + expect(rendered).not.toContain('this directory'); expect(rendered).not.toContain('Preferred route'); expect(rendered).not.toContain('Optional MCP:'); }); diff --git a/packages/cli/src/next-steps.ts b/packages/cli/src/next-steps.ts index c36c5591..06ef3412 100644 --- a/packages/cli/src/next-steps.ts +++ b/packages/cli/src/next-steps.ts @@ -43,7 +43,7 @@ function commandLines(commands: ReadonlyArray<{ command: string; description: st export function formatNextStepLines(indent = ' '): string[] { return [ - `${indent}KTX context is ready for agents. Open your coding agent in this directory and ask a data question.`, + `${indent}KTX context is ready for agents. Open your coding agent from the KTX project directory and ask a data question.`, `${indent}Verify with:`, ...commandLines(KTX_NEXT_STEP_DIRECT_COMMANDS, indent), ]; diff --git a/packages/cli/src/proxy-env.test.ts b/packages/cli/src/proxy-env.test.ts new file mode 100644 index 00000000..1da7bc91 --- /dev/null +++ b/packages/cli/src/proxy-env.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; +import { sanitizeChildProxyEnv } from './proxy-env.js'; + +describe('sanitizeChildProxyEnv', () => { + it('drops IPv6 CIDR no-proxy entries and normalizes both env keys', () => { + const env = sanitizeChildProxyEnv({ + NO_PROXY: 'localhost,127.0.0.1,127.0.0.0/8,fd07:b51a:cc66:f0::/64,*.orb.local', + no_proxy: '::1,0.250.250.0/24,fd00::/8,*.orb.internal', + }); + + expect(env.NO_PROXY).toBe('localhost,127.0.0.1,127.0.0.0/8,*.orb.local,::1,0.250.250.0/24,*.orb.internal'); + expect(env.no_proxy).toBe(env.NO_PROXY); + }); + + it('preserves the input object and leaves missing proxy env unset', () => { + const input = { PATH: '/usr/bin' }; + + expect(sanitizeChildProxyEnv(input)).toEqual({ PATH: '/usr/bin' }); + expect(input).toEqual({ PATH: '/usr/bin' }); + }); +}); diff --git a/packages/cli/src/proxy-env.ts b/packages/cli/src/proxy-env.ts new file mode 100644 index 00000000..dd47ad8e --- /dev/null +++ b/packages/cli/src/proxy-env.ts @@ -0,0 +1,27 @@ +const NO_PROXY_KEYS = ['NO_PROXY', 'no_proxy'] as const; + +function isIpv6CidrNoProxyEntry(entry: string): boolean { + return entry.includes('/') && entry.includes(':'); +} + +function cleanedNoProxyValue(env: NodeJS.ProcessEnv): string | undefined { + const entries = NO_PROXY_KEYS.flatMap((key) => (env[key] ?? '').split(',')) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0 && !isIpv6CidrNoProxyEntry(entry)); + + if (!NO_PROXY_KEYS.some((key) => env[key] !== undefined)) { + return undefined; + } + return [...new Set(entries)].join(','); +} + +export function sanitizeChildProxyEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const sanitized = { ...env }; + const noProxy = cleanedNoProxyValue(env); + if (noProxy === undefined) { + return sanitized; + } + sanitized.NO_PROXY = noProxy; + sanitized.no_proxy = noProxy; + return sanitized; +} diff --git a/packages/cli/src/runtime-requirements.test.ts b/packages/cli/src/runtime-requirements.test.ts index 7d36e86c..1a8f2d43 100644 --- a/packages/cli/src/runtime-requirements.test.ts +++ b/packages/cli/src/runtime-requirements.test.ts @@ -7,10 +7,10 @@ import { } from './runtime-requirements.js'; describe('runtime requirement detection', () => { - it('requires core for agent/MCP setup', () => { + it('does not require runtime for agent/MCP setup alone', () => { const config = buildDefaultKtxProjectConfig(); - expect(resolveProjectRuntimeRequirements(config, { agents: true }).features).toEqual(['core']); + expect(resolveProjectRuntimeRequirements(config).features).toEqual([]); }); it('requires core for Looker source ingest unless an external daemon is configured', () => { diff --git a/packages/cli/src/runtime-requirements.ts b/packages/cli/src/runtime-requirements.ts index 086f86af..1e35d90e 100644 --- a/packages/cli/src/runtime-requirements.ts +++ b/packages/cli/src/runtime-requirements.ts @@ -8,7 +8,6 @@ import type { KtxRuntimeFeature } from './managed-python-runtime.js'; import type { KtxPublicIngestPlan } from './public-ingest.js'; type KtxRuntimeRequirementReason = - | 'agent-mcp' | 'query-history' | 'looker-source' | 'database-introspection' @@ -26,7 +25,6 @@ export interface KtxRuntimeRequirements { } export interface KtxProjectRuntimeRequirementOptions { - agents?: boolean; databaseIntrospectionFallback?: boolean; env?: NodeJS.ProcessEnv | Record; } @@ -92,14 +90,6 @@ export function resolveProjectRuntimeRequirements( const env = options.env ?? process.env; const requirements: KtxRuntimeRequirement[] = []; - if (options.agents === true) { - requirements.push({ - feature: 'core', - reason: 'agent-mcp', - detail: 'Agent MCP setup uses semantic-layer query tools and SQL validation.', - }); - } - if (options.databaseIntrospectionFallback === true && !hasDaemonOverride(env)) { requirements.push({ feature: 'core', diff --git a/packages/cli/src/runtime.test.ts b/packages/cli/src/runtime.test.ts index 5af457de..01a529e7 100644 --- a/packages/cli/src/runtime.test.ts +++ b/packages/cli/src/runtime.test.ts @@ -52,6 +52,7 @@ describe('runKtxRuntime', () => { }, asset: { wheelPath: '/assets/python/kaelio_ktx-0.1.0-py3-none-any.whl', + requiresPython: { specifier: '>=3.13', minimumVersion: '3.13' }, manifest: { schemaVersion: 1, distributionName: 'kaelio-ktx', diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index 3a073ce3..bd9c9458 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -5,7 +5,8 @@ import { readKtxSetupState } from '@ktx/context/project'; import { strFromU8, unzipSync } from 'fflate'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { - formatInstallSummary, + createAgentNextActionsLineFormatter, + formatInstallSummaryLines, plannedKtxAgentFiles, readKtxAgentInstallManifest, removeKtxAgentInstall, @@ -82,7 +83,11 @@ describe('setup agents', () => { ]); expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-desktop', scope: 'global', mode: 'mcp' })).toEqual([ { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'), role: 'launcher' }, - { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-plugin.zip'), role: 'claude-plugin' }, + { + kind: 'file', + path: join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'), + role: 'claude-desktop-skill-bundle', + }, ]); expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'codex', scope: 'project', mode: 'mcp' })).toEqual([ { kind: 'file', path: join(tempDir, '.agents/skills/ktx-analytics/SKILL.md'), role: 'analytics-skill' }, @@ -123,7 +128,16 @@ describe('setup agents', () => { ]); expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-desktop', scope: 'global', mode: 'mcp-cli' })).toEqual([ { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'), role: 'launcher' }, - { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-plugin.zip'), role: 'claude-plugin' }, + { + kind: 'file', + path: join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'), + role: 'claude-desktop-skill-bundle', + }, + { + kind: 'file', + path: join(tempDir, '.ktx/agents/claude/ktx.zip'), + role: 'claude-desktop-skill-bundle', + }, ]); }); @@ -144,7 +158,7 @@ describe('setup agents', () => { }, io.io, ), - ).resolves.toEqual({ + ).resolves.toMatchObject({ status: 'ready', projectDir: tempDir, installs: [{ target: 'universal', scope: 'project', mode: 'mcp-cli' }], @@ -170,6 +184,118 @@ describe('setup agents', () => { expect(io.stderr()).toBe(''); }); + it('installs a specified target in non-interactive mode without --yes', async () => { + const io = makeIo(); + + await expect( + runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: false, + agents: true, + target: 'claude-code', + scope: 'project', + mode: 'mcp', + skipAgents: false, + }, + io.io, + ), + ).resolves.toMatchObject({ + status: 'ready', + projectDir: tempDir, + installs: [{ target: 'claude-code', scope: 'project', mode: 'mcp' }], + }); + + await expect(stat(join(tempDir, '.claude/skills/ktx-analytics/SKILL.md'))).resolves.toBeDefined(); + const mcpConfig = JSON.parse(await readFile(join(tempDir, '.mcp.json'), 'utf-8')) as { + mcpServers?: Record; + }; + expect(mcpConfig.mcpServers).toHaveProperty('ktx'); + expect(io.stderr()).toBe(''); + }); + + it('prints concrete target guidance when non-interactive agent setup has no target', async () => { + const io = makeIo(); + + await expect( + runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: false, + agents: true, + scope: 'project', + mode: 'mcp', + skipAgents: false, + }, + io.io, + ), + ).resolves.toEqual({ status: 'missing-input', projectDir: tempDir }); + + expect(io.stderr()).toBe('Run in a TTY, or pass --target .\n'); + }); + + it('prints standalone agent next actions after successful installation', async () => { + const io = makeIo(); + + const result = await runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: true, + agents: true, + target: 'claude-code', + scope: 'project', + mode: 'mcp-cli', + skipAgents: false, + }, + io.io, + ); + + expect(result).toMatchObject({ + status: 'ready', + nextActions: expect.stringContaining('Run this command before using Claude Code:'), + }); + expect(io.stdout()).toContain('Required before using agents'); + expect(io.stdout()).toContain('Run this command before using Claude Code:'); + expect(io.stdout()).toContain('RUN:'); + expect(io.stdout()).toContain(`ktx mcp start --project-dir ${tempDir}`); + expect(io.stdout()).toContain('If you need to stop MCP later:'); + expect(io.stdout()).toContain(`ktx mcp stop --project-dir ${tempDir}`); + expect(io.stdout()).toContain('All set.'); + expect(io.stdout()).not.toContain('Finish agent setup'); + expect(io.stdout()).not.toContain('Next actions'); + }); + + it('can return agent next actions without printing them', async () => { + const io = makeIo(); + + const result = await runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: true, + agents: true, + target: 'claude-code', + scope: 'project', + mode: 'mcp-cli', + skipAgents: false, + showNextActions: false, + }, + io.io, + ); + + expect(result).toMatchObject({ + status: 'ready', + nextActions: expect.stringContaining(`ktx mcp start --project-dir ${tempDir}`), + }); + expect(io.stdout()).toContain('Claude Code · Project scope'); + expect(io.stdout()).not.toContain('Agent integration complete'); + expect(io.stdout()).not.toContain('Required before using agents'); + expect(io.stdout()).not.toContain('All set.'); + }); + it('installs the analytics skill from the runtime asset', async () => { const io = makeIo(); @@ -252,7 +378,6 @@ describe('setup agents', () => { expect(await readKtxAgentInstallManifest(tempDir)).toMatchObject({ entries: expect.arrayContaining([{ kind: 'json-key', path: join(tempDir, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] }]), }); - expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.'); }); it('prompts for MCP-first client agent connection mode in interactive setup', async () => { @@ -283,10 +408,18 @@ describe('setup agents', () => { }); expect(prompts.select).toHaveBeenCalledWith({ - message: 'How should client agents connect to this KTX project?', + message: 'What should agents be allowed to do with this KTX project?', options: [ - { value: 'mcp', label: 'MCP tools + analytics skill' }, - { value: 'mcp-cli', label: 'MCP tools + analytics skill + admin CLI skill' }, + { + value: 'mcp', + label: 'Ask data questions with KTX MCP', + hint: 'Installs the MCP connection and analytics workflow skill. Best for normal use.', + }, + { + value: 'mcp-cli', + label: 'Ask data questions + manage KTX with CLI commands', + hint: 'Adds an admin CLI skill so agents can run ktx status, sl, wiki, and setup commands.', + }, ], }); expect(prompts.multiselect).toHaveBeenCalledWith( @@ -330,10 +463,18 @@ describe('setup agents', () => { }); expect(prompts.select).toHaveBeenCalledWith({ - message: 'Where should KTX install supported agent config?', + message: `Where should KTX install supported agent config?\n\nKTX project: ${tempDir}`, options: [ - { value: 'project', label: 'Project' }, - { value: 'global', label: 'Global' }, + { + value: 'project', + label: 'Project scope (KTX project directory)', + hint: 'Only agents opened from this KTX project path load the project-scoped config.', + }, + { + value: 'global', + label: 'Global scope (user config)', + hint: 'Agents can load this KTX project from any working directory.', + }, ], }); } finally { @@ -342,7 +483,7 @@ describe('setup agents', () => { } }); - it('registers Claude Desktop MCP via claude_desktop_config.json and ships a skills-only plugin', async () => { + it('registers Claude Desktop MCP and ships an uploadable analytics skill zip', async () => { const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); const previousHome = process.env.HOME; const envSnapshot = captureEnvKeys(process.env, ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY']); @@ -372,9 +513,11 @@ describe('setup agents', () => { installs: [{ target: 'claude-desktop', scope: 'global', mode: 'mcp' }], }); - const pluginPath = join(tempDir, '.ktx/agents/claude/ktx-plugin.zip'); + const analyticsSkillPath = join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'); + const adminSkillPath = join(tempDir, '.ktx/agents/claude/ktx.zip'); const launcherPath = join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'); - await expect(stat(pluginPath)).resolves.toBeDefined(); + await expect(stat(analyticsSkillPath)).resolves.toBeDefined(); + await expect(stat(adminSkillPath)).rejects.toThrow(); const launcherStat = await stat(launcherPath); expect(launcherStat.mode & 0o111).not.toBe(0); const launcher = await readFile(launcherPath, 'utf-8'); @@ -390,19 +533,24 @@ describe('setup agents', () => { args: ['--project-dir', tempDir, 'mcp', 'stdio'], }); - expect(await readZipText(pluginPath, '.claude-plugin/plugin.json')).toContain('"name": "ktx"'); - await expect(readZipText(pluginPath, '.mcp.json')).rejects.toThrow('Missing zip entry'); - expect(await readZipText(pluginPath, 'skills/ktx-analytics/SKILL.md')).toContain('KTX Analytics Workflow'); - const setupMd = await readZipText(pluginPath, 'SETUP.md'); - expect(setupMd).not.toContain('ktx mcp start'); - expect(setupMd).toContain('claude_desktop_config.json'); - await expect(readZipText(pluginPath, 'skills/ktx/SKILL.md')).rejects.toThrow('Missing zip entry'); + expect(await readZipText(analyticsSkillPath, 'ktx-analytics/SKILL.md')).toContain('KTX Analytics Workflow'); + await expect(readZipText(analyticsSkillPath, 'ktx/SKILL.md')).rejects.toThrow('Missing zip entry'); + await expect(readZipText(analyticsSkillPath, '.claude-plugin/plugin.json')).rejects.toThrow('Missing zip entry'); + await expect(readZipText(analyticsSkillPath, 'skills/ktx-analytics/SKILL.md')).rejects.toThrow( + 'Missing zip entry', + ); - expect(io.stdout()).toContain('Claude plugin generated'); - expect(io.stdout()).toContain('.ktx/agents/claude/ktx-plugin.zip'); - expect(io.stdout()).toContain('KTX MCP server registered'); + expect(io.stdout()).toContain('Claude Desktop'); + expect(io.stdout()).toContain(analyticsSkillPath); + expect(io.stdout()).not.toContain(adminSkillPath); expect(io.stdout()).toContain('claude_desktop_config.json'); - expect(io.stdout()).toContain('Restart Claude Desktop'); + expect(io.stdout()).toContain('Required before using agents'); + expect(io.stdout()).toContain('1. Restart Claude Desktop'); + expect(io.stdout()).toContain('Claude Desktop loads KTX MCP after restart.'); + expect(io.stdout()).toContain('2. Upload Claude Desktop skills'); + expect(io.stdout()).toContain('Customize > Skills > + > Create skill > Upload a skill'); + expect(io.stdout()).toContain('Upload this file:'); + expect(io.stdout()).toContain('Toggle the uploaded KTX skills on.'); expect(io.stdout()).not.toContain('Run `ktx mcp start`'); } finally { process.env.HOME = previousHome; @@ -459,7 +607,7 @@ describe('setup agents', () => { } }); - it('includes the admin CLI skill in the Claude Desktop plugin zip when requested', async () => { + it('includes an uploadable admin CLI skill zip for Claude Desktop when requested', async () => { const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); const previousHome = process.env.HOME; process.env.HOME = home; @@ -485,12 +633,18 @@ describe('setup agents', () => { installs: [{ target: 'claude-desktop', scope: 'global', mode: 'mcp-cli' }], }); - const pluginPath = join(tempDir, '.ktx/agents/claude/ktx-plugin.zip'); - const adminSkill = await readZipText(pluginPath, 'skills/ktx/SKILL.md'); + const analyticsSkillPath = join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'); + const adminSkillPath = join(tempDir, '.ktx/agents/claude/ktx.zip'); + expect(await readZipText(analyticsSkillPath, 'ktx-analytics/SKILL.md')).toContain('KTX Analytics Workflow'); + await expect(readZipText(analyticsSkillPath, 'ktx/SKILL.md')).rejects.toThrow('Missing zip entry'); + const adminSkill = await readZipText(adminSkillPath, 'ktx/SKILL.md'); expect(adminSkill).toContain(`--project-dir ${tempDir}`); expect(adminSkill).toContain('status --json'); - expect(await readZipText(pluginPath, 'SETUP.md')).toContain('admin CLI skill'); - await expect(readZipText(pluginPath, '.mcp.json')).rejects.toThrow('Missing zip entry'); + await expect(readZipText(adminSkillPath, '.mcp.json')).rejects.toThrow('Missing zip entry'); + await expect(readZipText(adminSkillPath, 'ktx-analytics/SKILL.md')).rejects.toThrow('Missing zip entry'); + expect(io.stdout()).toContain(analyticsSkillPath); + expect(io.stdout()).toContain(adminSkillPath); + expect(io.stdout()).toContain('Upload each file separately:'); } finally { process.env.HOME = previousHome; await rm(home, { recursive: true, force: true }); @@ -568,6 +722,9 @@ describe('setup agents', () => { ); expect(codexIo.stdout()).toContain('[mcp_servers.ktx]'); expect(codexIo.stdout()).toContain('url = "http://localhost:7878/mcp"'); + expect(codexIo.stdout()).toContain('1. Configure Codex'); + expect(codexIo.stdout()).toContain('Open ~/.codex/config.toml, then paste this block:'); + expect(codexIo.stdout()).toContain('PASTE:'); const opencodeIo = makeIo(); await runKtxSetupAgentsStep( @@ -585,6 +742,8 @@ describe('setup agents', () => { ); expect(opencodeIo.stdout()).toContain('"mcp"'); expect(opencodeIo.stdout()).toContain('"type": "remote"'); + expect(opencodeIo.stdout()).toContain('1. Configure OpenCode'); + expect(opencodeIo.stdout()).toContain('Open opencode.json, then paste this block:'); await expect(readFile(join(tempDir, 'opencode.json'), 'utf-8')).rejects.toThrow(); const universalIo = makeIo(); @@ -603,6 +762,8 @@ describe('setup agents', () => { ); expect(universalIo.stdout()).toContain('Universal MCP endpoint:'); expect(universalIo.stdout()).toContain('http://localhost:7878/mcp'); + expect(universalIo.stdout()).toContain('1. Configure unsupported MCP clients'); + expect(universalIo.stdout()).toContain('Use this endpoint when setting up unsupported MCP clients:'); }); it('uses MCP daemon state for port and token metadata without rendering literal tokens', async () => { @@ -648,7 +809,9 @@ describe('setup agents', () => { expect(rendered).toContain('http://127.0.0.1:8787/mcp'); expect(rendered).toContain('Bearer ${KTX_MCP_TOKEN}'); expect(rendered).not.toContain('secret-token'); - expect(io.stdout()).toContain('Run `ktx mcp start` to enable the configured KTX MCP server.'); + expect(io.stdout()).toContain('Run this command before using Claude Code:'); + expect(io.stdout()).toContain('RUN:'); + expect(io.stdout()).toContain(`ktx mcp start --project-dir ${tempDir}`); } finally { if (previousToken === undefined) { delete process.env.KTX_MCP_TOKEN; @@ -713,7 +876,7 @@ describe('setup agents', () => { await expect(readKtxAgentInstallManifest(tempDir)).resolves.toEqual(null); }); - it('removes generated Claude Desktop plugin from the manifest', async () => { + it('removes generated Claude Desktop skill zips from the manifest', async () => { const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); const previousHome = process.env.HOME; process.env.HOME = home; @@ -732,10 +895,12 @@ describe('setup agents', () => { }, io.io, ); - const pluginPath = join(tempDir, '.ktx/agents/claude/ktx-plugin.zip'); + const analyticsSkillPath = join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'); + const adminSkillPath = join(tempDir, '.ktx/agents/claude/ktx.zip'); const launcherPath = join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'); const configPath = join(home, 'Library/Application Support/Claude/claude_desktop_config.json'); - await expect(stat(pluginPath)).resolves.toBeDefined(); + await expect(stat(analyticsSkillPath)).resolves.toBeDefined(); + await expect(stat(adminSkillPath)).resolves.toBeDefined(); await expect(stat(launcherPath)).resolves.toBeDefined(); const beforeConfig = JSON.parse(await readFile(configPath, 'utf-8')) as { mcpServers: Record; @@ -744,7 +909,8 @@ describe('setup agents', () => { await expect(removeKtxAgentInstall(tempDir, io.io)).resolves.toBe(0); - await expect(stat(pluginPath)).rejects.toThrow(); + await expect(stat(analyticsSkillPath)).rejects.toThrow(); + await expect(stat(adminSkillPath)).rejects.toThrow(); await expect(stat(launcherPath)).rejects.toThrow(); const afterConfig = JSON.parse(await readFile(configPath, 'utf-8')) as { mcpServers: Record; @@ -782,7 +948,7 @@ describe('setup agents', () => { ).resolves.toEqual({ status: 'skipped', projectDir: tempDir }); }); - it('explains how to select multiple agent targets in interactive mode', async () => { + it('prints one navigation hint before interactive agent target prompts', async () => { const io = makeIo(); const prompts = { select: vi.fn(async () => 'mcp-cli'), @@ -806,10 +972,11 @@ describe('setup agents', () => { ), ).resolves.toEqual({ status: 'back', projectDir: tempDir }); + expect(io.stdout()).toContain('Space to select, Enter to confirm, Esc to go back.'); + expect(io.stdout().match(/Space to select/g)).toHaveLength(1); expect(prompts.multiselect).toHaveBeenCalledWith( expect.objectContaining({ - message: - 'Which agent targets should KTX install?\nUse Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + message: 'Which agent targets should KTX install?', }), ); }); @@ -832,36 +999,44 @@ describe('setup agents', () => { ); const output = io.stdout(); - expect(output).toContain('Agent integration complete'); - expect(output).toContain('Claude Code'); - expect(output).toContain('+ Analytics skill installed — teaches your agent the KTX MCP analytics workflow'); - expect(output).toContain('.claude/skills/ktx-analytics/SKILL.md'); - expect(output).toContain('+ Skill installed — teaches admin agents which KTX CLI commands to run'); - expect(output).toContain('.claude/skills/ktx/SKILL.md'); - expect(output).toContain('+ Rule installed — tells admin agents when to use KTX CLI'); - expect(output).toContain('.claude/rules/ktx.md'); + expect(output).toContain('Claude Code · Project scope'); + expect(output).toContain(join(tempDir, '.mcp.json')); + expect(output).toContain('Requires MCP to be started.'); + expect(output).toContain('Analytics skill installed.'); + expect(output).toContain('Admin CLI skill installed.'); + expect(output).not.toContain('Agent integration complete'); + expect(output).not.toContain(`KTX project\n ${tempDir}`); + expect(output).not.toContain('Installed agents'); + expect(output).not.toContain('.claude/skills/ktx-analytics/SKILL.md'); + expect(output).not.toContain('.claude/skills/ktx/SKILL.md'); + expect(output).not.toContain('.claude/rules/ktx.md'); }); - it('formats summary with relative paths for project scope', () => { - const summary = formatInstallSummary( + it('formats summary with explicit project-scoped config paths', () => { + const summary = formatInstallSummaryLines( [{ target: 'cursor', scope: 'project', mode: 'mcp-cli' }], [ { kind: 'file', path: join(tempDir, '.cursor/rules/ktx-analytics.mdc'), role: 'analytics-skill' }, { kind: 'file', path: join(tempDir, '.cursor/rules/ktx.mdc') }, + { kind: 'json-key', path: join(tempDir, '.cursor/mcp.json'), jsonPath: ['mcpServers', 'ktx'] }, ], tempDir, ); - expect(summary).toContain('Cursor'); - expect(summary).toContain('+ Analytics skill installed — teaches your agent the KTX MCP analytics workflow'); - expect(summary).toContain('.cursor/rules/ktx-analytics.mdc'); - expect(summary).toContain('+ Rule installed — tells admin agents when to use KTX CLI'); - expect(summary).toContain('.cursor/rules/ktx.mdc'); - expect(summary).not.toContain(tempDir); + expect(summary).toEqual([ + { + title: 'Cursor · Project scope', + lines: [ + join(tempDir, '.cursor/mcp.json'), + 'Requires MCP to be started.', + 'Cursor rules installed.', + ], + }, + ]); }); it('formats summary with multiple agent targets', () => { - const summary = formatInstallSummary( + const summary = formatInstallSummaryLines( [ { target: 'claude-code', scope: 'project', mode: 'mcp-cli' }, { target: 'codex', scope: 'project', mode: 'mcp-cli' }, @@ -870,6 +1045,7 @@ describe('setup agents', () => { { kind: 'file', path: join(tempDir, '.claude/skills/ktx-analytics/SKILL.md'), role: 'analytics-skill' }, { kind: 'file', path: join(tempDir, '.claude/skills/ktx/SKILL.md'), role: 'skill' }, { kind: 'file', path: join(tempDir, '.claude/rules/ktx.md'), role: 'rule' }, + { kind: 'json-key', path: join(tempDir, '.mcp.json'), jsonPath: ['mcpServers', 'ktx'] }, { kind: 'file', path: join(tempDir, '.agents/skills/ktx-analytics/SKILL.md'), role: 'analytics-skill' }, { kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md'), role: 'skill' }, { kind: 'file', path: join(tempDir, '.codex/instructions/ktx.md'), role: 'rule' }, @@ -877,12 +1053,232 @@ describe('setup agents', () => { tempDir, ); - expect(summary).toContain('Claude Code'); - expect(summary).toContain('+ Analytics skill installed — teaches your agent the KTX MCP analytics workflow'); - expect(summary).toContain('+ Skill installed — teaches admin agents which KTX CLI commands to run'); - expect(summary).toContain('+ Rule installed — tells admin agents when to use KTX CLI'); - expect(summary).toContain('Codex'); - expect(summary).toContain('.agents/skills/ktx-analytics/SKILL.md'); - expect(summary).toContain('.agents/skills/ktx/SKILL.md'); + expect(summary).toEqual([ + { + title: 'Claude Code · Project scope', + lines: [ + join(tempDir, '.mcp.json'), + 'Requires MCP to be started.', + 'Analytics skill installed.', + 'Admin CLI skill installed.', + ], + }, + { + title: 'Codex · Project scope', + lines: [ + 'Add the snippet shown below to ~/.codex/config.toml.', + 'Requires MCP to be started.', + 'Codex guidance installed.', + ], + }, + ]); + }); + + it('prints one target-aware next actions block for mixed agent targets', async () => { + const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); + const previousHome = process.env.HOME; + process.env.HOME = home; + try { + const io = makeIo(); + const prompts = { + select: vi.fn(async ({ message }: { message: string }) => + message.startsWith('Where should') ? 'project' : 'mcp', + ), + multiselect: vi.fn(async () => ['claude-code', 'claude-desktop']), + cancel: vi.fn(), + }; + + await expect( + runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'auto', + yes: false, + agents: true, + scope: 'project', + mode: 'mcp', + skipAgents: false, + }, + io.io, + { prompts }, + ), + ).resolves.toMatchObject({ + status: 'ready', + installs: [ + { target: 'claude-code', scope: 'project', mode: 'mcp' }, + { target: 'claude-desktop', scope: 'global', mode: 'mcp' }, + ], + }); + + const output = io.stdout(); + expect(output).toContain('Required before using agents'); + expect(output).not.toContain('Next actions'); + expect(output).toContain('1. Start MCP'); + expect(output).toContain('Run this command before using Claude Code:'); + expect(output).toContain(`ktx mcp start --project-dir ${tempDir}`); + expect(output).toContain(`ktx mcp stop --project-dir ${tempDir}\n\n2. Open Claude Code`); + expect(output).toContain('Open Claude Code from the KTX project directory'); + expect(output).toContain('RUN:'); + expect(output).toContain(`cd '${tempDir}'`); + expect(output).toContain('3. Restart Claude Desktop'); + expect(output).toContain('Claude Desktop loads KTX MCP after restart.'); + expect(output).toContain('4. Upload Claude Desktop skills'); + expect(output).toContain('Customize > Skills > + > Create skill > Upload a skill'); + expect(output).toContain(join(tempDir, '.ktx/agents/claude/ktx-analytics.zip')); + expect(output).not.toContain(join(tempDir, '.ktx/agents/claude/ktx.zip')); + expect(output).toContain('Upload this file:'); + expect(output).toContain('All set.'); + expect(output).not.toContain('Finish Claude Desktop setup'); + expect(output).not.toContain('Run `ktx mcp start` to enable the configured KTX MCP server.'); + } finally { + process.env.HOME = previousHome; + await rm(home, { recursive: true, force: true }); + } + }); + + it('does not tell global Claude Code installs to open from the project directory', async () => { + const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); + const previousHome = process.env.HOME; + process.env.HOME = home; + try { + const io = makeIo(); + + await expect( + runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: true, + agents: true, + target: 'claude-code', + scope: 'global', + mode: 'mcp', + skipAgents: false, + }, + io.io, + ), + ).resolves.toMatchObject({ + status: 'ready', + installs: [{ target: 'claude-code', scope: 'global', mode: 'mcp' }], + }); + + const output = io.stdout(); + expect(output).toContain('2. Open Claude Code'); + expect(output).toContain('RUN:'); + expect(output).toContain('claude'); + expect(output).not.toContain('Open Claude Code from the KTX project directory'); + expect(output).not.toContain(`cd '${tempDir}'`); + } finally { + process.env.HOME = previousHome; + await rm(home, { recursive: true, force: true }); + } + }); + + it('explains next actions for Codex, Cursor, OpenCode, and universal MCP targets', async () => { + const io = makeIo(); + const prompts = { + select: vi.fn(async () => 'mcp-cli'), + multiselect: vi.fn(async () => ['codex', 'cursor', 'opencode', 'universal']), + cancel: vi.fn(), + }; + + await expect( + runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'auto', + yes: false, + agents: true, + scope: 'project', + mode: 'mcp-cli', + skipAgents: false, + }, + io.io, + { prompts }, + ), + ).resolves.toMatchObject({ + status: 'ready', + installs: [ + { target: 'codex', scope: 'project', mode: 'mcp-cli' }, + { target: 'cursor', scope: 'project', mode: 'mcp-cli' }, + { target: 'opencode', scope: 'project', mode: 'mcp-cli' }, + { target: 'universal', scope: 'project', mode: 'mcp-cli' }, + ], + }); + + const output = io.stdout(); + expect(output).toContain('Required before using agents'); + expect(output).toContain('1. Configure Codex'); + expect(output).toContain('2. Configure OpenCode'); + expect(output).toContain('3. Configure unsupported MCP clients'); + expect(output).toContain('4. Start MCP'); + expect(output).toContain('Run this command before using Codex, Cursor, OpenCode, and Universal .agents:'); + expect(output).toContain('Open Cursor from the KTX project directory'); + expect(output).toContain('Open ~/.codex/config.toml, then paste this block:\n\n PASTE:\n [mcp_servers.ktx]'); + expect(output).toContain('Open opencode.json, then paste this block:'); + expect(output).toContain('Use this endpoint when setting up unsupported MCP clients:'); + expect(output).toContain('Codex guidance installed'); + expect(output).toContain('Cursor rules installed'); + expect(output).toContain('OpenCode commands installed'); + expect(output).toContain('.agents guidance installed'); + }); + + describe('createAgentNextActionsLineFormatter', () => { + function makeColorStdout(): { write: (chunk: string) => boolean; hasColors: () => boolean } { + return { write: () => true, hasColors: () => true }; + } + + function makePlainStdout(): { write: (chunk: string) => boolean; hasColors: () => boolean } { + return { write: () => true, hasColors: () => false }; + } + + const ESC = String.fromCharCode(27); + + it('returns the line untouched when the stream cannot render colors', () => { + const format = createAgentNextActionsLineFormatter(makePlainStdout()); + expect(format('2. Upload Claude Desktop skills')).toBe('2. Upload Claude Desktop skills'); + expect(format(' /tmp/ktx/.ktx/agents/claude/ktx.zip')).toBe(' /tmp/ktx/.ktx/agents/claude/ktx.zip'); + }); + + it('styles step headings and aligns sub-prose under the title', () => { + const format = createAgentNextActionsLineFormatter(makeColorStdout()); + const heading = format('2. Upload Claude Desktop skills'); + expect(heading).toContain(ESC); + expect(heading).toContain('2'); + expect(heading).toContain('Upload Claude Desktop skills'); + expect(heading).not.toMatch(/^2\. /); + + const sub = format(' Toggle the uploaded KTX skills on.'); + expect(sub).toMatch(/^ {3}/); + expect(sub).toContain('Toggle the uploaded KTX skills on.'); + }); + + it('renders skill bundle .zip paths as bullets and shortens HOME to ~', () => { + const previousHome = process.env.HOME; + process.env.HOME = '/tmp/test-home'; + try { + const format = createAgentNextActionsLineFormatter(makeColorStdout()); + const line = format(' /tmp/test-home/.ktx/agents/claude/ktx-analytics.zip'); + expect(line).toContain('•'); + expect(line).toContain('~/.ktx/agents/claude/ktx-analytics.zip'); + expect(line).not.toContain('/tmp/test-home/'); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + } + }); + + it('replaces breadcrumb separators with a typographic chevron', () => { + const format = createAgentNextActionsLineFormatter(makeColorStdout()); + const line = format(' Open Claude Desktop: Customize > Skills > + > Create skill > Upload a skill.'); + expect(line).toContain('›'); + expect(line).not.toContain(' > '); + }); + + it('leaves already-styled lines untouched to avoid double-wrapping', () => { + const format = createAgentNextActionsLineFormatter(makeColorStdout()); + const preStyled = `${ESC}[1m2. Already styled${ESC}[22m`; + expect(format(preStyled)).toBe(preStyled); + }); }); }); diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index be66b8a9..6fcd06c1 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -1,7 +1,10 @@ import { existsSync } from 'node:fs'; import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join, relative, resolve } from 'node:path'; +import type { Writable } from 'node:stream'; import { fileURLToPath } from 'node:url'; +import { styleText } from 'node:util'; +import { log, outro } from '@clack/prompts'; import { loadKtxProject, markKtxSetupStateStepComplete, @@ -9,8 +12,6 @@ import { } from '@ktx/context/project'; import { strToU8, zipSync } from 'fflate'; import type { KtxCliIo } from './cli-runtime.js'; -import { bold, dim, green } from './io/symbols.js'; -import { withMultiselectNavigation } from './prompt-navigation.js'; import { createKtxSetupPromptAdapter, createKtxSetupUiAdapter, @@ -31,6 +32,7 @@ export interface KtxSetupAgentsArgs { scope: KtxAgentScope; mode: KtxAgentInstallMode; skipAgents: boolean; + showNextActions?: boolean; } export type KtxSetupAgentsResult = @@ -38,6 +40,7 @@ export type KtxSetupAgentsResult = status: 'ready'; projectDir: string; installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>; + nextActions?: string; } | { status: 'skipped'; projectDir: string } | { status: 'back'; projectDir: string } @@ -50,7 +53,11 @@ export interface KtxAgentInstallManifest { installedAt: string; installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>; entries: Array< - | { kind: 'file'; path: string; role?: 'skill' | 'rule' | 'analytics-skill' | 'claude-plugin' | 'launcher' } + | { + kind: 'file'; + path: string; + role?: 'skill' | 'rule' | 'analytics-skill' | 'claude-desktop-skill-bundle' | 'launcher'; + } | { kind: 'json-key'; path: string; jsonPath: string[] } >; } @@ -69,11 +76,95 @@ interface KtxMcpClientInstallResult { notices: string[]; } +const MCP_DAEMON_REQUIRED_NOTICE = 'mcp-daemon-required'; + interface KtxCliLauncher { command: string; args: string[]; } +function isWritableTtyOutput(output: KtxCliIo['stdout']): output is KtxCliIo['stdout'] & Writable { + return ( + output.isTTY === true && + typeof (output as { on?: unknown }).on === 'function' && + typeof (output as { columns?: unknown }).columns !== 'undefined' + ); +} + +function writeSetupInfo(io: KtxCliIo, message: string): void { + if (isWritableTtyOutput(io.stdout)) { + log.info(message, { output: io.stdout }); + return; + } + io.stdout.write(`${message}\n`); +} + +function writeSetupStep(io: KtxCliIo, message: string): void { + if (isWritableTtyOutput(io.stdout)) { + log.step(message, { output: io.stdout }); + return; + } + io.stdout.write(`\n${message}\n`); +} + +function writeSetupOutro(io: KtxCliIo, message: string): void { + if (isWritableTtyOutput(io.stdout)) { + outro(message, { output: io.stdout }); + return; + } + io.stdout.write(`\n${message}\n`); +} + +const STEP_HEADING_RE = /^(\d+)\. (.+)$/; +const ACTION_MARKER_RE = /^(RUN|PASTE|USE|OPEN):$/; + +export function createAgentNextActionsLineFormatter( + stdout: KtxCliIo['stdout'], +): (line: string) => string { + const maybeHasColors = (stdout as { hasColors?: unknown }).hasColors; + const supportsColor = typeof maybeHasColors === 'function' && Boolean(maybeHasColors.call(stdout)); + if (!supportsColor) return (line) => line; + + const homeDir = process.env.HOME ? resolve(process.env.HOME) : ''; + const styleOptions = { validateStream: false } as const; + const dim = (s: string) => styleText('dim', s, styleOptions); + const bold = (s: string) => styleText('bold', s, styleOptions); + const cyanBold = (s: string) => styleText(['cyan', 'bold'], s, styleOptions); + const dimCyan = (s: string) => styleText(['dim', 'cyan'], s, styleOptions); + const shortenPath = (path: string): string => { + if (!homeDir) return path; + if (path === homeDir) return '~'; + if (path.startsWith(`${homeDir}/`)) return `~/${path.slice(homeDir.length + 1)}`; + return path; + }; + + return (rawLine: string): string => { + if (rawLine.length === 0 || rawLine.includes('[')) return rawLine; + + const heading = rawLine.match(STEP_HEADING_RE); + if (heading) { + return `${cyanBold(heading[1])} ${bold(heading[2])}`; + } + + if (!rawLine.startsWith(' ')) return rawLine; + const body = rawLine.slice(2); + + if (ACTION_MARKER_RE.test(body)) { + return ` ${dim(body)}`; + } + + if (body.endsWith('.zip') && (body.startsWith('/') || body.startsWith('~'))) { + return ` ${dimCyan('•')} ${shortenPath(body)}`; + } + + if (body.includes(' > ')) { + return ` ${body.replaceAll(' > ', ` ${dim('›')} `)}`; + } + + return ` ${dim(body)}`; + }; +} + async function readJsonObject(path: string): Promise> { if (!existsSync(path)) return {}; const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown; @@ -257,7 +348,7 @@ async function installMcpClientConfig(input: { const endpoint = await resolveMcpEndpoint(input.projectDir); if (!endpoint.running) { - notices.push('Run `ktx mcp start` to enable the configured KTX MCP server.'); + notices.push(MCP_DAEMON_REQUIRED_NOTICE); } if (input.target === 'claude-code') { @@ -269,15 +360,15 @@ async function installMcpClientConfig(input: { await writeJsonKey(config.path, config.jsonPath, cursorMcpEntry(endpoint)); entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath }); } else if (input.target === 'codex') { - snippets.push(`Codex MCP snippet for ~/.codex/config.toml:\n${codexSnippet(endpoint)}`); + snippets.push(`Add this Codex MCP snippet to ~/.codex/config.toml:\n${codexSnippet(endpoint)}`); } else if (input.target === 'opencode') { const path = input.scope === 'global' ? '~/.config/opencode/opencode.json' : relative(input.projectDir, join(input.projectDir, 'opencode.json')); - snippets.push(`opencode MCP snippet for ${path}:\n${opencodeSnippet(endpoint)}`); + snippets.push(`Add this OpenCode MCP snippet to ${path}:\n${opencodeSnippet(endpoint)}`); } else if (input.target === 'universal') { - snippets.push(universalMcpSnippet(endpoint)); + snippets.push(`Use this universal MCP endpoint with unsupported MCP clients:\n${universalMcpSnippet(endpoint)}`); } return { entries, snippets, notices }; @@ -307,8 +398,12 @@ export function agentInstallManifestPath(projectDir: string): string { return join(resolve(projectDir), '.ktx/agents/install-manifest.json'); } -function claudeDesktopPluginPath(projectDir: string): string { - return join(resolve(projectDir), '.ktx/agents/claude/ktx-plugin.zip'); +function claudeDesktopAnalyticsSkillBundlePath(projectDir: string): string { + return join(resolve(projectDir), '.ktx/agents/claude/ktx-analytics.zip'); +} + +function claudeDesktopAdminSkillBundlePath(projectDir: string): string { + return join(resolve(projectDir), '.ktx/agents/claude/ktx.zip'); } function claudeDesktopLauncherPath(projectDir: string): string { @@ -354,7 +449,20 @@ export function plannedKtxAgentFiles(input: { if (input.target === 'claude-desktop') { return [ { kind: 'file', path: claudeDesktopLauncherPath(input.projectDir), role: 'launcher' as const }, - { kind: 'file', path: claudeDesktopPluginPath(input.projectDir), role: 'claude-plugin' as const }, + { + kind: 'file', + path: claudeDesktopAnalyticsSkillBundlePath(input.projectDir), + role: 'claude-desktop-skill-bundle' as const, + }, + ...(withAdminCli + ? [ + { + kind: 'file' as const, + path: claudeDesktopAdminSkillBundlePath(input.projectDir), + role: 'claude-desktop-skill-bundle' as const, + }, + ] + : []), ]; } throw new Error(`Global ${input.target} installation is not supported; omit --global.`); @@ -484,43 +592,7 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun ].join('\n'); } -function claudePluginJsonContent(): string { - return `${JSON.stringify( - { - name: 'ktx', - version: '0.0.0-local', - description: 'KTX analytics workflow guidance and local MCP tools.', - }, - null, - 2, - )}\n`; -} - -function claudePluginVersionContent(): string { - return `${JSON.stringify({ version: '0.0.0-local' }, null, 2)}\n`; -} - -function claudePluginSetupContent(input: { projectDir: string; withAdminCli: boolean }): string { - return [ - '# KTX Claude Plugin', - '', - 'Install this plugin ZIP from Claude Desktop to load the KTX analytics skill.', - '', - `KTX project: \`${input.projectDir}\``, - '', - 'Included:', - '', - '- `ktx-analytics` skill for the MCP analytics workflow', - ...(input.withAdminCli ? ['- `ktx` admin CLI skill for KTX maintenance commands'] : []), - '', - 'The KTX MCP server is registered separately in `claude_desktop_config.json` by `ktx setup` and runs as a local stdio child of Claude Desktop — no daemon to start.', - '', - 'If this checkout or project directory moves, rerun `ktx setup --agents` and reinstall the regenerated plugin.', - '', - ].join('\n'); -} - -function claudePluginLauncherContent(input: { launcher: KtxCliLauncher }): string { +function claudeDesktopLauncherContent(input: { launcher: KtxCliLauncher }): string { const binPath = input.launcher.args[0]; if (!binPath) { throw new Error('Expected KTX CLI launcher to include a bin path.'); @@ -569,40 +641,45 @@ function claudePluginLauncherContent(input: { launcher: KtxCliLauncher }): strin ' run_with_node "$(command -v node)" "$@"', 'fi', '', - 'echo "KTX plugin could not find Node.js. Set KTX_NODE to a Node executable and reinstall the plugin." >&2', + 'echo "KTX Claude Desktop launcher could not find Node.js. Set KTX_NODE to a Node executable and rerun ktx setup --agents." >&2', 'exit 127', '', ].join('\n'); } -async function writeClaudeDesktopPlugin(input: { +async function writeClaudeDesktopSkillBundle(input: { projectDir: string; path: string; - mode: KtxAgentInstallMode; + skillName: 'ktx-analytics' | 'ktx'; launcher: KtxCliLauncher; }): Promise { - const withAdminCli = input.mode === 'mcp-cli'; + const content = + input.skillName === 'ktx-analytics' + ? await readAnalyticsSkillContent() + : cliInstructionContent({ projectDir: input.projectDir, launcher: input.launcher }); const files: Record = { - '.claude-plugin/plugin.json': strToU8(claudePluginJsonContent()), - 'version.json': strToU8(claudePluginVersionContent()), - 'skills/ktx-analytics/SKILL.md': strToU8(await readAnalyticsSkillContent()), - 'SETUP.md': strToU8(claudePluginSetupContent({ projectDir: input.projectDir, withAdminCli })), + [`${input.skillName}/SKILL.md`]: strToU8(content), }; - if (withAdminCli) { - files['skills/ktx/SKILL.md'] = strToU8( - cliInstructionContent({ projectDir: input.projectDir, launcher: input.launcher }), - ); - } await mkdir(dirname(input.path), { recursive: true }); await writeFile(input.path, Buffer.from(zipSync(files))); } +function claudeDesktopSkillNameForBundle(path: string): 'ktx-analytics' | 'ktx' { + if (path.endsWith('/ktx-analytics.zip')) { + return 'ktx-analytics'; + } + if (path.endsWith('/ktx.zip')) { + return 'ktx'; + } + throw new Error(`Unsupported Claude Desktop skill bundle path: ${path}`); +} + async function writeClaudeDesktopLauncher(input: { path: string; launcher: KtxCliLauncher; }): Promise { await mkdir(dirname(input.path), { recursive: true }); - await writeFile(input.path, claudePluginLauncherContent({ launcher: input.launcher }), 'utf-8'); + await writeFile(input.path, claudeDesktopLauncherContent({ launcher: input.launcher }), 'utf-8'); await chmod(input.path, 0o755); } @@ -719,17 +796,8 @@ const targetDisplayNames: Record = { universal: 'Universal .agents', }; -const fileEntryLabels: Record = { - 'claude-code': 'Skill installed', - 'claude-desktop': 'Skill installed', - codex: 'Skill installed', - cursor: 'Rule installed', - opencode: 'Command installed', - universal: 'Skill installed', -}; - -function mcpEntryLabel(entry: Extract): string { - return `MCP config installed — connects client agents to KTX MCP tools (${entry.jsonPath.join('.')})`; +export function targetDisplayName(target: string): string { + return Object.hasOwn(targetDisplayNames, target) ? targetDisplayNames[target as KtxAgentTarget] : target; } function targetSupportsGlobalScope(target: KtxAgentTarget): boolean { @@ -740,11 +808,72 @@ function effectiveInstallScope(target: KtxAgentTarget, requestedScope: KtxAgentS return target === 'claude-desktop' ? 'global' : requestedScope; } -export function formatInstallSummary( +function scopeDisplayName(scope: KtxAgentScope): string { + if (scope === 'project') return 'Project scope'; + if (scope === 'global') return 'Global scope'; + return 'Local scope'; +} + +function targetUsesHttpMcpDaemon(target: KtxAgentTarget): boolean { + return target !== 'claude-desktop'; +} + +function manualMcpConfigInstruction(target: KtxAgentTarget, scope: KtxAgentScope): string { + if (target === 'codex') { + return 'Add the snippet shown below to ~/.codex/config.toml.'; + } + if (target === 'opencode') { + return scope === 'global' + ? 'Add the snippet shown below to ~/.config/opencode/opencode.json.' + : 'Add the snippet shown below to opencode.json.'; + } + if (target === 'universal') { + return 'Use the printed endpoint with unsupported MCP clients.'; + } + return 'Add the printed snippet manually.'; +} + +function guidanceInstallLine(target: KtxAgentTarget): string { + if (target === 'codex') return 'Codex guidance installed'; + if (target === 'cursor') return 'Cursor rules installed'; + if (target === 'opencode') return 'OpenCode commands installed'; + if (target === 'universal') return '.agents guidance installed'; + return 'Agent guidance installed'; +} + +function hasEntryRole(entries: InstallEntry[], role: Extract['role']): boolean { + return entries.some((entry) => entry.kind === 'file' && entry.role === role); +} + +function hasAdminCliEntries(entries: InstallEntry[]): boolean { + return entries.some( + (entry) => + entry.kind === 'file' && + (entry.role === 'skill' || entry.role === 'rule' || entry.role === undefined), + ); +} + +export interface InstallSummaryEntry { + title: string; + lines: string[]; +} + +function formatInlinePath(path: string): string { + const home = process.env.HOME; + if (!home) return path; + const resolvedHome = resolve(home); + if (path === resolvedHome) return '~'; + if (path.startsWith(`${resolvedHome}/`)) { + return `~/${relative(resolvedHome, path)}`; + } + return path; +} + +export function formatInstallSummaryLines( installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>, entries: InstallEntry[], projectDir: string, -): string { +): InstallSummaryEntry[] { const entriesByTarget = new Map(); for (const install of installs) { const plannedFilePaths = new Set( @@ -766,48 +895,233 @@ export function formatInstallSummary( ); } - const fileHints: Record = { - skill: 'teaches admin agents which KTX CLI commands to run', - rule: 'tells admin agents when to use KTX CLI', - 'analytics-skill': 'teaches your agent the KTX MCP analytics workflow', - 'claude-plugin': 'bundles KTX skills for Claude Desktop (MCP server is registered in claude_desktop_config.json)', - launcher: 'runs the local KTX CLI with an available Node.js for Claude Desktop', - }; - - const lines: string[] = []; - for (const install of installs) { + return installs.map((install) => { const targetEntries = entriesByTarget.get(install.target) ?? []; - lines.push(` ${targetDisplayNames[install.target]}`); - for (const entry of targetEntries) { - if (entry.kind === 'file') { - const isRule = entry.role === 'rule' || (!entry.role && fileEntryLabels[install.target] === 'Rule installed'); - const label = - entry.role === 'analytics-skill' - ? 'Analytics skill installed' - : entry.role === 'claude-plugin' - ? 'Claude plugin generated' - : entry.role === 'launcher' - ? 'Launcher installed' - : isRule - ? 'Rule installed' - : fileEntryLabels[install.target]; - const hint = fileHints[isRule ? 'rule' : (entry.role ?? 'skill')] ?? ''; - lines.push(` + ${label} — ${hint}`); - if (entry.role !== 'claude-plugin') { - const displayPath = - install.scope === 'global' ? entry.path : relative(projectDir, entry.path); - lines.push(` ${displayPath}`); + const mcpEntry = mcpEntriesByTarget + .get(install.target) + ?.find((entry): entry is Extract => entry.kind === 'json-key'); + const lines: string[] = []; + + if (mcpEntry) { + lines.push(formatInlinePath(mcpEntry.path)); + } else if (install.target !== 'claude-desktop') { + lines.push(manualMcpConfigInstruction(install.target, install.scope)); + } + + if (targetUsesHttpMcpDaemon(install.target)) { + lines.push('Requires MCP to be started.'); + } + + const hasAnalytics = hasEntryRole(targetEntries, 'analytics-skill'); + const hasAdmin = hasAdminCliEntries(targetEntries); + const claudeDesktopSkillBundles = targetEntries.filter( + (entry): entry is Extract => + entry.kind === 'file' && entry.role === 'claude-desktop-skill-bundle', + ); + + if (install.target === 'claude-code') { + if (hasAnalytics) { + lines.push('Analytics skill installed.'); + } + if (hasAdmin) { + lines.push('Admin CLI skill installed.'); + } + } else if (install.target === 'claude-desktop') { + if (claudeDesktopSkillBundles.length > 0) { + lines.push('Skill bundles:'); + for (const bundle of claudeDesktopSkillBundles) { + lines.push(` ${bundle.path}`); } } + } else if (hasAnalytics || hasAdmin) { + lines.push(`${guidanceInstallLine(install.target)}.`); } - for (const entry of mcpEntriesByTarget - .get(install.target) - ?.filter((entry): entry is Extract => entry.kind === 'json-key') ?? []) { - const displayPath = install.scope === 'global' ? entry.path : relative(projectDir, entry.path); - lines.push(` + ${mcpEntryLabel(entry)}`); - lines.push(` ${displayPath}`); + + if (hasEntryRole(targetEntries, 'launcher')) { + lines.push('Starts KTX over stdio from Claude Desktop.'); + } + + return { + title: `${targetDisplayName(install.target)} · ${scopeDisplayName(install.scope)}`, + lines, + }; + }); +} + +function claudeDesktopSkillBundlePathsForInstalls( + projectDir: string, + installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>, +): string[] { + return installs + .filter((install) => install.target === 'claude-desktop') + .flatMap((install) => plannedKtxAgentFiles({ projectDir, ...install })) + .filter( + (entry): entry is Extract => + entry.kind === 'file' && entry.role === 'claude-desktop-skill-bundle', + ) + .map((entry) => entry.path); +} + +function humanList(values: string[]): string { + if (values.length <= 2) { + return values.join(' and '); + } + return `${values.slice(0, -1).join(', ')}, and ${values[values.length - 1]}`; +} + +function pushBlankLine(lines: string[]): void { + if (lines.length > 0 && lines[lines.length - 1] !== '') { + lines.push(''); + } +} + +function trimTrailingBlankLines(lines: string[]): void { + while (lines[lines.length - 1] === '') { + lines.pop(); + } +} + +function manualActionFromSnippet(snippet: string): { + title: string; + instruction: string; + marker: 'PASTE' | 'USE'; + body: string[]; +} { + const [label = '', ...body] = snippet.split('\n'); + const codexPrefix = 'Add this Codex MCP snippet to ~/.codex/config.toml:'; + if (label === codexPrefix) { + return { + title: 'Configure Codex', + instruction: 'Open ~/.codex/config.toml, then paste this block:', + marker: 'PASTE', + body, + }; + } + + const opencodeMatch = label.match(/^Add this OpenCode MCP snippet to (.+):$/); + if (opencodeMatch) { + return { + title: 'Configure OpenCode', + instruction: `Open ${opencodeMatch[1]}, then paste this block:`, + marker: 'PASTE', + body, + }; + } + + if (label === 'Use this universal MCP endpoint with unsupported MCP clients:') { + return { + title: 'Configure unsupported MCP clients', + instruction: 'Use this endpoint when setting up unsupported MCP clients:', + marker: 'USE', + body, + }; + } + + return { + title: 'Configure MCP client', + instruction: label, + marker: 'PASTE', + body, + }; +} + +function formatAgentNextActions(input: { + projectDir: string; + installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>; + notices: string[]; + snippets: string[]; +}): string { + const projectDir = resolve(input.projectDir); + const lines: string[] = []; + let step = 1; + + for (const snippet of input.snippets) { + const action = manualActionFromSnippet(snippet); + lines.push(`${step}. ${action.title}`); + lines.push(` ${action.instruction}`); + if (action.body.length > 0) { + lines.push('', ` ${action.marker}:`); + } + for (const line of action.body) { + lines.push(` ${line}`); + } + pushBlankLine(lines); + step += 1; + } + + const httpTargets = input.installs + .filter((install) => targetUsesHttpMcpDaemon(install.target)) + .map((install) => targetDisplayName(install.target)); + if (input.notices.length > 0 && httpTargets.length > 0) { + lines.push(`${step}. Start MCP`); + lines.push(` Run this command before using ${humanList(httpTargets)}:`); + lines.push(''); + lines.push(' RUN:'); + lines.push(` ktx mcp start --project-dir ${projectDir}`); + lines.push(''); + lines.push(' If you need to stop MCP later:'); + lines.push(` ktx mcp stop --project-dir ${projectDir}`); + pushBlankLine(lines); + step += 1; + } + + const claudeCodeInstall = input.installs.find((install) => install.target === 'claude-code'); + if (claudeCodeInstall) { + lines.push(`${step}. Open Claude Code`); + if (claudeCodeInstall.scope === 'project') { + lines.push(' Open Claude Code from the KTX project directory:'); + lines.push(''); + lines.push(' RUN:'); + lines.push(` cd ${shellScriptQuote(projectDir)}`); + lines.push(' claude'); + } else { + lines.push(' RUN:'); + lines.push(' claude'); + } + pushBlankLine(lines); + step += 1; + } + + const cursorInstall = input.installs.find((install) => install.target === 'cursor'); + if (cursorInstall) { + lines.push(`${step}. Open Cursor`); + if (cursorInstall.scope === 'project') { + lines.push(' Open Cursor from the KTX project directory:'); + lines.push(''); + lines.push(' OPEN:'); + lines.push(` ${projectDir}`); + } else { + lines.push(' Open Cursor.'); + } + pushBlankLine(lines); + step += 1; + } + + if (input.installs.some((install) => install.target === 'claude-desktop')) { + lines.push(`${step}. Restart Claude Desktop`); + lines.push(' Claude Desktop loads KTX MCP after restart.'); + pushBlankLine(lines); + step += 1; + + const skillBundlePaths = claudeDesktopSkillBundlePathsForInstalls(projectDir, input.installs); + if (skillBundlePaths.length > 0) { + lines.push(`${step}. Upload Claude Desktop skills`); + lines.push(' Open Claude Desktop: Customize > Skills > + > Create skill > Upload a skill.'); + lines.push(skillBundlePaths.length === 1 ? ' Upload this file:' : ' Upload each file separately:'); + for (const path of skillBundlePaths) { + lines.push(` ${path}`); + } + lines.push(' Toggle the uploaded KTX skills on.'); + pushBlankLine(lines); + step += 1; } } + + if (lines.length === 0) { + lines.push('Open your configured agent and ask a data question.'); + } + + trimTrailingBlankLines(lines); return lines.join('\n'); } @@ -825,11 +1139,11 @@ async function installTarget(input: { await writeClaudeDesktopLauncher({ path: entry.path, launcher }); continue; } - if (entry.role === 'claude-plugin') { - await writeClaudeDesktopPlugin({ + if (entry.role === 'claude-desktop-skill-bundle') { + await writeClaudeDesktopSkillBundle({ projectDir: input.projectDir, path: entry.path, - mode: input.mode, + skillName: claudeDesktopSkillNameForBundle(entry.path), launcher, }); continue; @@ -866,14 +1180,25 @@ export async function runKtxSetupAgentsStep( } const prompts = deps.prompts ?? createPromptAdapter(); + if (args.inputMode === 'auto' && args.target === undefined) { + writeSetupInfo(io, 'Space to select, Enter to confirm, Esc to go back.'); + } const mode = args.inputMode === 'disabled' ? args.mode : ((await prompts.select({ - message: 'How should client agents connect to this KTX project?', + message: 'What should agents be allowed to do with this KTX project?', options: [ - { value: 'mcp', label: 'MCP tools + analytics skill' }, - { value: 'mcp-cli', label: 'MCP tools + analytics skill + admin CLI skill' }, + { + value: 'mcp', + label: 'Ask data questions with KTX MCP', + hint: 'Installs the MCP connection and analytics workflow skill. Best for normal use.', + }, + { + value: 'mcp-cli', + label: 'Ask data questions + manage KTX with CLI commands', + hint: 'Adds an admin CLI skill so agents can run ktx status, sl, wiki, and setup commands.', + }, ], })) as KtxAgentInstallMode | 'back'); if (mode === 'back') return { status: 'skipped', projectDir: args.projectDir }; @@ -884,7 +1209,7 @@ export async function runKtxSetupAgentsStep( : args.inputMode === 'disabled' ? [] : ((await prompts.multiselect({ - message: withMultiselectNavigation('Which agent targets should KTX install?'), + message: 'Which agent targets should KTX install?', options: [ { value: 'claude-code', label: 'Claude Code' }, { value: 'claude-desktop', label: 'Claude Desktop' }, @@ -897,7 +1222,11 @@ export async function runKtxSetupAgentsStep( })) as KtxAgentTarget[]); if (targets.includes('back' as KtxAgentTarget)) return { status: 'back', projectDir: args.projectDir }; if (targets.length === 0) { - io.stderr.write('Missing agent target: pass --target or use interactive setup.\n'); + io.stderr.write( + args.inputMode === 'disabled' + ? 'Run in a TTY, or pass --target .\n' + : 'Missing agent target: pass --target or use interactive setup.\n', + ); return { status: 'missing-input', projectDir: args.projectDir }; } @@ -908,10 +1237,18 @@ export async function runKtxSetupAgentsStep( scopeTargets.length > 0 && scopeTargets.every(targetSupportsGlobalScope) ? ((await prompts.select({ - message: 'Where should KTX install supported agent config?', + message: `Where should KTX install supported agent config?\n\nKTX project: ${resolve(args.projectDir)}`, options: [ - { value: 'project', label: 'Project' }, - { value: 'global', label: 'Global' }, + { + value: 'project', + label: 'Project scope (KTX project directory)', + hint: 'Only agents opened from this KTX project path load the project-scoped config.', + }, + { + value: 'global', + label: 'Global scope (user config)', + hint: 'Agents can load this KTX project from any working directory.', + }, ], })) as KtxAgentScope | 'back') : args.scope; @@ -921,7 +1258,6 @@ export async function runKtxSetupAgentsStep( const entries: InstallEntry[] = []; const snippets: string[] = []; const notices = new Set(); - let claudeDesktopTutorial: string | undefined; try { for (const install of installs) { const targetEntries = await installTarget({ projectDir: args.projectDir, ...install }); @@ -934,25 +1270,6 @@ export async function runKtxSetupAgentsStep( entries.push(...mcpResult.entries); for (const snippet of mcpResult.snippets) snippets.push(snippet); for (const notice of mcpResult.notices) notices.add(notice); - if (install.target === 'claude-desktop') { - const pluginEntry = targetEntries.find( - (entry): entry is Extract => - entry.kind === 'file' && entry.role === 'claude-plugin', - ); - const pluginPath = pluginEntry?.path ?? ''; - const configPath = claudeDesktopConfigPath().path; - claudeDesktopTutorial = [ - `${green('✓')} ${bold('KTX MCP server registered')}`, - ` ${dim(configPath)}`, - '', - bold('1. Restart Claude Desktop'), - ' Quit and reopen so it picks up the new MCP server.', - '', - bold('2. Install the KTX plugin'), - ' Open Claude Desktop → Settings → Plugins and install from file:', - ` 📦 ${dim(pluginPath)}`, - ].join('\n'); - } } await writeManifest( args.projectDir, @@ -960,23 +1277,25 @@ export async function runKtxSetupAgentsStep( ); await markAgentsComplete(args.projectDir); const setupUi = createKtxSetupUiAdapter(); - setupUi.note( - formatInstallSummary(installs, entries, args.projectDir), - 'Agent integration complete', - io, - ); - if (claudeDesktopTutorial) { - setupUi.note(claudeDesktopTutorial, 'Finish Claude Desktop setup', io, { - format: (line) => line, + for (const summary of formatInstallSummaryLines(installs, entries, args.projectDir)) { + writeSetupStep( + io, + summary.lines.length > 0 ? `${summary.title}\n${summary.lines.join('\n')}` : summary.title, + ); + } + const nextActions = formatAgentNextActions({ + projectDir: args.projectDir, + installs, + notices: [...notices], + snippets, + }); + if (args.showNextActions !== false) { + setupUi.note(nextActions, 'Required before using agents', io, { + format: createAgentNextActionsLineFormatter(io.stdout), }); + writeSetupOutro(io, 'All set.'); } - const nextStepBlocks: string[] = []; - for (const notice of notices) nextStepBlocks.push(notice); - for (const snippet of snippets) nextStepBlocks.push(snippet); - if (nextStepBlocks.length > 0) { - setupUi.note(nextStepBlocks.join('\n\n'), 'Next steps', io, { format: bold }); - } - return { status: 'ready', projectDir: args.projectDir, installs }; + return { status: 'ready', projectDir: args.projectDir, installs, nextActions }; } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return { status: 'failed', projectDir: args.projectDir }; diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 12ab947f..7e22be26 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -46,9 +46,14 @@ function makePromptAdapter(options: { }; } -function managedDaemon(baseUrl = 'http://127.0.0.1:61234') { +function managedDaemon( + baseUrl = 'http://127.0.0.1:61234', + logs: { stdoutLog?: string; stderrLog?: string } = {}, +) { return { baseUrl, + stdoutLog: logs.stdoutLog ?? '/tmp/ktx-daemon.stdout.log', + stderrLog: logs.stderrLog ?? '/tmp/ktx-daemon.stderr.log', env: { KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl, }, @@ -324,12 +329,71 @@ describe('setup embeddings step', () => { expect(result.status).toBe('failed'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); - expect(config.ingest.embeddings.backend).toBe('deterministic'); + expect(config.ingest.embeddings.backend).toBe('none'); expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]'); expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings'); expect(io.stderr()).not.toContain('skip for now'); }); + it('prints the recent daemon stderr tail when local embedding health check fails', async () => { + const io = makeIo(); + const stderrLog = join(tempDir, '.ktx', 'runtime', 'daemon.stderr.log'); + await mkdir(join(tempDir, '.ktx', 'runtime'), { recursive: true }); + await writeFile( + stderrLog, + Array.from({ length: 45 }, (_value, index) => `daemon traceback line ${index + 1}`).join('\n'), + ); + + const result = await runKtxSetupEmbeddingsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: false, + }, + io.io, + { + env: {}, + ensureLocalEmbeddings: vi.fn(async () => managedDaemon('http://127.0.0.1:61234', { stderrLog })), + healthCheck: vi.fn(async () => ({ ok: false as const, message: 'HTTP 500' })), + }, + ); + + expect(result.status).toBe('failed'); + expect(io.stderr()).toContain('Recent local embeddings daemon stderr:'); + expect(io.stderr()).toContain('daemon traceback line 6'); + expect(io.stderr()).toContain('daemon traceback line 45'); + expect(io.stderr()).not.toContain('daemon traceback line 5'); + }); + + it('does not print daemon stderr diagnostics when the log is unavailable or empty', async () => { + const io = makeIo(); + + const result = await runKtxSetupEmbeddingsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: false, + }, + io.io, + { + env: {}, + ensureLocalEmbeddings: vi.fn(async () => + managedDaemon('http://127.0.0.1:61234', { + stderrLog: join(tempDir, '.ktx', 'runtime', 'missing.stderr.log'), + }), + ), + healthCheck: vi.fn(async () => ({ ok: false as const, message: 'HTTP 500' })), + }, + ); + + expect(result.status).toBe('failed'); + expect(io.stderr()).not.toContain('Recent local embeddings daemon stderr:'); + }); + it('uses fixed OpenAI defaults and only asks for credentials when OpenAI is selected', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); @@ -436,7 +500,7 @@ describe('setup embeddings step', () => { expect(result.status).toBe('skipped'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); - expect(config.ingest.embeddings.backend).toBe('deterministic'); + expect(config.ingest.embeddings.backend).toBe('none'); }); it('returns back without writing config when the local health check fails and Back is selected', async () => { @@ -460,7 +524,7 @@ describe('setup embeddings step', () => { expect(result.status).toBe('back'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.ingest.embeddings.backend).toBe('deterministic'); + expect(config.ingest.embeddings.backend).toBe('none'); }); it('preserves already completed embeddings setup when no embedding args request changes', async () => { diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 3e2e47db..475e5126 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -1,4 +1,4 @@ -import { writeFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import { resolveKtxConfigReference } from '@ktx/context/core'; import { type KtxProjectConfig, @@ -59,6 +59,7 @@ export interface KtxSetupEmbeddingsDeps { healthCheck?: (config: KtxEmbeddingConfig) => Promise; ensureLocalEmbeddings?: (options: { cliVersion: string; + projectDir: string; installPolicy: KtxManagedPythonInstallPolicy; io: KtxCliIo; }) => Promise; @@ -85,6 +86,7 @@ const EMBEDDING_OPTION_PROMPT_CONTEXT = 'KTX uses embeddings for semantic search over semantic-layer sources, wiki context, schema metadata, ' + 'and relationship evidence.'; const LOCAL_EMBEDDING_HEALTH_TIMEOUT_MS = 120_000; +const LOCAL_EMBEDDING_STDERR_TAIL_LINES = 40; function createPromptAdapter(): KtxSetupEmbeddingsPromptAdapter { return createKtxSetupPromptAdapter({ selectCancelValue: 'back' }); @@ -94,7 +96,6 @@ async function hasCompletedEmbeddings(projectDir: string, config: KtxProjectConf return ( (await readKtxSetupState(projectDir)).completed_steps.includes('embeddings') && config.ingest.embeddings.backend !== 'none' && - config.ingest.embeddings.backend !== 'deterministic' && typeof config.ingest.embeddings.model === 'string' && config.ingest.embeddings.model.length > 0 && config.ingest.embeddings.dimensions > 0 @@ -287,14 +288,33 @@ async function chooseEmbeddingBackend( return 'back'; } -function localEmbeddingSetupMessage(message: string): string { - return [ +async function readLocalEmbeddingDaemonStderrTail(stderrLog: string | undefined): Promise { + if (!stderrLog) { + return []; + } + try { + const lines = (await readFile(stderrLog, 'utf8')) + .split(/\r?\n/) + .map((line) => line.trimEnd()) + .filter((line) => line.trim().length > 0); + return lines.slice(-LOCAL_EMBEDDING_STDERR_TAIL_LINES); + } catch { + return []; + } +} + +function localEmbeddingSetupMessage(message: string, stderrTail: string[] = []): string { + const lines = [ `Local embedding health check failed: ${message}`, 'Local embeddings use the KTX-managed Python runtime.', 'Prepare the runtime with: ktx dev runtime start --feature local-embeddings', 'Use --yes with setup to install and start the runtime without prompting.', 'The first run may download Python packages and the all-MiniLM-L6-v2 model.', - ].join('\n'); + ]; + if (stderrTail.length > 0) { + lines.push('Recent local embeddings daemon stderr:', ...stderrTail); + } + return lines.join('\n'); } async function promptAfterLocalEmbeddingFailure( @@ -448,9 +468,13 @@ export async function runKtxSetupEmbeddingsStep( } progress.fail('Embedding test failed'); + const stderrTail = + selectedBackend === 'sentence-transformers' + ? await readLocalEmbeddingDaemonStderrTail(managedLocalEmbeddings?.stderrLog) + : []; io.stderr.write( selectedBackend === 'sentence-transformers' - ? `${localEmbeddingSetupMessage(health.message)}\n` + ? `${localEmbeddingSetupMessage(health.message, stderrTail)}\n` : `Embedding health check failed: ${health.message}\n`, ); if (args.inputMode === 'disabled') { diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index 4c25d092..82fc9461 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -307,7 +307,7 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -351,7 +351,7 @@ describe('setup Anthropic model step', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -658,7 +658,7 @@ describe('setup Anthropic model step', () => { llmBackend: 'vertex', vertexProject: 'kaelio-orbit-looker-20260430', vertexLocation: 'us-east5', - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -686,7 +686,7 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyFile: secretPath, - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -723,7 +723,7 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyFile: missingSecretPath, - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -1014,7 +1014,7 @@ describe('setup Anthropic model step', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }, io.io, @@ -1166,8 +1166,7 @@ describe('setup Anthropic model step', () => { ' default: claude-sonnet-4-6', 'ingest:', ' embeddings:', - ' backend: deterministic', - ' model: deterministic', + ' backend: none', ' dimensions: 8', ].join('\n'), 'utf-8', @@ -1209,8 +1208,7 @@ describe('setup Anthropic model step', () => { ` default: ${fixture.model}`, 'ingest:', ' embeddings:', - ' backend: deterministic', - ' model: deterministic', + ' backend: none', ' dimensions: 8', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index cd8aa848..e24800e4 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -37,7 +37,6 @@ export interface KtxSetupModelArgs { anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; llmModel?: string; - anthropicModel?: string; vertexProject?: string; vertexLocation?: string; forcePrompt?: boolean; @@ -478,14 +477,14 @@ function requestedBackend(args: KtxSetupModelArgs): KtxSetupLlmBackend | undefin if (args.vertexProject || args.vertexLocation) { return 'vertex'; } - if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile || args.llmModel || args.anthropicModel) { + if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile || args.llmModel) { return 'anthropic'; } return undefined; } function requestedModel(args: KtxSetupModelArgs): string | undefined { - return args.llmModel ?? args.anthropicModel; + return args.llmModel; } async function chooseBackend( @@ -929,7 +928,6 @@ export async function runKtxSetupAnthropicModelStep( !args.anthropicApiKeyEnv && !args.anthropicApiKeyFile && !args.llmModel && - !args.anthropicModel && !args.vertexProject && !args.vertexLocation ) { diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index 578beb8b..14cf18f7 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -55,12 +55,12 @@ describe('setup project step', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('creates a new project with --new and marks the project step complete', async () => { + it('creates a new project in non-interactive auto mode with --yes and marks the project step complete', async () => { const projectDir = join(tempDir, 'warehouse'); const testIo = makeIo(); const result = await runKtxSetupProjectStep( - { projectDir, mode: 'new', inputMode: 'disabled', yes: false }, + { projectDir, mode: 'auto', inputMode: 'disabled', yes: true }, testIo.io, ); @@ -74,7 +74,7 @@ describe('setup project step', () => { expect(testIo.stderr()).toBe(''); }); - it('loads an existing project with --existing and drops config setup progress', async () => { + it('loads an existing project in auto mode and drops config setup progress', async () => { const projectDir = join(tempDir, 'warehouse'); await initKtxProject({ projectDir }); await writeFile( @@ -91,7 +91,7 @@ describe('setup project step', () => { ); const result = await runKtxSetupProjectStep( - { projectDir, mode: 'existing', inputMode: 'disabled', yes: false }, + { projectDir, mode: 'auto', inputMode: 'disabled', yes: false }, makeIo().io, ); @@ -112,7 +112,7 @@ describe('setup project step', () => { await expect( runKtxSetupProjectStep({ projectDir, mode: 'auto', inputMode: 'disabled', yes: false }, rejectedIo.io), ).resolves.toMatchObject({ status: 'missing-input' }); - expect(rejectedIo.stderr()).toContain('Missing setup choice: pass --new or --yes'); + expect(rejectedIo.stderr()).toContain('Missing setup choice: pass --yes'); await expect(stat(join(projectDir, 'ktx.yaml'))).rejects.toThrow(); await expect( @@ -121,15 +121,15 @@ describe('setup project step', () => { await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined(); }); - it('fails --existing clearly when ktx.yaml is missing', async () => { + it('fails clearly in no-input auto mode when ktx.yaml is missing and --yes is absent', async () => { const projectDir = join(tempDir, 'warehouse'); const testIo = makeIo(); await expect( - runKtxSetupProjectStep({ projectDir, mode: 'existing', inputMode: 'disabled', yes: false }, testIo.io), + runKtxSetupProjectStep({ projectDir, mode: 'auto', inputMode: 'disabled', yes: false }, testIo.io), ).resolves.toMatchObject({ status: 'missing-input' }); - expect(testIo.stderr()).toContain(`No existing KTX project found at ${projectDir}`); + expect(testIo.stderr()).toContain('Missing setup choice: pass --yes'); }); it('prompts to use the current directory and creates a project in interactive auto mode', async () => { diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index 23884b0c..04bd54f5 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -18,7 +18,7 @@ import { type KtxSetupPromptOption, } from './setup-prompts.js'; -export type KtxSetupProjectMode = 'auto' | 'new' | 'existing' | 'prompt-new'; +export type KtxSetupProjectMode = 'auto' | 'prompt-new'; export type KtxSetupInputMode = 'auto' | 'disabled'; export interface KtxSetupProjectArgs { @@ -29,8 +29,18 @@ export interface KtxSetupProjectArgs { allowBack?: boolean; } +export type KtxSetupCreatedProjectCleanup = + | { kind: 'remove-project-dir'; projectDir: string } + | { kind: 'remove-ktx-scaffold'; projectDir: string }; + export type KtxSetupProjectResult = - | { status: 'ready'; projectDir: string; project: KtxLocalProject; confirmedCreation?: boolean } + | { + status: 'ready'; + projectDir: string; + project: KtxLocalProject; + confirmedCreation?: boolean; + createdProjectCleanup?: KtxSetupCreatedProjectCleanup; + } | { status: 'back'; projectDir: string } | { status: 'cancelled'; projectDir: string } | { status: 'missing-input'; projectDir: string }; @@ -49,7 +59,12 @@ export interface KtxSetupProjectDeps { } type PromptProjectDirResult = - | { status: 'selected'; projectDir: string; confirmedCreation: boolean } + | { + status: 'selected'; + projectDir: string; + confirmedCreation: boolean; + createdProjectCleanup?: KtxSetupCreatedProjectCleanup; + } | { status: 'cancelled'; projectDir: string } | { status: 'missing-input'; projectDir: string } | { status: 'back'; projectDir: string }; @@ -92,12 +107,29 @@ async function existingFolderState( } type ConfirmProjectDirResult = - | { status: 'confirmed'; confirmedCreation: boolean } + | { + status: 'confirmed'; + confirmedCreation: boolean; + createdProjectCleanup?: KtxSetupCreatedProjectCleanup; + } | { status: 'choose-another' } | { status: 'back' } | { status: 'cancelled' } | { status: 'not-directory' }; +function cleanupForFolderState( + projectDir: string, + state: Awaited>, +): KtxSetupCreatedProjectCleanup | undefined { + if (state === 'missing') { + return { kind: 'remove-project-dir', projectDir }; + } + if (state === 'empty-directory') { + return { kind: 'remove-ktx-scaffold', projectDir }; + } + return undefined; +} + async function confirmProjectDir( selectedDir: string, io: KtxCliIo, @@ -137,7 +169,7 @@ async function confirmProjectDir( if (action === 'choose-another') return { status: 'choose-another' }; if (action === 'back') return { status: 'back' }; if (action !== 'create') return { status: 'cancelled' }; - return { status: 'confirmed', confirmedCreation: true }; + return { status: 'confirmed', confirmedCreation: true, createdProjectCleanup: cleanupForFolderState(selectedDir, state) }; } async function normalizeSetupGitignore(projectDir: string): Promise { @@ -220,10 +252,28 @@ async function promptForNewProjectDir( if (confirmed.status === 'choose-another') continue; if (confirmed.status === 'back') return { status: 'back', projectDir }; if (confirmed.status === 'cancelled') return { status: 'cancelled', projectDir }; - return { status: 'selected', projectDir: selectedDir, confirmedCreation: confirmed.confirmedCreation }; + return { + status: 'selected', + projectDir: selectedDir, + confirmedCreation: confirmed.confirmedCreation, + ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}), + }; } } +async function createProjectWithCleanup( + projectDir: string, + deps: KtxSetupProjectDeps, +): Promise<{ project: KtxLocalProject; createdProjectCleanup?: KtxSetupCreatedProjectCleanup }> { + const state = await existingFolderState(projectDir); + const project = await createProject(projectDir, deps); + const createdProjectCleanup = cleanupForFolderState(projectDir, state); + return { + project, + ...(createdProjectCleanup ? { createdProjectCleanup } : {}), + }; +} + export async function runKtxSetupProjectStep( args: KtxSetupProjectArgs, io: KtxCliIo, @@ -233,30 +283,14 @@ export async function runKtxSetupProjectStep( const homeDir = deps.homeDir ?? homedir(); const exists = hasProjectConfig(projectDir); - if (args.mode === 'existing') { - if (!exists) { - io.stderr.write(`No existing KTX project found at ${projectDir}. Pass --new to create it.\n`); - return { status: 'missing-input', projectDir }; - } - const project = await loadExistingProject(projectDir, deps); - printProjectSummary(io, projectDir); - return { status: 'ready', projectDir, project }; - } - - if (args.mode === 'new') { - const project = await createProject(projectDir, deps); - printProjectSummary(io, projectDir); - return { status: 'ready', projectDir, project }; - } - if (args.mode === 'prompt-new') { if (args.inputMode === 'disabled') { - io.stderr.write('Missing new project folder: pass --new --project-dir to create a project without prompts.\n'); + io.stderr.write('Missing new project folder: pass --project-dir and --yes to create a project without prompts.\n'); return { status: 'missing-input', projectDir }; } if (!io.stdout.isTTY && !deps.prompts) { io.stderr.write( - 'Missing new project folder: pass --new --project-dir to create a project outside an interactive terminal.\n', + 'Missing new project folder: pass --project-dir and --yes to create a project outside an interactive terminal.\n', ); return { status: 'missing-input', projectDir }; } @@ -277,6 +311,7 @@ export async function runKtxSetupProjectStep( projectDir: selected.projectDir, project, confirmedCreation: selected.confirmedCreation, + ...(selected.createdProjectCleanup ? { createdProjectCleanup: selected.createdProjectCleanup } : {}), }; } @@ -288,16 +323,21 @@ export async function runKtxSetupProjectStep( if (args.inputMode === 'disabled') { if (!args.yes) { - io.stderr.write('Missing setup choice: pass --new or --yes to create a project in non-interactive setup.\n'); + io.stderr.write('Missing setup choice: pass --yes to create a project in non-interactive setup.\n'); return { status: 'missing-input', projectDir }; } - const project = await createProject(projectDir, deps); + const { project, createdProjectCleanup } = await createProjectWithCleanup(projectDir, deps); printProjectSummary(io, projectDir); - return { status: 'ready', projectDir, project }; + return { + status: 'ready', + projectDir, + project, + ...(createdProjectCleanup ? { createdProjectCleanup } : {}), + }; } if (!io.stdout.isTTY && !deps.prompts) { - io.stderr.write('Missing setup choice: pass --new or --yes to create a project outside an interactive terminal.\n'); + io.stderr.write('Missing setup choice: pass --yes to create a project outside an interactive terminal.\n'); return { status: 'missing-input', projectDir }; } @@ -332,9 +372,14 @@ export async function runKtxSetupProjectStep( } if (choice === 'current') { - const project = await createProject(projectDir, deps); + const { project, createdProjectCleanup } = await createProjectWithCleanup(projectDir, deps); printProjectSummary(io, projectDir); - return { status: 'ready', projectDir, project }; + return { + status: 'ready', + projectDir, + project, + ...(createdProjectCleanup ? { createdProjectCleanup } : {}), + }; } if (choice === 'new-default') { @@ -349,6 +394,7 @@ export async function runKtxSetupProjectStep( projectDir: defaultProjectDir, project, confirmedCreation: confirmed.confirmedCreation, + ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}), }; } @@ -372,7 +418,13 @@ export async function runKtxSetupProjectStep( if (confirmed.status === 'cancelled') return { status: 'cancelled', projectDir }; const project = await createProject(customDir, deps); printProjectSummary(io, customDir); - return { status: 'ready', projectDir: customDir, project, confirmedCreation: confirmed.confirmedCreation }; + return { + status: 'ready', + projectDir: customDir, + project, + confirmedCreation: confirmed.confirmedCreation, + ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}), + }; } prompts.cancel('Setup cancelled.'); diff --git a/packages/cli/src/setup-runtime.test.ts b/packages/cli/src/setup-runtime.test.ts index 0c1b129e..ee070fc7 100644 --- a/packages/cli/src/setup-runtime.test.ts +++ b/packages/cli/src/setup-runtime.test.ts @@ -4,7 +4,6 @@ import { join } from 'node:path'; import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context'; import { buildDefaultKtxProjectConfig, readKtxSetupState, type KtxProjectConfig } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { ManagedPythonCommandRuntime } from './managed-python-command.js'; import { runKtxSetupRuntimeStep } from './setup-runtime.js'; function makeIo() { @@ -43,9 +42,9 @@ describe('runKtxSetupRuntimeStep', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('ensures core runtime for agent setup and records the runtime step', async () => { + it('skips runtime setup when the project has no direct runtime requirements', async () => { const io = makeIo(); - const ensureRuntime = vi.fn(async (): Promise => ({} as ManagedPythonCommandRuntime)); + const ensureRuntime = vi.fn(); await expect( runKtxSetupRuntimeStep( @@ -54,7 +53,6 @@ describe('runKtxSetupRuntimeStep', () => { inputMode: 'auto', cliVersion: '0.2.0', runtimeInstallPolicy: 'prompt', - agents: true, }, io.io, { @@ -63,17 +61,11 @@ describe('runKtxSetupRuntimeStep', () => { env: {}, }, ), - ).resolves.toMatchObject({ status: 'ready' }); + ).resolves.toMatchObject({ status: 'skipped' }); - expect(ensureRuntime).toHaveBeenCalledWith( - expect.objectContaining({ - cliVersion: '0.2.0', - installPolicy: 'prompt', - feature: 'core', - }), - ); - expect((await readKtxSetupState(tempDir)).completed_steps).toContain('runtime'); - expect(io.stdout()).toContain('Runtime ready: yes (core)'); + expect(ensureRuntime).not.toHaveBeenCalled(); + expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime'); + expect(io.stdout()).toContain('Runtime setup skipped.'); }); it('fails fast when required runtime features cannot be installed in no-input mode', async () => { @@ -89,7 +81,7 @@ describe('runKtxSetupRuntimeStep', () => { inputMode: 'disabled', cliVersion: '0.2.0', runtimeInstallPolicy: 'never', - agents: true, + databaseIntrospectionFallback: true, }, io.io, { @@ -109,6 +101,8 @@ describe('runKtxSetupRuntimeStep', () => { const io = makeIo(); const ensureLocalEmbeddings = vi.fn(async () => ({ baseUrl: 'http://127.0.0.1:61234', + stdoutLog: join(tempDir, '.ktx', 'runtime', 'daemon.stdout.log'), + stderrLog: join(tempDir, '.ktx', 'runtime', 'daemon.stderr.log'), env: { KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: 'http://127.0.0.1:61234' }, })); const config: KtxProjectConfig = { @@ -131,7 +125,6 @@ describe('runKtxSetupRuntimeStep', () => { inputMode: 'auto', cliVersion: '0.2.0', runtimeInstallPolicy: 'auto', - agents: false, }, io.io, { diff --git a/packages/cli/src/setup-runtime.ts b/packages/cli/src/setup-runtime.ts index 07124fe8..cedcd3d0 100644 --- a/packages/cli/src/setup-runtime.ts +++ b/packages/cli/src/setup-runtime.ts @@ -24,7 +24,6 @@ export interface KtxSetupRuntimeArgs { inputMode: 'auto' | 'disabled'; cliVersion: string; runtimeInstallPolicy: KtxManagedPythonInstallPolicy; - agents: boolean; databaseIntrospectionFallback?: boolean; } @@ -62,7 +61,6 @@ export async function runKtxSetupRuntimeStep( const loadProjectForRuntime = deps.loadProject ?? loadKtxProject; const project = await loadProjectForRuntime({ projectDir: args.projectDir }); const requirements = resolveProjectRuntimeRequirements(project.config, { - agents: args.agents, databaseIntrospectionFallback: args.databaseIntrospectionFallback, env: deps.env ?? process.env, }); diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index a802fb03..9c6bf626 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process'; -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -9,7 +9,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js'; import { contextBuildCommands, writeKtxSetupContextState } from './setup-context.js'; import { runDemoTour } from './setup-demo-tour.js'; -import { formatKtxSetupStatus, readKtxSetupStatus, runKtxSetup } from './setup.js'; +import { formatKtxSetupCompletionSummary, formatKtxSetupStatus, readKtxSetupStatus, runKtxSetup } from './setup.js'; vi.mock('./setup-demo-tour.js', () => ({ runDemoTour: vi.fn(async () => 0), @@ -108,7 +108,7 @@ describe('setup status', () => { }); }); - it('reports deterministic default embeddings as not setup-ready', async () => { + it('reports disabled default embeddings as not setup-ready', async () => { await mkdir(tempDir, { recursive: true }); await writeFile( join(tempDir, 'ktx.yaml'), @@ -122,8 +122,7 @@ describe('setup status', () => { ' default: claude-sonnet-4-6', 'ingest:', ' embeddings:', - ' backend: deterministic', - ' model: deterministic', + ' backend: none', ' dimensions: 8', 'connections: {}', ].join('\n'), @@ -133,7 +132,7 @@ describe('setup status', () => { await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({ project: { path: tempDir, ready: true }, llm: { backend: 'anthropic', ready: true, model: 'claude-sonnet-4-6' }, - embeddings: { backend: 'deterministic', ready: false, model: 'deterministic', dimensions: 8 }, + embeddings: { backend: 'none', ready: false, dimensions: 8 }, }); }); @@ -373,8 +372,7 @@ describe('setup status', () => { ' default: claude-sonnet-4-6', 'ingest:', ' embeddings:', - ' backend: deterministic', - ' model: deterministic', + ' backend: none', ' dimensions: 8', '', ].join('\n'), @@ -424,6 +422,112 @@ describe('setup status', () => { expect(rendered).not.toContain('No KTX project found.'); }); + it('formats a concise ready summary for completed agent setup', () => { + const rendered = formatKtxSetupCompletionSummary( + { + project: { path: tempDir, ready: true }, + llm: { ready: true, model: 'sonnet' }, + embeddings: { ready: true, model: 'text-embedding-3-small' }, + databases: [{ connectionId: 'postgres-warehouse', ready: true }], + sources: [{ connectionId: 'dbt-main', type: 'dbt', ready: true }], + runtime: { required: true, ready: true, features: ['core'] }, + context: { ready: true, status: 'completed' }, + agents: [ + { target: 'claude-code', scope: 'project', ready: true }, + { target: 'claude-desktop', scope: 'global', ready: true }, + ], + }, + { + agentNextActions: [ + '1. Start MCP', + ' Run this command before using Claude Code:', + '', + ' RUN:', + ` ktx mcp start --project-dir ${tempDir}`, + '', + ' If you need to stop MCP later:', + ` ktx mcp stop --project-dir ${tempDir}`, + '', + '2. Open Claude Code', + ' Open Claude Code from the KTX project directory:', + '', + ' RUN:', + ` cd '${tempDir}'`, + ' claude', + ].join('\n'), + }, + ); + + expect(rendered).toContain(`Project\n ${tempDir}`); + expect(rendered).toContain('Context\n built'); + expect(rendered).toContain('Agents configured\n Claude Code, Claude Desktop'); + expect(rendered).toContain('REQUIRED BEFORE USING AGENTS\n\n 1. Start MCP'); + expect(rendered).toContain(' Run this command before using Claude Code:'); + expect(rendered).toContain(' RUN:'); + expect(rendered).toContain(' If you need to stop MCP later:'); + expect(rendered).toContain(`ktx mcp stop --project-dir ${tempDir}`); + expect(rendered).toContain('After that, try\n Ask your agent: "Use KTX to show me the available tables."'); + expect(rendered).not.toContain('Verify'); + expect(rendered).not.toContain('Project ready: yes'); + expect(rendered).not.toContain('What you can do next'); + }); + + it('prints agent next actions inside the final ready summary during full setup', async () => { + const testIo = makeIo(); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + target: 'claude-code', + skipAgents: false, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + skipDatabases: true, + skipSources: true, + databaseSchemas: [], + }, + testIo.io, + { + runtime: async () => runtimeReady(tempDir), + context: async () => { + await writeKtxSetupContextState(tempDir, { + runId: 'setup-context-local-test', + status: 'completed', + primarySourceConnectionIds: [], + contextSourceConnectionIds: [], + reportIds: [], + artifactPaths: [], + retryableFailedTargets: [], + commands: contextBuildCommands(tempDir, 'setup-context-local-test'), + }); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'context'] }); + return { status: 'ready', projectDir: tempDir, runId: 'setup-context-local-test' }; + }, + }, + ), + ).resolves.toBe(0); + + const output = testIo.stdout(); + expect(output).toContain('Claude Code · Project scope'); + expect(output).toContain(join(tempDir, '.mcp.json')); + expect(output).toContain('Requires MCP to be started.'); + expect(output).toContain('Analytics skill installed.'); + expect(output).not.toContain('Agent integration complete'); + expect(output).toContain('Finish KTX agent setup'); + expect(output).not.toContain('KTX project ready'); + expect(output).toContain('REQUIRED BEFORE USING AGENTS'); + expect(output).toContain('Run this command before using Claude Code:'); + expect(output).toContain(`ktx mcp start --project-dir ${tempDir}`); + expect(output).not.toContain('Finish agent setup'); + }); + it('prints the setup shell intro for auto-created run mode', async () => { const testIo = makeIo(); @@ -459,6 +563,112 @@ describe('setup status', () => { expect(testIo.stderr()).toBe(''); }); + it('removes a newly created missing project directory when a later runtime step fails', async () => { + const projectDir = join(tempDir, 'missing-project'); + const testIo = makeIo(); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir, + mode: 'auto', + agents: false, + skipAgents: true, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + databaseSchemas: [], + skipDatabases: true, + skipSources: true, + }, + testIo.io, + { + model: async () => ({ status: 'skipped', projectDir }), + embeddings: async () => ({ status: 'skipped', projectDir }), + databases: async () => ({ status: 'skipped', projectDir }), + sources: async () => ({ status: 'skipped', projectDir }), + runtime: async () => ({ status: 'failed', projectDir, requirements: { features: ['core'], requirements: [] } }), + }, + ), + ).resolves.toBe(1); + + await expect(stat(projectDir)).rejects.toThrow(); + }); + + it('removes KTX scaffold files from an initially empty project directory when runtime setup fails', async () => { + const testIo = makeIo(); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + skipAgents: true, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + databaseSchemas: [], + skipDatabases: true, + skipSources: true, + }, + testIo.io, + { + model: async () => ({ status: 'skipped', projectDir: tempDir }), + embeddings: async () => ({ status: 'skipped', projectDir: tempDir }), + databases: async () => ({ status: 'skipped', projectDir: tempDir }), + sources: async () => ({ status: 'skipped', projectDir: tempDir }), + runtime: async () => ({ status: 'failed', projectDir: tempDir, requirements: { features: ['core'], requirements: [] } }), + }, + ), + ).resolves.toBe(1); + + await expect(stat(tempDir)).resolves.toBeDefined(); + expect(await readdir(tempDir)).toEqual([]); + }); + + it('preserves a pre-existing non-empty project directory when runtime setup fails', async () => { + await writeFile(join(tempDir, 'notes.txt'), 'keep me\n', 'utf-8'); + const testIo = makeIo(); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + skipAgents: true, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + databaseSchemas: [], + skipDatabases: true, + skipSources: true, + }, + testIo.io, + { + model: async () => ({ status: 'skipped', projectDir: tempDir }), + embeddings: async () => ({ status: 'skipped', projectDir: tempDir }), + databases: async () => ({ status: 'skipped', projectDir: tempDir }), + sources: async () => ({ status: 'skipped', projectDir: tempDir }), + runtime: async () => ({ status: 'failed', projectDir: tempDir, requirements: { features: ['core'], requirements: [] } }), + }, + ), + ).resolves.toBe(1); + + await expect(readFile(join(tempDir, 'notes.txt'), 'utf-8')).resolves.toBe('keep me\n'); + await expect(stat(join(tempDir, 'ktx.yaml'))).resolves.toBeDefined(); + }); + it('shows demo near the bottom of the first setup intent menu before project creation', async () => { const testIo = makeIo(); const select = vi.fn(async (options: { options: Array<{ value: string; label: string }> }) => { @@ -843,7 +1053,7 @@ describe('setup status', () => { ); }); - it('creates a project through run mode when --new is selected', async () => { + it('creates a project through run mode when --yes is selected', async () => { const testIo = makeIo(); await expect( @@ -851,11 +1061,11 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', - yes: false, + yes: true, cliVersion: '0.2.0', skipLlm: true, skipEmbeddings: true, @@ -943,14 +1153,14 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', - yes: false, + yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -967,7 +1177,7 @@ describe('setup status', () => { projectDir: tempDir, inputMode: 'disabled', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }), testIo.io, @@ -983,16 +1193,16 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', - yes: false, + yes: true, cliVersion: '0.2.0', llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, skipEmbeddings: true, databaseSchemas: [], @@ -1011,7 +1221,7 @@ describe('setup status', () => { llmBackend: 'vertex', vertexProject: 'local-gcp-project', vertexLocation: 'us-east5', - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, }), testIo.io, @@ -1028,14 +1238,14 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, embeddingBackend: 'openai', embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret @@ -1066,13 +1276,14 @@ describe('setup status', () => { it('passes no-input runtime policy to the embeddings step', async () => { const io = makeIo(); const embeddings = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir })); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); await expect( runKtxSetup( { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, agentScope: 'project', skipAgents: true, @@ -1103,13 +1314,14 @@ describe('setup status', () => { const io = makeIo(); const embeddings = vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir })); const context = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir })); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); await expect( runKtxSetup( { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, agentScope: 'project', skipAgents: true, @@ -1148,6 +1360,7 @@ describe('setup status', () => { it('lets Back from embedding setup return to the model step instead of exiting', async () => { const testIo = makeIo(); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); const modelResults = [ { status: 'ready' as const, projectDir: tempDir }, { status: 'back' as const, projectDir: tempDir }, @@ -1160,7 +1373,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'auto', @@ -1184,6 +1397,7 @@ describe('setup status', () => { it('lets Back from database selection return to embedding setup', async () => { const testIo = makeIo(); + await writeFile(join(tempDir, 'ktx.yaml'), 'connections: {}\n', 'utf-8'); const modelResults = [ { status: 'ready' as const, projectDir: tempDir }, { status: 'back' as const, projectDir: tempDir }, @@ -1207,11 +1421,11 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'auto', - yes: false, + yes: true, cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: false, @@ -1254,7 +1468,7 @@ describe('setup status', () => { agents: false, skipAgents: true, inputMode: 'auto', - yes: false, + yes: true, cliVersion: '0.2.0', skipLlm: false, skipEmbeddings: true, @@ -1291,14 +1505,14 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', - yes: false, + yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, embeddingBackend: 'openai', embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret @@ -1349,7 +1563,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', @@ -1425,7 +1639,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', @@ -1475,7 +1689,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', @@ -1523,7 +1737,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: false, inputMode: 'disabled', yes: true, @@ -1584,7 +1798,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, inputMode: 'disabled', yes: true, @@ -1631,7 +1845,7 @@ describe('setup status', () => { expect(committedConfig.stdout).toContain('warehouse:'); }); - it('runs agent setup after context succeeds in --agents mode', async () => { + it('runs agent setup without runtime or context in --agents mode', async () => { const calls: string[] = []; const io = makeIo(); await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); @@ -1641,7 +1855,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: true, target: 'codex', agentScope: 'project', @@ -1663,11 +1877,11 @@ describe('setup status', () => { sources: async () => ({ status: 'skipped', projectDir: tempDir }), runtime: async () => { calls.push('runtime'); - return runtimeReady(tempDir); + throw new Error('runtime should not run'); }, context: async () => { calls.push('context'); - return { status: 'ready', projectDir: tempDir, runId: 'setup-context-local-test' }; + throw new Error('context should not run'); }, agents: async () => { calls.push('agents'); @@ -1681,11 +1895,13 @@ describe('setup status', () => { ), ).resolves.toBe(0); - expect(calls).toEqual(['runtime', 'context', 'agents']); + expect(calls).toEqual(['agents']); }); - it('does not install agents when non-interactive --agents finds context incomplete', async () => { + it('installs agents when non-interactive --agents finds context incomplete', async () => { const io = makeIo(); + const runtime = vi.fn(async () => runtimeReady(tempDir)); + const context = vi.fn(async () => ({ status: 'skipped' as const, projectDir: tempDir })); const agents = vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir, @@ -1698,7 +1914,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: true, target: 'codex', agentScope: 'project', @@ -1714,15 +1930,64 @@ describe('setup status', () => { }, io.io, { - runtime: async () => runtimeReady(tempDir), - context: async () => ({ status: 'skipped', projectDir: tempDir }), + runtime, + context, agents, }, ), - ).resolves.toBe(1); + ).resolves.toBe(0); - expect(agents).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('KTX context is not ready for agents.'); + expect(runtime).not.toHaveBeenCalled(); + expect(context).not.toHaveBeenCalled(); + expect(agents).toHaveBeenCalledTimes(1); + expect(io.stderr()).not.toContain('KTX context is not ready for agents.'); + }); + + it('runs non-TTY --agents with a target without requiring --no-input or --yes', async () => { + const io = makeIo(); + const agents = vi.fn(async () => ({ + status: 'ready' as const, + projectDir: tempDir, + installs: [{ target: 'claude-code' as const, scope: 'project' as const, mode: 'mcp' as const }], + })); + await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: true, + target: 'claude-code', + agentScope: 'project', + inputMode: 'auto', + yes: false, + cliVersion: '0.2.0', + skipLlm: false, + skipEmbeddings: false, + skipDatabases: false, + skipSources: false, + skipAgents: false, + databaseSchemas: [], + }, + io.io, + { agents }, + ), + ).resolves.toBe(0); + + expect(agents).toHaveBeenCalledWith( + expect.objectContaining({ + inputMode: 'disabled', + yes: false, + agents: true, + target: 'claude-code', + scope: 'project', + mode: 'mcp', + }), + io.io, + ); + expect(io.stderr()).not.toContain('Interactive setup requires a terminal'); }); it('routes a ready project menu selection to agent setup', async () => { @@ -1789,7 +2054,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: false, inputMode: 'auto', yes: false, @@ -1843,7 +2108,7 @@ describe('setup status', () => { } } - expect(calls).toEqual(['runtime', 'agents']); + expect(calls).toEqual(['agents']); }); it('skips to agent setup when context is ready but agents are not configured', async () => { @@ -1892,7 +2157,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'existing', + mode: 'auto', agents: false, inputMode: 'auto', yes: false, @@ -1940,10 +2205,10 @@ describe('setup status', () => { ).resolves.toBe(0); expect(readyMenuSelect).not.toHaveBeenCalled(); - expect(calls).toEqual(['runtime', 'agents']); + expect(calls).toEqual(['agents']); }); - it('runs only project resolution, runtime, context gate, and agent setup in --agents mode', async () => { + it('runs only project resolution and agent setup in --agents mode', async () => { const io = makeIo(); const runtime = vi.fn(async () => runtimeReady(tempDir)); const context = vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir, runId: 'setup-context-local-test' })); @@ -1958,7 +2223,7 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: true, target: 'universal', agentScope: 'project', @@ -1984,8 +2249,8 @@ describe('setup status', () => { ), ).resolves.toBe(0); - expect(runtime).toHaveBeenCalledTimes(1); - expect(context).toHaveBeenCalledTimes(1); + expect(runtime).not.toHaveBeenCalled(); + expect(context).not.toHaveBeenCalled(); expect(agents).toHaveBeenCalledTimes(1); }); @@ -1999,14 +2264,14 @@ describe('setup status', () => { { command: 'run', projectDir: tempDir, - mode: 'new', + mode: 'auto', agents: false, skipAgents: true, inputMode: 'disabled', - yes: false, + yes: true, cliVersion: '0.2.0', anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret - anthropicModel: 'claude-sonnet-4-6', + llmModel: 'claude-sonnet-4-6', skipLlm: false, skipEmbeddings: false, databaseSchemas: [], @@ -2017,6 +2282,7 @@ describe('setup status', () => { ), ).resolves.toBe(1); + expect(model).toHaveBeenCalledTimes(1); expect(embeddings).not.toHaveBeenCalled(); }); }); diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index c96e54e8..fbf40aa8 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -1,4 +1,5 @@ import { existsSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest'; import { @@ -19,6 +20,7 @@ import { type KtxSetupAgentsDeps, readKtxAgentInstallManifest, runKtxSetupAgentsStep, + targetDisplayName, } from './setup-agents.js'; import { type KtxSetupDatabaseDriver, @@ -32,7 +34,11 @@ import { isKtxSetupLlmConfigReady, runKtxSetupAnthropicModelStep, } from './setup-models.js'; -import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js'; +import { + type KtxSetupCreatedProjectCleanup, + type KtxSetupProjectDeps, + runKtxSetupProjectStep, +} from './setup-project.js'; import { isKtxPreAgentSetupReady, isKtxSetupReady, @@ -75,7 +81,7 @@ export type KtxSetupArgs = | { command: 'run'; projectDir: string; - mode: 'auto' | 'new' | 'existing'; + mode: 'auto'; agents: boolean; target?: KtxAgentTarget; agentScope?: KtxAgentScope; @@ -87,7 +93,6 @@ export type KtxSetupArgs = anthropicApiKeyEnv?: string; anthropicApiKeyFile?: string; llmModel?: string; - anthropicModel?: string; vertexProject?: string; vertexLocation?: string; skipLlm: boolean; @@ -237,7 +242,6 @@ function embeddingsReady(status: KtxSetupStatus['embeddings']): boolean { return ( status.backend !== undefined && status.backend !== 'none' && - status.backend !== 'deterministic' && typeof status.model === 'string' && status.model.length > 0 && typeof status.dimensions === 'number' && @@ -338,7 +342,6 @@ export async function readKtxSetupStatus( } const agents = [...agentMap.values()]; const runtimeRequirements = resolveProjectRuntimeRequirements(project.config, { - agents: agents.length > 0, env: options.env ?? process.env, }); let runtimeReady = runtimeRequirements.features.length === 0 || completedSteps.includes('runtime'); @@ -435,6 +438,35 @@ export function formatKtxSetupStatus(status: KtxSetupStatus): string { return `${lines.join('\n')}\n`; } +export function formatKtxSetupCompletionSummary( + status: KtxSetupStatus, + options: { agentNextActions?: string } = {}, +): string { + const readyAgents = status.agents.filter((agent) => agent.ready).map((agent) => targetDisplayName(agent.target)); + const lines = [ + 'Project', + ` ${status.project.path}`, + '', + 'Context', + ` ${status.context.ready ? 'built' : formatContextBuilt(status.context)}`, + '', + 'Agents configured', + ` ${readyAgents.length > 0 ? readyAgents.join(', ') : 'not installed'}`, + ]; + const agentNextActions = options.agentNextActions?.trim(); + if (agentNextActions) { + lines.push( + '', + 'REQUIRED BEFORE USING AGENTS', + '', + ...agentNextActions.split('\n').map((line) => (line ? ` ${line}` : '')), + ); + } + lines.push('', agentNextActions ? 'After that, try' : 'Try it'); + lines.push(' Ask your agent: "Use KTX to show me the available tables."'); + return lines.join('\n'); +} + function setupStatusReady(status: KtxSetupStatus): boolean { if (!status.project.ready) { return false; @@ -459,10 +491,8 @@ function setupContextReady(status: KtxSetupStatus): boolean { return status.context.ready; } -function writeContextNotReadyForAgents(projectDir: string, io: KtxCliIo): void { - io.stderr.write('KTX context is not ready for agents.\n\n'); - io.stderr.write(`Build context first:\n ktx setup --project-dir ${resolve(projectDir)}\n\n`); - io.stderr.write(`Then install agent integration:\n ktx setup --agents --project-dir ${resolve(projectDir)}\n`); +function shouldPrintConciseReadySummary(status: KtxSetupStatus): boolean { + return setupStatusReady(status) && setupContextReady(status) && status.agents.some((agent) => agent.ready); } function setupRuntimeInstallPolicy(args: Extract): 'prompt' | 'auto' | 'never' { @@ -477,6 +507,23 @@ async function commitSetupConfigChanges(projectDir: string): Promise { await project.git.commitFile('ktx.yaml', 'setup: update KTX project config', 'ktx setup', 'setup@ktx.local'); } +const KTX_SETUP_SCAFFOLD_PATHS = ['ktx.yaml', '.ktx', 'wiki', 'semantic-layer', 'raw-sources', '.git']; + +async function cleanupCreatedProjectScaffold(cleanup: KtxSetupCreatedProjectCleanup | undefined): Promise { + if (!cleanup) { + return; + } + if (cleanup.kind === 'remove-project-dir') { + await rm(cleanup.projectDir, { recursive: true, force: true }); + return; + } + await Promise.all( + KTX_SETUP_SCAFFOLD_PATHS.map((relativePath) => + rm(join(cleanup.projectDir, relativePath), { recursive: true, force: true }), + ), + ); +} + export async function runKtxSetup(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetupDeps = {}): Promise { try { return await runKtxSetupInner(args, io, deps); @@ -493,6 +540,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup setupUi.intro('KTX setup', io); let entryAction: KtxSetupEntryAction | undefined; let projectResult: Awaited>; + let agentNextActions: string | undefined; const canShowEntryMenu = args.showEntryMenu === true && args.inputMode !== 'disabled' && @@ -552,18 +600,19 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup } const runOnly = readyAction; + const agentOnlySetup = agentsRequested || runOnly === 'agents'; const shouldRunModels = !runOnly || runOnly === 'models'; const shouldRunEmbeddings = !runOnly || runOnly === 'embeddings'; const shouldRunDatabases = !runOnly || runOnly === 'databases'; const shouldRunSources = !runOnly || runOnly === 'sources'; const shouldRunRuntime = - agentsRequested || !runOnly || runOnly === 'runtime' || runOnly === 'context' || runOnly === 'agents'; - const shouldRunContext = agentsRequested || !runOnly || runOnly === 'context'; + !agentOnlySetup && (!runOnly || runOnly === 'runtime' || runOnly === 'context'); + const shouldRunContext = !agentOnlySetup && (!runOnly || runOnly === 'context'); const shouldRunAgents = agentsRequested || !runOnly || runOnly === 'agents'; const showPromptInstructions = projectResult.confirmedCreation !== true; - const setupSteps: KtxSetupFlowStep[] = agentsRequested - ? ['runtime', 'context'] + const setupSteps: KtxSetupFlowStep[] = agentOnlySetup + ? [] : ['models', 'embeddings', 'databases', 'sources', 'runtime', 'context']; if (shouldRunAgents && args.skipAgents !== true) { setupSteps.push('agents'); @@ -605,7 +654,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup ...(args.anthropicApiKeyEnv ? { anthropicApiKeyEnv: args.anthropicApiKeyEnv } : {}), ...(args.anthropicApiKeyFile ? { anthropicApiKeyFile: args.anthropicApiKeyFile } : {}), ...(args.llmModel ? { llmModel: args.llmModel } : {}), - ...(args.anthropicModel ? { anthropicModel: args.anthropicModel } : {}), ...(args.vertexProject ? { vertexProject: args.vertexProject } : {}), ...(args.vertexLocation ? { vertexLocation: args.vertexLocation } : {}), forcePrompt: forcePromptSteps.has('models') || runOnly === 'models', @@ -702,7 +750,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup inputMode: args.inputMode, cliVersion: args.cliVersion, runtimeInstallPolicy: setupRuntimeInstallPolicy(args), - agents: shouldRunAgents && args.skipAgents !== true, }, io, ); @@ -724,22 +771,34 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup } else { const agentsRunner = deps.agents ?? ((agentArgs, agentIo) => runKtxSetupAgentsStep(agentArgs, agentIo, deps.agentsDeps)); - stepResult = await agentsRunner( + const agentResult = await agentsRunner( { projectDir: projectResult.projectDir, - inputMode: args.inputMode, + inputMode: + args.inputMode === 'auto' && io.stdout.isTTY !== true && deps.agentsDeps?.prompts === undefined + ? 'disabled' + : args.inputMode, yes: args.yes, agents: true, ...(args.target ? { target: args.target } : {}), scope: args.agentScope ?? 'project', mode: 'mcp', skipAgents: false, + showNextActions: agentsRequested, }, io, ); + stepResult = agentResult; + if (agentResult.status === 'ready') { + agentNextActions = agentResult.nextActions; + } } - if (stepResult.status === 'failed' || stepResult.status === 'missing-input') { + if (stepResult.status === 'failed') { + await cleanupCreatedProjectScaffold(projectResult.createdProjectCleanup); + return 1; + } + if (stepResult.status === 'missing-input') { return 1; } if (stepResult.status === 'back') { @@ -759,10 +818,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup } if (step === 'context' && stepResult.status !== 'ready') { if (shouldRunAgents && args.skipAgents !== true) { - if (agentsRequested) { - writeContextNotReadyForAgents(projectResult.projectDir, io); - return args.inputMode === 'disabled' ? 1 : 0; - } return 0; } } @@ -779,19 +834,30 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup const status = await readKtxSetupStatus(projectResult.projectDir, { cliVersion: args.cliVersion }); const focusedOnAgents = args.agents || entryAction === 'agents'; if (!focusedOnAgents) { - setupUi.note(formatKtxSetupStatus(status).trimEnd(), 'Project status', io, { - format: (line) => line, - }); - setupUi.note( - formatSetupNextStepLines({ - setupReady: setupStatusReady(status), - hasContextTargets: setupHasContextTargets(status), - contextReady: setupContextReady(status), - agentIntegrationReady: status.agents.some((agent) => agent.ready), - }).join('\n'), - 'What you can do next', - io, - ); + if (shouldPrintConciseReadySummary(status)) { + setupUi.note( + formatKtxSetupCompletionSummary(status, { agentNextActions }), + agentNextActions ? 'Finish KTX agent setup' : 'KTX project ready', + io, + { + format: (line) => line, + }, + ); + } else { + setupUi.note(formatKtxSetupStatus(status).trimEnd(), 'Project status', io, { + format: (line) => line, + }); + setupUi.note( + formatSetupNextStepLines({ + setupReady: setupStatusReady(status), + hasContextTargets: setupHasContextTargets(status), + contextReady: setupContextReady(status), + agentIntegrationReady: status.agents.some((agent) => agent.ready), + }).join('\n'), + 'What you can do next', + io, + ); + } } return 0; } diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index 1bfef382..6486e536 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -123,7 +123,8 @@ describe('runKtxSl', () => { yaml: [ 'name: orders', 'table: public.orders', - 'description: Paid order facts', + 'descriptions:', + ' user: Paid order facts', 'grain: [order_id]', 'columns:', ' - name: order_id', diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index d63be434..688e69f2 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -93,7 +93,7 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich ' enrichment:', ' mode: deterministic', ' embeddings:', - ' backend: deterministic', + ' backend: none', ' dimensions: 6', ] : []), @@ -116,7 +116,6 @@ async function runSetupNewProject(projectDir: string): Promise { 'setup', '--project-dir', projectDir, - '--new', '--no-input', '--yes', '--skip-llm', @@ -166,7 +165,7 @@ describe('standalone built ktx CLI smoke', () => { }); it('runs status setup checks through the built binary', async () => { - const result = await runBuiltCli(['status', '--verbose', '--no-input']); + const result = await runBuiltCli(['status', '--verbose', '--no-input'], { cwd: tempDir }); expect(result.stdout).toMatch(/KTX status/); if (result.stdout.includes('No project here yet.')) { diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index 76d55851..297229de 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -242,15 +242,6 @@ function buildEmbeddingsStatus(config: KtxProjectEmbeddingConfig, env: NodeJS.Pr detail: 'disabled — semantic search will be skipped', }; } - if (backend === 'deterministic') { - return { - backend, - model, - dimensions, - status: 'warn', - detail: 'deterministic — semantic search degraded (lexical/dictionary lanes still work)', - }; - } if (backend === 'openai') { const ref = config.openai?.api_key; const resolved = resolveRef(ref, env); @@ -645,7 +636,7 @@ function buildVerdict( const reasons: string[] = []; if (llm.status === 'warn') reasons.push('LLM credentials missing'); if (embeddings.status === 'warn') { - if (embeddings.backend === 'deterministic' || embeddings.backend === 'none') { + if (embeddings.backend === 'none') { reasons.push('semantic search disabled'); } else { reasons.push('embedding credentials missing'); diff --git a/packages/connector-clickhouse/src/package-exports.test.ts b/packages/connector-clickhouse/src/package-exports.test.ts index 96ba11d4..644e6075 100644 --- a/packages/connector-clickhouse/src/package-exports.test.ts +++ b/packages/connector-clickhouse/src/package-exports.test.ts @@ -1,12 +1,16 @@ import { describe, expect, it } from 'vitest'; describe('@ktx/connector-clickhouse package exports', () => { - it('exports public connector APIs during package bootstrap', async () => { - const connector = await import('./index.js'); + it( + 'exports public connector APIs during package bootstrap', + async () => { + const connector = await import('./index.js'); - expect(connector.KtxClickHouseDialect).toBeTypeOf('function'); - expect(connector.KtxClickHouseScanConnector).toBeTypeOf('function'); - expect(connector.clickHouseClientConfigFromConfig).toBeTypeOf('function'); - expect(connector.createClickHouseLiveDatabaseIntrospection).toBeTypeOf('function'); - }); + expect(connector.KtxClickHouseDialect).toBeTypeOf('function'); + expect(connector.KtxClickHouseScanConnector).toBeTypeOf('function'); + expect(connector.clickHouseClientConfigFromConfig).toBeTypeOf('function'); + expect(connector.createClickHouseLiveDatabaseIntrospection).toBeTypeOf('function'); + }, + 20_000, + ); }); diff --git a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index 80e56a99..971a05ec 100644 --- a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -165,7 +165,7 @@ async function writeHistoricSqlProject(project: KtxLocalProject): Promise { ' adapters:', ' - fake', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', @@ -415,9 +415,11 @@ describe('canonical local ingest', () => { expect(result.result.failedWorkUnits).toEqual([]); const db = new Database(join(project.projectDir, '.ktx', 'db.sqlite'), { readonly: true }); try { - expect(db.prepare('SELECT key, summary, embedding_json IS NOT NULL AS has_embedding FROM knowledge_pages ORDER BY key').all()).toEqual([ - { key: 'orders_context', summary: 'Orders source context', has_embedding: 1 }, - ]); + expect( + db + .prepare('SELECT key, summary, embedding_json IS NOT NULL AS has_embedding FROM knowledge_pages ORDER BY key') + .all(), + ).toEqual([{ key: 'orders_context', summary: 'Orders source context', has_embedding: 0 }]); } finally { db.close(); } @@ -515,7 +517,7 @@ describe('canonical local ingest', () => { ' adapters:', ' - historic-sql', ' embeddings:', - ' backend: deterministic', + ' backend: none', 'storage:', ' state: sqlite', ' search: sqlite-fts5', @@ -599,7 +601,7 @@ describe('canonical local ingest', () => { ' adapters:', ' - metabase', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', @@ -677,7 +679,7 @@ describe('canonical local ingest', () => { ' adapters:', ' - metricflow', ' embeddings:', - ' backend: deterministic', + ' backend: none', 'storage:', ' state: sqlite', ' search: sqlite-fts5', @@ -805,7 +807,7 @@ describe('canonical local ingest', () => { ' adapters:', ' - looker', ' embeddings:', - ' backend: deterministic', + ' backend: none', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/packages/context/src/ingest/local-bundle-runtime.test.ts b/packages/context/src/ingest/local-bundle-runtime.test.ts index a8ec8c20..89dd2ce5 100644 --- a/packages/context/src/ingest/local-bundle-runtime.test.ts +++ b/packages/context/src/ingest/local-bundle-runtime.test.ts @@ -57,7 +57,7 @@ describe('createLocalBundleIngestRuntime', () => { ' adapters:', ' - fake', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', @@ -80,7 +80,7 @@ describe('createLocalBundleIngestRuntime', () => { 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:', ` ktx setup --project-dir ${project.projectDir} --llm-backend claude-code --no-input`, - ` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`, + ` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, ].join('\n'), ); }); @@ -303,7 +303,7 @@ describe('createLocalBundleIngestRuntime', () => { ' adapters:', ' - fake', ' embeddings:', - ' backend: deterministic', + ' backend: none', '', ].join('\n'), 'utf-8', diff --git a/packages/context/src/ingest/local-bundle-runtime.ts b/packages/context/src/ingest/local-bundle-runtime.ts index 8c3e20bc..bcb1258c 100644 --- a/packages/context/src/ingest/local-bundle-runtime.ts +++ b/packages/context/src/ingest/local-bundle-runtime.ts @@ -628,7 +628,7 @@ function localIngestLlmProviderGuardMessage(projectDir: string): string { 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:', ` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`, - ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`, + ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, ].join('\n'); } diff --git a/packages/context/src/ingest/local-embedding-provider.integration.test.ts b/packages/context/src/ingest/local-embedding-provider.integration.test.ts index af14fea9..34114e88 100644 --- a/packages/context/src/ingest/local-embedding-provider.integration.test.ts +++ b/packages/context/src/ingest/local-embedding-provider.integration.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createLocalKtxEmbeddingProviderFromConfig, KtxIngestEmbeddingPortAdapter } from '../llm/index.js'; +import type { KtxEmbeddingPort } from '../core/embedding.js'; import { CandidateDedupService } from './context-candidates/candidate-dedup.service.js'; import { ContextEvidenceIndexService } from './context-evidence/context-evidence-index.service.js'; import { SqliteContextEvidenceStore } from './context-evidence/sqlite-context-evidence-store.js'; @@ -43,16 +43,16 @@ describe('local ingest embedding providers with SQLite ingest stores', () => { await rm(tempDir, { recursive: true, force: true }); }); - function embeddings() { - const provider = createLocalKtxEmbeddingProviderFromConfig({ - backend: 'deterministic', - dimensions: 8, - batchSize: 4, - }); - if (!provider) { - throw new Error('deterministic local embedding provider was not created'); - } - return new KtxIngestEmbeddingPortAdapter(provider); + function embeddings(): KtxEmbeddingPort { + return { + maxBatchSize: 4, + async computeEmbedding() { + return [1, 0, 0]; + }, + async computeEmbeddingsBulk(texts) { + return texts.map(() => [1, 0, 0]); + }, + }; } it('indexes and searches context evidence using a package-owned local embedding provider', async () => { diff --git a/packages/context/src/llm/local-config.test.ts b/packages/context/src/llm/local-config.test.ts index 539afe45..23487e36 100644 --- a/packages/context/src/llm/local-config.test.ts +++ b/packages/context/src/llm/local-config.test.ts @@ -221,21 +221,15 @@ describe('local KTX embedding config', () => { }); }); - it('constructs deterministic embeddings from the default project config', () => { + it('returns null for the default disabled project embedding config', () => { const createKtxEmbeddingProvider = vi.fn(() => ({}) as never); const provider = createLocalKtxEmbeddingProviderFromConfig( buildDefaultKtxProjectConfig().ingest.embeddings, { createKtxEmbeddingProvider }, ); - expect(provider).not.toBeNull(); - expect(createKtxEmbeddingProvider).toHaveBeenCalledWith( - expect.objectContaining({ - backend: 'deterministic', - model: 'deterministic', - dimensions: 8, - }), - ); + expect(provider).toBeNull(); + expect(createKtxEmbeddingProvider).not.toHaveBeenCalled(); }); it('returns null when embeddings are disabled', () => { diff --git a/packages/context/src/llm/local-config.ts b/packages/context/src/llm/local-config.ts index 4b04e99b..e63a5ed1 100644 --- a/packages/context/src/llm/local-config.ts +++ b/packages/context/src/llm/local-config.ts @@ -184,26 +184,13 @@ export function resolveLocalKtxEmbeddingConfig( } return { backend: config.backend, - model: config.model ?? 'deterministic', + model: config.model ?? 'text-embedding-3-small', dimensions: config.dimensions, openai, batchSize: config.batchSize, }; } - return { - backend: config.backend, - model: config.model ?? 'deterministic', - dimensions: config.dimensions, - ...(config.sentenceTransformers - ? { - sentenceTransformers: { - baseURL: config.sentenceTransformers.base_url, - pathPrefix: config.sentenceTransformers.pathPrefix, - }, - } - : {}), - batchSize: config.batchSize, - }; + throw new Error(`Unsupported KTX embedding backend: ${String((config as { backend?: string }).backend)}`); } export function createLocalKtxEmbeddingProviderFromConfig( diff --git a/packages/context/src/project/config.test.ts b/packages/context/src/project/config.test.ts index 3967b363..f8faad58 100644 --- a/packages/context/src/project/config.test.ts +++ b/packages/context/src/project/config.test.ts @@ -42,8 +42,7 @@ connections: ingest: { adapters: [], embeddings: { - backend: 'deterministic', - model: 'deterministic', + backend: 'none', dimensions: 8, }, workUnits: { @@ -87,13 +86,10 @@ connections: expect(serialized).not.toContain('project:'); expect(serialized).not.toContain('live-database'); - expect(serialized).toContain( - ' embeddings:\n backend: deterministic\n model: deterministic\n dimensions: 8', - ); + expect(serialized).toContain(' embeddings:\n backend: none\n dimensions: 8'); expect(parsed.ingest.adapters).toEqual([]); expect(parsed.ingest.embeddings).toEqual({ - backend: 'deterministic', - model: 'deterministic', + backend: 'none', dimensions: 8, }); }); @@ -404,8 +400,7 @@ scan: expect(config).toEqual(buildDefaultKtxProjectConfig()); expect(config.ingest.embeddings).toEqual({ - backend: 'deterministic', - model: 'deterministic', + backend: 'none', dimensions: 8, }); }); diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index 912c31de..b639c922 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -4,7 +4,7 @@ import * as z from 'zod'; import { connectionConfigSchema } from './driver-schemas.js'; const KTX_LLM_BACKENDS = ['none', 'anthropic', 'vertex', 'gateway', 'claude-code'] as const; -const KTX_EMBEDDING_BACKENDS = ['none', 'deterministic', 'openai', 'sentence-transformers'] as const; +const KTX_EMBEDDING_BACKENDS = ['none', 'openai', 'sentence-transformers'] as const; const KTX_PROMPT_CACHE_TTLS = ['5m', '1h'] as const; const KTX_ENRICHMENT_MODES = ['none', 'deterministic', 'llm'] as const; const KTX_WORK_UNIT_FAILURE_MODES = ['abort', 'continue'] as const; @@ -80,9 +80,9 @@ const embeddingSchema = z .strictObject({ backend: z .enum(KTX_EMBEDDING_BACKENDS) - .default('deterministic') - .describe('Embedding backend. "deterministic" is a built-in hash-based vector for offline use; "openai" and "sentence-transformers" call out to those providers; "none" disables embeddings.'), - model: z.string().min(1).optional().describe('Provider-specific embedding model identifier (e.g. "text-embedding-3-small"). Ignored by the "deterministic" backend.'), + .default('none') + .describe('Embedding backend. "openai" and "sentence-transformers" call out to those providers; "none" disables embeddings.'), + model: z.string().min(1).optional().describe('Provider-specific embedding model identifier (e.g. "text-embedding-3-small").'), dimensions: z.int().positive().default(8).describe('Embedding vector dimensionality. Must match the chosen model when using a real provider.'), openai: apiCredentialsSchema.optional().describe('OpenAI credentials, used when backend is "openai".'), sentenceTransformers: sentenceTransformersSchema.optional().describe('Sentence-transformers server config, used when backend is "sentence-transformers".'), @@ -108,7 +108,7 @@ const ingestSchema = z .default([]) .describe('Ingest adapter identifiers to run (e.g. "metabase", "looker", "historic-sql"). Empty array means no adapters are run.'), embeddings: embeddingSchema - .prefault({ backend: 'deterministic', model: 'deterministic' }) + .prefault({ backend: 'none' }) .describe('Embedding configuration used when ingest adapters need to embed documents.'), workUnits: workUnitsSchema.prefault({}).describe('Concurrency and failure handling for ingest work units.'), }) diff --git a/packages/context/src/scan/enrichment-state.test.ts b/packages/context/src/scan/enrichment-state.test.ts index 036db607..4ae597c6 100644 --- a/packages/context/src/scan/enrichment-state.test.ts +++ b/packages/context/src/scan/enrichment-state.test.ts @@ -58,13 +58,13 @@ describe('scan enrichment state', () => { snapshot, mode: 'enriched', detectRelationships: true, - providerIdentity: { provider: 'deterministic', embeddingDimensions: 8, llmModel: 'a' }, + providerIdentity: { provider: 'local-heuristic', llmModel: 'a' }, }); const second = computeKtxScanEnrichmentInputHash({ snapshot: { ...snapshot, metadata: {} }, mode: 'enriched', detectRelationships: true, - providerIdentity: { llmModel: 'a', embeddingDimensions: 8, provider: 'deterministic' }, + providerIdentity: { llmModel: 'a', provider: 'local-heuristic' }, }); const firstTable = snapshot.tables[0]; if (!firstTable) { @@ -74,7 +74,7 @@ describe('scan enrichment state', () => { snapshot: { ...snapshot, tables: [{ ...firstTable, name: 'orders_v2' }] }, mode: 'enriched', detectRelationships: true, - providerIdentity: { provider: 'deterministic', embeddingDimensions: 8, llmModel: 'a' }, + providerIdentity: { provider: 'local-heuristic', llmModel: 'a' }, }); expect(first).toMatch(/^[a-f0-9]{64}$/); @@ -87,7 +87,7 @@ describe('scan enrichment state', () => { snapshot, mode: 'enriched', detectRelationships: true, - providerIdentity: { provider: 'deterministic', embeddingDimensions: 8 }, + providerIdentity: { provider: 'local-heuristic' }, }); await store.saveCompletedStage({ diff --git a/packages/context/src/scan/index.ts b/packages/context/src/scan/index.ts index 1eecdfeb..75e3b27a 100644 --- a/packages/context/src/scan/index.ts +++ b/packages/context/src/scan/index.ts @@ -96,7 +96,6 @@ export type { KtxStructuralSyncPlan, } from './enrichment-types.js'; export type { - DeterministicLocalScanEnrichmentProviderOptions, KtxLocalScanEnrichmentInput, KtxLocalScanEnrichmentProviders, KtxLocalScanEnrichmentResult, diff --git a/packages/context/src/scan/local-enrichment.test.ts b/packages/context/src/scan/local-enrichment.test.ts index 72307bc4..c45589bd 100644 --- a/packages/context/src/scan/local-enrichment.test.ts +++ b/packages/context/src/scan/local-enrichment.test.ts @@ -17,11 +17,24 @@ import { createKtxConnectorCapabilities, type KtxQueryResult, type KtxReadOnlyQueryInput, + type KtxEmbeddingPort, type KtxScanConnector, type KtxScanContext, type KtxSchemaSnapshot, } from './types.js'; +function fakeScanEmbedding(options: { dimensions: number; maxBatchSize?: number }): KtxEmbeddingPort { + return { + dimensions: options.dimensions, + maxBatchSize: options.maxBatchSize ?? 64, + async embedBatch(texts) { + return texts.map((_, textIndex) => + Array.from({ length: options.dimensions }, (__, dimensionIndex) => textIndex + dimensionIndex), + ); + }, + }; +} + const snapshot: KtxSchemaSnapshot = { connectionId: 'warehouse', driver: 'postgres', @@ -355,7 +368,7 @@ describe('local scan enrichment', () => { }); it('honors scan relationship config when LLM proposals are disabled', async () => { - const providers = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 3 }); + const providers = createDeterministicLocalScanEnrichmentProviders(); const generateObject = vi.fn(); const result = await runLocalScanEnrichment({ connectionId: 'warehouse', @@ -424,7 +437,7 @@ describe('local scan enrichment', () => { detectRelationships: false, connector: failingConnector, context: { runId: 'scan-run-warnings', logger }, - providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }), + providers: createDeterministicLocalScanEnrichmentProviders(), }); const codes = result.warnings.map((warning) => warning.code); @@ -439,25 +452,24 @@ describe('local scan enrichment', () => { expect(failingConnector.sampleTable).toHaveBeenCalledTimes(6); }); - it('runs configured deterministic enrichment with descriptions and embeddings', async () => { + it('runs configured deterministic enrichment with descriptions and no embeddings', async () => { const result = await runLocalScanEnrichment({ connectionId: 'warehouse', mode: 'enriched', detectRelationships: true, connector: connector(), context: { runId: 'scan-run-2' }, - providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }), + providers: createDeterministicLocalScanEnrichmentProviders(), }); expect(result.summary).toMatchObject({ dataDictionary: 'completed', tableDescriptions: 'completed', columnDescriptions: 'completed', - embeddings: 'completed', + embeddings: 'skipped', deterministicRelationships: 'completed', }); - expect(result.embeddingUpdates).toHaveLength(3); - expect(result.embeddingUpdates[0]?.embedding).toHaveLength(6); + expect(result.embeddingUpdates).toEqual([]); expect(result.snapshot).toEqual(snapshot); expect(result.relationships).toEqual({ accepted: 0, review: 1, rejected: 0, skipped: 0 }); }); @@ -518,7 +530,7 @@ describe('local scan enrichment', () => { mode: 'enriched', connector: scanConnector, context: { runId: 'scan-run-concurrent-descriptions' }, - providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 3 }), + providers: createDeterministicLocalScanEnrichmentProviders(), relationshipSettings: settings, }); @@ -542,7 +554,10 @@ describe('local scan enrichment', () => { detectRelationships: true, connector: connector(), context: { runId: 'scan-run-progress', progress }, - providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }), + providers: { + ...createDeterministicLocalScanEnrichmentProviders(), + embedding: fakeScanEmbedding({ dimensions: 6 }), + }, }); expect(events).toEqual( @@ -613,7 +628,7 @@ describe('local scan enrichment', () => { ...connector(), introspect: vi.fn(async () => manyColumnSnapshot), }; - const deterministicProviders = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 3 }); + const deterministicProviders = createDeterministicLocalScanEnrichmentProviders(); const embedBatch = vi.fn(async (texts: string[]) => { if (texts.length > 2) { throw new Error(`Embedding batch size ${texts.length} exceeds maximum 2`); @@ -644,7 +659,10 @@ describe('local scan enrichment', () => { it('reuses completed description and embedding stages for the same run id and snapshot hash', async () => { const stateStore = memoryEnrichmentStateStore(); const scanConnector = connector(); - const providers = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }); + const providers = { + ...createDeterministicLocalScanEnrichmentProviders(), + embedding: fakeScanEmbedding({ dimensions: 6 }), + }; const first = await runLocalScanEnrichment({ connectionId: 'warehouse', @@ -655,7 +673,7 @@ describe('local scan enrichment', () => { providers, stateStore, syncId: 'sync-resume-1', - providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 }, + providerIdentity: { provider: 'fake', embeddingDimensions: 6 }, }); const generateText = vi.spyOn(providers.llmRuntime, 'generateText'); @@ -669,7 +687,7 @@ describe('local scan enrichment', () => { providers, stateStore, syncId: 'sync-resume-1', - providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 }, + providerIdentity: { provider: 'fake', embeddingDimensions: 6 }, }); expect(first.state.completedStages).toEqual(['descriptions', 'embeddings', 'relationships']); @@ -685,7 +703,10 @@ describe('local scan enrichment', () => { it('does not reuse completed stages when the snapshot changes', async () => { const stateStore = memoryEnrichmentStateStore(); - const providers = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }); + const providers = { + ...createDeterministicLocalScanEnrichmentProviders(), + embedding: fakeScanEmbedding({ dimensions: 6 }), + }; const scanConnector = connector(); await runLocalScanEnrichment({ @@ -697,7 +718,7 @@ describe('local scan enrichment', () => { providers, stateStore, syncId: 'sync-resume-hash', - providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 }, + providerIdentity: { provider: 'fake', embeddingDimensions: 6 }, }); const firstTable = snapshot.tables[0]; @@ -722,7 +743,7 @@ describe('local scan enrichment', () => { providers, stateStore, syncId: 'sync-resume-hash', - providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 }, + providerIdentity: { provider: 'fake', embeddingDimensions: 6 }, }); expect(result.state.resumedStages).toEqual([]); @@ -828,8 +849,8 @@ describe('local scan enrichment', () => { }, ); - expect(providers?.embedding.dimensions).toBe(1536); - expect(providers?.embedding.maxBatchSize).toBe(8); + expect(providers?.embedding?.dimensions).toBe(1536); + expect(providers?.embedding?.maxBatchSize).toBe(8); expect(createKtxLlmProvider).toHaveBeenCalledWith( expect.objectContaining({ backend: 'gateway', modelSlots: { default: 'provider/language-model' } }), ); diff --git a/packages/context/src/scan/local-enrichment.ts b/packages/context/src/scan/local-enrichment.ts index 839b6fc9..381f3cb0 100644 --- a/packages/context/src/scan/local-enrichment.ts +++ b/packages/context/src/scan/local-enrichment.ts @@ -43,14 +43,9 @@ import type { const DESCRIPTION_TABLE_CONCURRENCY = 6; -export interface DeterministicLocalScanEnrichmentProviderOptions { - embeddingDimensions?: number; - maxBatchSize?: number; -} - export interface KtxLocalScanEnrichmentProviders { llmRuntime: KtxLlmRuntimePort; - embedding: KtxEmbeddingPort; + embedding?: KtxEmbeddingPort | null; } export interface KtxLocalScanEnrichmentInput { @@ -173,31 +168,9 @@ function providerlessEnrichedWarning(relationshipDetection: boolean): KtxScanWar }; } -function hashEmbedding(text: string, dimensions: number): number[] { - const values = Array.from({ length: dimensions }, (_, index) => { - let hash = index + 17; - for (const char of text) { - hash = (hash * 31 + char.charCodeAt(0) + index) % 1009; - } - return Number(((hash % 200) / 100 - 1).toFixed(4)); - }); - return values; -} - -export function createDeterministicLocalScanEnrichmentProviders( - options: DeterministicLocalScanEnrichmentProviderOptions = {}, -): KtxLocalScanEnrichmentProviders { - const dimensions = options.embeddingDimensions ?? 8; - const maxBatchSize = options.maxBatchSize ?? 64; +export function createDeterministicLocalScanEnrichmentProviders(): KtxLocalScanEnrichmentProviders { return { llmRuntime: deterministicLlmRuntime(), - embedding: { - dimensions, - maxBatchSize, - async embedBatch(texts) { - return texts.map((text) => hashEmbedding(text, dimensions)); - }, - }, }; } @@ -370,7 +343,7 @@ async function generateDescriptions(input: { async function buildEmbeddings(input: { snapshot: KtxSchemaSnapshot; - providers: KtxLocalScanEnrichmentProviders; + embedding: KtxEmbeddingPort; descriptions: KtxLocalScanEnrichmentResult['descriptionUpdates']; progress?: KtxProgressPort; }): Promise<{ updates: KtxEmbeddingUpdate[]; byColumnId: Map }> { @@ -400,7 +373,7 @@ async function buildEmbeddings(input: { } const embeddings: number[][] = []; - const maxBatchSize = embeddingBatchSize(input.providers.embedding.maxBatchSize); + const maxBatchSize = embeddingBatchSize(input.embedding.maxBatchSize); const embeddingTexts = texts.map((item) => item.text); const batchCount = Math.ceil(embeddingTexts.length / maxBatchSize); if (batchCount === 0) { @@ -412,7 +385,7 @@ async function buildEmbeddings(input: { transient: true, }); const batch = embeddingTexts.slice(offset, offset + maxBatchSize); - const batchEmbeddings = await input.providers.embedding.embedBatch(batch); + const batchEmbeddings = await input.embedding.embedBatch(batch); if (batchEmbeddings.length !== batch.length) { throw new Error(`expected ${batch.length} embeddings, received ${batchEmbeddings.length}`); } @@ -560,34 +533,38 @@ export async function runLocalScanEnrichment( warnings, }), }); - const embeddingProgress = progress?.startPhase(0.2); - embeddingUpdates = await runEnrichmentStage({ - stateStore: input.stateStore, - runId: input.context.runId, - connectionId: input.connectionId, - syncId, - mode: input.mode, - stage: 'embeddings', - inputHash, - now, - resumedStages: state.resumedStages, - completedStages: state.completedStages, - failedStages: state.failedStages, - compute: async () => { - const embeddings = await buildEmbeddings({ - snapshot, - providers, - descriptions, - progress: embeddingProgress, - }); - return embeddings.updates; - }, - }); - schema = snapshotToKtxEnrichedSchema(snapshot, embeddingsByColumnId(embeddingUpdates)); summary.dataDictionary = input.connector.sampleColumn ? 'completed' : 'skipped'; summary.tableDescriptions = 'completed'; summary.columnDescriptions = 'completed'; - summary.embeddings = 'completed'; + + const embeddingProgress = progress?.startPhase(0.2); + const embedding = providers.embedding; + if (embedding) { + embeddingUpdates = await runEnrichmentStage({ + stateStore: input.stateStore, + runId: input.context.runId, + connectionId: input.connectionId, + syncId, + mode: input.mode, + stage: 'embeddings', + inputHash, + now, + resumedStages: state.resumedStages, + completedStages: state.completedStages, + failedStages: state.failedStages, + compute: async () => { + const embeddings = await buildEmbeddings({ + snapshot, + embedding, + descriptions, + progress: embeddingProgress, + }); + return embeddings.updates; + }, + }); + schema = snapshotToKtxEnrichedSchema(snapshot, embeddingsByColumnId(embeddingUpdates)); + summary.embeddings = 'completed'; + } } let relationshipUpdate: KtxRelationshipUpdate | null = null; diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/context/src/scan/local-scan.test.ts index 16fab098..d04f9953 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/context/src/scan/local-scan.test.ts @@ -1017,7 +1017,7 @@ describe('local scan', () => { expect(persistedReport).not.toContain('postgres://reader:secret@example.test/db'); // pragma: allowlist secret }); - it('runs enriched scans when deterministic standalone enrichment is configured', async () => { + it('runs enriched scans when deterministic standalone enrichment is configured without embeddings', async () => { await writeFile( join(project.projectDir, 'ktx.yaml'), [ @@ -1103,10 +1103,9 @@ describe('local scan', () => { expect(result.report.mode).toBe('enriched'); expect(result.report.enrichment.tableDescriptions).toBe('completed'); expect(result.report.enrichment.columnDescriptions).toBe('completed'); - expect(result.report.enrichment.embeddings).toBe('completed'); + expect(result.report.enrichment.embeddings).toBe('skipped'); expect(result.report.artifactPaths.enrichmentArtifacts).toEqual([ 'raw-sources/warehouse/live-database/2026-04-29-091500-scan-enriched-1/enrichment/descriptions.json', - 'raw-sources/warehouse/live-database/2026-04-29-091500-scan-enriched-1/enrichment/embeddings.json', 'raw-sources/warehouse/live-database/2026-04-29-091500-scan-enriched-1/enrichment/relationships.json', 'raw-sources/warehouse/live-database/2026-04-29-091500-scan-enriched-1/enrichment/relationship-profile.json', 'raw-sources/warehouse/live-database/2026-04-29-091500-scan-enriched-1/enrichment/relationship-diagnostics.json', diff --git a/packages/llm/src/embedding-provider.test.ts b/packages/llm/src/embedding-provider.test.ts index 3a0a6f9a..c649a948 100644 --- a/packages/llm/src/embedding-provider.test.ts +++ b/packages/llm/src/embedding-provider.test.ts @@ -3,19 +3,16 @@ import { createKtxEmbeddingProvider } from './embedding-provider.js'; import type { KtxEmbeddingConfig } from './types.js'; describe('createKtxEmbeddingProvider', () => { - it('creates deterministic embeddings with stable dimensions', async () => { - const provider = createKtxEmbeddingProvider({ - backend: 'deterministic', - model: 'sha256', - dimensions: 6, - batchSize: 4, - }); + it('rejects deterministic embeddings', () => { + const config = JSON.parse( + JSON.stringify({ + backend: 'deterministic', + model: 'sha256', + dimensions: 6, + }), + ) as KtxEmbeddingConfig; - await expect(provider.embed('Revenue policy')).resolves.toHaveLength(6); - await expect(provider.embed('Revenue policy')).resolves.toEqual(await provider.embed('Revenue policy')); - await expect(provider.embed('Revenue policy')).resolves.not.toEqual(await provider.embed('Approval policy')); - await expect(provider.embedMany(['a', 'b'])).resolves.toHaveLength(2); - expect(provider.maxBatchSize).toBe(4); + expect(() => createKtxEmbeddingProvider(config)).toThrow('Unsupported KTX embedding backend: deterministic'); }); it('rejects gateway embeddings', () => { @@ -114,12 +111,12 @@ describe('createKtxEmbeddingProvider', () => { ); }); - it('falls back to one-shot ktx-daemon inference when the local HTTP daemon is unavailable', async () => { - const fetch = vi.fn().mockRejectedValue(new TypeError('fetch failed')); - const runSentenceTransformersJson = vi + it('reports local HTTP daemon failures without a ktx-daemon spawn fallback cascade', async () => { + const fetch = vi .fn() - .mockResolvedValueOnce({ embedding: [0.1, 0.2] }) - .mockResolvedValueOnce({ embeddings: [[0.3, 0.4], [0.5, 0.6]] }); + .mockResolvedValue( + new Response('Embedding compute failed: httpx.InvalidURL: Invalid port', { status: 500 }), + ); const provider = createKtxEmbeddingProvider( { @@ -128,19 +125,13 @@ describe('createKtxEmbeddingProvider', () => { dimensions: 2, sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, }, - { fetch, runSentenceTransformersJson }, + { fetch }, ); - await expect(provider.embedMany(['hello', 'world'])).resolves.toEqual([ - [0.3, 0.4], - [0.5, 0.6], - ]); + await expect(provider.embed('hello')).rejects.toThrow( + 'Embedding provider sentence-transformers request failed with HTTP 500: Embedding compute failed: httpx.InvalidURL: Invalid port', + ); + await expect(provider.embed('hello')).rejects.not.toThrow('ktx-daemon fallback failed'); expect(fetch).toHaveBeenCalledTimes(1); - expect(runSentenceTransformersJson).toHaveBeenNthCalledWith(1, 'embedding-compute', { - text: '__ktx_embedding_probe__', - }); - expect(runSentenceTransformersJson).toHaveBeenNthCalledWith(2, 'embedding-compute-bulk', { - texts: ['hello', 'world'], - }); }); }); diff --git a/packages/llm/src/embedding-provider.ts b/packages/llm/src/embedding-provider.ts index fa16b561..5290c044 100644 --- a/packages/llm/src/embedding-provider.ts +++ b/packages/llm/src/embedding-provider.ts @@ -1,16 +1,7 @@ -import { createHash } from 'node:crypto'; -import { spawn } from 'node:child_process'; -import { join } from 'node:path'; import OpenAI from 'openai'; import type { KtxEmbeddingConfig, KtxEmbeddingProvider } from './types.js'; type FetchFn = typeof fetch; -type SentenceTransformersCommand = 'embedding-compute' | 'embedding-compute-bulk'; -type SentenceTransformersJsonRunner = ( - subcommand: SentenceTransformersCommand, - payload: Record, -) => Promise>; -type SentenceTransformersProcessCommand = { command: string; args: string[] }; export interface KtxEmbeddingProviderDeps { createOpenAIClient?: (options: { apiKey?: string; baseURL?: string }) => { @@ -24,22 +15,10 @@ export interface KtxEmbeddingProviderDeps { }; }; fetch?: FetchFn; - runSentenceTransformersJson?: SentenceTransformersJsonRunner; - sentenceTransformersCommand?: string; - sentenceTransformersArgs?: string[]; - sentenceTransformersCwd?: string; - sentenceTransformersEnv?: NodeJS.ProcessEnv; } const DEFAULT_BATCH_SIZE = 100; - -function deterministicVector(text: string, dimensions: number): number[] { - const digest = createHash('sha256').update(text).digest(); - return Array.from({ length: dimensions }, (_, index) => { - const byte = digest[index % digest.length]; - return Number(((byte / 255) * 2 - 1).toFixed(6)); - }); -} +const HTTP_ERROR_BODY_MAX_LENGTH = 2_000; function assertNonEmptyText(text: string): void { if (!text.trim()) { @@ -78,128 +57,12 @@ function joinUrl(baseURL: string, pathPrefix: string, path: string): string { return prefix ? `${base}/${prefix}/${suffix}` : `${base}/${suffix}`; } -function errorText(error: unknown): string { - if (error instanceof Error) { - return error.cause - ? `${error.name}: ${error.message}; cause: ${errorText(error.cause)}` - : `${error.name}: ${error.message}`; - } - return String(error); -} - -function parseJsonObject(raw: string, subcommand: string): Record { - const parsed = JSON.parse(raw) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error(`ktx-daemon ${subcommand} returned non-object JSON`); - } - return parsed as Record; -} - -function isCommandNotFound(error: unknown): boolean { - return ( - error instanceof Error && - ('code' in error || 'errno' in error) && - ((error as { code?: unknown }).code === 'ENOENT' || (error as { errno?: unknown }).errno === 'ENOENT') - ); -} - -function defaultSentenceTransformersProcessCommands(): SentenceTransformersProcessCommand[] { - const venvBin = - process.platform === 'win32' ? join('.venv', 'Scripts', 'ktx-daemon.exe') : join('.venv', 'bin', 'ktx-daemon'); - const repoVenvBin = - process.platform === 'win32' - ? join('ktx', '.venv', 'Scripts', 'ktx-daemon.exe') - : join('ktx', '.venv', 'bin', 'ktx-daemon'); - return [ - { command: 'ktx-daemon', args: [] }, - { command: venvBin, args: [] }, - { command: repoVenvBin, args: [] }, - ]; -} - -function runSentenceTransformersProcessCommand( - options: SentenceTransformersProcessCommand & { - cwd?: string; - env?: NodeJS.ProcessEnv; - }, -): SentenceTransformersJsonRunner { - return async ( - subcommand: SentenceTransformersCommand, - payload: Record, - ): Promise> => - new Promise((resolve, reject) => { - const child = spawn(options.command, [...options.args, subcommand], { - cwd: options.cwd, - env: { ...process.env, ...options.env }, - stdio: ['pipe', 'pipe', 'pipe'], - }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - - child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); - child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); - child.on('error', reject); - child.on('close', (code) => { - const stdoutText = Buffer.concat(stdout).toString('utf8').trim(); - const stderrText = Buffer.concat(stderr).toString('utf8').trim(); - if (code !== 0) { - reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`)); - return; - } - try { - resolve(parseJsonObject(stdoutText, subcommand)); - } catch (error) { - reject(error); - } - }); - child.stdin.end(`${JSON.stringify(payload)}\n`); - }); -} - -function runSentenceTransformersProcessJson(options: { - commands: SentenceTransformersProcessCommand[]; - cwd?: string; - env?: NodeJS.ProcessEnv; -}): SentenceTransformersJsonRunner { - return async ( - subcommand: SentenceTransformersCommand, - payload: Record, - ): Promise> => { - const errors: string[] = []; - for (const command of options.commands) { - try { - return await runSentenceTransformersProcessCommand({ - ...command, - cwd: options.cwd, - env: options.env, - })(subcommand, payload); - } catch (error) { - errors.push(`${command.command}: ${errorText(error)}`); - if (!isCommandNotFound(error)) { - break; - } - } - } - throw new Error(`ktx-daemon ${subcommand} failed: ${errors.join('; ')}`); - }; -} - -class DeterministicEmbeddingProvider implements KtxEmbeddingProvider { - readonly maxBatchSize: number; - - constructor(readonly dimensions: number, batchSize = DEFAULT_BATCH_SIZE) { - this.maxBatchSize = batchSize; - } - - async embed(text: string): Promise { - assertNonEmptyText(text); - return deterministicVector(text, this.dimensions); - } - - async embedMany(texts: string[]): Promise { - assertBatchSize(texts, this.maxBatchSize); - return texts.map((text) => deterministicVector(text, this.dimensions)); +function boundedHttpBody(text: string): string { + const normalized = text.trim(); + if (normalized.length <= HTTP_ERROR_BODY_MAX_LENGTH) { + return normalized; } + return `${normalized.slice(0, HTTP_ERROR_BODY_MAX_LENGTH)}...`; } class OpenAIEmbeddingProvider implements KtxEmbeddingProvider { @@ -255,9 +118,7 @@ class SentenceTransformersEmbeddingProvider implements KtxEmbeddingProvider { private readonly fetch: FetchFn; private readonly baseURL: string; private readonly pathPrefix: string; - private readonly runJson: SentenceTransformersJsonRunner; private readonly startupProbe: Promise; - private useProcessRunner = false; constructor(config: KtxEmbeddingConfig, deps: KtxEmbeddingProviderDeps) { if (!config.sentenceTransformers?.baseURL) { @@ -268,15 +129,6 @@ class SentenceTransformersEmbeddingProvider implements KtxEmbeddingProvider { this.fetch = deps.fetch ?? fetch; this.baseURL = config.sentenceTransformers.baseURL; this.pathPrefix = config.sentenceTransformers.pathPrefix ?? '/api'; - this.runJson = - deps.runSentenceTransformersJson ?? - runSentenceTransformersProcessJson({ - commands: deps.sentenceTransformersCommand - ? [{ command: deps.sentenceTransformersCommand, args: deps.sentenceTransformersArgs ?? [] }] - : defaultSentenceTransformersProcessCommands(), - cwd: deps.sentenceTransformersCwd, - env: deps.sentenceTransformersEnv, - }); this.startupProbe = this.requestSingle('__ktx_embedding_probe__').then((embedding) => { assertVectorDimensions(embedding, this.dimensions, 'sentence-transformers'); }); @@ -291,7 +143,7 @@ class SentenceTransformersEmbeddingProvider implements KtxEmbeddingProvider { async embedMany(texts: string[]): Promise { assertBatchSize(texts, this.maxBatchSize); await this.startupProbe; - const response = await this.requestJson('embedding-compute-bulk', '/embeddings/compute-bulk', { texts }); + const response = await this.requestJson('/embeddings/compute-bulk', { texts }); if ( !response || typeof response !== 'object' || @@ -312,37 +164,15 @@ class SentenceTransformersEmbeddingProvider implements KtxEmbeddingProvider { } private async requestSingle(text: string): Promise { - const response = await this.requestJson('embedding-compute', '/embeddings/compute', { text }); + const response = await this.requestJson('/embeddings/compute', { text }); if (!response || typeof response !== 'object' || !('embedding' in response) || !Array.isArray(response.embedding)) { throw new Error('Embedding provider sentence-transformers returned malformed single response'); } return response.embedding; } - private async requestJson( - command: SentenceTransformersCommand, - path: string, - body: Record, - ): Promise> { - if (this.useProcessRunner) { - return this.runJson(command, body); - } - - try { - return await this.postJson(path, body); - } catch (httpError) { - try { - const response = await this.runJson(command, body); - this.useProcessRunner = true; - return response; - } catch (processError) { - throw new Error( - `Embedding provider sentence-transformers local HTTP request failed (${errorText( - httpError, - )}) and ktx-daemon fallback failed (${errorText(processError)})`, - ); - } - } + private async requestJson(path: string, body: Record): Promise> { + return await this.postJson(path, body); } private async postJson(path: string, body: Record): Promise> { @@ -352,7 +182,12 @@ class SentenceTransformersEmbeddingProvider implements KtxEmbeddingProvider { body: JSON.stringify(body), }); if (!response.ok) { - throw new Error(`Embedding provider sentence-transformers request failed with HTTP ${response.status}`); + const bodyText = boundedHttpBody(await response.text()); + throw new Error( + `Embedding provider sentence-transformers request failed with HTTP ${response.status}${ + bodyText ? `: ${bodyText}` : '' + }`, + ); } const parsed = (await response.json()) as unknown; if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { @@ -367,8 +202,6 @@ export function createKtxEmbeddingProvider( deps: KtxEmbeddingProviderDeps = {}, ): KtxEmbeddingProvider { switch (config.backend) { - case 'deterministic': - return new DeterministicEmbeddingProvider(config.dimensions, config.batchSize); case 'openai': return new OpenAIEmbeddingProvider(config, deps); case 'sentence-transformers': diff --git a/packages/llm/src/types.ts b/packages/llm/src/types.ts index b91aec25..bc928e08 100644 --- a/packages/llm/src/types.ts +++ b/packages/llm/src/types.ts @@ -62,7 +62,7 @@ export interface KtxLlmProvider { activeBackend(): KtxLlmBackend; } -export type KtxEmbeddingBackend = 'openai' | 'deterministic' | 'sentence-transformers'; +export type KtxEmbeddingBackend = 'openai' | 'sentence-transformers'; export interface KtxEmbeddingTokenUsageEvent { backend: KtxEmbeddingBackend; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 094c1deb..5dbc325a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,9 @@ importers: '@semantic-release/github': specifier: ^12.0.8 version: 12.0.8(semantic-release@25.0.3(typescript@6.0.3)) + '@semantic-release/npm': + specifier: ^13.1.5 + version: 13.1.5(semantic-release@25.0.3(typescript@6.0.3)) '@semantic-release/release-notes-generator': specifier: ^14.1.1 version: 14.1.1(semantic-release@25.0.3(typescript@6.0.3)) @@ -57,6 +60,9 @@ importers: docs-site: dependencies: + '@xyflow/react': + specifier: ^12.10.2 + version: 12.10.2(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) fumadocs-core: specifier: 16.8.10 version: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) @@ -2748,6 +2754,24 @@ packages: '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + '@types/debug@4.1.13': resolution: {integrity: sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==} @@ -2853,6 +2877,15 @@ packages: '@vitest/utils@4.1.6': resolution: {integrity: sha512-FxIY+U81R3LGKCxaHHFRQ5+g6/iRgGLmeHWdp2Amj4ljQRrEIWHmZyDfDYBRZlpyqA7qKxtS9DD1dhk8RnRIVQ==} + '@xyflow/react@12.10.2': + resolution: {integrity: sha512-CgIi6HwlcHXwlkTpr0fxLv/0sRVNZ8IdwKLzzeCscaYBwpvfcH1QFOCeaTCuEn1FQEs/B8CjnTSjhs8udgmBgQ==} + peerDependencies: + react: '>=17' + react-dom: '>=17' + + '@xyflow/system@0.0.76': + resolution: {integrity: sha512-hvwvnRS1B3REwVDlWexsq7YQaPZeG3/mKo1jv38UmnpWmxihp14bW6VtEOuHEwJX2FvzFw8k77LyKSk/wiZVNA==} + abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -3139,6 +3172,9 @@ packages: class-variance-authority@0.7.1: resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} + classcat@5.0.5: + resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + clean-stack@2.2.0: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} @@ -3325,6 +3361,44 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} @@ -5937,6 +6011,11 @@ packages: '@types/react': optional: true + use-sync-external-store@1.6.0: + resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -6158,6 +6237,21 @@ packages: zod@4.4.3: resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + zustand@4.5.7: + resolution: {integrity: sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==} + engines: {node: '>=12.7.0'} + peerDependencies: + '@types/react': '>=16.8' + immer: '>=9.0.6' + react: '>=16.8' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -9061,6 +9155,27 @@ snapshots: '@types/deep-eql': 4.0.2 assertion-error: 2.0.1 + '@types/d3-color@3.1.3': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/debug@4.1.13': dependencies: '@types/ms': 2.1.0 @@ -9191,6 +9306,29 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 + '@xyflow/react@12.10.2(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + dependencies: + '@xyflow/system': 0.0.76 + classcat: 5.0.5 + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) + zustand: 4.5.7(@types/react@19.2.14)(react@19.2.6) + transitivePeerDependencies: + - '@types/react' + - immer + + '@xyflow/system@0.0.76': + dependencies: + '@types/d3-drag': 3.0.7 + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -9459,6 +9597,8 @@ snapshots: dependencies: clsx: 2.1.1 + classcat@5.0.5: {} + clean-stack@2.2.0: {} clean-stack@5.3.0: @@ -9631,6 +9771,42 @@ snapshots: csstype@3.2.3: {} + d3-color@3.1.0: {} + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-selection@3.0.0: {} + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + data-uri-to-buffer@4.0.1: {} debug@4.4.3: @@ -12730,6 +12906,10 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 + use-sync-external-store@1.6.0(react@19.2.6): + dependencies: + react: 19.2.6 + util-deprecate@1.0.2: {} validate-npm-package-license@3.0.4: @@ -12907,4 +13087,11 @@ snapshots: zod@4.4.3: {} + zustand@4.5.7(@types/react@19.2.14)(react@19.2.6): + dependencies: + use-sync-external-store: 1.6.0(react@19.2.6) + optionalDependencies: + '@types/react': 19.2.14 + react: 19.2.6 + zwitch@2.0.4: {} diff --git a/scripts/build-public-npm-package.mjs b/scripts/build-public-npm-package.mjs index 0e34ae6d..63551d38 100644 --- a/scripts/build-public-npm-package.mjs +++ b/scripts/build-public-npm-package.mjs @@ -146,12 +146,12 @@ export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUB license: cliPackageJson.license ?? 'Apache-2.0', repository: { type: 'git', - url: 'git+https://github.com/kaelio/ktx.git', + url: 'https://github.com/Kaelio/ktx', }, bugs: { - url: 'https://github.com/kaelio/ktx/issues', + url: 'https://github.com/Kaelio/ktx/issues', }, - homepage: 'https://github.com/kaelio/ktx#readme', + homepage: 'https://github.com/Kaelio/ktx#readme', }; } diff --git a/scripts/build-public-npm-package.test.mjs b/scripts/build-public-npm-package.test.mjs index 4afe1de5..c78ae164 100644 --- a/scripts/build-public-npm-package.test.mjs +++ b/scripts/build-public-npm-package.test.mjs @@ -217,6 +217,14 @@ describe('publicNpmPackageJson', () => { assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' }); assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES); assert.deepEqual(packageJson.files, ['dist', 'assets']); + assert.deepEqual(packageJson.repository, { + type: 'git', + url: 'https://github.com/Kaelio/ktx', + }); + assert.deepEqual(packageJson.bugs, { + url: 'https://github.com/Kaelio/ktx/issues', + }); + assert.equal(packageJson.homepage, 'https://github.com/Kaelio/ktx#readme'); }); }); diff --git a/scripts/check-boundaries.mjs b/scripts/check-boundaries.mjs index f51fd223..3093fd42 100644 --- a/scripts/check-boundaries.mjs +++ b/scripts/check-boundaries.mjs @@ -10,7 +10,7 @@ const identifierSkipPrefixes = ['docs/', 'docs-site/', 'examples/', 'python/ktx- const identifierAllowPatterns = [ /^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|release-version|runtime)(?:\.test)?\.ts$/, /^python\/ktx-daemon\/src\/ktx_daemon\/__init__\.py$/, - /^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|publish-public-npm-package|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, + /^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, ]; const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_']; diff --git a/scripts/check-boundaries.test.mjs b/scripts/check-boundaries.test.mjs index 952b5614..832dc1c2 100644 --- a/scripts/check-boundaries.test.mjs +++ b/scripts/check-boundaries.test.mjs @@ -79,7 +79,6 @@ describe('scanFileContent', () => { assert.equal(scanFileContent('scripts/local-embeddings-runtime-smoke.mjs', `@${name}/ktx`).length, 0); assert.equal(scanFileContent('scripts/package-artifacts.test.mjs', `${name}-ktx`).length, 0); assert.equal(scanFileContent('scripts/public-npm-release-metadata.mjs', `@${name}/ktx`).length, 0); - assert.equal(scanFileContent('scripts/publish-public-npm-package.test.mjs', `@${name}/ktx`).length, 0); assert.equal(scanFileContent('packages/cli/src/release-version.ts', `@${name}/ktx`).length, 0); assert.equal(scanFileContent('packages/cli/src/managed-python-runtime.ts', `${name}_ktx`).length, 0); assert.equal(scanFileContent('python/ktx-daemon/src/ktx_daemon/__init__.py', `${name}-ktx`).length, 0); diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 2f6c9ef6..bebe4965 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -151,7 +151,6 @@ describe('standalone example docs', () => { assert.match(contributing, /context\/\s+# Core context engine/); assert.match(contributing, /llm\/\s+# LLM client abstraction/); assert.match(contributing, /connector-bigquery\/\s+# BigQuery connector/); - assert.match(contributing, /connector-clickhouse\/\s+# ClickHouse connector/); assert.match(contributing, /connector-mysql\/\s+# MySQL connector/); assert.match(contributing, /connector-postgres\/\s+# PostgreSQL connector/); assert.match(contributing, /connector-snowflake\/\s+# Snowflake connector/); @@ -192,7 +191,7 @@ describe('standalone example docs', () => { const quickstart = await readText('docs-site/content/docs/getting-started/quickstart.mdx'); const packageArtifacts = await readText('examples/package-artifacts/README.md'); - assert.match(rootReadme, publicPackagePattern('pnpm add --global {package}')); + assert.match(rootReadme, publicPackagePattern('npm install -g {package}')); assert.match(quickstart, publicPackagePattern('npm install -g {package}')); assert.match(quickstart, /ktx dev runtime install --feature local-embeddings --yes/); assert.match(quickstart, /ktx dev runtime start --feature local-embeddings/); diff --git a/scripts/local-embeddings-runtime-smoke.mjs b/scripts/local-embeddings-runtime-smoke.mjs index 8b0243a0..652a039c 100644 --- a/scripts/local-embeddings-runtime-smoke.mjs +++ b/scripts/local-embeddings-runtime-smoke.mjs @@ -106,7 +106,6 @@ export function localEmbeddingsSmokeCommands(input) { 'setup', '--project-dir', input.projectDir, - '--new', '--no-input', '--yes', '--skip-llm', diff --git a/scripts/local-embeddings-runtime-smoke.test.mjs b/scripts/local-embeddings-runtime-smoke.test.mjs index 95ee119c..f059f7a2 100644 --- a/scripts/local-embeddings-runtime-smoke.test.mjs +++ b/scripts/local-embeddings-runtime-smoke.test.mjs @@ -114,7 +114,6 @@ describe('localEmbeddingsSmokeCommands', () => { 'setup', '--project-dir', '/tmp/ktx-local-embedding-smoke/project', - '--new', '--no-input', '--yes', '--skip-llm', diff --git a/scripts/normalize-lcov-paths.mjs b/scripts/normalize-lcov-paths.mjs new file mode 100644 index 00000000..62df4f1a --- /dev/null +++ b/scripts/normalize-lcov-paths.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node + +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +function toPosix(filePath) { + return filePath.replace(/\\/g, '/'); +} + +export function normalizeLcovContent(content, packagePath) { + const normalizedPackagePath = toPosix(packagePath).replace(/\/$/, ''); + + return content.replace(/^SF:(.+)$/gm, (line, sourcePath) => { + const normalizedSourcePath = toPosix(sourcePath); + + if ( + path.isAbsolute(sourcePath) || + normalizedSourcePath.startsWith(`${normalizedPackagePath}/`) || + normalizedSourcePath.startsWith('../') + ) { + return line; + } + + return `SF:${normalizedPackagePath}/${normalizedSourcePath}`; + }); +} + +export async function normalizeLcovFile(rootDir, reportPath) { + const absoluteReportPath = path.resolve(rootDir, reportPath); + const packagePath = toPosix(path.relative(rootDir, path.dirname(path.dirname(absoluteReportPath)))); + const content = await readFile(absoluteReportPath, 'utf8'); + const normalizedContent = normalizeLcovContent(content, packagePath); + + if (normalizedContent !== content) { + await writeFile(absoluteReportPath, normalizedContent); + } +} + +async function main() { + const scriptDir = path.dirname(fileURLToPath(import.meta.url)); + const rootDir = path.resolve(scriptDir, '..'); + const reportPaths = process.argv.slice(2); + + if (reportPaths.length === 0) { + throw new Error('Usage: normalize-lcov-paths.mjs [...]'); + } + + for (const reportPath of reportPaths) { + await normalizeLcovFile(rootDir, reportPath); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/normalize-lcov-paths.test.mjs b/scripts/normalize-lcov-paths.test.mjs new file mode 100644 index 00000000..44ab9fe6 --- /dev/null +++ b/scripts/normalize-lcov-paths.test.mjs @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { normalizeLcovContent } from './normalize-lcov-paths.mjs'; + +describe('normalizeLcovContent', () => { + it('prefixes relative LCOV source paths with the package path', () => { + const input = ['TN:', 'SF:src/index.ts', 'SF:src\\windows.ts', 'DA:1,1', 'end_of_record'].join('\n'); + + assert.equal( + normalizeLcovContent(input, 'packages/context'), + [ + 'TN:', + 'SF:packages/context/src/index.ts', + 'SF:packages/context/src/windows.ts', + 'DA:1,1', + 'end_of_record', + ].join('\n'), + ); + }); + + it('leaves already-normalized and absolute paths unchanged', () => { + const input = [ + 'SF:packages/cli/src/index.ts', + 'SF:/tmp/repo/packages/cli/src/index.ts', + 'SF:../shared/source.ts', + ].join('\n'); + + assert.equal(normalizeLcovContent(input, 'packages/cli'), input); + }); +}); diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 8e1f174d..3c885ac9 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -481,13 +481,15 @@ export function npmRuntimeSmokeSource() { return ` import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { promisify } from 'node:util'; const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); async function run(command, args, options = {}) { process.stdout.write('$ ' + command + ' ' + args.join(' ') + '\\n'); @@ -547,6 +549,15 @@ function requireOutput(label, result, text) { assert.match(result.stdout, text, label + ' output did not match ' + text); } +function escapeRegExp(value) { + return value.replace(/[|\\\\{}()[\\]^$+*?.]/g, '\\\\$&'); +} + +async function installedPackageVersionPattern() { + const packageJson = JSON.parse(await readFile(require.resolve('@kaelio/ktx/package.json'), 'utf8')); + return new RegExp('^' + escapeRegExp(packageJson.name) + ' ' + escapeRegExp(packageJson.version) + '$', 'm'); +} + function parseJsonResult(label, result) { requireSuccess(label, result); return JSON.parse(result.stdout); @@ -592,7 +603,7 @@ try { const version = await run('pnpm', ['exec', 'ktx', '--version']); requireSuccess('ktx public package version', version); - requireOutput('ktx public package version', version, /@kaelio\\/ktx 0\\.1\\.0/); + requireOutput('ktx public package version', version, await installedPackageVersionPattern()); const runtimeStatusBefore = parseJsonResultWithExitCode( 'ktx dev runtime status missing', @@ -609,7 +620,6 @@ try { 'setup', '--project-dir', projectDir, - '--new', '--no-input', '--yes', '--skip-llm', @@ -627,7 +637,6 @@ try { 'setup', '--project-dir', emptyProjectDir, - '--new', '--no-input', '--yes', '--skip-llm', diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index bbffe06c..89d1a760 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -467,7 +467,8 @@ describe('verification snippets', () => { const source = npmRuntimeSmokeSource(); assert.match(source, /ktx public package version/); - assert.match(source, /@kaelio\\\/ktx 0\\\.1\\\.0/); + assert.match(source, /installedPackageVersionPattern/); + assert.doesNotMatch(source, /@kaelio\\\/ktx 0\\\.1\\\.0/); assert.match(source, /'ktx', 'sl', 'query'/); assert.doesNotMatch(source, /@ktx\/context/); assert.doesNotMatch(source, /@modelcontextprotocol/); diff --git a/scripts/publish-public-npm-package.mjs b/scripts/publish-public-npm-package.mjs deleted file mode 100644 index c8c5b6e3..00000000 --- a/scripts/publish-public-npm-package.mjs +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env node - -import { spawn } from 'node:child_process'; -import { access } from 'node:fs/promises'; -import { pathToFileURL } from 'node:url'; - -import { packageArtifactLayout } from './package-artifacts.mjs'; -import { releaseReadinessReport } from './release-readiness.mjs'; - -export const NPM_PUBLISH_TIMEOUT_MS = 180_000; - -export function resolvePublishMode(args = process.argv.slice(2)) { - return { live: args.includes('--publish') }; -} - -export function requireNpmPublicReleaseReady(report) { - if (report.releaseMode !== 'npm-public-release-ready' || report.npmPublishEnabled !== true || !report.npmPublish) { - throw new Error('release-policy.json must use npm-public-release-ready before publishing'); - } - return report.npmPublish; -} - -export function buildNpmPublishCommand(tarballPath, publish, mode) { - return { - command: 'npm', - args: [ - 'publish', - tarballPath, - '--access', - publish.access, - '--tag', - publish.tag, - ...(mode.live ? [] : ['--dry-run']), - ], - env: publish.registry ? { npm_config_registry: publish.registry } : {}, - }; -} - -async function assertFileExists(path) { - try { - await access(path); - } catch { - throw new Error(`Missing npm tarball: ${path}. Run pnpm run artifacts:check first.`); - } -} - -async function runPublishCommand(command) { - process.stdout.write(`$ ${command.command} ${command.args.join(' ')}\n`); - - await new Promise((resolvePromise, reject) => { - let settled = false; - const child = spawn(command.command, command.args, { - env: { ...process.env, ...command.env }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - const settle = (callback, value) => { - if (settled) { - return; - } - settled = true; - clearTimeout(timeout); - callback(value); - }; - const timeout = setTimeout(() => { - child.kill('SIGTERM'); - settle(reject, new Error(`Timed out after ${NPM_PUBLISH_TIMEOUT_MS}ms while publishing npm package`)); - }, NPM_PUBLISH_TIMEOUT_MS); - - child.stdout.on('data', (chunk) => { - process.stdout.write(chunk); - }); - child.stderr.on('data', (chunk) => { - process.stderr.write(chunk); - }); - child.on('error', (error) => { - settle(reject, error); - }); - child.on('close', (code, signal) => { - if (code === 0) { - settle(resolvePromise); - return; - } - settle(reject, new Error(`npm publish failed with ${signal ? `signal ${signal}` : `exit code ${code}`}`)); - }); - }); -} - -export async function publishPublicNpmPackage(options = {}) { - const rootDir = options.rootDir; - const mode = options.mode ?? resolvePublishMode(options.args); - const report = await releaseReadinessReport(rootDir); - const publish = requireNpmPublicReleaseReady(report); - const layout = packageArtifactLayout(rootDir); - const tarballPath = layout.cliTarball; - - await assertFileExists(tarballPath); - const command = buildNpmPublishCommand(tarballPath, publish, mode); - await runPublishCommand(command); - - process.stdout.write( - mode.live - ? `Published ${publish.packageName}@${publish.version} with tag ${publish.tag}\n` - : `Dry-run verified ${publish.packageName}@${publish.version} with tag ${publish.tag}\n`, - ); -} - -async function main() { - await publishPublicNpmPackage({ args: process.argv.slice(2) }); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - try { - await main(); - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); - process.exitCode = 1; - } -} diff --git a/scripts/publish-public-npm-package.test.mjs b/scripts/publish-public-npm-package.test.mjs deleted file mode 100644 index d3b5b016..00000000 --- a/scripts/publish-public-npm-package.test.mjs +++ /dev/null @@ -1,108 +0,0 @@ -import assert from 'node:assert/strict'; -import { readFile } from 'node:fs/promises'; -import { describe, it } from 'node:test'; - -import { - buildNpmPublishCommand, - requireNpmPublicReleaseReady, - resolvePublishMode, -} from './publish-public-npm-package.mjs'; - -const readyReport = { - releaseMode: 'npm-public-release-ready', - npmPublishEnabled: true, - npmPublish: { - packageName: '@kaelio/ktx', - version: '0.1.0-rc.1', - access: 'public', - tag: 'next', - registry: null, - }, -}; - -describe('resolvePublishMode', () => { - it('dry-runs by default', () => { - assert.deepEqual(resolvePublishMode([]), { live: false }); - }); - - it('requires an explicit flag for live publish', () => { - assert.deepEqual(resolvePublishMode(['--publish']), { live: true }); - }); -}); - -describe('requireNpmPublicReleaseReady', () => { - it('accepts the npm public release ready report', () => { - assert.equal(requireNpmPublicReleaseReady(readyReport), readyReport.npmPublish); - }); - - it('rejects artifact-only reports', () => { - assert.throws( - () => - requireNpmPublicReleaseReady({ - releaseMode: 'ci-artifact-only', - npmPublishEnabled: false, - npmPublish: null, - }), - /release-policy.json must use npm-public-release-ready before publishing/, - ); - }); -}); - -describe('buildNpmPublishCommand', () => { - it('builds a dry-run npm publish command by default', () => { - assert.deepEqual( - buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', readyReport.npmPublish, { - live: false, - }), - { - command: 'npm', - args: [ - 'publish', - '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', - '--access', - 'public', - '--tag', - 'next', - '--dry-run', - ], - env: {}, - }, - ); - }); - - it('omits dry-run only for explicit live publish', () => { - assert.deepEqual( - buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', readyReport.npmPublish, { - live: true, - }).args, - [ - 'publish', - '/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', - '--access', - 'public', - '--tag', - 'next', - ], - ); - }); - - it('uses npm_config_registry when a registry is configured', () => { - const publish = { - ...readyReport.npmPublish, - registry: 'https://registry.npmjs.org/', - }; - - assert.deepEqual( - buildNpmPublishCommand('/repo/ktx/dist/artifacts/npm/kaelio-ktx-0.1.0-rc.1.tgz', publish, { live: false }).env, - { npm_config_registry: 'https://registry.npmjs.org/' }, - ); - }); -}); - -describe('package script', () => { - it('registers release:npm-publish', async () => { - const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); - - assert.equal(packageJson.scripts['release:npm-publish'], 'node scripts/publish-public-npm-package.mjs'); - }); -}); diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs index 60a82011..e6f8350a 100644 --- a/scripts/release-workflow.test.mjs +++ b/scripts/release-workflow.test.mjs @@ -9,16 +9,22 @@ describe('release workflow', () => { assert.match(workflow, /^name: KTX Release$/m); assert.match(workflow, /^ workflow_dispatch:$/m); assert.match(workflow, /release_kind:/); - assert.match(workflow, /options:\n - rc\n - stable/); + assert.match(workflow, /release_kind:[\s\S]*?default: "stable"/); + assert.match(workflow, /options:\n - stable\n - rc/); assert.match(workflow, /force_release:/); assert.match(workflow, /publish_live:/); - assert.match(workflow, /default: false/); + assert.match(workflow, /publish_live:[\s\S]*?default: true/); assert.match(workflow, /^ contents: write$/m); assert.match(workflow, /^ id-token: write$/m); assert.match(workflow, /fetch-depth: 0/); assert.match(workflow, /registry-url: "https:\/\/registry\.npmjs\.org"/); + assert.doesNotMatch(workflow, /Prepare first stable release floor/); + assert.doesNotMatch(workflow, /git tag v0\.0\.0/); + assert.doesNotMatch(workflow, /KTX_STABLE_RELEASE_FLOOR_TAG/); assert.match(workflow, /Prepare next prerelease branch/); assert.match(workflow, /git checkout -B "\$\{KTX_PRERELEASE_BRANCH\}"/); + assert.match(workflow, /Prepare npm package root for release verification/); + assert.match(workflow, /dist\/public-npm-package\/package\.json/); assert.match(workflow, /GITHUB_REF="refs\/heads\/\$\{KTX_PRERELEASE_BRANCH\}"/); assert.match(workflow, /pnpm run semantic-release:dry-run/); assert.match(workflow, /pnpm run semantic-release$/m); diff --git a/scripts/semantic-release-config.cjs b/scripts/semantic-release-config.cjs index 66b9aad7..e9ec9cc5 100644 --- a/scripts/semantic-release-config.cjs +++ b/scripts/semantic-release-config.cjs @@ -90,6 +90,26 @@ function releaseTag(kind) { return kind === 'rc' ? 'next' : 'latest'; } +function releaseChangelogPlugins(kind) { + return kind === 'rc' ? ['@semantic-release/changelog'] : []; +} + +function releaseGitPlugins(kind) { + if (kind !== 'rc') { + return []; + } + + return [ + [ + '@semantic-release/git', + { + assets: ['CHANGELOG.md', 'package.json', 'release-policy.json'], + message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}', + }, + ], + ]; +} + function releaseBranches(env = process.env) { const branch = currentBranch(env); const kind = releaseKind(env); @@ -137,7 +157,7 @@ function createReleaseConfig(env = process.env) { }, }, ], - '@semantic-release/changelog', + ...releaseChangelogPlugins(kind), [ '@semantic-release/exec', { @@ -146,19 +166,22 @@ function createReleaseConfig(env = process.env) { 'pnpm run artifacts:check', 'pnpm run release:readiness', ].join(' && '), - publishCmd: [ - 'pnpm run release:npm-publish -- --publish', - 'pnpm run release:published-smoke', - ].join(' && '), }, ], [ - '@semantic-release/git', + '@semantic-release/npm', { - assets: ['CHANGELOG.md', 'package.json', 'release-policy.json'], - message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}', + pkgRoot: 'dist/public-npm-package', + tarballDir: 'dist/artifacts/npm', }, ], + [ + '@semantic-release/exec', + { + publishCmd: 'pnpm run release:published-smoke', + }, + ], + ...releaseGitPlugins(kind), [ '@semantic-release/github', { diff --git a/scripts/semantic-release-config.test.mjs b/scripts/semantic-release-config.test.mjs index 0a267d77..99cd53c4 100644 --- a/scripts/semantic-release-config.test.mjs +++ b/scripts/semantic-release-config.test.mjs @@ -9,6 +9,14 @@ function releaseExecOptions(config) { return config.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/exec' && plugin[1].prepareCmd)[1]; } +function releaseExecIndex(config) { + return config.plugins.findIndex((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/exec' && plugin[1].prepareCmd); +} + +function pluginNames(config) { + return config.plugins.map((plugin) => (Array.isArray(plugin) ? plugin[0] : plugin)); +} + describe('semantic-release config', () => { it('configures rc releases on a dedicated next prerelease branch', () => { assert.equal(releaseKind({ KTX_RELEASE_KIND: 'rc' }), 'rc'); @@ -19,10 +27,29 @@ describe('semantic-release config', () => { ]); const config = createReleaseConfig({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'main' }); + assert.deepEqual( + config.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/npm'), + [ + '@semantic-release/npm', + { + pkgRoot: 'dist/public-npm-package', + tarballDir: 'dist/artifacts/npm', + }, + ], + ); assert.match( releaseExecOptions(config).prepareCmd, /update-public-release-version\.mjs "\$\{nextRelease\.version\}" "next"/, ); + assert.doesNotMatch(JSON.stringify(config.plugins), /release:npm-publish/); + const releaseFilePluginNames = pluginNames(config).filter( + (plugin) => plugin === '@semantic-release/changelog' || plugin === '@semantic-release/git', + ); + assert.deepEqual(releaseFilePluginNames, ['@semantic-release/changelog', '@semantic-release/git']); + + const names = pluginNames(config); + assert.ok(names.indexOf('@semantic-release/changelog') < releaseExecIndex(config)); + assert.ok(names.indexOf('@semantic-release/git') > releaseExecIndex(config)); }); it('configures stable releases only from main with latest tag', () => { @@ -35,6 +62,14 @@ describe('semantic-release config', () => { releaseExecOptions(config).prepareCmd, /update-public-release-version\.mjs "\$\{nextRelease\.version\}" "latest"/, ); + assert.equal(config.plugins.includes('./scripts/semantic-release-version-policy.cjs'), false); + }); + + it('does not commit release files back to protected main during stable releases', () => { + const config = createReleaseConfig({ KTX_RELEASE_KIND: 'stable', GITHUB_REF_NAME: 'main' }); + + assert.equal(pluginNames(config).includes('@semantic-release/git'), false); + assert.equal(pluginNames(config).includes('@semantic-release/changelog'), false); }); it('rejects stable releases from non-main branches', () => { @@ -51,4 +86,23 @@ describe('semantic-release config', () => { ); assert.match(analyzeExec[1].analyzeCommitsCmd, /FORCE_RELEASE === 'true' \? 'patch' : ''/); }); + + it('does not configure any commit type to create an automatic major release', () => { + const config = createReleaseConfig({ KTX_RELEASE_KIND: 'stable', GITHUB_REF_NAME: 'main' }); + const analyzer = config.plugins.find( + (plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/commit-analyzer', + ); + + assert.equal( + analyzer[1].releaseRules.some((rule) => rule.release === 'major'), + false, + ); + assert.deepEqual( + analyzer[1].releaseRules.filter((rule) => rule.breaking || rule.type === 'major'), + [ + { breaking: true, release: 'minor' }, + { type: 'major', release: 'minor' }, + ], + ); + }); });