diff --git a/.github/workflows/star-history.yml b/.github/workflows/star-history.yml
new file mode 100644
index 00000000..b7d90c43
--- /dev/null
+++ b/.github/workflows/star-history.yml
@@ -0,0 +1,72 @@
+name: Refresh star history chart
+
+on:
+ schedule:
+ # Twice daily at 06:00 and 18:00 UTC.
+ - cron: "0 6,18 * * *"
+ workflow_dispatch:
+
+permissions:
+ contents: write
+
+env:
+ DO_NOT_TRACK: "1"
+ KTX_TELEMETRY_DISABLED: "1"
+ NEXT_TELEMETRY_DISABLED: "1"
+
+concurrency:
+ group: star-history-refresh
+ cancel-in-progress: true
+
+jobs:
+ refresh:
+ name: Regenerate assets/star-history.svg
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ # RELEASE_PAT can push to the protected main branch; the default
+ # GITHUB_TOKEN is rejected by the branch-protection hook (GH006).
+ token: ${{ secrets.RELEASE_PAT }}
+
+ - name: Fetch fresh star-history SVG
+ run: |
+ set -euo pipefail
+ # cachebust forces star-history to regenerate instead of serving its
+ # own server-side cache; --location follows the slug-normalizing 301.
+ url="https://api.star-history.com/svg?repos=Kaelio/ktx&type=Date&cachebust=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ curl --fail --location --silent --show-error \
+ --retry 3 --retry-delay 5 --max-time 60 \
+ -o assets/star-history.svg.new "$url"
+ # Guard against error pages / truncated responses before overwriting.
+ if ! grep -q "" assets/star-history.svg.new; then
+ echo "Downloaded file is not a valid SVG; aborting." >&2
+ exit 1
+ fi
+ if [ "$(wc -c < assets/star-history.svg.new)" -lt 1000 ]; then
+ echo "Downloaded SVG is suspiciously small; aborting." >&2
+ exit 1
+ fi
+ # The star-history API returns the SVG without a trailing newline,
+ # which end-of-file-fixer rewrites whenever pre-commit runs
+ # --all-files on a PR. Because the refresh commit below uses [skip ci],
+ # the hook never runs against it here, so an un-normalized file
+ # silently breaks the pre-commit check on every open PR. Normalize to
+ # exactly one trailing newline before committing.
+ printf '%s\n' "$(cat assets/star-history.svg.new)" > assets/star-history.svg
+ rm -f assets/star-history.svg.new
+
+ - name: Commit if changed
+ run: |
+ set -euo pipefail
+ if git diff --quiet -- assets/star-history.svg; then
+ echo "Star-history chart unchanged; nothing to commit."
+ exit 0
+ fi
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git add assets/star-history.svg
+ # [skip ci] keeps this housekeeping commit from triggering KTX CI.
+ git commit -m "chore: refresh star history chart [skip ci]"
+ git push
diff --git a/AGENTS.md b/AGENTS.md
index 4bdecf89..20f9bcdf 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -64,25 +64,6 @@ When rules conflict, follow this order:
4. Code quality: types, readable boundaries, focused modules
5. Performance where it matters
-## Opinionated Product Defaults
-
-- **MUST**: Prefer one canonical behavior over configurable alternatives. A new
- flag, config field, environment variable, mode, strategy option, adapter hook,
- or fallback path is a product feature and must be justified by an explicit
- user request or a real correctness requirement.
-- **MUST NOT**: Add speculative flexibility for imagined users, migrations,
- review preferences, local workflows, or "just in case" scenarios. If the
- requested behavior can work with one solid default, implement that default.
-- **MUST NOT**: Add boolean switches that create two runtime paths unless both
- paths are essential and the user explicitly asked for the choice. Boolean
- policy knobs are especially suspect because they double the state space and
- test surface.
-- **MUST**: When a design seems to need a new option, first try to remove the
- need by choosing the stronger default, tightening the invariant, or failing
- clearly. Ask the user before adding the option if it still seems necessary.
-- **MUST**: Delete obsolete branches, tests, docs, and config when removing a
- behavior. Do not preserve dormant compatibility paths.
-
## Repository Shape
**ktx** is a pnpm + uv workspace.
@@ -211,19 +192,6 @@ autonomously — without being asked the leading question — is the bar.
next stack. The only acceptable static patterns are genuinely universal
invariants (e.g. DB-engine system catalogs) and ktx's own self-emitted
signatures.
-- **MUST**: Give each capability one implementation and route every caller
- through it. When some behavior — running a query, resolving a credential or
- config reference, authenticating, selecting a dialect, loading config —
- already has a working implementation that some call sites use, make new or
- divergent call sites depend on that path instead of standing up a second one.
- Parallel implementations of one capability drift apart silently: a fix, a
- newly supported input, or an added case lands on one path and not the other,
- so one entry point (a CLI command, an MCP tool, an ingest stage) succeeds
- while another fails on the same input. When two paths already do the same
- job, collapse onto the shared one and delete the duplicate instead of
- keeping both. When fixing a defect that lives on one path, fix the shared
- implementation; do not patch the symptom on a forked branch, which preserves
- the divergence you set out to remove.
- **SHOULD**: Before inventing an abstraction or hand-rolling structural logic,
search for what already exists and reuse it — the codebase's canonical
representation (a structured ref/key type) instead of a parallel string scheme,
@@ -244,54 +212,12 @@ Before presenting a design, answer these explicitly:
instead of building or parsing my own?
5. Am I discarding the better option on a weak or misapplied constraint
(one-time vs recurring cost, "more surface area", "more work now")?
-6. Does another entry point already perform this operation through a shared
- implementation? If so, am I routing through that path instead of forking a
- parallel one — and if I'm fixing a bug, am I fixing the shared layer rather
- than one branch?
-7. Am I adding a user-visible option or alternate runtime path that the user did
- not ask for? If yes, can one opinionated default solve the problem instead?
-8. Does this option multiply behavior by caller path, config value, or local
- state? If yes, remove it unless it is explicitly required.
A user question that nudges toward an alternative ("would X help?", "should I
always do Y?", "will you hardcode Z?") is a signal that a better option exists.
Investigate the implied direction and reason it through *before* defending the
original proposal — and prefer to have asked yourself the question first.
-Example: If generated context changes should be saved, choose one save policy
-and route ingest, setup, memory, indexing, and docs through it. Do not add an
-`auto_commit`-style switch unless the user explicitly asks for staged-only runs
-and accepts the extra runtime path.
-
-## Code Comments and Docstrings
-
-Code must be self-explanatory. Clear names, types, and signatures do the
-documenting; a comment or docstring exists only to state what the code cannot
-show. Everything else belongs in the PR description or nowhere.
-
-- **MUST**: Keep each comment to 1-3 lines stating only what the code cannot
- show: a cross-file invariant ("error-severity issues never reach here — the
- doctor exits on them first"), a required ordering ("ktx.yaml is written
- before git init, so a crash cannot leave a bare `.git`"), or a library quirk
- ("zod reports unknown record keys as `invalid_key`").
-- **MUST**: Hold docstrings (Python `"""..."""`, JSDoc/TSDoc) to the same bar.
- A docstring states a function's purpose or contract in 1-3 lines; when a real
- quirk or invariant motivates the code, note it once and briefly. Let
- self-explanatory code carry the rest — a well-named, well-typed function
- often needs no docstring at all.
-- **MUST**: State each invariant once, at the public entry point. Do not repeat
- the same guarantee across a module docstring, a helper, its wrapper, and the
- call site.
-- **MUST NOT**: Write multi-paragraph docstrings or prose comment blocks —
- design rationale, alternatives considered, change narration ("is now written
- before…"), caller enumerations ("shared by X, Y, and Z"), worked examples
- that restate the code, or the same explanation repeated in a module docstring
- and the function it describes. That is the author addressing the reviewer; it
- belongs in the PR description and rots once merged.
-- **MAY**: Open a regression test with a 1-3 line comment stating the scenario
- it guards when the test name cannot carry it. Omit design history and
- references to removed designs.
-
## TypeScript Standards
- Use Node 22+ and pnpm workspace commands.
@@ -411,8 +337,7 @@ use `PascalCase` without the suffix.
## Telemetry
-**ktx** ships PostHog usage telemetry. Catalog telemetry events use strict
-schemas. When adding commands or events:
+**ktx** ships PostHog usage telemetry. When adding commands or events:
- **MUST NOT**: Add fields that carry user data — file paths, hostnames,
environment values, SQL text, schema/table/column names, error messages,
@@ -429,24 +354,6 @@ schemas. When adding commands or events:
of collected data changes. Adding another event with no new field types
needs no docs change.
-### Error reports
-
-**ktx** also sends PostHog Error Tracking `$exception` events when telemetry is
-enabled. This channel is separate from the strict catalog event schema and is
-used only for exception diagnostics.
-
-`$exception` events may include stack frames, error class names, raw error
-messages, cause chains, `source`, `handled`, `fatal`, runtime version fields,
-OS/runtime fields, and the hashed `projectId` when known. Stack frames may
-include local file paths and the local username when those appear in paths.
-
-`$exception` events must never intentionally include secrets, credentials,
-database URLs, auth headers, raw argv, raw environment values, SQL text,
-schema/table/column names as explicit properties, customer row data, user prompt
-text, or raw MCP arguments. Reporters must redact call-site-provided secret
-snapshots and common static credential patterns before the SDK serializes the
-exception.
-
## Documentation and Specs
- Keep public documentation in `README.md`, package READMEs, example READMEs,
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 212dd9e6..a4fb3040 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
-# Contributing to ktx
+# Contributing to KTX
-Thanks for your interest in **ktx**. This page covers **how to contribute** and
+Thanks for your interest in KTX. This page covers **how to contribute** and
the **contributor rewards program**. For development setup, repository
layout, and verification commands, see the
[Contributing guide in the docs](https://docs.kaelio.com/ktx/docs/community/contributing).
@@ -23,7 +23,7 @@ layout, and verification commands, see the
## Contributor rewards program
We send merch to contributors whose pull requests get merged. The goal is
-to thank the people building **ktx** with us, not to drive volume.
+to thank the people building KTX with us, not to drive volume.
### How it works
@@ -76,7 +76,7 @@ See the [Community & Support](https://docs.kaelio.com/ktx/docs/community/support
page for the full guide. The short version:
- **Questions, "how do I...", setup help, sharing patterns**: join the
- [**ktx** Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ).
+ [KTX Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ).
- **Bugs**: use the [Bug report](.github/ISSUE_TEMPLATE/bug_report.yml)
template.
- **Feature requests**: use the
@@ -87,7 +87,7 @@ page for the full guide. The short version:
## Code of conduct
-**ktx** follows the
+KTX follows the
[Contributor Covenant](https://www.contributor-covenant.org/version/2/1/code_of_conduct/).
Be respectful, assume good intent, and keep discussion focused on the
project. Report concerns to the maintainers in Slack or by email at
diff --git a/README.md b/README.md
index 618e9c81..2c433e0d 100644
--- a/README.md
+++ b/README.md
@@ -13,20 +13,16 @@
-
+
- At runtime, agents reach ktx through MCP. ktx searches the context
- layer, returns approved metrics, and compiles them into read-only SQL
- the warehouse runs.
-
-
-
-
-
-
- Serving flow
-
-
- From an agent request to a governed answer
-
-
- The agent asks in plain language. ktx is the only thing that touches
- the context layer and the warehouse, and every database connection
- is read-only.
-
-
-
-
-
-
-
- );
-}
diff --git a/docs-site/components/theme-toggle.tsx b/docs-site/components/theme-toggle.tsx
deleted file mode 100644
index 29dead0b..00000000
--- a/docs-site/components/theme-toggle.tsx
+++ /dev/null
@@ -1,157 +0,0 @@
-"use client";
-
-import {
- useEffect,
- useRef,
- useState,
- type ComponentProps,
- type KeyboardEvent,
- type SVGProps,
-} from "react";
-import { useTheme } from "fumadocs-ui/provider/base";
-
-/**
- * Three-icon theme switcher (light / system / dark) rendered as a radio group —
- * each icon selects its own theme, unlike fumadocs' default "light-dark"
- * switcher, which is a single blind toggle that flips on any click. Reads
- * `theme`, not `resolvedTheme`, so the "system" option can show as selected
- * (resolvedTheme collapses system to light/dark). Dropped into the sidebar
- * footer pill via `slots.themeSwitch`, so fumadocs passes the container
- * className (left divider, `ms-auto`, rounded inner buttons); we merge it onto
- * our own base.
- *
- * Icons are inlined (the project doesn't depend on `lucide-react` directly);
- * `useTheme` is re-exported by fumadocs so we avoid a bare `next-themes` import.
- */
-function SunIcon(props: SVGProps) {
- return (
-
- );
-}
-
-function MonitorIcon(props: SVGProps) {
- return (
-
- );
-}
-
-function MoonIcon(props: SVGProps) {
- return (
-
- );
-}
-
-const OPTIONS = [
- ["light", SunIcon],
- ["system", MonitorIcon],
- ["dark", MoonIcon],
-] as const;
-
-function cx(...classes: (string | false | undefined)[]): string {
- return classes.filter(Boolean).join(" ");
-}
-
-export function ThemeToggle({ className, ...props }: ComponentProps<"div">) {
- const { theme, setTheme } = useTheme();
- const [mounted, setMounted] = useState(false);
- useEffect(() => setMounted(true), []);
- const active = mounted ? theme : null;
-
- const buttonsRef = useRef<(HTMLButtonElement | null)[]>([]);
-
- // Pre-mount nothing is selected, so keep the first control tabbable.
- const selectedIndex = OPTIONS.findIndex(([key]) => key === active);
- const rovingIndex = selectedIndex === -1 ? 0 : selectedIndex;
-
- // Radio-group keyboard model: arrows move focus and pick that theme.
- function onKeyDown(event: KeyboardEvent, index: number) {
- const delta =
- event.key === "ArrowRight" || event.key === "ArrowDown"
- ? 1
- : event.key === "ArrowLeft" || event.key === "ArrowUp"
- ? -1
- : 0;
- if (delta === 0) return;
- event.preventDefault();
- const next = (index + delta + OPTIONS.length) % OPTIONS.length;
- setTheme(OPTIONS[next][0]);
- buttonsRef.current[next]?.focus();
- }
-
- return (
-
- );
-}
diff --git a/docs-site/content/docs/ai-resources/agent-instructions.mdx b/docs-site/content/docs/ai-resources/agent-instructions.mdx
new file mode 100644
index 00000000..37ee6ba9
--- /dev/null
+++ b/docs-site/content/docs/ai-resources/agent-instructions.mdx
@@ -0,0 +1,40 @@
+---
+title: Agent Instructions
+description: Suggested instructions for coding assistants that need to read and cite ktx docs.
+---
+
+Use these instructions when a coding assistant needs to answer questions from the **ktx** documentation.
+
+```text
+When answering ktx docs questions:
+
+1. Start with https://docs.kaelio.com/ktx/llms.txt.
+2. Fetch the smallest relevant Markdown page from the index.
+3. Prefer /docs/.md over rendered HTML.
+4. Use https://docs.kaelio.com/ktx/llms-full.txt only when the task needs broad docs context.
+5. Quote commands exactly from docs pages.
+6. If docs and local repository behavior disagree, say what differs and prefer local verified output for code changes.
+```
+
+## What this is for
+
+This page is for documentation consumption only:
+
+- answering questions about **ktx**
+- finding the right docs page
+- citing setup or CLI guidance
+- helping an assistant avoid stale or invented commands
+
+It does not describe local tool configuration.
+
+## Minimal project prompt
+
+```text
+You are helping with ktx. Read https://docs.kaelio.com/ktx/llms.txt first, then fetch only the Markdown pages needed for the task. Do not scrape the rendered docs site when a .md route exists.
+```
+
+## Repository prompt
+
+```text
+Before editing ktx docs, read /llms.txt and the affected .md docs pages. Keep AI Resources focused on docs consumption. After editing, verify /llms.txt, /llms-full.txt, and any changed .md routes.
+```
diff --git a/docs-site/content/docs/ai-resources/agent-quickstart.mdx b/docs-site/content/docs/ai-resources/agent-quickstart.mdx
new file mode 100644
index 00000000..6a47f56f
--- /dev/null
+++ b/docs-site/content/docs/ai-resources/agent-quickstart.mdx
@@ -0,0 +1,54 @@
+---
+title: Agent Quickstart
+description: A task-first route for coding agents that need to understand ktx docs.
+---
+
+This page is for coding assistants reading or citing the **ktx** docs. It is intentionally limited to documentation lookup, docs navigation, and safe command discovery.
+
+For Markdown endpoints, use [Markdown Access](/docs/ai-resources/markdown-access).
+For reusable task prompts, use [Prompt Recipes](/docs/ai-resources/prompt-recipes).
+To install **ktx** into an agent client, use [Agent Clients](/docs/integrations/agent-clients).
+
+## First read
+
+Agents should start with the smallest source that answers the task:
+
+1. [`/llms.txt`](/llms.txt) - discover the docs and preferred entry points.
+2. The relevant per-page Markdown URL, for example `/docs/getting-started/quickstart.md`.
+3. [`/llms-full.txt`](/llms-full.txt) - use only when the task needs broad context across many pages.
+
+## Task router
+
+| User asks the agent to explain... | Read first | Then read |
+|------------------------------------|------------|-----------|
+| What **ktx** does | [Introduction](/docs/getting-started/introduction) | [The Context Layer](/docs/concepts/the-context-layer) |
+| How to start from a checkout | [Quickstart](/docs/getting-started/quickstart) | [ktx setup](/docs/cli-reference/ktx-setup) |
+| How to check project readiness | [ktx status](/docs/cli-reference/ktx-status) | [Quickstart](/docs/getting-started/quickstart) |
+| How context gets built | [Building Context](/docs/guides/building-context) | [ktx ingest](/docs/cli-reference/ktx-ingest) |
+| How semantic YAML works | [Writing Context](/docs/guides/writing-context) | [ktx sl](/docs/cli-reference/ktx-sl) |
+| How machine-readable CLI output is shaped | [ktx sl](/docs/cli-reference/ktx-sl) | [ktx wiki](/docs/cli-reference/ktx-wiki) |
+
+## Operating workflow
+
+Use this workflow when the user asks an assistant to answer a **ktx** docs question:
+
+1. Read [`/llms.txt`](/llms.txt).
+2. Pick the smallest relevant `.md` page.
+3. Use [`/llms-full.txt`](/llms-full.txt) only if the answer needs multiple sections of the docs.
+4. Quote commands exactly from the docs page.
+5. If a command affects a local project, ask the user before assuming credentials or live services are available.
+
+## Docs lookup from a shell
+
+```bash
+curl https://docs.kaelio.com/ktx/llms.txt
+curl https://docs.kaelio.com/ktx/docs/getting-started/quickstart.md
+```
+
+## Guardrails
+
+- Do not invent CLI flags. Fetch the relevant CLI reference page.
+- Do not scrape rendered HTML when a `.md` route exists.
+- Do not assume docs lookup requires agent-client configuration.
+- Do not include credentials or secrets in prompts, URLs, or copied docs snippets.
+- When docs and local CLI behavior disagree, prefer the local CLI output and mention the mismatch.
diff --git a/docs-site/content/docs/ai-resources/markdown-access.mdx b/docs-site/content/docs/ai-resources/markdown-access.mdx
new file mode 100644
index 00000000..99fb1295
--- /dev/null
+++ b/docs-site/content/docs/ai-resources/markdown-access.mdx
@@ -0,0 +1,76 @@
+---
+title: Markdown Access
+description: Fetch ktx docs as llms.txt, llms-full.txt, or per-page Markdown.
+---
+
+**ktx** docs are available as plain Markdown so assistants do not need to parse the rendered HTML site.
+
+## Index
+
+Fetch the curated index:
+
+```text
+https://docs.kaelio.com/ktx/llms.txt
+```
+
+Use this file to discover high-value pages, task-specific entry points, and Markdown URLs.
+
+## Full corpus
+
+Fetch the complete docs corpus:
+
+```text
+https://docs.kaelio.com/ktx/llms-full.txt
+```
+
+Use this when an assistant needs broad context across setup, concepts, CLI reference, integrations, and troubleshooting. Prefer the smaller per-page Markdown route for narrow tasks.
+
+## Per-page Markdown
+
+Every docs page has a Markdown route:
+
+```text
+https://docs.kaelio.com/ktx/docs/getting-started/quickstart.md
+https://docs.kaelio.com/ktx/docs/cli-reference/ktx-sl.md
+https://docs.kaelio.com/ktx/docs/cli-reference/ktx-wiki.md
+https://docs.kaelio.com/ktx/docs/guides/building-context.md
+```
+
+Requests that ask for Markdown can also use the normal docs URL with `Accept: text/markdown`:
+
+```bash
+curl -H "Accept: text/markdown" https://docs.kaelio.com/ktx/docs/getting-started/quickstart
+```
+
+## Recommended retrieval order
+
+1. Fetch `/llms.txt`.
+2. Select one or two relevant page Markdown URLs.
+3. Fetch `/llms-full.txt` only when page-level docs are not enough.
+
+## Output contract
+
+Markdown responses are designed for agent consumption:
+
+- Frontmatter is removed.
+- Each page includes a title, description, canonical URL, and Markdown URL.
+- Code blocks stay as code blocks.
+- Tables stay as Markdown tables.
+- Missing docs pages return a plain-text `404` instead of silently falling back to HTML.
+
+## Page actions
+
+Rendered docs pages include page-level actions near the title:
+
+- **Copy MD** copies the generated Markdown for the current page.
+- **View MD** opens the generated Markdown route.
+- **Copy MDX** copies the source MDX for the current page.
+
+## Common mistakes
+
+| Mistake | Better path |
+|---------|-------------|
+| Scraping the HTML page for a docs answer | Fetch the `.md` route instead |
+| Loading `/llms-full.txt` for a single CLI flag lookup | Fetch the relevant CLI reference page |
+| Treating `/llms.txt` as complete documentation | Use it as an index, then fetch linked pages |
+| Copying rendered text by hand | Use **Copy MD** or **Copy MDX** from the page actions |
diff --git a/docs-site/content/docs/ai-resources/meta.json b/docs-site/content/docs/ai-resources/meta.json
new file mode 100644
index 00000000..a925bf37
--- /dev/null
+++ b/docs-site/content/docs/ai-resources/meta.json
@@ -0,0 +1,10 @@
+{
+ "title": "AI Resources",
+ "defaultOpen": true,
+ "pages": [
+ "agent-quickstart",
+ "markdown-access",
+ "agent-instructions",
+ "prompt-recipes"
+ ]
+}
diff --git a/docs-site/content/docs/ai-resources/prompt-recipes.mdx b/docs-site/content/docs/ai-resources/prompt-recipes.mdx
new file mode 100644
index 00000000..9ba8e3b8
--- /dev/null
+++ b/docs-site/content/docs/ai-resources/prompt-recipes.mdx
@@ -0,0 +1,55 @@
+---
+title: Prompt Recipes
+description: Copyable prompts for common ktx agent workflows.
+---
+
+Use these prompts when asking a coding assistant to work with **ktx**. Replace project names, connection ids, and business terms with your own values.
+
+## Learn the docs
+
+```text
+Read https://docs.kaelio.com/ktx/llms.txt first. Then fetch only the ktx Markdown pages needed for this task. Do not scrape rendered HTML unless no Markdown route exists.
+```
+
+## Set up a project
+
+```text
+Run npx skills add Kaelio/ktx --skill ktx and use the ktx skill to install
+and configure ktx in this project.
+```
+
+## Find a command
+
+```text
+Find the correct ktx command for this task: . Start with /llms.txt, then fetch the smallest relevant CLI reference .md page. Quote the exact command and flags from the docs.
+```
+
+## Explain setup
+
+```text
+Explain how to set up ktx for this repo. Read /docs/getting-started/quickstart.md and the relevant CLI reference pages. Summarize prerequisites, commands, generated files, and any credentials the user must provide manually.
+```
+
+## Compare concepts
+
+```text
+Explain the difference between these ktx concepts: . Start from /llms.txt, fetch the relevant concept and guide pages as Markdown, and answer with links to the source pages.
+```
+
+## Review semantic changes
+
+```text
+Review the ktx semantic-layer and knowledge changes in this branch. Check that measures have clear definitions, joins use valid keys, hidden/internal columns are not exposed to agents, and validation passes. List concrete file and line issues first.
+```
+
+## Copy exact docs source
+
+```text
+Open the relevant ktx docs page and use the page action to copy the generated Markdown or source MDX. Preserve code fences and tables exactly.
+```
+
+## Update docs
+
+```text
+Update the ktx docs for agent readability. Keep AI Resources focused on docs consumption. After editing, verify /llms.txt, /llms-full.txt, and the affected .md routes.
+```
diff --git a/docs-site/content/docs/cli-reference/ktx-admin.mdx b/docs-site/content/docs/cli-reference/ktx-admin.mdx
index 031c05ce..54ac5f84 100644
--- a/docs-site/content/docs/cli-reference/ktx-admin.mdx
+++ b/docs-site/content/docs/cli-reference/ktx-admin.mdx
@@ -48,11 +48,6 @@ directory. Use it from any directory to generate editor or agent schema files.
| `stop` | Stop the **ktx** daemon |
| `status` | Show managed Python runtime status and readiness checks |
-`install` is self-contained: **ktx** downloads its own pinned, checksum-verified
-`uv` build under the runtime root and uses it to provision Python and the
-runtime wheel. Nothing needs to be installed on `PATH` first; the host only
-needs network access to `github.com` during the first install.
-
## `admin runtime` Options
| Flag | Description | Default |
diff --git a/docs-site/content/docs/cli-reference/ktx-ingest.mdx b/docs-site/content/docs/cli-reference/ktx-ingest.mdx
index 0ab6a274..80820efa 100644
--- a/docs-site/content/docs/cli-reference/ktx-ingest.mdx
+++ b/docs-site/content/docs/cli-reference/ktx-ingest.mdx
@@ -8,7 +8,7 @@ can also capture free-form text into **ktx** memory. Database connections build
enriched context — schema plus AI-generated descriptions, embeddings, and
relationship evidence — and require a configured model and embeddings.
Context-source connections ingest metadata from tools such as dbt, Looker,
-Metabase, MetricFlow, LookML, Notion, and Sigma. Pass `--text` or `--file` to capture
+Metabase, MetricFlow, LookML, and Notion. Pass `--text` or `--file` to capture
inline text or text files into memory instead.
## Command signature
@@ -34,10 +34,8 @@ connection is selected.
| `--query-history` | Include database query-history usage patterns | Stored connection default |
| `--no-query-history` | Skip database query-history usage patterns for this run | Stored connection default |
| `--query-history-window-days ` | BigQuery/Snowflake query-history lookback window for this run | Stored connection default |
-| `--stages ` | Comma-separated enrichment stages to (re)run: `descriptions`, `embeddings`, `relationships` | All three |
| `--text ` | Capture inline text into **ktx** memory; repeatable | `[]` |
| `--file ` | Capture a text file into **ktx** memory; use `-` for stdin; repeatable | `[]` |
-| `--verbatim` | Store each `--text`/`--file` document body unchanged as a `GLOBAL` wiki page; the LLM derives metadata only | `false` |
| `--connection-id ` | **ktx** connection id to tag captured text/file notes | - |
| `--user-id ` | Memory user id for text/file capture attribution | `local-cli` |
| `--fail-fast` | Stop after the first failed text/file item | `false` |
@@ -65,65 +63,6 @@ use `--no-input` to fail fast with install guidance.
`--text` and `--file` cannot be combined with a positional `connectionId` or
`--all`; pass `--connection-id ` instead to tag captured notes.
-### Verbatim ingest
-
-By default, captured text is routed through the memory agent, which decides what
-to persist and may rewrite, condense, split, or re-title it. For *authoritative*
-documents — metric definitions, formula specs, runbooks, compliance text — that
-paraphrasing is a defect. Add `--verbatim` to store each `--text`/`--file`
-document body **unchanged** as a `GLOBAL` wiki page:
-
-- The stored body is the input document, written by code; the LLM never edits it.
- It is used only to derive page metadata (`summary`, `tags`, `sl_refs`), and even
- that is skipped for fields the document's own frontmatter already sets.
-- The page key is deterministic: a `--file` derives it from the filename, inline
- `--text` from the document's leading Markdown heading (inline text without a
- heading is rejected — pass it as `--file` instead).
-- Ingest is idempotent. Re-running the same document is a safe no-op; a different
- body at the same key fails loudly rather than overwriting.
-- `--verbatim` works with `llm.provider.backend: none` — the only ingest path that
- does. With no backend the `summary` is derived from the heading or first
- sentence and `tags`/`sl_refs` are left empty; the full body is still stored.
-- Existing frontmatter passes through untouched (including fields **ktx** does not
- model, such as `effective_date` or `version`); generated metadata only fills
- absent fields. `--connection-id ` scopes the page to that connection by
- setting its `connections` frontmatter.
-
-### Selecting enrichment stages
-
-Database enrichment runs three stages: `descriptions` (one LLM call per table),
-`embeddings` (vectors over the schema and descriptions), and `relationships`
-(join detection, optionally LLM-proposed). Each stage is cached on a **per-stage
-hash of only its own inputs**, so changing one stage's inputs invalidates only
-that stage. Switching the description LLM re-runs only `descriptions`; upgrading
-the embeddings model re-runs only `embeddings`; turning on
-`scan.relationships.llmProposals` re-runs only `relationships`. The expensive
-per-table descriptions are never thrown away because an unrelated setting moved.
-
-`--stages ` re-runs a chosen subset on an already-ingested connection. A
-named stage is **force-recomputed** (it bypasses the completed-stage cache),
-while unselected stages are left exactly as they are on disk:
-
-- `ktx ingest warehouse --stages embeddings` — re-embed on a new model, keeping
- descriptions and joins.
-- `ktx ingest --all --stages relationships --no-query-history` — backfill joins
- across every database after enabling `llmProposals`, without re-paying for
- descriptions.
-- `ktx ingest warehouse --stages descriptions` — re-run thin descriptions (for
- example after raising `KTX_ENRICH_LLM_TIMEOUT_MS`). When nothing the
- descriptions depend on changed, the per-table resume record means only the
- tables that previously failed are re-sent to the LLM.
-
-Stage names are validated: an unknown or empty name (`--stages foo`, `--stages
-descriptions,foo`, `--stages ""`) is a hard parse error. Naming all three
-(`--stages descriptions,embeddings,relationships`) forces a full enrichment
-recompute, which is **not** the same as omitting the flag (omitting resumes
-whatever is already done). After a selective run, **ktx** warns
-(`enrichment_stage_stale`) when an unselected stage's inputs no longer match what
-it was last built from — for example, re-running `descriptions` flags
-`embeddings` as stale until you re-run `--stages embeddings`. The warning is
-informational; **ktx** never silently cascades the extra work.
-
## Examples
```bash
@@ -138,11 +77,6 @@ ktx ingest warehouse --query-history
# Set the lookback window for BigQuery or Snowflake query history
ktx ingest warehouse --query-history-window-days 30
-# Re-embed one connection on a new embeddings model (descriptions/joins untouched)
-ktx ingest warehouse --stages embeddings
-# Backfill LLM-proposed joins across every database without re-describing
-ktx ingest --all --stages relationships --no-query-history
-
# Build a context-source connection
ktx ingest notion
@@ -157,12 +91,6 @@ ktx ingest --file docs/revenue-notes.md --connection-id warehouse
# Capture one stdin item
printf "Refunds are excluded from net revenue." | ktx ingest --file -
-
-# Store an authoritative document verbatim (body preserved exactly)
-ktx ingest --file docs/rfm-bucket-definitions.md --verbatim
-
-# Store it verbatim and scope it to one connection
-ktx ingest --file docs/haversine-formula.md --verbatim --connection-id warehouse
```
## Output
@@ -180,21 +108,6 @@ notion skipped skipped done done
Use `--json` when a script or agent needs the selected plan and per-target
results.
-## Final validation pruning
-
-At the end of a context-source ingest, **ktx** validates the composed semantic
-layer and wiki before saving it. If the final validation finds dangling
-references, **ktx** removes the reference instead of failing accepted work. This
-can remove joins that point at missing semantic sources, wiki `refs`, wiki
-`sl_refs`, and inline wiki body references. If a generated semantic source is
-invalid, **ktx** drops that source from the final save.
-
-The stored ingest report records these changes as `finalGatePrunedReferences`
-and `finalGateDroppedSources`. The trace emits `final_gate_reference_pruned`,
-`final_gate_source_dropped`, `final_gate_prune_committed`, and
-`final_gate_prune_finished` events when pruning runs. If validation still fails
-after pruning, the ingest fails and the report keeps the final validation error.
-
## Inspect context-source ingest traces
Context-source ingest writes persistent JSONL traces for postmortem debugging.
@@ -264,9 +177,7 @@ Slowest phase: reconciliation (2m 05s, 48% of wall time). 2 work units (1 failed
Work units run serially by default (`ingest.workUnits.maxConcurrency` is `1`);
raise it in `ktx.yaml` if the profile shows the run is bound by serialized
-work-unit agent loops. If the provider reports an LLM rate limit, **ktx** shows
-a transient wait message and temporarily reduces effective work-unit concurrency
-according to `ingest.rateLimit`.
+work-unit agent loops.
## Common errors
@@ -278,7 +189,3 @@ according to `ingest.rateLimit`.
| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command |
| Context-source options were ignored | Query-history flags were supplied for a context-source connection | Omit database-only flags when ingesting context-source connections |
| Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures |
-| `--verbatim requires --text or --file` | `--verbatim` was passed without a document to store | Add `--text` or `--file`, or drop `--verbatim` |
-| Inline verbatim text needs a leading heading | `--text --verbatim` content has no `# Heading` to derive a stable key | Add a leading Markdown heading, or pass the content as `--file ` |
-| A different page already exists at key | A verbatim re-run targeted an existing key with a different body | Use a distinct document name/key, or remove the existing page first |
-| Connection scope conflict | Frontmatter `connections` disagrees with `--connection-id` | Remove one so the intended scope is unambiguous |
diff --git a/docs-site/content/docs/cli-reference/ktx-mcp.mdx b/docs-site/content/docs/cli-reference/ktx-mcp.mdx
index 54f666f5..79c6c949 100644
--- a/docs-site/content/docs/cli-reference/ktx-mcp.mdx
+++ b/docs-site/content/docs/cli-reference/ktx-mcp.mdx
@@ -68,4 +68,3 @@ hosts and origins for browser clients.
| No **ktx** project found | Current directory has no `ktx.yaml` and `KTX_PROJECT_DIR` is unset | Run from a **ktx** project or pass `--project-dir ` |
| Non-loopback host rejected | The server needs token auth before binding beyond localhost | Pass `--token ` or set `KTX_MCP_TOKEN` |
| Client cannot connect | Host, port, token, allowed host, or allowed origin does not match the client | Check `ktx mcp status`, then restart with explicit `--host`, `--port`, `--allowed-host`, and `--allowed-origin` values |
-| A Python-backed tool reports a runtime install failure | A tool that needs the managed Python runtime (metric compute, query-history SQL analysis) ran on a host that cannot reach `github.com` to download the pinned `uv` and Python | The server still starts and serves catalog and search tools. Restore network access and retry, or pre-build the runtime where network is available: `ktx admin runtime install --yes` |
diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx
index d127525a..0e6cb57c 100644
--- a/docs-site/content/docs/cli-reference/ktx-setup.mdx
+++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx
@@ -29,18 +29,9 @@ below.
| `--agents` | Install agent configuration and rules only | `false` |
| `--target ` | Agent target: `claude-code`, `claude-desktop`, `codex`, `cursor`, `opencode`, or `universal` | - |
| `--global` | Install agent integration into the global target scope for `claude-code` or `codex` | `false` |
-| `--install-dir ` | Install project-scoped agent configuration | ktx project dir |
| `--yes` | Accept project creation and runtime install defaults where setup asks for confirmation | `false` |
| `--no-input` | Disable interactive terminal input | - |
-> **`--install-dir `**
->
-> Installs project-scoped agent configuration into the specified directory.
-> The path is resolved against the current directory and created if it doesn't
-> exist. Use it to install `.claude/`, `.mcp.json`, and rules where you open
-> your agent (for example, `--install-dir .`). This option is mutually exclusive
-> with `--global` and `--local`.
-
Use the global `--project-dir ` option when setup should target a
specific directory.
@@ -63,6 +54,7 @@ prompts.
| `--llm-backend ` | LLM backend: `anthropic`, `vertex`, `claude-code`, or `codex` |
| `--llm-backend claude-code` | Use the local Claude Code session for **ktx** LLM calls |
| `--llm-backend codex` | Use local Codex authentication for **ktx** LLM calls |
+| `--llm-model ` | LLM model ID or backend model alias to validate and save |
| `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key |
| `--anthropic-api-key-file ` | File containing the Anthropic API key |
| `--vertex-project ` | Vertex AI project ID, `env:NAME`, or `file:/path` reference |
@@ -72,16 +64,13 @@ prompts.
Choose only one Anthropic credential source. Anthropic credential flags are only
valid with the Anthropic backend; Vertex flags are only valid with the Vertex
backend. The `claude-code` and `codex` backends use local authentication instead
-of Anthropic API key or Vertex flags. After you choose a backend, `ktx setup`
-writes that backend's per-role model preset to `ktx.yaml`. To change a model,
-edit the matching `llm.models.` value in `ktx.yaml`.
-
-With `--no-input`, `ktx setup` does not assume a default LLM provider, because
-every backend needs credentials only you can supply. Pass `--llm-backend`
-explicitly. Note that `--target` selects the agent integration, not the LLM
-provider: `ktx setup --target claude-code --no-input` still needs
-`--llm-backend claude-code` to use your Claude subscription for **ktx** LLM
-calls.
+of Anthropic API key or Vertex flags. For Claude Code, `--llm-model` accepts
+`sonnet`, `opus`, `haiku`, or a full Claude model ID. For Codex, `--llm-model`
+accepts `codex`, `default`, or a `gpt-*` / `codex-*` model ID such as
+`gpt-5.5`; any other value is rejected before the auth probe. Run `codex` to
+see the models available to your login, and pick a `gpt-*` / `codex-*` id from
+that list. Note that `*-codex` API-billing model IDs (for example
+`gpt-5.3-codex`) are not available to ChatGPT-subscription logins.
### Embeddings
@@ -120,9 +109,9 @@ runtime features are missing.
| Flag | Description |
|------|-------------|
-| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `duckdb`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` |
+| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` |
| `--database-connection-id ` | Existing selected connection id; repeatable. With `--database` or `--database-url`, connection id for the new connection. |
-| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite or DuckDB path |
+| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite path |
| `--database-schema ` | Database schema or dataset to include; repeatable |
| `--skip-databases` | Leave database setup incomplete |
@@ -134,21 +123,6 @@ incomplete.
MySQL, and SQL Server; `schema_names` for Snowflake; `dataset_ids` for
BigQuery; and `databases` for ClickHouse.
-A BigQuery `--database-schema` value may be qualified as `project.dataset` to
-scan a dataset hosted in another project (such as
-`bigquery-public-data.austin_311`); a bare value stays in the credentials'
-project. Setup does not discover foreign-project datasets, so supply qualified
-entries explicitly. See
-[Primary sources → BigQuery](/docs/integrations/primary-sources#cross-project-datasets).
-
-With `--no-input`, scope for a scope-bearing driver (PostgreSQL, MySQL,
-ClickHouse, SQL Server, BigQuery, Snowflake) must come from `--database-schema`
-or from existing connection config in `ktx.yaml` (for example
-`connections..dataset_ids`). When neither is set, the database step fails
-fast and prints the missing scope flag and config key — non-interactive setup
-never auto-discovers and scans every schema. SQLite has no scope and is
-unaffected.
-
### Query History
| Flag | Description |
@@ -193,7 +167,7 @@ sources. This is equivalent to passing `--skip-sources` in scripted setup.
| Flag | Description |
|------|-------------|
-| `--source ` | Context-source connector type: `dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`, or `sigma` |
+| `--source ` | Context-source connector type: `dbt`, `metricflow`, `metabase`, `looker`, `lookml`, or `notion` |
| `--source-connection-id ` | Connection id for context-source setup |
| `--source-path ` | Local source path for dbt, MetricFlow, or LookML |
| `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML |
@@ -224,13 +198,14 @@ ktx setup
# Run setup for a specific project directory
ktx setup --project-dir ./analytics
-# Use Claude Code for ktx LLM calls
+# Use Claude Code with Opus for ktx LLM calls
ktx setup \
--project-dir ./analytics \
- --llm-backend claude-code
+ --llm-backend claude-code \
+ --llm-model opus
# Configure **ktx** to use local Codex authentication for LLM work
-ktx setup --llm-backend codex --no-input
+ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input
```
When you choose `--llm-backend codex`, setup prints a warning if the public
@@ -278,13 +253,6 @@ ktx setup \
--notion-crawl-mode selected_roots \
--notion-root-page-id abc123def456
-# Add a Sigma source
-ktx setup \
- --source sigma \
- --source-connection-id sigma-main \
- --source-client-id your-client-id \
- --source-client-secret-ref env:SIGMA_CLIENT_SECRET
-
# Install project-scoped agent integration for Codex
ktx setup --agents --target codex
```
@@ -314,7 +282,6 @@ Use `ktx status` for repeatable readiness checks after setup exits.
|-------|-------|----------|
| Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly |
| Setup cannot run in CI | Required values are missing and `--no-input` disables prompts | Provide the relevant automation flags or create a fixture `ktx.yaml` |
-| `Missing LLM backend: pass --llm-backend …` | `--no-input` setup ran without an LLM backend; `--target` does not select one | Pass `--llm-backend claude-code`, `codex`, `anthropic`, or `vertex` (with that backend's credential flags) |
| Provider health check fails | Provider key, model id, Vertex project, or Vertex location is invalid | Fix the `env:` or `file:` reference and rerun setup |
| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command |
| `--enable-query-history` is rejected | The selected database driver does not support query history | Use Postgres, BigQuery, or Snowflake, or rerun without query-history flags |
diff --git a/docs-site/content/docs/cli-reference/ktx-status.mdx b/docs-site/content/docs/cli-reference/ktx-status.mdx
index 7bf498be..66e4964c 100644
--- a/docs-site/content/docs/cli-reference/ktx-status.mdx
+++ b/docs-site/content/docs/cli-reference/ktx-status.mdx
@@ -94,6 +94,6 @@ stats, and are always shown (they do not require external communication).
|-------|-------|----------|
| No **ktx** project found | Current directory has no `ktx.yaml` and `KTX_PROJECT_DIR` is unset | `ktx status` runs setup checks; run from a **ktx** project or set `KTX_PROJECT_DIR` for project checks |
| Project config check fails | The project directory is missing or has an invalid `ktx.yaml` | Run `ktx setup` to resume setup |
-| Schema validation fails | A field **ktx** recognizes has an invalid value. Unrecognized keys are reported as non-blocking warnings (exit `0`), not failures | Run `ktx status --validate --json` for structured issue details, then edit `ktx.yaml` or rerun `ktx setup` |
+| Schema validation fails | `ktx.yaml` does not match the current config schema | Run `ktx status --validate --json` for structured issue details, then edit `ktx.yaml` or rerun `ktx setup` |
| Semantic search check warns | Embeddings are not configured or the provider probe failed | Run `ktx setup` or inspect the check's `fix` field in JSON output |
| Query history check warns | A database has query history enabled but the warehouse prerequisites are missing | Fix the warehouse extension, grants, or history access, then rerun `ktx status` |
diff --git a/docs-site/content/docs/cli-reference/ktx-wiki.mdx b/docs-site/content/docs/cli-reference/ktx-wiki.mdx
index 15461fed..7887a463 100644
--- a/docs-site/content/docs/cli-reference/ktx-wiki.mdx
+++ b/docs-site/content/docs/cli-reference/ktx-wiki.mdx
@@ -28,17 +28,10 @@ Edit the Markdown files under `wiki/` directly, or ingest source content with
| Flag | Description | Default |
|------|-------------|---------|
| `--user-id ` | Local user id | `local` |
-| `-c, --connection ` | Scope results to one connection: unscoped pages plus pages tagged with that connection | - |
| `--limit ` | Maximum search results (search mode only) | - |
| `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` |
| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` |
-`-c, --connection ` takes a connection id from the `connections` map in
-`ktx.yaml` (an unknown id is rejected). It narrows both list and search to
-pages that are not tied to any connection plus pages tagged with that
-connection, so an agent working against one database sees only the wiki
-knowledge relevant to it.
-
`ktx wiki ` uses hybrid search when `storage.search` is `sqlite-fts5`.
**ktx** combines lexical SQLite FTS5 matches, token matches, and semantic matches
from wiki page embeddings stored in `.ktx/db.sqlite`. If embeddings are not
@@ -57,12 +50,6 @@ ktx wiki --json
# Search wiki pages
ktx wiki "monthly recurring revenue"
-# List pages scoped to one connection (unscoped + connection-tagged)
-ktx wiki --connection warehouse
-
-# Search within one connection's scope
-ktx wiki "monthly recurring revenue" -c warehouse
-
# Search wiki pages as JSON
ktx wiki "monthly recurring revenue" --json --limit 10
diff --git a/docs-site/content/docs/cli-reference/ktx.mdx b/docs-site/content/docs/cli-reference/ktx.mdx
index dd02fade..8b9a2cc5 100644
--- a/docs-site/content/docs/cli-reference/ktx.mdx
+++ b/docs-site/content/docs/cli-reference/ktx.mdx
@@ -74,56 +74,6 @@ The public context-build entrypoint is `ktx ingest [connectionId]` or
| `-v`, `--version` | Show the CLI package name and version. |
| `-h`, `--help` | Show help for the current command. |
-## Update notices
-
-> **Note:** The update notifier writes only to stderr and keeps command stdout
-> unchanged.
-
-When a newer package is available on your installed release channel, `ktx`
-prints a short notice after the command finishes:
-
-```text
-↑ Update available: ktx 0.9.0 → 0.10.0
- npm i -g @kaelio/ktx
-```
-
-Stable installs compare against the npm `latest` dist-tag.
-Release-candidate installs compare against the `next` dist-tag and show:
-
-```text
-npm i -g @kaelio/ktx@next
-```
-
-The check is skipped for JSON output, CI, non-TTY stdout, and hidden completion
-commands. To opt out explicitly, set any of these environment variables:
-
-```bash
-KTX_NO_UPDATE_CHECK=1
-NO_UPDATE_NOTIFIER=1
-DO_NOT_TRACK=1
-```
-
-The `ktx` CLI prints one npm command because globally installed binaries don't
-expose a reliable runtime package-manager signal. If you prefer another global
-package manager, use the equivalent command:
-
-```bash
-pnpm add -g @kaelio/ktx
-yarn global add @kaelio/ktx
-```
-
-## Build-view star prompt
-
-During an interactive context build, `ktx setup` and `ktx ingest` can show a dim
-GitHub star reminder above the `Ctrl+C to stop` hint. **ktx** skips this prompt
-for CI, non-TTY output, and `DO_NOT_TRACK=1`.
-
-To suppress only this prompt while keeping other notices enabled, set:
-
-```bash
-KTX_NO_STAR=1
-```
-
## Project resolution
Most commands are project-aware. Pass `--project-dir ` when scripting or
diff --git a/docs-site/content/docs/community/ai-resources.mdx b/docs-site/content/docs/community/ai-resources.mdx
deleted file mode 100644
index 3bb9336d..00000000
--- a/docs-site/content/docs/community/ai-resources.mdx
+++ /dev/null
@@ -1,111 +0,0 @@
----
-title: AI Resources
-description: How coding agents read, cite, and act on the ktx docs - Markdown endpoints, a task router, and copy-paste prompts.
----
-
-This page is for coding assistants that read or cite the **ktx** docs. It covers
-the machine-readable endpoints, a task router for common questions, and
-copy-paste prompts. It is scoped to documentation lookup and safe command
-discovery - to wire **ktx** into an agent client, see
-[Agent Clients](/docs/integrations/agent-clients).
-
-## Markdown endpoints
-
-**ktx** docs are available as plain Markdown so assistants never have to parse
-the rendered HTML site.
-
-- [`/llms.txt`](/llms.txt) - a curated index of high-value pages and agent entry
- points. **Start here.**
-- [`/llms-full.txt`](/llms-full.txt) - the entire docs corpus in one response.
- Use only when a task needs broad context across many pages.
-- **Per-page Markdown** - append `.md` to any docs URL:
-
-```text
-https://docs.kaelio.com/ktx/docs/getting-started/quickstart.md
-https://docs.kaelio.com/ktx/docs/cli-reference/ktx-sl.md
-https://docs.kaelio.com/ktx/docs/guides/building-context.md
-```
-
-A request for any docs URL with an `Accept: text/markdown` header returns the
-same Markdown without the `.md` suffix:
-
-```bash
-curl -H "Accept: text/markdown" https://docs.kaelio.com/ktx/docs/getting-started/quickstart
-```
-
-Each Markdown response leads with the page title, description, canonical URL, and
-Markdown URL; frontmatter is stripped; code blocks and tables are preserved; and
-missing pages return a plain-text `404` instead of falling back to HTML. Rendered
-pages also expose a **Copy as Markdown** action near the title.
-
-### Retrieval order
-
-1. Fetch [`/llms.txt`](/llms.txt).
-2. Pick one or two relevant per-page `.md` URLs.
-3. Fetch [`/llms-full.txt`](/llms-full.txt) only when page-level docs are not
- enough.
-
-```bash
-curl https://docs.kaelio.com/ktx/llms.txt
-curl https://docs.kaelio.com/ktx/docs/getting-started/quickstart.md
-```
-
-## Task router
-
-| User asks the agent to explain... | Read first | Then read |
-|------------------------------------|------------|-----------|
-| What **ktx** does | [Introduction](/docs/getting-started/introduction) | [The Context Layer](/docs/concepts/the-context-layer) |
-| How to start from a checkout | [Quickstart](/docs/getting-started/quickstart) | [ktx setup](/docs/cli-reference/ktx-setup) |
-| How to check project readiness | [ktx status](/docs/cli-reference/ktx-status) | [Quickstart](/docs/getting-started/quickstart) |
-| How context gets built | [Building Context](/docs/guides/building-context) | [ktx ingest](/docs/cli-reference/ktx-ingest) |
-| How semantic YAML works | [Writing Context](/docs/guides/writing-context) | [ktx sl](/docs/cli-reference/ktx-sl) |
-| How machine-readable CLI output is shaped | [ktx sl](/docs/cli-reference/ktx-sl) | [ktx wiki](/docs/cli-reference/ktx-wiki) |
-
-## Agent instructions
-
-Paste this into a project or system prompt when an assistant needs to answer
-from the **ktx** docs:
-
-```text
-When answering ktx docs questions:
-
-1. Start with https://docs.kaelio.com/ktx/llms.txt.
-2. Fetch the smallest relevant Markdown page (append .md to its docs URL).
-3. Prefer the .md route over rendered HTML.
-4. Use https://docs.kaelio.com/ktx/llms-full.txt only when the task needs broad docs context.
-5. Quote commands exactly from docs pages.
-6. If docs and local CLI behavior disagree, say what differs and prefer local verified output.
-```
-
-## Prompts
-
-Replace project names, connection ids, and business terms with your own values.
-
-**Install and configure ktx in a project**
-
-```text
-Run npx skills add Kaelio/ktx --skill ktx and use the ktx skill to install and configure ktx
-```
-
-**Find the right command**
-
-```text
-Find the correct ktx command for this task: . Start with /llms.txt, then fetch the smallest relevant CLI reference .md page. Quote the exact command and flags from the docs.
-```
-
-**Review semantic changes**
-
-```text
-Review the ktx semantic-layer and wiki changes in this branch. Check that measures have clear definitions, joins use valid keys, hidden or internal columns are not exposed to agents, and validation passes. List concrete file and line issues first.
-```
-
-## Guardrails
-
-- Do not invent CLI flags - fetch the relevant CLI reference page.
-- Do not scrape rendered HTML when a `.md` route exists.
-- Do not treat `/llms.txt` as complete documentation - use it as an index, then
- fetch the linked pages.
-- Do not include credentials or secrets in prompts, URLs, or copied docs
- snippets.
-- When docs and local CLI behavior disagree, prefer the local CLI output and
- mention the mismatch.
diff --git a/docs-site/content/docs/community/meta.json b/docs-site/content/docs/community/meta.json
index fec59882..199bc1b8 100644
--- a/docs-site/content/docs/community/meta.json
+++ b/docs-site/content/docs/community/meta.json
@@ -1,5 +1,5 @@
{
- "title": "Community & Resources",
+ "title": "Community",
"defaultOpen": true,
- "pages": ["support", "contributing", "telemetry", "ai-resources"]
+ "pages": ["support", "contributing", "telemetry"]
}
diff --git a/docs-site/content/docs/community/telemetry.mdx b/docs-site/content/docs/community/telemetry.mdx
index 78bdb3e5..a3a10564 100644
--- a/docs-site/content/docs/community/telemetry.mdx
+++ b/docs-site/content/docs/community/telemetry.mdx
@@ -46,33 +46,6 @@ an operation errors, the detail we record is the error as your tools reported
it, which can include identifiers from your setup. If you'd rather send nothing
at all, turn telemetry off using any of the options above.
-## Error reports
-
-When telemetry is enabled, **ktx** sends PostHog Error Tracking `$exception`
-events for CLI and daemon exceptions. Error reports help group crashes and
-handled failures into PostHog issues.
-
-Error reports can include:
-
-- Stack frames, including function names, local file paths, line numbers, and
- SDK-provided source context.
-- Error class names and raw error messages.
-- Cause chains when the runtime exposes them.
-- `source`, `handled`, and `fatal` diagnostic fields.
-- Runtime version, OS, architecture, and CI fields.
-- The hashed `projectId` when **ktx** knows the project.
-
-Error reports never intentionally include:
-
-- Secrets, credentials, API keys, tokens, cookies, signed URLs, or auth headers.
-- Database URLs, connection strings, DSNs, raw argv, or raw environment values.
-- SQL text, schema names, table names, or column names as explicit payload
- properties.
-- Customer row data.
-- User prompt text or raw MCP arguments.
-
-The same opt-out controls listed above disable error reports.
-
## Storage and retention
Telemetry is sent to PostHog, a third-party product-analytics service used by
diff --git a/docs-site/content/docs/concepts/cross-database-federation.mdx b/docs-site/content/docs/concepts/cross-database-federation.mdx
deleted file mode 100644
index 78717ff4..00000000
--- a/docs-site/content/docs/concepts/cross-database-federation.mdx
+++ /dev/null
@@ -1,161 +0,0 @@
----
-title: Cross-database federation
-description: How ktx federates postgres, mysql, sqlite, and duckdb connections so a single read-only SQL query can join across them without copying data.
----
-
-Cross-database federation lets a single read-only SQL query join tables that
-live in different databases. **ktx** achieves this by embedding DuckDB and
-using its `ATTACH` mechanism to connect each member database read-only. The
-join executes inside DuckDB at query time — live data, no ETL, no copy.
-
-You run federated queries as raw SQL against the `_ktx_federated` connection
-(see [Querying the federated connection
-directly](#querying-the-federated-connection-directly)). Semantic-layer queries
-(`ktx sl query` / the `sl_query` tool) stay per-connection; pointing one at
-`_ktx_federated` returns an error telling you to use raw SQL instead.
-
-Federation activates automatically when a `ktx.yaml` file declares two or more
-attach-compatible connections. There is nothing to configure and no federation
-block to add. With zero or one compatible connection the behavior is unchanged.
-
-## Which connections participate
-
-The v1 federation engine supports four drivers:
-
-| Driver | Participates in federation |
-|--------|---------------------------|
-| `postgres` | Yes |
-| `mysql` | Yes |
-| `sqlite` | Yes |
-| `duckdb` | Yes |
-| `snowflake` | No — standalone connection |
-| `bigquery` | No — standalone connection |
-| `clickhouse` | No — standalone connection |
-| `sqlserver` | No — standalone connection |
-
-Non-participating connections continue to work exactly as they did. They are
-queried independently; they do not appear as federation members.
-
-## How it activates
-
-**ktx** inspects the connections in `ktx.yaml` at startup. When it finds two or
-more connections whose driver is `postgres`, `mysql`, `sqlite`, or `duckdb`, it
-instantiates the DuckDB federation engine and attaches each one read-only.
-There is no `federation:` key, no opt-in flag, and no connection-level setting
-to enable. The engine is derived entirely from what is already declared.
-
-A minimal `ktx.yaml` that triggers federation:
-
-```yaml
-connections:
- pg_books:
- driver: postgres
- url: "postgres://user:pass@localhost:5432/books" # pragma: allowlist secret
- sqlite_reviews:
- driver: sqlite
- path: ./data/reviews.db
-```
-
-Two attach-compatible connections are present, so federation is active.
-
-## Table naming in federated queries
-
-Inside a federated query, postgres and mysql tables use a three-part name:
-`connectionId.schema.table`. SQLite and DuckDB tables use the two-part form
-`connectionId.table`, since ktx addresses both as single-namespace members. In
-both cases the connection's `id` field in `ktx.yaml` becomes the catalog name
-inside DuckDB.
-
-If a connection `id` is not a bare SQL identifier — for example it contains a
-hyphen, like `books-db` — double-quote it in the query the same way DuckDB
-quotes any identifier: `"books-db".public.books`. Writing it unquoted
-(`books-db.public.books`) is a SQL syntax error, not a federation feature.
-
-For the example above:
-
-- `pg_books.public.books` — the `books` table in the `public` schema of the
- postgres connection
-- `sqlite_reviews.reviews` — the `reviews` table in the sqlite connection
-
-These fully qualified names are what you write when you query the federated
-connection with raw SQL (see [Querying the federated connection
-directly](#querying-the-federated-connection-directly)). A source file's own
-`table:` field is not prefixed this way — see [Source files keep member-native
-table refs](#source-files-keep-member-native-table-refs) below.
-
-## Source names in the federated view
-
-When you list or search semantic-layer sources under the federated connection,
-each source's `name` is prefixed with its member connection id — for example
-`pg_books.books` and `sqlite_reviews.reviews`. The prefix keeps names unique
-when two members own a table with the same name: a `users` table in each of
-`pg_app` and `sqlite_app` surfaces as `pg_app.users` and `sqlite_app.users`
-rather than colliding on a bare `users`.
-
-## Source files keep member-native table refs
-
-A source file's physical `table:` field is not prefixed with the connection id.
-It stays the member-native reference the connector uses on its own —
-`public.books` for the postgres member, `reviews` for the sqlite member —
-because the same file backs a per-connection semantic-layer query against that
-member, which runs on the member's own driver where a `pg_books.` catalog prefix
-would point at a database that does not exist. The connection-id prefix is a
-DuckDB catalog name that appears only in raw federated SQL; the member prefix on
-the source `name` (above) is independent of it.
-
-## Cross-database joins
-
-Write a cross-database join as raw SQL against `_ktx_federated` — see
-[Querying the federated connection
-directly](#querying-the-federated-connection-directly) below for a runnable
-example. DuckDB attaches both members and resolves the join live at query time.
-
-Declaring the join in a source file's `joins:` block is not supported yet. The
-semantic layer plans each connection on its own, so a `joins:` entry whose `to:`
-points at a table in another member is not resolved across the federation
-boundary. Until that lands, express cross-database joins as raw SQL.
-
-## Querying the federated connection directly
-
-The federated connection is addressable by its id,
-`_ktx_federated`, anywhere **ktx** runs read-only SQL. The same id works for the
-`ktx sql` command and for a data agent calling the `sql_execution` MCP tool, so
-both surfaces can run a cross-database query without a source file:
-
-```bash
-ktx sql -c _ktx_federated \
- "SELECT b.title, avg(r.rating) AS avg_rating
- FROM pg_books.public.books b
- JOIN sqlite_reviews.reviews r ON b.id = r.book_id
- GROUP BY b.title"
-```
-
-Table names follow the rules from
-[Table naming in federated queries](#table-naming-in-federated-queries):
-three-part `connectionId.schema.table` for postgres and mysql, two-part
-`connectionId.table` for sqlite and duckdb. The `_ktx_federated` id is virtual —
-it is never written to `ktx.yaml` and only exists when two or more attach-compatible
-connections are declared. It surfaces in `ktx connection` and in the agent's
-connection list so the id is discoverable. Querying a single member database
-directly with its own connection id (`ktx sql -c pg_books ...`) is unchanged.
-
-If any member connection sets
-[`query_policy: semantic-layer-only`](/docs/configuration/ktx-yaml#query-policy),
-raw SQL against `_ktx_federated` is rejected as a whole: a federated query can
-touch any member's tables, so one restricted member restricts the federation.
-
-## Federated queries are read-only
-
-DuckDB attaches every member database with read-only access. Federated queries
-are `SELECT`/`WITH` only. No writes, no DDL, and no mutations reach any member
-database through the federation engine.
-
-## Current limitations
-
-- **Raw SQL joins only.** Cross-database joins are written as raw SQL; declaring
- them in a source's `joins:` block and automatic discovery of cross-database
- relationships are not available yet. Intra-database relationship discovery for
- each member connection is unchanged.
-- **postgres, mysql, sqlite, and duckdb only.** Other drivers (snowflake,
- bigquery, clickhouse, sqlserver) do not participate in federation in this
- version. They remain usable as standalone connections.
diff --git a/docs-site/content/docs/concepts/meta.json b/docs-site/content/docs/concepts/meta.json
index 3936328a..bf4de9d6 100644
--- a/docs-site/content/docs/concepts/meta.json
+++ b/docs-site/content/docs/concepts/meta.json
@@ -1,5 +1,5 @@
{
"title": "Concepts",
"defaultOpen": true,
- "pages": ["the-context-layer", "semantic-layer-internals", "cross-database-federation", "wiki-retrieval"]
+ "pages": ["the-context-layer", "semantic-layer-internals", "wiki-retrieval"]
}
diff --git a/docs-site/content/docs/configuration/ktx-yaml.mdx b/docs-site/content/docs/configuration/ktx-yaml.mdx
index 05ebea3e..17a04c53 100644
--- a/docs-site/content/docs/configuration/ktx-yaml.mdx
+++ b/docs-site/content/docs/configuration/ktx-yaml.mdx
@@ -46,7 +46,7 @@ read, how to think, and where to put the results.
llm - provider, models, prompt cache
-
ingest - connectors, embeddings, work units
+
ingest - adapters, embeddings, work units
scan - enrichment, relationships
agent - research-agent feature flags
@@ -109,7 +109,6 @@ context-source drivers share the map.
| `postgres` | Warehouse | `driver` | `url`, `enabled_tables`, `historicSql`, `context.queryHistory` |
| `mysql` | Warehouse | `driver` | `url`, `enabled_tables` |
| `sqlite` | Warehouse | `driver` | `url` or `path`, `enabled_tables` |
-| `duckdb` | Warehouse | `driver` | `url` or `path`, `enabled_tables` |
| `sqlserver` | Warehouse | `driver` | `url`, `enabled_tables` |
| `bigquery` | Warehouse | `driver` | `credentials_json`, `dataset_ids`, `enabled_tables`, `historicSql` |
| `snowflake` | Warehouse | `driver` | `schema_names`, `enabled_tables`, `historicSql` |
@@ -120,16 +119,13 @@ context-source drivers share the map.
| `dbt` | Context source | `driver`, one of `source_dir` or `repo_url` | `branch`, `path`, `profiles_path`, `target`, `project_name` |
| `metricflow` | Context source | `driver`, `metricflow.repoUrl` | `metricflow.branch`, `metricflow.path`, `metricflow.auth_token_ref` |
| `notion` | Context source | `driver`, `auth_token_ref` | `crawl_mode`, `root_*_ids`, `max_*_per_run` |
-| `sigma` | Context source | `driver`, `client_id`, `client_secret_ref` | `api_url` |
### Warehouse drivers
Warehouse connections are open objects: the listed fields are validated, and
any other field is preserved and passed through to the connector. Use
-`enabled_tables` to scope ingest to a specific list of objects - useful for
-smoke tests. Each entry accepts a `catalog.db.name`, `db.name`, or bare `name`
-qualifier. ktx restricts the scan to the listed objects and fails with a clear
-error (naming the available objects) if none match.
+`enabled_tables` to scope ingest to a specific list of
+`schema.table` names - useful for smoke tests.
```yaml
connections:
@@ -141,18 +137,6 @@ connections:
- public.customers
```
-For SQLite, which exposes a single `main` schema, the qualified `main.`
-and the bare `` forms select the same object:
-
-```yaml
-connections:
- local-db:
- driver: sqlite
- path: ./warehouse.db
- enabled_tables:
- - customers # equivalent to main.customers
-```
-
Connector-specific scope fields let setup and scan use the same warehouse
boundary:
@@ -174,12 +158,6 @@ connections:
dataset_ids: [analytics, mart]
```
-A BigQuery `dataset_ids` / `dataset_id` entry may be written `project.dataset`
-to introspect a dataset hosted in another project (for example
-`bigquery-public-data.austin_311`); jobs still bill to the `project_id` in
-`credentials_json`. A bare `dataset` keeps using your own project. See
-[Primary sources → BigQuery](/docs/integrations/primary-sources#cross-project-datasets).
-
For Postgres, MySQL, SQL Server, and Snowflake connections, set
`maxConnections` when scan or ingest work needs to stay below the target's
connection cap. Postgres, MySQL, and SQL Server default to `10`; Snowflake
@@ -217,41 +195,6 @@ connections:
observed in-scope query history. The block uses `mode: exclude` and remains
hand-editable.
-### Query policy
-
-Set `query_policy: semantic-layer-only` on a warehouse connection to stop
-agents from authoring SQL against it. The default, `read-only-sql`, allows
-parser-validated read-only SQL through `ktx sql` and the `sql_execution` MCP
-tool alongside semantic-layer queries.
-
-```yaml
-connections:
- warehouse:
- driver: snowflake
- query_policy: semantic-layer-only
-```
-
-With `semantic-layer-only`:
-
-- `ktx sql` and the `sql_execution` MCP tool reject the connection with a
- clear error. When every SQL connection in the project is restricted, the
- `sql_execution` tool is not registered at all.
-- Raw SQL against the federated connection (`_ktx_federated`) is rejected
- when any member connection is restricted.
-- Semantic-layer queries (`ktx sl query`, the `sl_query` tool) accept only
- measures predefined in the semantic-layer sources. Composed aggregate
- expressions such as `sum(orders.amount)` are rejected wherever they appear,
- including inside `filters` (a `HAVING`-style clause may only compare a
- predefined measure by name, e.g. `orders.revenue > 100`). Grouping by
- declared dimensions, filtering on columns, and segments remain available.
-- `connection_list` marks the connection as restricted so agents route to
- `sl_query` instead of burning a failed call.
-
-The policy governs agent-facing query authorship, not data access: **ktx**'s
-own scan, ingest, and semantic-layer-generated SQL still run, and context
-tools such as `entity_details` and `dictionary_search` still expose schema
-metadata and sampled values.
-
### Metabase
```yaml
@@ -382,31 +325,6 @@ connections:
| `max_knowledge_creates_per_run` | Max new wiki pages created per run (0-25). |
| `max_knowledge_updates_per_run` | Max existing wiki pages updated per run (0-100). |
-### Sigma
-
-```yaml
-connections:
- sigma-main:
- driver: sigma
- api_url: https://api.sigmacomputing.com
- client_id: ""
- client_secret_ref: env:SIGMA_CLIENT_SECRET
- workbookFilter:
- includeArchived: false
- includeExplorations: false
- updatedSince: "2026-01-01T00:00:00Z"
-```
-
-| Field | Purpose |
-|-------|---------|
-| `api_url` | Sigma API base URL. Defaults to `https://api.sigmacomputing.com` (GCP US). Override for AWS US (`https://aws-api.sigmacomputing.com`) or other regions. |
-| `client_id` | Sigma OAuth client ID. Required. |
-| `client_secret` / `client_secret_ref` | Literal secret or reference. Prefer the `_ref`. |
-| `connectionMappings` | Maps Sigma internal connection UUIDs to **ktx** warehouse connection IDs. Enables `sl_validate` for projected semantic-layer sources. |
-| `workbookFilter.includeArchived` | Include archived workbooks during ingest. Default: `false`. |
-| `workbookFilter.includeExplorations` | Include exploration workbooks during ingest. Default: `false`. |
-| `workbookFilter.updatedSince` | ISO 8601 date string. Only workbooks updated on or after this date are fetched. Useful for limiting ingest scope at large scale. |
-
## `setup`
Captured by the setup wizard. The only field **ktx** still reads is
@@ -426,14 +344,15 @@ setup:
## `storage`
-`storage` controls where **ktx** keeps its own state and search index. Defaults
-work for a single-user local project.
+`storage` controls where **ktx** keeps its own state and search index, and how
+state changes are committed. Defaults work for a single-user local project.
```yaml
storage:
state: sqlite # sqlite | postgres
search: sqlite-fts5 # sqlite-fts5 | postgres-hybrid
git:
+ auto_commit: true
author: "ktx "
```
@@ -441,7 +360,8 @@ storage:
|-------|------|---------|---------|
| `state` | `sqlite` \| `postgres` | `sqlite` | Backend for ktx state. `sqlite` uses `.ktx/db.sqlite`; `postgres` expects a configured Postgres connection. |
| `search` | `sqlite-fts5` \| `postgres-hybrid` | `sqlite-fts5` | Backend for search indexes. `postgres-hybrid` combines lexical and vector search in Postgres. |
-| `git.author` | `string` | `ktx ` | Git author identity for commits. Standard `Name ` form. |
+| `git.auto_commit` | `boolean` | `true` | When `true`, ktx auto-commits changes to the git-backed state store. |
+| `git.author` | `string` | `ktx ` | Git author identity for auto-commits. Standard `Name ` form. |
## `llm`
@@ -457,10 +377,6 @@ llm:
models:
default: claude-sonnet-4-6
triage: claude-haiku-4-5
- candidateExtraction: claude-sonnet-4-6
- curator: claude-opus-4-7
- reconcile: claude-opus-4-7
- repair: claude-haiku-4-5
promptCaching:
enabled: true
systemTtl: 1h
@@ -488,11 +404,6 @@ llm:
backend: codex
models:
default: gpt-5.5
- triage: gpt-5.5
- candidateExtraction: gpt-5.5
- curator: gpt-5.5
- reconcile: gpt-5.5
- repair: gpt-5.5
```
### Model roles
@@ -522,7 +433,7 @@ provider-specific model identifiers.
## `ingest`
`ingest` controls how **ktx** builds context from your stack. It lists the
-connectors to run, the embedding provider used when connectors embed documents,
+adapters to run, the embedding provider used when adapters embed documents,
and the concurrency and failure policy for work units.
```yaml
@@ -541,24 +452,14 @@ ingest:
stepBudget: 40
maxConcurrency: 2
failureMode: continue
- rateLimit:
- enabled: true
- throttleThreshold: 0.8
- minConcurrencyUnderPressure: 1
- maxWaitMs: 600000
- retry:
- maxAttempts: 6
- baseDelayMs: 1000
- maxDelayMs: 60000
- jitter: true
```
-### Connectors
+### Adapters
-`adapters` is a list of connector IDs that should run. Each ID matches a
+`adapters` is a list of adapter IDs that should run. Each ID matches a
connector that **ktx** ships locally:
-| Connector ID | What it ingests |
+| Adapter ID | What it ingests |
|------------|-----------------|
| `live-database` | Live warehouse introspection (schemas, tables, columns, samples). |
| `historic-sql` | Query history from Postgres `pg_stat_statements`, BigQuery `INFORMATION_SCHEMA.JOBS`, or Snowflake query history. |
@@ -568,7 +469,7 @@ connector that **ktx** ships locally:
| `looker` | Looker dashboards and looks via the API. |
| `metabase` | Metabase cards, dashboards, and database mappings. |
| `notion` | Notion pages and databases for wiki context. |
-| `fake` | Test/demo connector. Useful in fixtures. |
+| `fake` | Test/demo adapter. Useful in fixtures. |
### Embeddings
@@ -597,24 +498,6 @@ handles failures.
| `workUnits.maxConcurrency` | `int > 0` | `1` | How many work units run in parallel. |
| `workUnits.failureMode` | `abort` \| `continue` | `continue` | `abort` stops the whole ingest run on the first failure; `continue` records it and keeps going. |
-### Rate limits
-
-`rateLimit` controls provider-neutral pacing for LLM calls during ingest. When a
-provider reports a subscription window, retry-after delay, or HTTP 429,
-**ktx** pauses new work-unit model calls, shows a transient wait in the CLI,
-and reduces work-unit concurrency while the provider is under pressure.
-
-| Field | Type | Default | Purpose |
-|-------|------|---------|---------|
-| `rateLimit.enabled` | `boolean` | `true` | Master switch for ingest LLM rate-limit pacing and visible waits. |
-| `rateLimit.throttleThreshold` | `number between 0 and 1` | `0.8` | Fraction of a known provider window at which **ktx** starts reducing concurrency. |
-| `rateLimit.minConcurrencyUnderPressure` | `int > 0` | `1` | Effective work-unit concurrency while a provider is under rate-limit pressure. |
-| `rateLimit.maxWaitMs` | `int > 0` | unset | Caps how long a single provider-reset wait can last. This bounds each wait, not the whole run: after a capped wait elapses **ktx** retries and may pause again. Omit to wait until the provider's reset time. |
-| `rateLimit.retry.maxAttempts` | `int > 0` | `6` | Maximum attempts for a single rate-limited LLM call before the failure surfaces (counts the first try). Also bounds how far opaque backoff grows for responses without a reset time or retry-after value. |
-| `rateLimit.retry.baseDelayMs` | `int > 0` | `1000` | Initial opaque retry delay in milliseconds. |
-| `rateLimit.retry.maxDelayMs` | `int > 0` | `60000` | Maximum opaque retry delay in milliseconds. |
-| `rateLimit.retry.jitter` | `boolean` | `true` | Add jitter to opaque retry delays. |
-
## `scan`
`scan` configures how schema-level inputs become structured context:
@@ -636,7 +519,6 @@ scan:
profileConcurrency: 4
validationConcurrency: 4
validationBudget: all
- detectionBudgetMs: 600000
```
### Enrichment
@@ -665,7 +547,6 @@ the manifest.
| `relationships.profileConcurrency` | `int > 0` | `4` | Parallel relationship-profile queries against the database. For pooled connectors, effective database concurrency is also bounded by the connection's `maxConnections`. |
| `relationships.validationConcurrency` | `int > 0` | `4` | Parallel relationship validation queries against the database. |
| `relationships.validationBudget` | `all` \| `int ≥ 0` | runtime default | Cap on validation queries per scan. `all` means unlimited. |
-| `relationships.detectionBudgetMs` | `int > 0` | `600000` | Wall-clock budget (ms) for the whole relationship-detection stage, checked at table-profile, candidate-validation, and composite-probe boundaries. On exhaustion the stage stops scheduling new work and writes the joins found so far, marked partial; descriptions and embeddings are already durable. Sits above the per-query deadline. Raise it to trigger a fresher, fuller run. |
## `agent`
@@ -690,6 +571,19 @@ agent:
| `run_research.max_iterations` | `int ≥ 0` | `20` | Maximum tool-call iterations per research run. |
| `run_research.default_toolset` | `string[]` | `[sl_query, wiki_search, sl_read_source]` | Tool identifiers exposed to the research agent. |
+## `memory`
+
+`memory` controls the agent memory subsystem.
+
+```yaml
+memory:
+ auto_commit: true
+```
+
+| Field | Type | Default | Purpose |
+|-------|------|---------|---------|
+| `auto_commit` | `boolean` | `true` | When `true`, ktx auto-commits memory updates to the git-backed store. |
+
## A full example
Combining the blocks above:
@@ -714,17 +608,13 @@ storage:
state: sqlite
search: sqlite-fts5
git:
+ auto_commit: true
author: "ktx "
llm:
provider:
backend: claude-code
models:
default: sonnet
- triage: haiku
- candidateExtraction: sonnet
- curator: opus
- reconcile: opus
- repair: haiku
ingest:
adapters:
- live-database
@@ -746,25 +636,17 @@ scan:
agent:
run_research:
enabled: true
+memory:
+ auto_commit: true
```
## Validating your config
-**ktx** validates `ktx.yaml` when it loads, and treats two kinds of problems
-differently:
-
-- **An invalid value on a field ktx recognizes** (for example
- `llm.provider.backend: nope`) is a hard error. Setup and CLI commands stop and
- report the exact path so you can fix it.
-- **An unrecognized key** — one left over from a different **ktx** version, or a
- typo such as `scan.relationships.acceptThreshhold` — is tolerated, not fatal.
- **ktx** ignores the key and keeps running, so a misspelled field quietly falls
- back to its default instead of taking effect. `ktx status` lists each ignored
- key as a warning (and exits `0`) so you can remove or correct it when
- convenient.
-
-Warehouse connections accept extra driver-specific fields, so passthrough values
-like `historicSql` and `context.queryHistory` are allowed.
+**ktx** validates `ktx.yaml` strictly: unknown keys at the top level or inside
+strict blocks cause setup and CLI commands to fail with a precise path
+(`scan.relationships.acceptThreshhold: Unrecognized key`). Warehouse
+connections accept extra driver-specific fields, so passthrough values like
+`historicSql` and `context.queryHistory` are allowed.
To re-validate without running anything else:
diff --git a/docs-site/content/docs/getting-started/introduction.mdx b/docs-site/content/docs/getting-started/introduction.mdx
index 283a023a..cc3b0ca8 100644
--- a/docs-site/content/docs/getting-started/introduction.mdx
+++ b/docs-site/content/docs/getting-started/introduction.mdx
@@ -4,7 +4,6 @@ description: ktx is an open-source, self-improving context layer for data agents
---
import { ProductMechanics } from "@/components/product-mechanics";
-import { ProductRuntime } from "@/components/product-runtime";
@@ -24,7 +23,7 @@ import { ProductRuntime } from "@/components/product-runtime";
>
Make analytics context usable by agents
-
+
{'ktx is an open-source context layer for data agents. It turns warehouse metadata, BI tool definitions, query history, docs, and approved metric definitions into reviewable files agents can search and execute.'}
@@ -60,8 +59,6 @@ serves that context to agents at runtime.
-
-
## Use it for
Use **ktx** when agents need more than raw database access. Agents can search wiki
@@ -95,8 +92,8 @@ best first step for users; contributor setup lives in the community docs.
Complete flag and subcommand reference for every **ktx** command.
-
- Machine-readable docs, a task router, and copy-paste agent prompts.
+
+ Machine-readable docs and agent-facing setup notes.
diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx
index 16669794..abd6044d 100644
--- a/docs-site/content/docs/getting-started/quickstart.mdx
+++ b/docs-site/content/docs/getting-started/quickstart.mdx
@@ -9,8 +9,8 @@ This guide takes a local analytics project from empty to agent-ready. You'll
install the CLI, run one guided setup command, and hand the context to a
coding assistant.
-If you're a coding assistant choosing a docs route, start with
-[AI Resources](/docs/community/ai-resources) instead.
+If you're a coding assistant choosing a docs route, start with the
+[Agent Quickstart](/docs/ai-resources/agent-quickstart) instead.
` to see the error, fix the connection, then retry.
```
@@ -339,16 +332,6 @@ separate `ktx` binary on `PATH`. If the CLI path changes, rerun
## What setup writes
**ktx** writes plain files so people and agents can review changes in git.
-**ktx** initializes a git repository at the project directory and writes context
-changes there. If the project directory is nested inside another repository,
-**ktx** still keeps its own repo and does not commit to the parent repo.
-
-Because **ktx** owns that repository, it will not adopt one it did not create. If
-you point setup at a directory that is already a git repository's root - such as
-an existing application checkout - **ktx** stops and asks you to pick a dedicated
-directory instead. In the setup wizard choose the **New subfolder** option (for
-example `ktx-project`), or pass a fresh `--project-dir` when running setup
-non-interactively.
| Path | Purpose |
|------|---------|
diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx
index f494abf7..9bcf2659 100644
--- a/docs-site/content/docs/guides/building-context.mdx
+++ b/docs-site/content/docs/guides/building-context.mdx
@@ -43,7 +43,7 @@ Local-auth backends keep provider credentials out of `ktx.yaml`:
```bash
ktx setup --llm-backend claude-code --no-input
-ktx setup --llm-backend codex --no-input
+ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input
```
With `claude-code`, **ktx** agent loops can invoke only the **ktx** MCP tools
@@ -102,7 +102,6 @@ Supported source types:
| `looker` | Looker API | Explores, looks, dashboards, and model metadata |
| `metabase` | Metabase API | Questions, dashboards, table metadata, and mappings |
| `notion` | Notion API | Wiki pages and business knowledge |
-| `sigma` | Sigma API | Data model specs, pages, element metadata, and workbook metadata |
Context-source ingest writes semantic source YAML and wiki Markdown, reconciling
with local edits.
diff --git a/docs-site/content/docs/guides/llm-configuration.mdx b/docs-site/content/docs/guides/llm-configuration.mdx
index 776cb275..71ab9d80 100644
--- a/docs-site/content/docs/guides/llm-configuration.mdx
+++ b/docs-site/content/docs/guides/llm-configuration.mdx
@@ -30,19 +30,19 @@ llm:
default: sonnet
triage: haiku
candidateExtraction: sonnet
- curator: opus
- reconcile: opus
- repair: haiku
+ curator: sonnet
+ reconcile: sonnet
+ repair: sonnet
```
-During setup, choose the backend interactively or pass it in automation:
+During setup, choose the backend interactively or pass the model in automation:
```bash
-ktx setup --llm-backend claude-code --no-input
+ktx setup --llm-backend claude-code --llm-model opus --no-input
```
-Setup writes `sonnet`, `haiku`, and `opus` aliases into `llm.models`. You can
-edit any role to another alias or a full Claude model ID after setup.
+For Claude Code, `sonnet`, `opus`, and `haiku` map to **ktx** defaults. Full Claude
+model IDs are also accepted.
`claude-code` exposes only **ktx** MCP tools for the current agent loop. SDK init
metadata may still list host slash commands, skills, and subagents; **ktx** does not
@@ -59,17 +59,12 @@ llm:
backend: codex
models:
default: gpt-5.5
- triage: gpt-5.5
- candidateExtraction: gpt-5.5
- curator: gpt-5.5
- reconcile: gpt-5.5
- repair: gpt-5.5
```
Configure it non-interactively:
```bash
-ktx setup --llm-backend codex --no-input
+ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input
```
This is separate from Codex agent-client setup. `ktx setup --agents --target
diff --git a/docs-site/content/docs/guides/reviewing-context.mdx b/docs-site/content/docs/guides/reviewing-context.mdx
index 26f191b6..63d4fceb 100644
--- a/docs-site/content/docs/guides/reviewing-context.mdx
+++ b/docs-site/content/docs/guides/reviewing-context.mdx
@@ -61,14 +61,11 @@ committing the file.
## A typical review session
-The loop above describes the shape. Run these commands from the **ktx** project
-directory. **ktx** keeps that directory as its own git repository, even when the
-directory lives inside another repository, so reviewing context changes never
-requires committing to a parent application repo.
+The loop above describes the shape. In practice, one review session looks like
+this:
```bash
# 1. Run ingest on a branch
-cd /path/to/ktx-project
git checkout -b ingest/2026-05-21
ktx ingest --all
diff --git a/docs-site/content/docs/guides/writing-context.mdx b/docs-site/content/docs/guides/writing-context.mdx
index 33cb90e1..e4ac70c0 100644
--- a/docs-site/content/docs/guides/writing-context.mdx
+++ b/docs-site/content/docs/guides/writing-context.mdx
@@ -44,17 +44,12 @@ Use this order for most context changes:
Semantic sources are YAML files for queryable tables or custom SQL. They define
agent-facing measures, dimensions, segments, joins, and grain.
-Semantic source files live under:
+Semantic source files live at:
```text
-semantic-layer//
+semantic-layer//.yaml
```
-The file's `name:` field is the source's identity — it carries the warehouse
-identifier verbatim, including case. The filename is a derived label: simple
-lowercase names get `.yaml`, anything else gets a slugged
-filename. Renaming a file does not rename the source.
-
### Minimal source
```yaml
@@ -157,7 +152,7 @@ joins:
| Field | Required | Description |
|-------|----------|-------------|
-| `name` | Yes | Source identity (not the filename). When overlaying an ingested table, match the manifest identifier verbatim, including case (e.g. `SIGNED_UP`); for a new standalone source, lowercase words and underscores are recommended. |
+| `name` | Yes | Source identifier. Use lowercase words and underscores. |
| `descriptions` | No | Description map keyed by source, such as `user`, `dbt`, or `ai`. |
| `table` or `sql` | Yes | Database table or custom SQL expression. Use exactly one. |
| `grain` | Yes | Columns that uniquely identify a row at the source grain. |
@@ -321,23 +316,6 @@ Useful frontmatter:
5. Add `sl_refs` for relevant semantic sources.
6. Search again with a user-like phrase.
-### Ingest an authoritative document verbatim
-
-When the document is already the source of truth — a metric-definition sheet, a
-formula spec, a runbook, compliance text — you want **ktx** to index and surface
-it, not re-author it. Instead of hand-copying the file into `wiki/global/`, ingest
-it verbatim:
-
-```bash
-ktx ingest --file docs/rfm-bucket-definitions.md --verbatim
-```
-
-The body is stored byte-for-byte (the LLM only derives `summary`, `tags`, and
-`sl_refs` for the absent frontmatter fields), the page key is derived from the
-filename, and re-running is a safe no-op. Existing frontmatter — including fields
-**ktx** does not model, like `effective_date` — passes through unchanged. See
-[`ktx ingest`](/docs/cli-reference/ktx-ingest) for the full flag reference.
-
## Review context changes
Before accepting agent-written context:
diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx
index 1ef75d22..46a1ec8b 100644
--- a/docs-site/content/docs/integrations/agent-clients.mdx
+++ b/docs-site/content/docs/integrations/agent-clients.mdx
@@ -68,30 +68,19 @@ If you choose an install mode, it then asks which targets to install:
└
```
-When at least one selected target supports project-scoped setup, the command
-asks where to install agent config:
+When every selected target supports both project and global setup, the command
+also asks where to install supported agent config:
```txt
-◆ Where should ktx install agent config?
+◆ Where should ktx install supported agent config?
│
│ ktx project: /path/to/your/ktx-project
│
-│ ○ ktx project directory /path/to/your/ktx-project
-│ ○ Current directory /path/to/where/you/ran/ktx
-│ ○ Custom directory… (enter a path)
+│ ○ Project scope (ktx project directory)
│ ○ Global scope (user config)
└
```
-The first three choices write project-scoped files (`.claude/`, `.mcp.json`,
-`.cursor/`, skills, and rules) into the chosen directory while still pointing
-them at this ktx project. Use **Current directory** or **Custom directory…**
-when you open your coding agent from somewhere other than the ktx project
-directory. **Current directory** is hidden when it is already the ktx project
-directory, and **Global scope** appears only when every selected target
-supports global setup. Non-interactive runs pass `--install-dir ` (for
-example `--install-dir .`) for the same result.
-
## Generated files
**ktx** writes MCP client configuration and analytics guidance by default. It writes
diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx
index 3bfc3ff3..213f5a3d 100644
--- a/docs-site/content/docs/integrations/context-sources.mdx
+++ b/docs-site/content/docs/integrations/context-sources.mdx
@@ -1,6 +1,6 @@
---
title: Context Sources
-description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, Notion, Sigma, and Google Drive.
+description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, and Notion.
---
Context sources feed your existing analytics tooling into **ktx**. During ingestion, **ktx** extracts metadata from each source and uses a reconciliation agent to reconcile it with your existing semantic layer and knowledge base - preserving accepted edits rather than overwriting.
@@ -27,7 +27,7 @@ LookML uses top-level `repoUrl`, and MetricFlow uses nested
| Field | Required | Description |
|-------|----------|-------------|
-| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, `notion`, `sigma`, or `gdrive` |
+| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` |
| `source_dir` | For local file sources | Absolute or project-relative source directory |
| `repo_url` | For Git-hosted dbt sources | Git repository URL |
| `repoUrl` | For Git-hosted LookML sources | Git repository URL |
@@ -38,16 +38,15 @@ LookML uses top-level `repoUrl`, and MetricFlow uses nested
## dbt
-Ingests schema definitions, model descriptions, column metadata, and column test definitions from a dbt project.
+Ingests schema definitions, model descriptions, column metadata, and test coverage from a dbt project.
### What it provides
- Model and source definitions from `schema.yml` files
-- Column names, descriptions, and data types
-- Column tests, mapped to semantic facts — `not_null` / `unique` become column constraints, `accepted_values` becomes enum value lists, and `relationships` becomes join / foreign-key edges
-- Model and source tags, and source freshness settings
-
-MetricFlow `semantic_models:` and `metrics:` are ingested through the separate [MetricFlow](#metricflow) source, not the dbt driver.
+- Column descriptions and types
+- Test coverage signals
+- Semantic model references (if using dbt semantic layer)
+- Data lineage between models
### Connection config
@@ -88,9 +87,9 @@ connections:
### What gets ingested
-- **Semantic-layer overlays** (`semantic-layer/*.yaml`): descriptions, constraints, enum values, and joins from the dbt YAML are written onto the semantic source for the matching warehouse table. Overlays land on the warehouse connection that owns the table, which is usually a different connection than the dbt source itself.
-- **Wiki pages** (`wiki/`): for definitions or relationships that don't map to a confirmed physical table.
-- **Work units** for parallel processing: one per schema file under `models/` when the project has more than 25 YAML files, otherwise a single combined unit.
+- YAML semantic sources generated from dbt schema files
+- One work unit per semantic source (for projects with >25 YAML files) or all at once for smaller projects
+- Column descriptions, tests, and relationships are preserved
---
@@ -102,7 +101,7 @@ Ingests MetricFlow semantic models and metric definitions. Useful when your team
- Semantic model definitions (entities, dimensions, measures)
- Cross-model metric definitions
-- Entity relationships between models, inferred from matching foreign and primary entities
+- Dimension and entity relationships between models
### Connection config
@@ -134,7 +133,7 @@ For a local path:
### What gets ingested
-- Semantic models with their entities, dimensions, measures, and the join edges inferred from entity relationships
+- Semantic models with their entities, dimensions, and measures
- Metric definitions with their expressions and filters
- Work units organized by connected component (metrics + related semantic models grouped together)
@@ -179,10 +178,10 @@ For a local path:
### What gets ingested
-- One work unit per model, plus a unit for orphan views and one per dashboard
-- Semantic-layer sources per view — overlays for thin `sql_table_name` wrappers, standalone sources for `derived_table` views
-- Measures, joins (with their Looker `relationship:`), and field types mapped to column types (`yesno` → boolean, date/timestamp → time)
-- Wiki pages for relationships and descriptions, with warehouse identifiers verified before writing
+- View and model definitions organized by connected component
+- LookML field types mapped to semantic layer column types
+- Join definitions and relationship cardinalities
+- SQL table references for warehouse mapping validation
### Warehouse mapping
@@ -193,19 +192,19 @@ Optionally validate that LookML references match your expected Looker connection
expectedLookerConnectionName: postgres_connection
```
-This compares each model's `connection:` declaration against the expected name. Mismatched models are flagged, and semantic-layer writes are disabled for them during that ingest while wiki extraction still proceeds.
+This validates that LookML model `connection:` declarations match expectations, flagging mismatches during ingestion.
---
## Metabase
-Ingests collections, questions, models, and metrics — with their underlying SQL — from a Metabase instance. Maps Metabase databases to your **ktx** warehouse connections.
+Ingests dashboards, questions, and their underlying SQL queries from a Metabase instance. Maps Metabase databases to your **ktx** warehouse connections.
### What it provides
-- Collections and their hierarchy, used to organize ingested context
-- Questions, models, and metrics — resolved SQL for both native and structured (MBQL) queries
-- Each card's output schema: column types and primary/foreign-key hints
+- Dashboard metadata and organization
+- Question/query definitions (native SQL and structured queries)
+- Table and column usage patterns from queries
- Database-to-warehouse relationship mapping
### Connection config
@@ -234,9 +233,9 @@ Generate an API key in Metabase: **Admin > Settings > Authentication > API Keys*
### What gets ingested
-- Semantic-layer sources generated from each card's resolved SQL and column metadata, written to the mapped warehouse connection
-- Fallback wiki notes only when a referenced table can't be mapped or an identifier can't be verified
-- One work unit per Metabase collection; re-syncs reprocess only collections with changed cards
+- Semantic sources generated from SQL queries in questions
+- Wiki pages for dashboards (purpose, key metrics, relationships)
+- Work units per dashboard and per question
### Warehouse mapping
@@ -290,10 +289,10 @@ Generate API credentials in Looker: **Admin > Users > Edit > API Keys**.
### What gets ingested
-- Semantic-layer sources from explore fields, written to the mapped warehouse connection (mapped explores only)
-- Wiki pages capturing reusable metric, segment, and domain knowledge from dashboards and Looks
-- Usage and recency signals that drive a triage gate, focusing processing on high-value content
-- Work units per explore, per dashboard, and per Look
+- Semantic sources from explore field definitions
+- Wiki pages for dashboards (purpose, audience, key metrics)
+- Triage signals for automated content classification
+- Work units per explore and per dashboard
### Warehouse mapping
@@ -315,10 +314,10 @@ Ingests pages and databases from a Notion workspace as wiki pages. Useful for ca
### What it provides
-- Notion pages crawled from selected roots or all accessible content
-- Page bodies and blocks normalized to Markdown
-- Page hierarchy and cross-page links (child pages, mentions, relations)
-- Notion databases and their data-source rows as individual pages
+- Wiki pages synthesized from Notion content
+- Page hierarchy and relationships
+- Database schemas (when Notion databases describe primary sources)
+- Semantic clustering for organized ingestion
### Connection config
@@ -357,7 +356,6 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in
| `crawl_mode` | `all_accessible` or `selected_roots` | - |
| `root_page_ids` | Page IDs to crawl from (for `selected_roots`) | `[]` |
| `root_database_ids` | Database IDs to include | `[]` |
-| `root_data_source_ids` | Data-source IDs to include (for `selected_roots`) | `[]` |
| `max_pages_per_run` | Pages processed per sync | `1000` |
| `max_knowledge_creates_per_run` | New pages created per sync | `25` |
| `max_knowledge_updates_per_run` | Pages updated per sync | `20` |
@@ -365,178 +363,17 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in
### What gets ingested
- Wiki pages synthesized from Notion content (not raw copies)
-- Semantic-layer sources when a page defines a reusable dataset or metric mapped to a confirmed non-Notion target; otherwise the fact stays wiki-only
-- Page-relevance triage that skips transient content (task lists, status updates, date-titled snapshots)
-- Work units clustered by embedding similarity for efficient synthesis
+- Domain context extracted and organized by topic
+- Triage signals for classifying page relevance
+- Work units clustered by semantic similarity for efficient processing
### Notes
-- Notion is wiki-first: it writes durable wiki pages by default and only emits semantic-layer sources for content mapped to a confirmed non-Notion target; unmapped facts stay wiki-only
+- Notion is knowledge-only - it does not produce semantic layer sources
- Rate limits apply; large workspaces may require multiple ingestion runs
- Incremental sync cursors are stored in `.ktx/db.sqlite`; don't add
`last_successful_cursor` to `ktx.yaml`
----
-
-## Sigma
-
-Ingests data model definitions and workbook metadata from a Sigma workspace as semantic context. Uses the Sigma REST API to fetch data model specs and workbook summaries.
-
-### What it provides
-
-- Data model names, folder paths, and ownership metadata
-- Page and element definitions within each data model
-- Column identifiers and data types where available
-- Workbook names, paths, descriptions, and version metadata
-
-### Connection config
-
-```yaml title="ktx.yaml"
-connections:
- sigma-main:
- driver: sigma
- api_url: https://api.sigmacomputing.com # Omit for GCP US (default)
- client_id: ""
- client_secret_ref: env:SIGMA_CLIENT_SECRET
-```
-
-For the AWS US region, override `api_url`:
-
-```yaml title="ktx.yaml"
-connections:
- sigma-main:
- driver: sigma
- api_url: https://aws-api.sigmacomputing.com
- client_id: ""
- client_secret_ref: env:SIGMA_CLIENT_SECRET
-```
-
-### Authentication
-
-| Method | Config |
-|--------|--------|
-| OAuth client credentials | `client_id` + `client_secret_ref: env:SIGMA_CLIENT_SECRET` |
-
-Generate a client in Sigma: **Administration → Developer Access → Add New Client**.
-
-### What gets ingested
-
-- Active data model specs, organized by folder into work units
-- Workbook metadata (name, path, description, version) — archived and exploration workbooks excluded by default
-- Models backed by CSV uploads or unsupported connector subtypes are listed in the manifest but skipped during spec fetch (a Sigma API limitation)
-
-### Warehouse connection mapping
-
-`connectionMappings` is optional. Without it, **ktx** produces wiki knowledge only — no semantic-layer sources are written and warehouse validation is skipped. To get semantic-layer output and enable `sl_validate`, map each Sigma internal connection UUID to a **ktx** warehouse connection ID:
-
-```yaml title="ktx.yaml"
-connections:
- sigma-main:
- driver: sigma
- client_id: ""
- client_secret_ref: env:SIGMA_CLIENT_SECRET
- connectionMappings:
- "": snowflake-prod # data models using this connection get SL sources
-```
-
-Find the Sigma connection UUID in **Administration → Connections** or from the `source.connectionId` field in a fetched data model spec. Data model elements whose `connectionId` has no mapping are ingested as wiki-only.
-
-### Workbook filter
-
-At large scale, you can limit which workbooks are fetched during ingest using `workbookFilter`:
-
-```yaml title="ktx.yaml"
-connections:
- sigma-main:
- driver: sigma
- client_id: ""
- client_secret_ref: env:SIGMA_CLIENT_SECRET
- workbookFilter:
- includeArchived: false # default
- includeExplorations: false # default
- updatedSince: "2026-01-01T00:00:00Z" # only recently updated workbooks
-```
-
-| Field | Default | Description |
-|-------|---------|-------------|
-| `includeArchived` | `false` | Include archived workbooks |
-| `includeExplorations` | `false` | Include exploration workbooks |
-| `updatedSince` | — | ISO 8601 date; only workbooks updated on or after this date are fetched |
-
-### Notes
-
-- `connectionMappings` is optional for wiki-only ingest; it is required to generate semantic-layer sources and run warehouse validation
-- Context ingest (`ktx ingest sigma-main`) fetches from the Sigma API directly
-- Ingest is incremental: items whose `updatedAt` timestamp is unchanged since the last run are skipped
-- Models backed by CSV uploads or unsupported connector subtypes cannot have their spec exported; these are skipped with a warning (a Sigma API limitation)
-- Joins are not projected from Sigma data models in this release; `joins: []` is always written by the projection step. Lookup relationships visible in data model specs are captured as wiki knowledge instead.
-
----
-
-## Google Drive
-
-Ingests Google Docs from a shared Google Drive folder as wiki-ready knowledge content. This v1 implementation is knowledge-only and ingests Google Docs MIME types only.
-
-### What it provides
-
-- Wiki pages synthesized from Google Docs content
-- Folder-scoped knowledge ingestion from a specific Drive folder
-- Markdown normalization for headings, lists, paragraphs, links, common inline formatting, and Google Docs tables
-
-### Connection config
-
-```yaml title="ktx.yaml"
-connections:
- company-docs:
- driver: gdrive
- service_account_key_ref: file:/absolute/path/to/google-service-account.json
- folder_id: your-google-drive-folder-id
- recursive: false
-```
-
-### Authentication
-
-| Method | Config |
-|--------|--------|
-| Service account JSON key file | `service_account_key_ref: file:/absolute/path/to/key.json` |
-
-### Google Cloud setup
-
-1. Create a Google Cloud project.
-2. Enable the Google Drive API.
-3. Enable the Google Docs API.
-4. Create a service account.
-5. Download the service account JSON key.
-6. Share the target Drive folder with the service account email.
-7. Reference the key in `ktx.yaml` with `service_account_key_ref`.
-
-### Required scopes
-
-- `https://www.googleapis.com/auth/drive.readonly`
-- `https://www.googleapis.com/auth/documents.readonly`
-
-### Configuration options
-
-| Field | Description | Default |
-|-------|-------------|---------|
-| `service_account_key_ref` | File reference to the service account JSON key | - |
-| `folder_id` | Google Drive folder ID to ingest | - |
-| `recursive` | Traverse subfolders under `folder_id` | `false` |
-
-### What gets ingested
-
-- Google Docs documents only
-- Wiki-oriented knowledge content
-- One work unit per staged Google Doc
-
-### Notes
-
-- `gdrive` is knowledge-only in v1; it does not produce semantic layer sources
-- `ktx setup` supports Google Drive configuration, including the service-account key ref, folder id, and recursive crawl flag
-- `ktx connection test ` supports `gdrive`: it verifies that `folder_id` resolves to a folder the service account can read, then reports the number of Google Docs visible in it. A wrong or unshared `folder_id` fails the test instead of reporting zero docs
-- Only Google Docs are ingested in v1; other file types (Sheets, Slides, PDFs) in the folder are skipped and recorded in the staged manifest
-- The service account must be granted access to the target folder explicitly
-
## Common errors
| Error or symptom | Likely cause | Recovery |
diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx
index b553a3a7..6cb2d26f 100644
--- a/docs-site/content/docs/integrations/primary-sources.mdx
+++ b/docs-site/content/docs/integrations/primary-sources.mdx
@@ -1,6 +1,6 @@
---
title: Primary Sources
-description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, MongoDB, or Amazon Athena.
+description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, or SQLite.
---
**ktx** connects to your data warehouse or database to build schema context,
@@ -26,23 +26,15 @@ Agents should prefer environment or file references over literal secrets.
| Field | Required | Applies to | Description |
|-------|----------|------------|-------------|
-| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, `mongodb`, or `athena` |
+| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, or `sqlite` |
| `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` |
| `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values |
| `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan |
-| `databases` | No | ClickHouse, MongoDB, Athena | List of databases to scan |
-| `sample_size`, `order_by` | No | MongoDB | Schema-inference sampling controls (recent documents, sort field) |
| `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it |
-| `path` | Yes for path-style SQLite/DuckDB | SQLite, DuckDB | Local SQLite or DuckDB database path or `env:NAME` reference |
+| `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference |
| `max_bytes_billed` | No | BigQuery | Maximum bytes billed per query job |
-| `query_timeout_ms` | No | all warehouses | Maximum execution time for a single read-only query, in milliseconds (default 30000). A query exceeding it is cancelled server-side (or, for SQLite, by terminating the off-process executor) and returns a `query exceeded Ns` error so the agent can revise. |
+| `job_timeout_ms` | No | BigQuery | BigQuery query job timeout in milliseconds |
| `project_id` | No | BigQuery | Optional local descriptor and mapping metadata; not used for BigQuery authentication |
-| `region` | Yes | Athena | AWS region where the Athena workgroup and Glue catalog reside (e.g. `us-east-1`) |
-| `s3_staging_dir` | Yes | Athena | S3 URI for Athena query result storage (e.g. `s3://my-bucket/athena-results/`) |
-| `workgroup` | No | Athena | Athena workgroup name (default `primary`) |
-| `catalog` | No | Athena | Glue Data Catalog name (default `AwsDataCatalog`) |
-| `database` | No | Athena | Default Glue database name passed as the query execution context |
-| `databases` | No | Athena | Glue databases to include in schema scans; written by `ktx setup` and read by `ktx ingest` |
## PostgreSQL
@@ -226,37 +218,6 @@ BigQuery dataset scope is stored in `connections..dataset_ids`. Interactive
setup discovers datasets from credentials plus location, then writes the chosen
dataset ids as the scan scope.
-### Cross-project datasets
-
-To introspect a dataset hosted in a **different project** than the one your
-credentials bill to — for example Google's `bigquery-public-data`, a partner's
-shared project, or an organization's central data project — qualify the entry
-as `project.dataset`:
-
-```yaml title="ktx.yaml"
-connections:
- public-bq:
- driver: bigquery
- credentials_json: file:~/.config/gcloud/bq-service-account.json
- location: US
- dataset_ids:
- - bigquery-public-data.austin_311
- - bigquery-public-data.census_bureau_usa
- - analytics
-```
-
-**ktx** introspects each dataset in its host project while every query job still
-bills to the `project_id` inside your `credentials_json`. A bare `dataset` entry
-(no prefix) is scanned in your own project, exactly as before. A single
-connection may mix datasets from several projects, and two projects may host
-datasets with the same name without colliding.
-
-Interactive setup does not enumerate datasets in projects your credentials don't
-own, so hand-write `project.dataset` entries for foreign datasets. The wizard's
-table picker also only lists datasets in your connection's `location` region;
-this affects table selection only — ingest and `discover_data` introspect a
-cross-project dataset regardless of region.
-
### Authentication
| Method | Config |
@@ -306,77 +267,7 @@ staged artifact shape as Postgres and Snowflake.
- Parameter binding uses named `@param` syntax
- Arrays flattened to comma-separated strings in results
- Location specified at query execution time
-- Supports the `max_bytes_billed` limit from `ktx.yaml`; the shared `query_timeout_ms` field maps to the query job's `jobTimeoutMs`
-
----
-
-## Amazon Athena
-
-Connects to Amazon Athena using the AWS Glue Data Catalog for schema introspection and the Athena query API for read-only SQL execution. Authentication uses the standard AWS credential chain — no credentials are embedded in `ktx.yaml`.
-
-### Connection config
-
-```yaml title="ktx.yaml"
-connections:
- my-athena:
- driver: athena
- region: us-east-1
- s3_staging_dir: s3://my-bucket/athena-results/
-```
-
-With optional fields:
-
-```yaml title="ktx.yaml"
-connections:
- my-athena:
- driver: athena
- region: us-east-1
- s3_staging_dir: env:ATHENA_S3_STAGING_DIR
- workgroup: analytics
- catalog: AwsDataCatalog
- database: my_default_database
- databases:
- - analytics
- - raw
-```
-
-`ktx setup` writes the `databases` array when you select Glue databases during setup. `ktx scan` reads it to limit introspection to those databases.
-
-### Authentication
-
-**ktx** uses the AWS SDK default credential chain — no credentials appear in `ktx.yaml`. The chain resolves credentials in this order:
-
-| Method | How to configure |
-|--------|-----------------|
-| Environment variables | Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN` |
-| Shared credentials file | Configure `~/.aws/credentials` with a `[default]` or named profile; set `AWS_PROFILE` to select a non-default profile |
-| IAM instance profile | Attach an IAM role to the EC2 instance or ECS task — no local configuration needed |
-| IAM roles for service accounts (EKS) | Annotate the pod's service account with the IAM role ARN |
-
-The IAM principal must have `athena:StartQueryExecution`, `athena:GetQueryExecution`, `athena:GetQueryResults`, `glue:GetDatabases`, and `glue:GetTables` permissions, plus read access to the S3 results bucket.
-
-### Features
-
-| Feature | Supported | Notes |
-|---------|-----------|-------|
-| Tables & views | Yes | Via AWS Glue Data Catalog |
-| Primary keys | No | Glue does not expose constraint metadata |
-| Foreign keys | No | Not available in Glue/Athena |
-| Row count estimates | No | Glue table statistics are often stale |
-| Column statistics | No | - |
-| Query history | No | - |
-| Table sampling | Yes | `SELECT ... LIMIT n` |
-
-### Dialect notes
-
-- SQL dialect is Presto/Trino; identifiers are quoted with double-quotes
-- Table names use three-part format: `"catalog"."database"."table"` (e.g. `"AwsDataCatalog"."analytics"."orders"`)
-- Partition columns (`PartitionKeys` in Glue) are included after regular columns in the schema and are fully queryable
-- Athena does not support `TABLESAMPLE`; random sampling uses `ORDER BY rand()`
-- Query execution is asynchronous: **ktx** starts the query, polls until completion, then fetches results from S3
-- Results are stored in `s3_staging_dir`; the IAM principal needs write access to that bucket
-- Use `workgroup` to apply per-workgroup cost controls and result configuration
-- The connector always uses your account's default Glue Data Catalog; cross-account catalog access (`CatalogId` pointing to another account) is not supported
+- Supports `max_bytes_billed` and `job_timeout_ms` limits from `ktx.yaml`
---
@@ -619,141 +510,12 @@ No authentication required - SQLite is file-based. The file must be readable by
- Foreign key enforcement requires explicit `PRAGMA foreign_keys = ON`
- Database file must exist before `ktx connection test` or ingest runs
----
-
-## DuckDB
-
-File-based connector using the DuckDB Node.js API. Ideal for local analytics, embedded warehouses, and cross-database federation.
-
-### Connection config
-
-```yaml title="ktx.yaml"
-connections:
- warehouse:
- driver: duckdb
- path: data/warehouse.duckdb
-```
-
-`path` is resolved relative to the project directory. The `.duckdb` file must already exist — **ktx** never creates a missing database file.
-
-### Authentication
-
-No authentication required — DuckDB is file-based. The `.duckdb` file must be readable by the process running **ktx**.
-
-### Features
-
-| Feature | Supported | Notes |
-|---------|-----------|-------|
-| Tables & views | Yes | Via `information_schema` on the `main` schema |
-| Primary keys | Yes | Via `information_schema.table_constraints` |
-| Foreign keys | Yes | Via DuckDB's `duckdb_constraints()` catalog function |
-| Row count estimates | Yes | Exact count via `SELECT COUNT(*)` |
-| Column statistics | No | - |
-| Query history | No | - |
-| Table sampling | Yes | - |
-| Nested analysis | No | - |
-
-### Dialect notes
-
-- Introspection scans the `main` schema only
-- Execution is read-only; **ktx** opens the file without write access
-- Parameter binding uses positional `?` placeholders
-- Uses `LIMIT X OFFSET Y` for pagination
-- Database file must exist before `ktx connection test` or ingest runs
-
-### Cross-database federation
-
-When a project declares two or more attach-compatible connections — any combination of `postgres`, `mysql`, `sqlite`, and `duckdb` — **ktx** derives a cross-database federation connection. That connection can ATTACH a native `.duckdb` file, allowing semantic queries to join across sources without manually copying data.
-
----
-
-## MongoDB
-
-Connects to MongoDB as a primary context source. **ktx** treats each collection
-as a table and each inferred top-level field as a column. MongoDB is a non-SQL
-source: `ktx sql` and semantic-layer metric compilation do not apply to a MongoDB
-connection, but its collections still flow through `ktx ingest`, descriptions, and
-relationship discovery.
-
-### Connection config
-
-```yaml title="ktx.yaml"
-connections:
- mongo-prod:
- driver: mongodb
- url: env:MONGO_URL
- databases: [app]
- enabled_tables: [app.users, app.orders] # optional collection allowlist
- sample_size: 1000
- # order_by: createdAt # only when _id is not an ObjectId
-```
-
-Standard `mongodb://` and `mongodb+srv://` connection strings are supported,
-including TLS and MongoDB Atlas — pass the full connection string (with its
-query parameters) as `url`. The `databases` list selects which databases to
-introspect; if omitted, **ktx** uses the database in the URL path. `ktx setup`
-also offers MongoDB and stores the selected databases under
-`connections..databases`.
-
-### Authentication
-
-| Method | Config |
-|--------|--------|
-| Connection string | `url: env:MONGO_URL` or `url: file:/path/to/secret` |
-| Atlas / TLS | Use a `mongodb+srv://` URL with the credentials and TLS options Atlas provides |
-
-### Schema inference
-
-MongoDB has no fixed schema, so **ktx** infers one by sampling the most recent
-`sample_size` documents per collection (default 1000), sorted by `_id`
-descending. Because an ObjectId embeds its creation time, this captures the
-collection's current shape with zero configuration. When `_id` is not an
-ObjectId (custom string or UUID keys), set `order_by` to a timestamp field such
-as `createdAt` so "most recent" is well-defined. A custom `order_by` field
-should be indexed — an unindexed sort hits MongoDB's in-memory sort limit and
-fails on large collections (`_id`, the default, is always indexed).
-
-For each top-level field, **ktx** unions the BSON types seen and derives
-nullability from how often the field is present:
-
-- Scalar BSON types map to `string`, `number`, `time`, or `boolean`
-- A field seen with more than one type is recorded as `mixed` and treated as a string
-- Sub-documents and arrays become a single opaque `json` column (no dotted-path
- columns); their sampled values are stringified, not faithfully serialized
-- `_id` is the primary key
-
-### Features
-
-| Feature | Supported | Notes |
-|---------|-----------|-------|
-| Collections (as tables) | Yes | Via `listCollections`; `system.*` collections are excluded |
-| Primary keys | Yes | `_id` |
-| Foreign keys | No | MongoDB has no formal foreign keys |
-| Row count estimates | Yes | Via `estimatedDocumentCount` |
-| Column statistics | No | - |
-| Query history | No | - |
-| Table sampling | Yes | Reads the most recent documents |
-| Nested analysis | Yes | Sub-documents and arrays modeled as opaque `json` |
-| Read-only SQL (`ktx sql`) | No | MongoDB is not a SQL source |
-
-### Dialect notes
-
-- Strictly read-only: the connector only issues `find`, `listCollections`,
- `estimatedDocumentCount`, and read aggregations
-- Sampling rides the `_id` index and uses a server-side time limit so large
- collections do not stall a run; a custom `order_by` must be indexed for the
- same guarantee
-- `sample_size` trades inference coverage for speed; raise it for collections
- with highly variable documents
-
## Common errors
| Error or symptom | Likely cause | Recovery |
|------------------|--------------|----------|
| Connection URL appears in git diff | A literal credential URL was written to `ktx.yaml` | Replace it with `env:NAME` or `file:/path/to/secret` and rotate exposed credentials |
-| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions. For Athena, confirm the IAM principal has `glue:GetDatabases` and `glue:GetTables` permissions |
+| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions |
| Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest --query-history` or `ktx setup` |
| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on schema-level context without column statistics |
| Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags |
-| Athena query fails with `ACCESS_DENIED` | IAM principal lacks `athena:StartQueryExecution` or S3 write access to `s3_staging_dir` | Attach a policy granting Athena query permissions and `s3:PutObject` on the staging bucket |
-| Athena ingest finds databases but no tables | IAM principal has `glue:GetDatabases` but not `glue:GetTables` | Grant `glue:GetTables` on the relevant Glue catalog resources |
diff --git a/docs-site/content/docs/meta.json b/docs-site/content/docs/meta.json
index c872cf4b..7be8bc90 100644
--- a/docs-site/content/docs/meta.json
+++ b/docs-site/content/docs/meta.json
@@ -8,6 +8,7 @@
"integrations",
"configuration",
"cli-reference",
+ "ai-resources",
"community"
]
}
diff --git a/docs-site/lib/llm-docs.ts b/docs-site/lib/llm-docs.ts
index 1f5766e1..fd6c8dd1 100644
--- a/docs-site/lib/llm-docs.ts
+++ b/docs-site/lib/llm-docs.ts
@@ -54,7 +54,9 @@ ktx provides semantic-layer files, warehouse scans, wiki pages, provenance, and
- Installable setup skill: run \`npx skills add Kaelio/ktx --skill ktx\` from
the project you want to configure.
-${link("/docs/community/ai-resources", "AI Resources", "How coding agents read, cite, and act on the ktx docs")}
+${link("/docs/ai-resources/agent-quickstart", "Agent Quickstart", "Task-first route for coding assistants using ktx")}
+${link("/docs/ai-resources/markdown-access", "Markdown Access", "Fetch ktx docs as llms.txt, llms-full.txt, or per-page Markdown")}
+${link("/docs/ai-resources/agent-instructions", "Agent Instructions", "Suggested instructions for coding assistants that need to read and cite ktx docs")}
## Start Here
@@ -65,7 +67,7 @@ ${link("/docs/guides/writing-context", "Writing Context", "Write semantic source
## Machine-Readable Documentation
- [Full documentation](${absoluteUrl("/llms-full.txt")}): All docs pages in one plain-text markdown response
-- [AI Resources guide](${absoluteUrl("/docs/community/ai-resources.md")}): How agents fetch llms.txt, llms-full.txt, and per-page Markdown
+- [Markdown access guide](${absoluteUrl("/docs/ai-resources/markdown-access.md")}): How to fetch llms.txt, llms-full.txt, and per-page Markdown
- [Quickstart markdown](${absoluteUrl("/docs/getting-started/quickstart.md")}): Human setup walkthrough
- [Semantic-layer CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-sl.md")}): Semantic-layer commands and JSON output
- [Wiki CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-wiki.md")}): Wiki page commands and JSON output
@@ -145,8 +147,8 @@ function absoluteUrl(path: string) {
function formatCategoryName(category: string) {
const labels: Record = {
+ "ai-resources": "AI Resources",
"cli-reference": "CLI Reference",
- community: "Community & Resources",
};
if (labels[category]) {
diff --git a/docs-site/next.config.mjs b/docs-site/next.config.mjs
index f800bcf5..380dba85 100644
--- a/docs-site/next.config.mjs
+++ b/docs-site/next.config.mjs
@@ -30,36 +30,7 @@ const config = {
};
},
async redirects() {
- // Alias-host canonicalization MUST come before the generic root/docs
- // redirects below. Those generic rules have no host guard, so if they ran
- // first they would inject a "/ktx" basePath into the path on the alias
- // hosts, which the alias catch-alls would then prepend a second time —
- // producing https://docs.kaelio.com/ktx/ktx/docs/... Redirects also run
- // before beforeFiles rewrites, so the ktx.sh catch-all must exclude
- // /stars* to let the stars dashboard rewrite proxy through.
return [
- {
- source: "/slack",
- has: [{ type: "host", value: "ktx.sh" }],
- destination:
- "https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ",
- permanent: false,
- basePath: false,
- },
- {
- source: "/:path*",
- has: [{ type: "host", value: "docs.ktx.sh" }],
- destination: "https://docs.kaelio.com/ktx/:path*",
- permanent: true,
- basePath: false,
- },
- {
- source: "/:path((?!stars(?:/|$)).*)",
- has: [{ type: "host", value: "ktx.sh" }],
- destination: "https://docs.kaelio.com/ktx/:path",
- permanent: true,
- basePath: false,
- },
{
source: "/",
destination: "/ktx/docs/getting-started/introduction",
@@ -73,30 +44,26 @@ const config = {
basePath: false,
},
{
- // AI Resources collapsed from four pages to one and now lives under the
- // Community & Resources section. Redirect the old top-level URL and the
- // retired per-page slugs to the new home. Redirects run before the .md
- // rewrite, so the Markdown variants must be matched first and keep their
- // .md suffix; otherwise a cached Markdown URL would 308 to the HTML page
- // and break the agent Markdown contract.
- source: "/docs/ai-resources.md",
- destination: "/docs/community/ai-resources.md",
+ source: "/:path*",
+ has: [{ type: "host", value: "docs.ktx.sh" }],
+ destination: "https://docs.kaelio.com/ktx/:path*",
permanent: true,
+ basePath: false,
},
{
- source: "/docs/ai-resources/:slug([^/]+\\.md)",
- destination: "/docs/community/ai-resources.md",
- permanent: true,
+ source: "/slack",
+ has: [{ type: "host", value: "ktx.sh" }],
+ destination:
+ "https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ",
+ permanent: false,
+ basePath: false,
},
{
- source: "/docs/ai-resources",
- destination: "/docs/community/ai-resources",
- permanent: true,
- },
- {
- source: "/docs/ai-resources/:slug",
- destination: "/docs/community/ai-resources",
+ source: "/:path((?!stars(?:/|$)).*)",
+ has: [{ type: "host", value: "ktx.sh" }],
+ destination: "https://docs.kaelio.com/ktx/:path",
permanent: true,
+ basePath: false,
},
];
},
diff --git a/docs-site/tests/docs-index-route.test.mjs b/docs-site/tests/docs-index-route.test.mjs
index e2ab24f0..fdd8ec81 100644
--- a/docs-site/tests/docs-index-route.test.mjs
+++ b/docs-site/tests/docs-index-route.test.mjs
@@ -2,8 +2,6 @@ import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { once } from "node:events";
import { readFile, writeFile } from "node:fs/promises";
-import http from "node:http";
-import https from "node:https";
import { dirname, join } from "node:path";
import { createServer } from "node:net";
import { after, before, test } from "node:test";
@@ -102,37 +100,6 @@ after(async () => {
}
});
-// Node's fetch (undici) overwrites the Host header with the connection host,
-// so the alias-host redirect rules never match. The low-level http(s) client
-// sends Host verbatim, which is what the alias canonicalization keys off of.
-function requestWithHost(hostHeader, path) {
- const target = new URL(docsSiteUrl);
- const client = target.protocol === "https:" ? https : http;
- const port =
- target.port || (target.protocol === "https:" ? "443" : "80");
-
- return new Promise((resolve, reject) => {
- const request = client.request(
- {
- hostname: target.hostname,
- port,
- path,
- method: "GET",
- headers: { Host: hostHeader },
- },
- (response) => {
- response.resume();
- resolve({
- status: response.statusCode,
- location: response.headers.location,
- });
- },
- );
- request.on("error", reject);
- request.end();
- });
-}
-
test("/ktx/docs redirects to the docs introduction", async () => {
const response = await fetch(`${docsSiteUrl}${docsBasePath}/docs`, {
redirect: "manual",
@@ -145,53 +112,6 @@ test("/ktx/docs redirects to the docs introduction", async () => {
);
});
-test("retired AI Resources URLs redirect to the page under Community", async () => {
- // The former top-level URL.
- const bare = await fetch(
- `${docsSiteUrl}${docsBasePath}/docs/ai-resources`,
- { redirect: "manual" },
- );
-
- assert.equal(bare.status, 308);
- assert.equal(
- bare.headers.get("location"),
- `${docsBasePath}/docs/community/ai-resources`,
- );
-
- // A retired per-page slug.
- const slug = await fetch(
- `${docsSiteUrl}${docsBasePath}/docs/ai-resources/agent-quickstart`,
- { redirect: "manual" },
- );
-
- assert.equal(slug.status, 308);
- assert.equal(
- slug.headers.get("location"),
- `${docsBasePath}/docs/community/ai-resources`,
- );
-
- // A retired per-page Markdown URL must stay Markdown: it has to redirect to
- // the new .md route, not fall through to the HTML page.
- const markdown = await fetch(
- `${docsSiteUrl}${docsBasePath}/docs/ai-resources/agent-quickstart.md`,
- { redirect: "manual" },
- );
-
- assert.equal(markdown.status, 308);
- assert.equal(
- markdown.headers.get("location"),
- `${docsBasePath}/docs/community/ai-resources.md`,
- );
-
- // Following that redirect end to end must land on Markdown, not HTML.
- const followed = await fetch(
- `${docsSiteUrl}${docsBasePath}/docs/ai-resources/agent-quickstart.md`,
- );
-
- assert.equal(followed.status, 200);
- assert.match(followed.headers.get("content-type") ?? "", /text\/markdown/);
-});
-
test("/ redirects into the /ktx docs site", async () => {
const response = await fetch(`${docsSiteUrl}/`, {
redirect: "manual",
@@ -221,51 +141,3 @@ test("/ktx/api/search returns docs search results", async () => {
"search should return at least one docs result",
);
});
-
-test("ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => {
- const root = await requestWithHost("ktx.sh", "/");
- assert.equal(root.status, 308);
- assert.equal(root.location, "https://docs.kaelio.com/ktx/");
- assert.ok(
- !root.location.includes("/ktx/ktx"),
- "the basePath must not be doubled",
- );
-
- const page = await requestWithHost(
- "ktx.sh",
- "/docs/getting-started/quickstart",
- );
- assert.equal(page.status, 308);
- assert.equal(
- page.location,
- "https://docs.kaelio.com/ktx/docs/getting-started/quickstart",
- );
-});
-
-test("docs.ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => {
- const root = await requestWithHost("docs.ktx.sh", "/");
- assert.equal(root.status, 308);
- assert.equal(root.location, "https://docs.kaelio.com/ktx");
- assert.ok(
- !root.location.includes("/ktx/ktx"),
- "the basePath must not be doubled",
- );
-
- const page = await requestWithHost("docs.ktx.sh", "/llms.txt");
- assert.equal(page.status, 308);
- assert.equal(page.location, "https://docs.kaelio.com/ktx/llms.txt");
-});
-
-test("ktx.sh keeps the /slack and /stars exceptions", async () => {
- const slack = await requestWithHost("ktx.sh", "/slack");
- assert.equal(slack.status, 307);
- assert.match(slack.location, /^https:\/\/join\.slack\.com\//);
-
- // /stars is proxied by a beforeFiles rewrite, so the apex catch-all must not
- // canonicalize it to the docs host.
- const stars = await requestWithHost("ktx.sh", "/stars");
- assert.ok(
- !(stars.location ?? "").startsWith("https://docs.kaelio.com"),
- "the stars dashboard must not be redirected to the docs host",
- );
-});
diff --git a/docs-site/tests/product-mechanics-content.test.mjs b/docs-site/tests/product-mechanics-content.test.mjs
index d0c9471c..5cce9001 100644
--- a/docs-site/tests/product-mechanics-content.test.mjs
+++ b/docs-site/tests/product-mechanics-content.test.mjs
@@ -85,7 +85,7 @@ test("product mechanics component explains ingestion outputs", async () => {
"compile into SQL",
'"use client"',
"@xyflow/react",
- " {
);
}
- // The ReactFlow canvas config lives in the shared FlowCanvas wrapper, which
- // product-mechanics renders. Assert the static read-only behavior there.
- const flowCanvas = await readDocsFile("components/flow-canvas.tsx");
- for (const guard of [
+ assert.match(
+ component,
/nodesDraggable=\{false\}/,
- /nodesConnectable=\{false\}/,
+ "ReactFlow canvas should disable node dragging",
+ );
+ assert.match(
+ component,
+ /panOnDrag=\{false\}/,
+ "ReactFlow canvas should disable panning",
+ );
+ assert.match(
+ component,
/zoomOnScroll=\{false\}/,
- /elementsSelectable=\{false\}/,
- ]) {
- assert.match(
- flowCanvas,
- guard,
- `shared FlowCanvas should enforce static read-only behavior: ${guard}`,
- );
- }
+ "ReactFlow canvas should disable scroll zoom",
+ );
assert.doesNotMatch(component, /raw-sources/);
assert.doesNotMatch(component, /\.ktx/);
diff --git a/docs-site/tests/product-runtime-content.test.mjs b/docs-site/tests/product-runtime-content.test.mjs
deleted file mode 100644
index ac643faa..00000000
--- a/docs-site/tests/product-runtime-content.test.mjs
+++ /dev/null
@@ -1,74 +0,0 @@
-import assert from "node:assert/strict";
-import { readFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { test } from "node:test";
-import { fileURLToPath } from "node:url";
-
-const docsSiteDir = join(dirname(fileURLToPath(import.meta.url)), "..");
-
-async function readDocsFile(path) {
- return readFile(join(docsSiteDir, path), "utf8");
-}
-
-test("docs introduction renders the serving phase after ingestion", async () => {
- const introduction = await readDocsFile(
- "content/docs/getting-started/introduction.mdx",
- );
-
- assert.match(
- introduction,
- /import\s+\{\s*ProductRuntime\s*\}\s+from\s+"@\/components\/product-runtime";/,
- );
- assert.match(introduction, //);
-
- const mechanicsIndex = introduction.indexOf("");
- const runtimeIndex = introduction.indexOf("");
- const useCaseIndex = introduction.indexOf("## Use it for");
-
- assert.ok(
- runtimeIndex > mechanicsIndex,
- "serving diagram should appear after the ingestion diagram",
- );
- assert.ok(
- runtimeIndex < useCaseIndex,
- "serving diagram should appear before use-case sections",
- );
-});
-
-test("product runtime component explains the serving cycle", async () => {
- const component = await readDocsFile("components/product-runtime.tsx");
-
- for (const expectedText of [
- "How serving works",
- "Serving flow",
- "From an agent request to a governed answer",
- "Your agent",
- "Claude Code",
- "Cursor",
- "Codex",
- "Search wiki + semantic layer",
- "Return approved metrics",
- "Compile metrics → SQL",
- "Context layer",
- "Database",
- "search + read",
- "read-only",
- "wiki/*.md",
- "semantic-layer/*.yaml",
- '"use client"',
- "@xyflow/react",
- "FlowCanvas",
- "getSmoothStepPath",
- "animateMotion",
- "runtime-particle",
- "buildCyclePath",
- ]) {
- assert.ok(
- component.includes(expectedText),
- `component should include: ${expectedText}`,
- );
- }
-
- assert.doesNotMatch(component, /raw-sources/);
- assert.doesNotMatch(component, /"
ingest:
adapters:
@@ -17,3 +18,5 @@ agent:
- sl_query
- wiki_search
- sl_read_source
+memory:
+ auto_commit: true
diff --git a/examples/mongodb/README.md b/examples/mongodb/README.md
deleted file mode 100644
index f51bf0bd..00000000
--- a/examples/mongodb/README.md
+++ /dev/null
@@ -1,127 +0,0 @@
-# MongoDB Connector Example
-
-A manual, self-contained example for the **ktx** MongoDB connector. It starts a
-local MongoDB, seeds a representative dataset, and exercises the connector both
-as a fast no-LLM introspection smoke and as a full `ktx ingest` run.
-
-MongoDB is a **context-only** primary source: collections become tables and
-inferred top-level fields become columns, but `ktx sql` and semantic-layer
-metric compilation do not apply. See
-[`docs-site/content/docs/integrations/primary-sources.mdx`](../../docs-site/content/docs/integrations/primary-sources.mdx).
-
-## Prerequisites
-
-- Docker with Compose v2, or Podman with `podman compose`
-- Node and pnpm matching the **ktx** workspace
-- The built CLI: `pnpm --filter @kaelio/ktx run build`
-- For the full ingest only: `uv` on `PATH` and a usable local Claude Code
- session (the keyless `claude-code` LLM backend)
-
-## What the seed contains
-
-[`init/seed.js`](init/seed.js) creates the `app` database with:
-
-- `users` — `_id` (ObjectId), scalar fields, a nested `address`, an array
- `tags`, a `Decimal128` `balance`, a `ref` field that holds more than one type
- (inferred `mixed`), and an `age` field absent from one document (nullable)
-- `orders` — an ObjectId `user_id` reference for relationship discovery
-- `active_users` — a **view** (to confirm introspection never runs a count
- command on a view)
-
-MongoDB applies the script once on first container start. Apply it by hand with:
-
-```bash
-mongosh "mongodb://localhost:27117" < examples/mongodb/init/seed.js
-```
-
-## Smoke (no LLM credentials)
-
-From the **ktx** repository root:
-
-```bash
-examples/mongodb/scripts/smoke.sh
-```
-
-It starts MongoDB on `127.0.0.1:27117`, seeds it, and asserts the connector's
-inferred schema (collections → tables, nested → `json`, `mixed`, nullability,
-`_id` primary key, and a view introspected with `estimatedRows: null`). This
-drives the same entry point `ktx ingest`'s "database schema" stage uses, without
-needing an LLM or embeddings.
-
-Podman:
-
-```bash
-KTX_MONGODB_COMPOSE="podman compose" examples/mongodb/scripts/smoke.sh
-```
-
-Set `KTX_MONGODB_KEEP=1` to leave the container running after the script exits.
-
-## Full `ktx ingest`
-
-The public database-ingest path requires a configured model and embeddings.
-This runs entirely locally with the keyless `claude-code` LLM backend and the
-**ktx**-managed `sentence-transformers` embedding daemon — no API keys.
-
-Start MongoDB and create a project:
-
-```bash
-docker compose -f examples/mongodb/docker-compose.yml up -d --wait # or: podman compose
-node packages/cli/dist/bin.js admin init /tmp/ktx-mongodb-example
-```
-
-Add the connection and a keyless enrichment stack to
-`/tmp/ktx-mongodb-example/ktx.yaml`:
-
-```yaml
-connections:
- mongo-prod:
- driver: mongodb
- url: mongodb://localhost:27117/app
- databases:
- - app
-llm:
- provider:
- backend: claude-code
- models:
- default: sonnet
-scan:
- enrichment:
- mode: llm
- embeddings:
- backend: sentence-transformers
- model: all-MiniLM-L6-v2
- dimensions: 384
- sentenceTransformers:
- base_url: ""
-```
-
-Test the connection and ingest:
-
-```bash
-node packages/cli/dist/bin.js connection test mongo-prod --project-dir /tmp/ktx-mongodb-example
-node packages/cli/dist/bin.js ingest mongo-prod --project-dir /tmp/ktx-mongodb-example --yes --plain
-```
-
-The first ingest starts the **ktx** embedding daemon and downloads the
-`all-MiniLM-L6-v2` model. Expected final state: `Database schema: done`.
-
-Inspect the result:
-
-- `raw-sources/mongo-prod/live-database//tables/*.json` — one per
- collection, including the `active_users` view with `estimatedRows: null`
-- `raw-sources/mongo-prod/live-database//enrichment/relationships.json` —
- inferred relationships sit in `review` (a non-SQL source has no read-only SQL
- coverage validation), with `accepted: []`
-- `semantic-layer/mongo-prod/_schema/app.yaml` — the schema with per-column AI
- descriptions
-
-`ktx sql -c mongo-prod "SELECT 1"` is refused by the read-only SQL capability
-gate, and `ktx sl query -c mongo-prod ...` is refused because MongoDB is not a
-SQL source.
-
-## Cleanup
-
-```bash
-docker compose -f examples/mongodb/docker-compose.yml down -v # or: podman compose
-rm -rf /tmp/ktx-mongodb-example
-```
diff --git a/examples/mongodb/docker-compose.yml b/examples/mongodb/docker-compose.yml
deleted file mode 100644
index d753ec43..00000000
--- a/examples/mongodb/docker-compose.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-services:
- mongodb:
- image: mongo:7
- ports:
- # Non-default host port so the example does not clash with a local MongoDB.
- - "27117:27017"
- healthcheck:
- test: ["CMD-SHELL", "mongosh --quiet --eval \"db.runCommand({ ping: 1 }).ok\" | grep -q 1"]
- interval: 2s
- timeout: 5s
- retries: 30
- volumes:
- # MongoDB runs *.js here once, on first start, against an empty data dir.
- - ./init:/docker-entrypoint-initdb.d:ro
diff --git a/examples/mongodb/init/seed.js b/examples/mongodb/init/seed.js
deleted file mode 100644
index 0e76928a..00000000
--- a/examples/mongodb/init/seed.js
+++ /dev/null
@@ -1,64 +0,0 @@
-// Seed a representative MongoDB dataset for the ktx connector example.
-//
-// MongoDB runs this once on first container start (it is mounted into
-// /docker-entrypoint-initdb.d). It can also be applied by hand:
-// mongosh "mongodb://localhost:27117" < examples/mongodb/init/seed.js
-//
-// The shapes here exercise the connector's schema inference end to end:
-// scalar BSON types, a nested sub-document, an array, Decimal128, dates, a
-// field with more than one type (-> "mixed"), an absent field (-> nullable),
-// an ObjectId reference for relationship discovery, and a view (to confirm
-// introspection never runs a count command on a view).
-const app = db.getSiblingDB('app');
-
-app.users.drop();
-app.orders.drop();
-
-app.users.insertMany([
- {
- email: 'ada@example.com',
- age: 31,
- active: true,
- created: new Date('2026-01-04T10:00:00Z'),
- balance: NumberDecimal('120.50'),
- address: { city: 'NY', zip: '10001' },
- tags: ['admin', 'early-access'],
- ref: 'abc',
- },
- {
- email: 'grace@example.com',
- active: false,
- created: new Date('2026-02-11T08:30:00Z'),
- balance: NumberDecimal('0.00'),
- address: { city: 'SF', zip: '94016' },
- tags: [],
- ref: 42, // a second type for this field -> inferred "mixed"
- // age intentionally absent -> inferred nullable
- },
- {
- email: 'linus@example.com',
- age: 27,
- active: true,
- created: new Date('2026-03-01T12:00:00Z'),
- balance: NumberDecimal('9.99'),
- address: { city: 'Austin', zip: '73301' },
- tags: ['beta'],
- ref: null,
- },
-]);
-
-const userIds = app.users.find({}, { _id: 1 }).toArray().map((u) => u._id);
-
-app.orders.insertMany([
- { user_id: userIds[0], total: 120.5, status: 'paid', placed: new Date('2026-03-02T09:00:00Z') },
- { user_id: userIds[0], total: 9.99, status: 'pending', placed: new Date('2026-03-05T14:00:00Z') },
- { user_id: userIds[1], total: 50.25, status: 'paid', placed: new Date('2026-03-06T16:00:00Z') },
-]);
-
-// A view, to confirm introspection does not issue a count command on it
-// (MongoDB rejects count on a view with CommandNotSupportedOnView).
-app.createView('active_users', 'users', [{ $match: { active: true } }]);
-
-print('users: ' + app.users.countDocuments());
-print('orders: ' + app.orders.countDocuments());
-print('collections: ' + app.getCollectionNames().join(', '));
diff --git a/examples/mongodb/scripts/introspect-smoke.mjs b/examples/mongodb/scripts/introspect-smoke.mjs
deleted file mode 100644
index 511681fc..00000000
--- a/examples/mongodb/scripts/introspect-smoke.mjs
+++ /dev/null
@@ -1,53 +0,0 @@
-// Deterministic, no-LLM smoke for the MongoDB connector. Drives the same
-// introspection entry point ktx ingest's "database schema" stage uses, against
-// the seeded example database, and asserts the inferred schema.
-//
-// Usage: node introspect-smoke.mjs [mongoUrl]
-import { fileURLToPath } from 'node:url';
-import { dirname, resolve } from 'node:path';
-
-const here = dirname(fileURLToPath(import.meta.url));
-const ktxRoot = resolve(here, '../../..');
-const connectorUrl = `file://${resolve(
- ktxRoot,
- 'packages/cli/dist/connectors/mongodb/live-database-introspection.js',
-)}`;
-
-const mongoUrl = process.argv[2] ?? 'mongodb://localhost:27117/app';
-
-const { createMongoDbLiveDatabaseIntrospection } = await import(connectorUrl);
-
-function assert(condition, message) {
- if (!condition) {
- throw new Error(`assertion failed: ${message}`);
- }
-}
-
-const port = createMongoDbLiveDatabaseIntrospection({
- connections: { 'mongo-example': { driver: 'mongodb', url: mongoUrl, databases: ['app'] } },
-});
-
-const snapshot = await port.extractSchema('mongo-example');
-const tables = new Map(snapshot.tables.map((table) => [table.name, table]));
-
-assert(snapshot.driver === 'mongodb', 'snapshot driver is mongodb');
-assert(['orders', 'users'].every((name) => tables.has(name)), 'users and orders collections introspected');
-
-const users = tables.get('users');
-const columns = new Map(users.columns.map((column) => [column.name, column]));
-assert(columns.get('_id')?.primaryKey === true && columns.get('_id')?.nullable === false, '_id is the non-null primary key');
-assert(columns.get('age')?.nullable === true, 'age is nullable (absent in one document)');
-assert(columns.get('email')?.nullable === false, 'email is non-nullable (present in every document)');
-assert(columns.get('address')?.normalizedType === 'json', 'nested address maps to opaque json');
-assert(columns.get('tags')?.normalizedType === 'json', 'array tags maps to opaque json');
-assert(columns.get('ref')?.nativeType === 'mixed', 'ref with two types is inferred as mixed');
-
-const view = tables.get('active_users');
-assert(view?.kind === 'view', 'active_users is a view');
-assert(view?.estimatedRows === null, 'a view is introspected without a count (estimatedRows null)');
-
-console.log(`OK: introspected ${snapshot.tables.length} collections from ${mongoUrl}`);
-for (const table of snapshot.tables) {
- console.log(` - ${table.db}.${table.name} (${table.kind}, ${table.columns.length} columns)`);
-}
-process.exit(0);
diff --git a/examples/mongodb/scripts/smoke.sh b/examples/mongodb/scripts/smoke.sh
deleted file mode 100755
index 4c747083..00000000
--- a/examples/mongodb/scripts/smoke.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Manual smoke for the MongoDB connector: start MongoDB, seed it, and assert the
-# connector's schema introspection (the deterministic, no-LLM half of ktx ingest's
-# "database schema" stage). The full enrichment ingest is documented in README.md.
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-EXAMPLE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
-KTX_ROOT="$(cd "$EXAMPLE_DIR/../.." && pwd)"
-COMPOSE_FILE="$EXAMPLE_DIR/docker-compose.yml"
-CONNECTOR="$KTX_ROOT/packages/cli/dist/connectors/mongodb/live-database-introspection.js"
-MONGO_URL="${KTX_MONGODB_URL:-mongodb://localhost:27117/app}"
-
-# Compose engine: docker by default, override for podman:
-# KTX_MONGODB_COMPOSE="podman compose" examples/mongodb/scripts/smoke.sh
-COMPOSE="${KTX_MONGODB_COMPOSE:-docker compose}"
-
-cleanup() {
- if [[ "${KTX_MONGODB_KEEP:-0}" != "1" ]]; then
- $COMPOSE -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true
- fi
-}
-trap cleanup EXIT
-
-if [[ ! -f "$CONNECTOR" ]]; then
- echo "Build the CLI first: pnpm --filter @kaelio/ktx run build" >&2
- exit 1
-fi
-
-echo "Starting MongoDB and seeding (${COMPOSE})…"
-$COMPOSE -f "$COMPOSE_FILE" up -d --wait
-
-echo "Asserting connector introspection against ${MONGO_URL}…"
-node "$SCRIPT_DIR/introspect-smoke.mjs" "$MONGO_URL"
-
-echo "Smoke passed."
diff --git a/examples/orbit-relationship-verification/README.md b/examples/orbit-relationship-verification/README.md
index d99c8fea..126488a2 100644
--- a/examples/orbit-relationship-verification/README.md
+++ b/examples/orbit-relationship-verification/README.md
@@ -1,11 +1,11 @@
# Orbit-style relationship discovery verification
-This **ktx** project backs the default `relationships:verify-orbit` command. It uses
+This KTX project backs the default `relationships:verify-orbit` command. It uses
the checked-in Orbit-style SQLite fixture from the relationship discovery
benchmark corpus, with no declared primary keys or foreign keys in the database
schema.
-Run from the **ktx** workspace root:
+Run from the KTX workspace root:
```bash
pnpm run relationships:verify-orbit
diff --git a/examples/orbit-relationship-verification/ktx.yaml b/examples/orbit-relationship-verification/ktx.yaml
index 4bb605e2..b1d30961 100644
--- a/examples/orbit-relationship-verification/ktx.yaml
+++ b/examples/orbit-relationship-verification/ktx.yaml
@@ -6,6 +6,7 @@ storage:
state: sqlite
search: sqlite-fts5
git:
+ auto_commit: true
author: "ktx "
ingest:
adapters: []
diff --git a/examples/package-artifacts/README.md b/examples/package-artifacts/README.md
index b5813b0c..7fa39fb3 100644
--- a/examples/package-artifacts/README.md
+++ b/examples/package-artifacts/README.md
@@ -14,7 +14,7 @@ generated local project.
The managed Python runtime smoke requires `uv` on `PATH`, isolates
`KTX_RUNTIME_ROOT`, verifies `ktx admin runtime status`, runs `ktx sl query --yes` to
install the core runtime from the bundled wheel, checks `ktx admin runtime status`,
-starts and reuses the **ktx** daemon, and stops it.
+starts and reuses the KTX daemon, and stops it.
The artifact manifest contains the public `@kaelio/ktx` npm tarball and the
bundled `kaelio-ktx` runtime wheel. The smoke does not install standalone
diff --git a/examples/postgres-historic/README.md b/examples/postgres-historic/README.md
index aadd3cd4..64fc2593 100644
--- a/examples/postgres-historic/README.md
+++ b/examples/postgres-historic/README.md
@@ -17,19 +17,19 @@ unchanged bounded pattern shards do not schedule LLM work.
## Prerequisites
- Docker with Compose v2
-- Node and pnpm matching the **ktx** workspace
-- `uv` on `PATH` so the **ktx**-managed Python runtime can install the bundled
+- Node and pnpm matching the KTX workspace
+- `uv` on `PATH` so the KTX-managed Python runtime can install the bundled
runtime wheel
## Run
-From the **ktx** repository root:
+From the KTX repository root:
```bash
examples/postgres-historic/scripts/smoke.sh
```
-The smoke creates a temporary **ktx** project, isolates the managed Python runtime
+The smoke creates a temporary KTX project, isolates the managed Python runtime
under the temporary project parent, starts Postgres on `127.0.0.1:55432`, and
uses this connection URL:
@@ -41,7 +41,7 @@ Set `KTX_POSTGRES_HISTORIC_KEEP_DOCKER=1` to leave the container running after
the script exits.
The smoke validates the query-history raw snapshot path without requiring LLM
-credentials. It uses **ktx**'s local stage-only ingest API after `ktx setup`, so the
+credentials. It uses KTX's local stage-only ingest API after `ktx setup`, so the
deterministic reader, batch SQL parser, stable artifact writer, and diff-based
WorkUnit planning are checked independently from curation.
@@ -124,6 +124,6 @@ table.
- Missing grants: confirm `GRANT pg_read_all_stats TO ktx_reader;`.
- Empty snapshot: rerun `scripts/generate-workload.sh base` and keep
`--query-history-min-executions 2` for the smoke.
-- SQL-analysis failures: run `pnpm run ktx -- dev runtime status` from the **ktx**
+- SQL-analysis failures: run `pnpm run ktx -- dev runtime status` from the KTX
repository root and confirm `uv`, the bundled Python wheel, and the managed
runtime all pass.
diff --git a/knip.json b/knip.json
index 507cef0d..65b1a0a2 100644
--- a/knip.json
+++ b/knip.json
@@ -17,9 +17,7 @@
"test/**/*.test-utils.ts",
"test/**/acceptance-fixtures.ts",
"src/context/scan/relationship-benchmarks.ts!",
- "src/context/scan/relationship-benchmark-report.ts!",
- "src/connectors/sqlite/read-query-child.ts!",
- "src/context/llm/subprocess-generate-object-child.ts!"
+ "src/context/scan/relationship-benchmark-report.ts!"
]
},
"docs-site": {
@@ -40,8 +38,7 @@
"conventional-changelog-conventionalcommits"
],
"ignore": [
- ".context/**",
- "examples/**"
+ ".context/**"
],
"ignoreBinaries": [
"uv",
diff --git a/package.json b/package.json
index e9f94149..e7714634 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "ktx-workspace",
- "version": "0.16.0",
+ "version": "0.9.0",
"description": "Workspace root for ktx packages",
"private": true,
"type": "module",
@@ -69,6 +69,11 @@
"typescript": "^6.0.3",
"yaml": "^2.9.0"
},
+ "pnpm": {
+ "onlyBuiltDependencies": [
+ "better-sqlite3"
+ ]
+ },
"license": "Apache-2.0",
"repository": {
"type": "git",
diff --git a/packages/cli/package.json b/packages/cli/package.json
index af4bf65c..939a8b9c 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,11 +1,7 @@
{
"name": "@kaelio/ktx",
- "version": "0.16.0",
+ "version": "0.9.0",
"description": "Standalone ktx context layer for data agents",
- "author": {
- "name": "Kaelio",
- "url": "https://www.kaelio.com"
- },
"type": "module",
"engines": {
"node": ">=22.0.0"
@@ -51,15 +47,10 @@
"@ai-sdk/devtools": "0.0.18",
"@ai-sdk/google-vertex": "^4.0.134",
"@anthropic-ai/claude-agent-sdk": "0.3.146",
- "@aws-sdk/client-athena": "^3.1068.0",
- "@aws-sdk/client-glue": "^3.1068.0",
- "@clack/core": "1.3.1",
"@clack/prompts": "1.4.0",
"@clickhouse/client": "^1.18.5",
"@commander-js/extra-typings": "14.0.0",
- "@duckdb/node-api": "1.5.3-r.3",
"@google-cloud/bigquery": "^8.3.1",
- "google-auth-library": "10.6.2",
"@looker/sdk": "^26.8.0",
"@looker/sdk-node": "^26.8.0",
"@looker/sdk-rtl": "^21.6.5",
@@ -74,17 +65,13 @@
"ink": "^7.0.3",
"lookml-parser": "7.1.0",
"minimatch": "^10.2.5",
- "mongodb": "^6.12.0",
"mssql": "^12.5.4",
"mysql2": "^3.22.3",
"openai": "^6.38.0",
"p-limit": "^7.3.0",
"pg": "^8.21.0",
- "pino": "^10.3.1",
- "pino-pretty": "^13.1.3",
"posthog-node": "^5.34.9",
"react": "^19.2.6",
- "semver": "^7.8.1",
"simple-git": "3.36.0",
"snowflake-sdk": "^2.4.2",
"yaml": "^2.9.0",
@@ -98,7 +85,6 @@
"@types/node": "^25.9.1",
"@types/pg": "^8.20.0",
"@types/react": "^19.2.15",
- "@types/semver": "^7.7.1",
"@vitest/coverage-v8": "^4.1.7",
"ajv": "8.20.0",
"ink-testing-library": "^4.0.0",
diff --git a/packages/cli/scripts/copy-runtime-assets.mjs b/packages/cli/scripts/copy-runtime-assets.mjs
index 47502c21..579cb8e8 100644
--- a/packages/cli/scripts/copy-runtime-assets.mjs
+++ b/packages/cli/scripts/copy-runtime-assets.mjs
@@ -7,17 +7,10 @@ const promptsSource = join(packageRoot, 'src', 'prompts');
const promptsTarget = join(packageRoot, 'dist', 'prompts');
const skillsSource = join(packageRoot, 'src', 'skills');
const skillsTarget = join(packageRoot, 'dist', 'skills');
-// Per-dialect SQL notes are markdown served by the sql_dialect_notes MCP tool;
-// tsc does not emit non-.ts files, so copy them next to their compiled module.
-const dialectNotesSource = join(packageRoot, 'src', 'context', 'sql-analysis', 'dialects');
-const dialectNotesTarget = join(packageRoot, 'dist', 'context', 'sql-analysis', 'dialects');
await rm(promptsTarget, { recursive: true, force: true });
await rm(skillsTarget, { recursive: true, force: true });
-await rm(dialectNotesTarget, { recursive: true, force: true });
await mkdir(dirname(promptsTarget), { recursive: true });
await mkdir(dirname(skillsTarget), { recursive: true });
-await mkdir(dirname(dialectNotesTarget), { recursive: true });
await cp(promptsSource, promptsTarget, { recursive: true });
await cp(skillsSource, skillsTarget, { recursive: true });
-await cp(dialectNotesSource, dialectNotesTarget, { recursive: true });
diff --git a/packages/cli/src/admin.ts b/packages/cli/src/admin.ts
index d4b63ac1..6c04f82f 100644
--- a/packages/cli/src/admin.ts
+++ b/packages/cli/src/admin.ts
@@ -24,7 +24,7 @@ export function registerAdminCommands(program: Command, context: KtxCliCommandCo
admin
.command('init')
- .description('Initialize a Git-backed ktx project directory for maintenance scripts')
+ .description('Initialize a Git-backed KTX project directory for maintenance scripts')
.argument('[directory]', 'Project directory')
.option('--force', 'Rewrite ktx.yaml and scaffold files in an existing project', false)
.action(
diff --git a/packages/cli/src/clack.ts b/packages/cli/src/clack.ts
index 94999476..2ad51e6c 100644
--- a/packages/cli/src/clack.ts
+++ b/packages/cli/src/clack.ts
@@ -3,30 +3,6 @@ import type { KtxCliIo } from './cli-runtime.js';
const ESC = String.fromCharCode(0x1b);
-export interface CliStyleEnv {
- NO_COLOR?: string;
- TERM?: string;
-}
-
-function ansiEnabled(env: CliStyleEnv = process.env): boolean {
- return !env.NO_COLOR && env.TERM !== 'dumb';
-}
-
-function ansiColor(text: string, open: number, close: number, env?: CliStyleEnv): string {
- if (!ansiEnabled(env)) {
- return text;
- }
- return `${ESC}[${open}m${text}${ESC}[${close}m`;
-}
-
-export function dim(text: string, env?: CliStyleEnv): string {
- return ansiColor(text, 2, 22, env);
-}
-
-export function cyan(text: string, env?: CliStyleEnv): string {
- return ansiColor(text, 36, 39, env);
-}
-
export interface RailBufferedSource {
stdoutText(): string;
stderrText(): string;
@@ -81,39 +57,27 @@ class KtxCliPromptCancelledError extends Error {
}
export function createClackSpinner(): KtxCliSpinner {
- // clack colors the animated spinner frame magenta by default; styleFrame
- // (typed in SpinnerOptions, absent from the README) recolors it ktx orange.
- return spinner({ styleFrame: orange });
+ return spinner();
}
-// ktx mascot orange (#FF8A4C) via 24-bit truecolor.
-function orange(text: string): string {
- if (!ansiEnabled()) {
- return text;
- }
- return `${ESC}[38;2;255;138;76m${text}${ESC}[39m`;
+function magenta(text: string): string {
+ return `${ESC}[35m${text}${ESC}[39m`;
}
function red(text: string): string {
- return ansiColor(text, 31, 39);
+ return `${ESC}[31m${text}${ESC}[39m`;
}
-/**
- * Stderr-only, non-animated spinner. Use this instead of {@link createCliSpinner}
- * when the next step reads stdin in raw mode (an Ink TUI or a keypress wait):
- * the animated clack spinner seizes stdin via `@clack/core`'s `block()` and
- * leaves it dirty, which the following raw-mode reader misreads as a stray key.
- */
export function createStaticCliSpinner(io: KtxCliSpinnerIo): KtxCliSpinner {
return {
start(message) {
- io.stderr.write(`${orange('◐')} ${message}\n`);
+ io.stderr.write(`${magenta('◐')} ${message}\n`);
},
message(message) {
- io.stderr.write(`${orange('│')} ${message}\n`);
+ io.stderr.write(`${magenta('│')} ${message}\n`);
},
stop(message) {
- io.stderr.write(`${orange('◇')} ${message}\n`);
+ io.stderr.write(`${magenta('◇')} ${message}\n`);
},
error(message) {
io.stderr.write(`${red('■')} ${message}\n`);
@@ -121,30 +85,6 @@ export function createStaticCliSpinner(io: KtxCliSpinnerIo): KtxCliSpinner {
};
}
-/**
- * Animated spinner in an interactive terminal, static `◐/◇/■` lines otherwise
- * (scripts, CI, piped output) so logs stay clean and uncluttered by frames.
- */
-export function createCliSpinner(io: KtxCliIo): KtxCliSpinner {
- return io.stdout.isTTY === true ? createClackSpinner() : createStaticCliSpinner(io);
-}
-
-export async function runWithCliSpinner(
- spinner: KtxCliSpinner,
- text: { start: string; success: string; failure: string },
- run: () => Promise,
-): Promise {
- spinner.start(text.start);
- try {
- const value = await run();
- spinner.stop(text.success);
- return value;
- } catch (error) {
- spinner.error(text.failure);
- throw error;
- }
-}
-
export function createClackPromptAdapter(): KtxCliPromptAdapter {
return {
async confirm(options) {
diff --git a/packages/cli/src/claude-code-prompt-caching.ts b/packages/cli/src/claude-code-prompt-caching.ts
index 0d696579..a7c0fa54 100644
--- a/packages/cli/src/claude-code-prompt-caching.ts
+++ b/packages/cli/src/claude-code-prompt-caching.ts
@@ -21,7 +21,7 @@ export function formatClaudeCodePromptCachingWarning(fields: string[]): string |
if (fields.length === 0) {
return null;
}
- return `claude-code ignores ${fields.join(', ')} because the Claude Agent SDK does not expose ktx prompt-cache TTL, tool, or history markers.`;
+ return `claude-code ignores ${fields.join(', ')} because the Claude Agent SDK does not expose KTX prompt-cache TTL, tool, or history markers.`;
}
export function formatClaudeCodePromptCachingFix(): string {
diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts
index c30711c8..31ab8a03 100644
--- a/packages/cli/src/cli-program.ts
+++ b/packages/cli/src/cli-program.ts
@@ -2,7 +2,6 @@ import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { Command, type CommandUnknownOpts, InvalidArgumentError } from '@commander-js/extra-typings';
import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js';
-import { SLACK_HELP_FOOTER, writeErrorCommunityHint } from './community-cta.js';
import { registerCompletionCommands } from './commands/completion-commands.js';
import { registerConnectionCommands } from './commands/connection-commands.js';
import { registerIngestCommands } from './commands/ingest-commands.js';
@@ -17,7 +16,6 @@ import { renderMissingProjectMessage } from './doctor.js';
import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js';
import { profileMark, profileSpan } from './startup-profile.js';
import type { CommandOutcome } from './telemetry/index.js';
-import { prepareUpdateCheckNotice, type PrepareUpdateCheckNoticeOptions } from './update-check/update-check.js';
profileMark('module:cli-program');
@@ -41,8 +39,6 @@ interface KtxCommanderProgramOptions {
runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise;
}
-type KtxCliUpdateCheckOptions = Pick;
-
export interface BuildKtxProgramOptions {
io: KtxCliIo;
deps: KtxCliDeps;
@@ -51,7 +47,6 @@ export interface BuildKtxProgramOptions {
setExitCode?: (code: number) => void;
argv?: string[];
setTelemetryModule?: (telemetry: typeof import('./telemetry/index.js')) => void;
- updateCheck?: KtxCliUpdateCheckOptions;
}
type CommanderExitLike = { exitCode: number; code: string; message: string };
@@ -133,7 +128,7 @@ export function parseBooleanStringOption(value: string): boolean {
}
export function parseSafeConnectionIdOption(value: string): string {
- if (!/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(value)) {
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
@@ -252,14 +247,13 @@ export function resolveCommandProjectDirOverride(command: CommandWithGlobalOptio
function createBaseProgram(info: KtxCliPackageInfo, io: KtxCliIo): Command {
return new Command()
.name('ktx')
- .description('ktx data agent context layer CLI')
- .option('--project-dir ', 'ktx project directory (default: KTX_PROJECT_DIR, nearest ktx.yaml, or cwd)')
+ .description('KTX data agent context layer CLI')
+ .option('--project-dir ', 'KTX project directory (default: KTX_PROJECT_DIR, nearest ktx.yaml, or cwd)')
.option('--debug', 'Enable diagnostic logging to stderr')
.version(`${info.name} ${info.version}`, '-v, --version', 'Show CLI version')
.helpOption('-h, --help', 'Show this help text')
.configureHelp({ showGlobalOptions: true })
.showHelpAfterError()
- .addHelpText('after', `\n${SLACK_HELP_FOOTER}`)
.exitOverride()
.configureOutput({
writeOut: (chunk) => io.stdout.write(chunk),
@@ -437,36 +431,23 @@ export function collectCommandFlagsPresent(command: CommandUnknownOpts): Record<
export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
const program = createBaseProgram(options.packageInfo, options.io);
- let pendingUpdateNotice: string | null = null;
-
program.hook('preAction', async (_thisCommand, actionCommand) => {
// The hidden completion command must stay silent and side-effect free: skip
- // the telemetry notice, command span, project checks, and update checks entirely.
+ // the telemetry notice, command span, and project checks entirely.
if (commandPath(actionCommand as CommandPathNode).includes('__complete')) {
return;
}
- const commandNode = actionCommand as CommandPathNode;
- const updateCheck = await prepareUpdateCheckNotice({
- io: options.io,
- env: options.updateCheck?.env,
- fetchDistTags: options.updateCheck?.fetchDistTags,
- homeDir: options.updateCheck?.homeDir,
- installedVersion: options.packageInfo.version,
- now: options.updateCheck?.now,
- commandOptions: commandOptions(commandNode),
- });
- pendingUpdateNotice = updateCheck.notice;
-
const telemetry = await import('./telemetry/index.js');
options.setTelemetryModule?.(telemetry);
await telemetry.showTelemetryNoticeIfNeeded(options.io, options.packageInfo);
+ const commandNode = actionCommand as CommandPathNode;
const path = commandPath(commandNode);
const projectDir = resolveCommandProjectDir(commandNode);
const hasProject = ktxYamlExists(projectDir);
const attachProjectGroup = shouldAttachCommandProjectGroup(path, hasProject);
telemetry.beginCommandSpan({
commandPath: path,
- flagsPresent: collectCommandFlagsPresent(actionCommand),
+ flagsPresent: collectCommandFlagsPresent(commandNode as unknown as CommandUnknownOpts),
projectDir: attachProjectGroup ? projectDir : undefined,
hasProject,
attachProjectGroup,
@@ -476,13 +457,6 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
ensureProjectAvailable(options.io, commandNode);
});
- program.hook('postAction', () => {
- if (pendingUpdateNotice) {
- options.io.stderr.write(pendingUpdateNotice);
- pendingUpdateNotice = null;
- }
- });
-
const context: KtxCliCommandContext = {
io: options.io,
deps: options.deps,
@@ -555,15 +529,7 @@ export async function runCommanderKtxCli(
try {
return await runBareInteractiveCommand(program, io, context);
} catch (error) {
- const telemetry = await import('./telemetry/index.js');
- await telemetry.reportException({
- error,
- context: { source: 'bare-interactive', handled: true, fatal: false },
- packageInfo: info,
- io,
- });
io.stderr.write(`${formatCliError(error)}\n`);
- writeErrorCommunityHint(io, 'error');
return 1;
}
}
@@ -588,7 +554,6 @@ export async function runCommanderKtxCli(
exitCode = error.exitCode === 0 ? 0 : 1;
} else {
io.stderr.write(`${formatCliError(error)}\n`);
- writeErrorCommunityHint(io, 'error');
exitCode = 1;
}
} finally {
@@ -598,23 +563,6 @@ export async function runCommanderKtxCli(
outcome: commandOutcomeForParseResult(parseError, exitCode),
error: parseError,
});
- if (
- parseError &&
- !isCommanderExit(parseError) &&
- !isKtxProjectMissingAbortError(parseError)
- ) {
- await telemetryModule.reportException({
- error: parseError,
- context: {
- source: completed?.commandPath.join(' ') ?? 'commander parseAsync',
- handled: true,
- fatal: false,
- },
- projectDir: completed?.projectGroupAttached ? completed.projectDir : undefined,
- packageInfo: info,
- io,
- });
- }
await telemetryModule.emitCompletedCommand({ completed, packageInfo: info, io });
await telemetryModule.shutdownTelemetryEmitter();
}
diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts
index 89c7c11d..7043143b 100644
--- a/packages/cli/src/cli-runtime.ts
+++ b/packages/cli/src/cli-runtime.ts
@@ -12,7 +12,6 @@ import type { KtxSqlArgs } from './sql.js';
import { profileMark, profileSpan } from './startup-profile.js';
import type { KtxTextIngestArgs } from './text-ingest.js';
import { assertCliVersion } from './release-version.js';
-import { writeErrorCommunityHint } from './community-cta.js';
profileMark('module:cli-runtime');
@@ -61,7 +60,7 @@ export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo {
typeof packageJson.name !== 'string' ||
typeof packageJson.version !== 'string'
) {
- throw new Error('Invalid ktx CLI package metadata');
+ throw new Error('Invalid KTX CLI package metadata');
}
return {
@@ -77,7 +76,7 @@ async function runInit(args: { projectDir: string; force: boolean }, io: KtxCliI
force: args.force,
});
- io.stdout.write(`Initialized ktx project at ${result.projectDir}\n`);
+ io.stdout.write(`Initialized KTX project at ${result.projectDir}\n`);
io.stdout.write(`Config: ${result.configPath}\n`);
io.stdout.write(`Commit: ${result.commitHash ?? 'none'}\n`);
return 0;
@@ -130,54 +129,6 @@ function installTelemetrySignalFlush(io: KtxCliIo, info: KtxCliPackageInfo): ()
};
}
-/** @internal */
-export function createGlobalExceptionReporter(io: KtxCliIo, info: KtxCliPackageInfo) {
- return async (source: 'uncaughtException' | 'unhandledRejection', error: unknown): Promise => {
- const { reportException, shutdownTelemetryEmitter } = await import('./telemetry/index.js');
- await reportException({
- error,
- context: { source, handled: false, fatal: true },
- io,
- packageInfo: info,
- immediate: true,
- });
- await shutdownTelemetryEmitter();
- };
-}
-
-/** @internal */
-export function writeGlobalExceptionToStderr(io: KtxCliIo, error: unknown): void {
- if (error instanceof Error && error.stack) {
- io.stderr.write(`${error.stack}\n`);
- } else {
- io.stderr.write(`${String(error)}\n`);
- }
- writeErrorCommunityHint(io, 'crash');
-}
-
-export function installGlobalExceptionHandlers(io: KtxCliIo, info: KtxCliPackageInfo): () => void {
- const report = createGlobalExceptionReporter(io, info);
- const handle = (source: 'uncaughtException' | 'unhandledRejection', error: unknown): void => {
- void (async () => {
- try {
- await report(source, error);
- } catch {
- // Best-effort: preserve Node's process termination behavior.
- }
- writeGlobalExceptionToStderr(io, error);
- process.exit(1);
- })();
- };
- const onUncaught = (error: Error): void => handle('uncaughtException', error);
- const onUnhandled = (reason: unknown): void => handle('unhandledRejection', reason);
- process.on('uncaughtException', onUncaught);
- process.on('unhandledRejection', onUnhandled);
- return () => {
- process.off('uncaughtException', onUncaught);
- process.off('unhandledRejection', onUnhandled);
- };
-}
-
export async function runKtxCli(
argv = process.argv.slice(2),
io: KtxCliIo = process,
@@ -190,14 +141,11 @@ export async function runKtxCli(
// Real-process entry only: flush telemetry if interrupted. Test/programmatic
// callers pass their own `io`, so they never install process-level handlers.
const removeSignalFlush = (io as unknown) === process ? installTelemetrySignalFlush(io, info) : undefined;
- const removeGlobalExceptionHandlers =
- (io as unknown) === process ? installGlobalExceptionHandlers(io, info) : undefined;
try {
return await runCommanderKtxCli(argv, io, deps, info, {
runInit: runInitForCommander,
});
} finally {
- removeGlobalExceptionHandlers?.();
removeSignalFlush?.();
}
}
diff --git a/packages/cli/src/commands/connection-commands.ts b/packages/cli/src/commands/connection-commands.ts
index 579575d1..213bf608 100644
--- a/packages/cli/src/commands/connection-commands.ts
+++ b/packages/cli/src/commands/connection-commands.ts
@@ -37,7 +37,7 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm
connection
.command('test')
.description('Test one or all configured connections (default: all)')
- .argument('[connectionId]', 'ktx connection id to test (omit to test all)')
+ .argument('[connectionId]', 'KTX connection id to test (omit to test all)')
.option('--all', 'Test every configured connection and print a summary list')
.action(async (connectionId: string | undefined, options: { all?: boolean }, command) => {
if (options.all === true && connectionId !== undefined) {
diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts
index 9d1c3af5..b5efe443 100644
--- a/packages/cli/src/commands/ingest-commands.ts
+++ b/packages/cli/src/commands/ingest-commands.ts
@@ -1,12 +1,10 @@
-import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
+import { type Command, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KtxCliCommandContext,
parsePositiveIntegerOption,
resolveCommandProjectDir,
} from '../cli-program.js';
-import { KTX_SCAN_ENRICHMENT_STAGES } from '../context/scan/enrichment-state.js';
-import type { KtxScanEnrichmentStage } from '../context/scan/types.js';
import type { KtxCliDeps, KtxCliIo } from '../index.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxPublicIngestArgs } from '../public-ingest.js';
@@ -16,36 +14,6 @@ import { resolveConnectionSelection } from './connection-selection.js';
profileMark('module:commands/ingest-commands');
-/**
- * Parses `--stages` into an ordered, de-duplicated subset of the canonical
- * enrichment-stage registry. An unknown or empty name is a hard parse error so
- * a typo never silently degrades to "run everything."
- *
- * @internal
- */
-export function parseEnrichmentStagesOption(value: string): KtxScanEnrichmentStage[] {
- const names = value
- .split(',')
- .map((name) => name.trim())
- .filter((name) => name.length > 0);
- if (names.length === 0) {
- throw new InvalidArgumentError(
- `must be a non-empty comma-separated list of stages (${KTX_SCAN_ENRICHMENT_STAGES.join(', ')})`,
- );
- }
- const valid = new Set(KTX_SCAN_ENRICHMENT_STAGES);
- const selected = new Set();
- for (const name of names) {
- if (!valid.has(name)) {
- throw new InvalidArgumentError(
- `unknown stage "${name}"; valid stages are ${KTX_SCAN_ENRICHMENT_STAGES.join(', ')}`,
- );
- }
- selected.add(name as KtxScanEnrichmentStage);
- }
- return KTX_SCAN_ENRICHMENT_STAGES.filter((stage) => selected.has(stage));
-}
-
interface IngestCommandOptions {
runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise;
}
@@ -57,26 +25,16 @@ export function registerIngestCommands(
): void {
const ingest = program
.command('ingest')
- .description('Build or inspect ktx context, or capture text into memory')
+ .description('Build or inspect KTX context, or capture text into memory')
.usage('[options] [connectionId]')
.argument('[connectionId]', 'Configured connection id to ingest (omit to ingest all)')
.option('--all', 'Ingest all configured connections', false)
.addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory'))
.addOption(new Option('--no-query-history', 'Skip database query-history usage patterns'))
.option('--query-history-window-days ', 'Query-history lookback window for this run', parsePositiveIntegerOption)
- .option(
- '--stages ',
- 'Comma-separated enrichment stages to (re)run (descriptions,embeddings,relationships); omit to run all',
- parseEnrichmentStagesOption,
- )
- .option('--text ', 'Capture inline text into ktx memory; repeatable', collectOption, [])
- .option('--file ', 'Capture a text file into ktx memory; use - for stdin; repeatable', collectOption, [])
- .option(
- '--verbatim',
- 'Store each --text/--file document body unchanged as a GLOBAL wiki page; the LLM derives only metadata',
- false,
- )
- .option('--connection-id ', 'ktx connection id to tag captured text/file notes')
+ .option('--text ', 'Capture inline text into KTX memory; repeatable', collectOption, [])
+ .option('--file ', 'Capture a text file into KTX memory; use - for stdin; repeatable', collectOption, [])
+ .option('--connection-id ', 'KTX connection id to tag captured text/file notes')
.option('--user-id ', 'Memory user id for text/file capture attribution', 'local-cli')
.option('--fail-fast', 'Stop after the first failed text/file item', false)
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json']))
@@ -89,14 +47,6 @@ export function registerIngestCommands(
const projectDir = resolveCommandProjectDir(command);
const hasTextCapture = options.text.length > 0 || options.file.length > 0;
- if (options.verbatim === true && !hasTextCapture) {
- command.error('error: --verbatim requires --text or --file');
- }
-
- if (options.stages !== undefined && hasTextCapture) {
- command.error('error: --stages applies to database ingest only; it cannot be combined with --text or --file');
- }
-
if (hasTextCapture) {
if (connectionId !== undefined) {
command.error(
@@ -116,7 +66,6 @@ export function registerIngestCommands(
userId: options.userId,
json: options.json === true,
failFast: options.failFast === true,
- ...(options.verbatim === true ? { verbatim: true } : {}),
},
context.io,
context.deps,
@@ -138,7 +87,6 @@ export function registerIngestCommands(
inputMode: options.input === false ? 'disabled' : 'auto',
queryHistory,
...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
- ...(options.stages ? { stages: options.stages } : {}),
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
};
diff --git a/packages/cli/src/commands/knowledge-commands.ts b/packages/cli/src/commands/knowledge-commands.ts
index ee537458..b601b688 100644
--- a/packages/cli/src/commands/knowledge-commands.ts
+++ b/packages/cli/src/commands/knowledge-commands.ts
@@ -27,7 +27,6 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
.usage('[options] [query...]')
.argument('[query...]', 'Search query; omit to list all pages')
.option('--user-id ', 'Local user id', 'local')
- .option('-c, --connection ', 'Scope results to one connection (unscoped pages plus pages tagged with it)')
.option('--limit ', 'Maximum search results (search mode only)', parsePositiveIntegerOption)
.addOption(
new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
@@ -47,7 +46,6 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
query: string[],
options: {
userId: string;
- connection?: string;
limit?: number;
output?: 'pretty' | 'plain' | 'json';
json?: boolean;
@@ -59,7 +57,6 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
command: 'list',
projectDir: resolveCommandProjectDir(command),
userId: options.userId,
- ...(options.connection !== undefined ? { connectionId: options.connection } : {}),
output: options.output,
json: options.json,
cliVersion: context.packageInfo.version,
@@ -71,7 +68,6 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
projectDir: resolveCommandProjectDir(command),
query: query.join(' '),
userId: options.userId,
- ...(options.connection !== undefined ? { connectionId: options.connection } : {}),
output: options.output,
json: options.json,
...(isDebugEnabled(command) ? { debug: true } : {}),
diff --git a/packages/cli/src/commands/mcp-commands.ts b/packages/cli/src/commands/mcp-commands.ts
index 6978d2b7..94b17498 100644
--- a/packages/cli/src/commands/mcp-commands.ts
+++ b/packages/cli/src/commands/mcp-commands.ts
@@ -27,11 +27,11 @@ function binPath(): string {
function formatMcpStartResultMessage(input: { status: 'started' | 'already-running'; url: string }): string {
return [
- input.status === 'started' ? `ktx MCP daemon started: ${input.url}` : `ktx MCP daemon already running: ${input.url}`,
+ input.status === 'started' ? `KTX MCP daemon started: ${input.url}` : `KTX MCP daemon already running: ${input.url}`,
'',
- 'ktx is ready for configured agents.',
- 'Open your agent for this ktx project and ask a data question, for example:',
- ' "Use ktx to show me the available tables and metrics."',
+ 'KTX is ready for configured agents.',
+ 'Open your agent for this KTX project and ask a data question, for example:',
+ ' "Use KTX to show me the available tables and metrics."',
'',
].join('\n');
}
@@ -50,14 +50,14 @@ async function printMcpStatus(context: KtxCliCommandContext, projectDir: string)
export function registerMcpCommands(program: Command, context: KtxCliCommandContext): void {
const mcp = program
.command('mcp')
- .description('Manage the ktx MCP HTTP server (bare command: show status)')
+ .description('Manage the KTX MCP HTTP server (bare command: show status)')
.action(async (_options, command) => {
await printMcpStatus(context, resolveCommandProjectDir(command));
});
mcp
.command('stdio')
- .description('Run the ktx MCP server over stdio')
+ .description('Run the KTX MCP server over stdio')
.action(async (_options, command) => {
await (context.deps.mcp?.runStdioServer ?? runKtxMcpStdioServer)({
projectDir: resolveCommandProjectDir(command),
@@ -68,7 +68,7 @@ export function registerMcpCommands(program: Command, context: KtxCliCommandCont
mcp
.command('start')
- .description('Start the ktx MCP HTTP server')
+ .description('Start the KTX MCP HTTP server')
.option('--host ', 'Host to bind', '127.0.0.1')
.option('--port ', 'Port to bind', parsePositiveIntegerOption, 7878)
.option('--token ', 'Bearer token required for non-loopback binding')
@@ -96,7 +96,7 @@ export function registerMcpCommands(program: Command, context: KtxCliCommandCont
allowedOrigins: options.allowedOrigin,
io: context.io,
});
- context.io.stdout.write(`ktx MCP server listening at http://${options.host}:${options.port}/mcp\n`);
+ context.io.stdout.write(`KTX MCP server listening at http://${options.host}:${options.port}/mcp\n`);
return;
}
const result = await (context.deps.mcp?.startDaemon ?? startKtxMcpDaemon)({
@@ -114,24 +114,24 @@ export function registerMcpCommands(program: Command, context: KtxCliCommandCont
mcp
.command('stop')
- .description('Stop the ktx MCP daemon')
+ .description('Stop the KTX MCP daemon')
.action(async (_options, command) => {
const result = await (context.deps.mcp?.stopDaemon ?? stopKtxMcpDaemon)({
projectDir: resolveCommandProjectDir(command),
});
- context.io.stdout.write(result.status === 'stopped' ? 'ktx MCP daemon stopped.\n' : 'ktx MCP daemon is not running.\n');
+ context.io.stdout.write(result.status === 'stopped' ? 'KTX MCP daemon stopped.\n' : 'KTX MCP daemon is not running.\n');
});
mcp
.command('status')
- .description('Show ktx MCP daemon status')
+ .description('Show KTX MCP daemon status')
.action(async (_options, command) => {
await printMcpStatus(context, resolveCommandProjectDir(command));
});
mcp
.command('logs')
- .description('Print the ktx MCP daemon log')
+ .description('Print the KTX MCP daemon log')
.option('--follow', 'Follow log output', false)
.action(async (options, command) => {
const logPath = mcpDaemonLayout(resolveCommandProjectDir(command)).logPath;
diff --git a/packages/cli/src/commands/runtime-commands.ts b/packages/cli/src/commands/runtime-commands.ts
index 441cceda..ef0d06cc 100644
--- a/packages/cli/src/commands/runtime-commands.ts
+++ b/packages/cli/src/commands/runtime-commands.ts
@@ -18,7 +18,7 @@ async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArg
export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void {
const runtime = program
.command('runtime')
- .description('Install, start, stop, and inspect the ktx-managed Python runtime')
+ .description('Install, start, stop, and inspect the KTX-managed Python runtime')
.showHelpAfterError();
runtime
@@ -38,7 +38,7 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand
runtime
.command('start')
- .description('Start the ktx daemon')
+ .description('Start the KTX daemon')
.addOption(createRuntimeFeatureOption())
.option('--force', 'Restart even when a matching daemon is already running', false)
.action(async (options: { feature: RuntimeFeature; force?: boolean }, command: CommandWithGlobalOptions) => {
@@ -53,8 +53,8 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand
runtime
.command('stop')
- .description('Stop the ktx daemon')
- .option('--all', 'Stop all ktx daemon processes recorded or discoverable on this machine', false)
+ .description('Stop the KTX daemon')
+ .option('--all', 'Stop all KTX daemon processes recorded or discoverable on this machine', false)
.action(async (options: { all?: boolean }, command: CommandWithGlobalOptions) => {
await runRuntimeArgs(context, {
command: 'stop',
diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts
index d838e472..1619a80a 100644
--- a/packages/cli/src/commands/setup-commands.ts
+++ b/packages/cli/src/commands/setup-commands.ts
@@ -2,7 +2,7 @@ import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-
import type { KtxCliCommandContext } from '../cli-program.js';
import { resolveCommandProjectDir } from '../cli-program.js';
import type { KtxSetupDatabaseDriver } from '../setup-databases.js';
-import { isKtxSetupLlmBackend, type KtxSetupLlmBackend } from '../setup-models.js';
+import type { KtxSetupLlmBackend } from '../setup-models.js';
import type { KtxSetupSourceType } from '../setup-sources.js';
async function runSetupArgs(
@@ -16,7 +16,7 @@ async function runSetupArgs(
function positiveInteger(value: string): number {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
- throw new InvalidArgumentError(`Expected a positive integer, received ${value}`);
+ throw new Error(`Expected a positive integer, received ${value}`);
}
return parsed;
}
@@ -29,7 +29,7 @@ function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
}
function llmBackend(value: string): KtxSetupLlmBackend {
- if (isKtxSetupLlmBackend(value)) {
+ if (value === 'anthropic' || value === 'vertex' || value === 'claude-code' || value === 'codex') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
@@ -38,7 +38,6 @@ function llmBackend(value: string): KtxSetupLlmBackend {
function databaseDriver(value: string): KtxSetupDatabaseDriver {
if (
value === 'sqlite' ||
- value === 'duckdb' ||
value === 'postgres' ||
value === 'mysql' ||
value === 'clickhouse' ||
@@ -58,9 +57,7 @@ function sourceType(value: string): KtxSetupSourceType {
value === 'metabase' ||
value === 'looker' ||
value === 'lookml' ||
- value === 'notion' ||
- value === 'sigma' ||
- value === 'gdrive'
+ value === 'notion'
) {
return value;
}
@@ -92,13 +89,13 @@ function shouldShowSetupEntryMenu(
target?: string;
global?: boolean;
local?: boolean;
- installDir?: string;
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
llmBackend?: KtxSetupLlmBackend;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
+ llmModel?: string;
vertexProject?: string;
vertexLocation?: string;
skipLlm?: boolean;
@@ -135,9 +132,6 @@ function shouldShowSetupEntryMenu(
metabaseDatabaseId?: number;
notionCrawlMode?: string;
notionRootPageId?: string[];
- gdriveServiceAccountKeyRef?: string;
- gdriveFolderId?: string;
- gdriveRecursive?: boolean;
skipSources?: boolean;
},
command: Command,
@@ -166,13 +160,13 @@ function shouldShowSetupEntryMenu(
'target',
'global',
'local',
- 'installDir',
'skipAgents',
'yes',
'input',
'llmBackend',
'anthropicApiKeyEnv',
'anthropicApiKeyFile',
+ 'llmModel',
'vertexProject',
'vertexLocation',
'skipLlm',
@@ -203,9 +197,6 @@ function shouldShowSetupEntryMenu(
'sourceTarget',
'metabaseDatabaseId',
'notionCrawlMode',
- 'gdriveServiceAccountKeyRef',
- 'gdriveFolderId',
- 'gdriveRecursive',
'skipSources',
].some((optionName) => optionWasSpecified(command, optionName));
}
@@ -213,8 +204,8 @@ function shouldShowSetupEntryMenu(
export function registerSetupCommands(program: Command, context: KtxCliCommandContext): void {
const setup = program
.command('setup')
- .description('Set up or resume a local ktx project')
- .addOption(new Option('--project-dir ', 'ktx project directory').hideHelp())
+ .description('Set up or resume a local KTX project')
+ .addOption(new Option('--project-dir ', 'KTX project directory').hideHelp())
.option('--agents', 'Install agent integration only', false)
.addOption(
new Option('--target ', 'Agent target').choices([
@@ -228,10 +219,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
)
.option('--global', 'Install agent integration into the global target scope', false)
.option('--local', 'Install Claude Code MCP config into the private per-project ~/.claude.json scope', false)
- .option(
- '--install-dir ',
- 'Directory to install project-scoped agent config into (defaults to the ktx project directory)',
- )
.addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false))
.option('--yes', 'Accept project creation and runtime install defaults where setup confirms', false)
.option('--no-input', 'Disable interactive terminal input')
@@ -242,6 +229,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
.addOption(
new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(),
)
+ .addOption(new Option('--llm-model ', 'LLM model ID or backend model alias').hideHelp())
.addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp())
.addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp())
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
@@ -310,7 +298,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
.hideHelp(),
)
.addOption(
- new Option('--skip-databases', 'Leave database setup incomplete; ktx cannot work until a database is added')
+ new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a database is added')
.hideHelp()
.default(false),
)
@@ -346,12 +334,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
.default([] as string[])
.hideHelp(),
)
- .addOption(
- new Option('--gdrive-service-account-key-ref ', 'file: reference to a Google service account JSON key')
- .hideHelp(),
- )
- .addOption(new Option('--gdrive-folder-id ', 'Google Drive folder id to ingest').hideHelp())
- .addOption(new Option('--gdrive-recursive', 'Recursively traverse Google Drive subfolders').hideHelp().default(false))
.addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false))
.showHelpAfterError();
@@ -415,16 +397,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
context.setExitCode(1);
return;
}
- if (options.installDir && (options.global || options.local)) {
- context.io.stderr.write('Choose either --install-dir or a scope flag (--global / --local), not both.\n');
- context.setExitCode(1);
- return;
- }
- if (options.installDir && options.target === 'claude-desktop') {
- context.io.stderr.write('--install-dir does not apply to --target claude-desktop, which is always global.\n');
- context.setExitCode(1);
- return;
- }
const creatingDatabaseConnection = options.database.length > 0 || options.databaseUrl !== undefined;
if (creatingDatabaseConnection && options.databaseConnectionId.length > 1) {
@@ -434,8 +406,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
}
const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project';
- const debugEnabled =
- ((command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as { debug?: unknown }).debug === true;
await runSetupArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),
@@ -443,15 +413,14 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
agents: options.agents === true,
...(options.target ? { target: options.target } : {}),
agentScope: resolvedAgentScope,
- ...(options.installDir ? { installRoot: options.installDir } : {}),
skipAgents: options.skipAgents === true,
inputMode: options.input === false ? 'disabled' : 'auto',
- ...(debugEnabled ? { debug: true } : {}),
yes: options.yes === true,
cliVersion: context.packageInfo.version,
...(options.llmBackend ? { llmBackend: options.llmBackend } : {}),
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
+ ...(options.llmModel ? { llmModel: options.llmModel } : {}),
...(options.vertexProject ? { vertexProject: options.vertexProject } : {}),
...(options.vertexLocation ? { vertexLocation: options.vertexLocation } : {}),
skipLlm: options.skipLlm === true,
@@ -501,11 +470,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
...(options.metabaseDatabaseId !== undefined ? { metabaseDatabaseId: options.metabaseDatabaseId } : {}),
...(options.notionCrawlMode ? { notionCrawlMode: options.notionCrawlMode } : {}),
...(options.notionRootPageId.length > 0 ? { notionRootPageIds: options.notionRootPageId } : {}),
- ...(options.gdriveServiceAccountKeyRef
- ? { gdriveServiceAccountKeyRef: options.gdriveServiceAccountKeyRef }
- : {}),
- ...(options.gdriveFolderId ? { gdriveFolderId: options.gdriveFolderId } : {}),
- ...(options.gdriveRecursive ? { gdriveRecursive: true } : {}),
runInitialSourceIngest: false,
skipSources: options.skipSources === true,
showEntryMenu: shouldShowSetupEntryMenu(options, command),
diff --git a/packages/cli/src/commands/sl-commands.ts b/packages/cli/src/commands/sl-commands.ts
index 7b99e593..8f2f05a3 100644
--- a/packages/cli/src/commands/sl-commands.ts
+++ b/packages/cli/src/commands/sl-commands.ts
@@ -44,7 +44,7 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
.description('List, search, validate, or query local semantic-layer sources')
.usage('[options] [query...]')
.argument('[query...]', 'Search query; omit to list all sources')
- .option('--connection-id ', 'ktx connection id')
+ .option('--connection-id ', 'KTX connection id')
.option('--limit ', 'Maximum search results (search mode only)', parsePositiveIntegerOption)
.addOption(
new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
diff --git a/packages/cli/src/commands/sql-commands.ts b/packages/cli/src/commands/sql-commands.ts
index 213ab059..0c73df6a 100644
--- a/packages/cli/src/commands/sql-commands.ts
+++ b/packages/cli/src/commands/sql-commands.ts
@@ -26,7 +26,7 @@ export function registerSqlCommands(program: Command, context: KtxCliCommandCont
.command('sql')
.description('Execute parser-validated read-only SQL against a configured connection')
.argument('', 'SQL query to execute')
- .requiredOption('-c, --connection ', 'ktx connection id')
+ .requiredOption('-c, --connection ', 'KTX connection id')
.option('--max-rows ', 'Maximum rows to return', parseSqlMaxRowsOption, DEFAULT_MAX_ROWS)
.addOption(
new Option('--output ', 'Output mode: pretty (default), plain (TSV), or json').choices([
diff --git a/packages/cli/src/commands/status-commands.ts b/packages/cli/src/commands/status-commands.ts
index 1f72a385..ec429576 100644
--- a/packages/cli/src/commands/status-commands.ts
+++ b/packages/cli/src/commands/status-commands.ts
@@ -15,7 +15,7 @@ function inputMode(options: { input?: boolean }): { inputMode?: 'disabled' } {
export function registerStatusCommands(program: Command, context: KtxCliCommandContext): void {
program
.command('status')
- .description('Check current ktx setup and project readiness')
+ .description('Check current KTX setup and project readiness')
.option('--json', 'Print JSON output', false)
.option('-v, --verbose', 'Show every check, including passing ones', false)
.option('--validate', 'Only validate the ktx.yaml schema; skip readiness checks', false)
diff --git a/packages/cli/src/community-cta.ts b/packages/cli/src/community-cta.ts
deleted file mode 100644
index b4702542..00000000
--- a/packages/cli/src/community-cta.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import type { KtxCliIo } from './cli-runtime.js';
-import { isWritableTtyOutput } from './io/tty.js';
-import { dim } from './io/symbols.js';
-import { SLACK_URL } from './links.js';
-
-type ErrorCtaVariant = 'error' | 'crash';
-
-/** @internal */
-export const SLACK_HELP_FOOTER = `Community & support: ${SLACK_URL}`;
-
-/** @internal */
-export const SLACK_SETUP_NOTE = {
- title: 'Community',
- body: `Questions or feedback? Join the ktx Slack: ${SLACK_URL}`,
-} as const;
-
-export function writeErrorCommunityHint(io: KtxCliIo, variant: ErrorCtaVariant): void {
- if (!isWritableTtyOutput(io.stderr)) {
- return;
- }
-
- const line =
- variant === 'crash'
- ? `This may be a bug - report it or ask in the ktx community: ${SLACK_URL}`
- : `Stuck? The ktx community can help: ${SLACK_URL}`;
-
- io.stderr.write(`${dim(line)}\n`);
-}
diff --git a/packages/cli/src/connection-drivers.ts b/packages/cli/src/connection-drivers.ts
index b7b1857d..4f10e663 100644
--- a/packages/cli/src/connection-drivers.ts
+++ b/packages/cli/src/connection-drivers.ts
@@ -1,21 +1,14 @@
import type { KtxProjectConnectionConfig } from './context/project/config.js';
-/** @internal Canonical SQL-warehouse driver ids; the dialect-notes coverage test derives its required coverage from this set. */
-export const KTX_DATABASE_DRIVER_IDS = [
+const KTX_DATABASE_DRIVER_IDS = new Set([
'sqlite',
- 'duckdb',
'postgres',
'mysql',
'clickhouse',
'sqlserver',
'bigquery',
'snowflake',
- 'athena',
-] as const;
-
-// mongodb is a database driver but has no SQL dialect, so it sits outside the
-// dialect-notes coverage set above.
-const databaseDriverIds = new Set([...KTX_DATABASE_DRIVER_IDS, 'mongodb']);
+]);
export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string {
return String(connection.driver ?? '')
@@ -24,5 +17,5 @@ export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig
}
export function isDatabaseDriver(driver: string): boolean {
- return databaseDriverIds.has(driver.trim().toLowerCase());
+ return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
}
diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts
index 809eea96..96281e82 100644
--- a/packages/cli/src/connection.ts
+++ b/packages/cli/src/connection.ts
@@ -3,13 +3,9 @@ import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js';
-import { parseGdriveConnectionConfig, resolveGdriveServiceAccountKey } from './context/connections/gdrive-config.js';
import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js';
import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js';
-import { createGoogleDocsClients, verifyGdriveFolderAndCountDocs } from './context/ingest/adapters/gdrive/gdrive-client.js';
-import { gdriveServiceAccountKeySchema } from './context/ingest/adapters/gdrive/types.js';
import { testRepoConnection } from './context/ingest/repo-fetch.js';
-import { federatedConnectionListing } from './context/connections/federation.js';
import { getDriverRegistration } from './context/connections/drivers.js';
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
import { resolveKtxConfigReference } from './context/core/config-reference.js';
@@ -20,8 +16,7 @@ import { bold, dim, green, red, SYMBOLS } from './io/symbols.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { profileMark } from './startup-profile.js';
import { isDemoConnection } from './telemetry/demo-detect.js';
-import { emitTelemetryEvent, reportException } from './telemetry/index.js';
-import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
+import { emitTelemetryEvent } from './telemetry/index.js';
import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
profileMark('module:connection');
@@ -34,10 +29,6 @@ export type KtxConnectionArgs =
type MetabaseTestPort = Pick;
type LookerTestPort = Pick;
type NotionTestPort = Pick;
-type GdriveTestPort = Pick<
- ReturnType['drive'],
- 'listFiles' | 'getFile'
->;
type TestRepoConnection = typeof testRepoConnection;
export interface KtxConnectionDeps {
@@ -45,13 +36,11 @@ export interface KtxConnectionDeps {
createMetabaseClient?: (project: KtxLocalProject, connectionId: string) => Promise;
createLookerClient?: (project: KtxLocalProject, connectionId: string) => Promise;
createNotionClient?: (project: KtxLocalProject, connectionId: string) => Promise;
- createGdriveClient?: (project: KtxLocalProject, connectionId: string) => Promise;
testRepoConnection?: TestRepoConnection;
}
const SUPPORTED_TEST_DRIVERS = [
'sqlite',
- 'duckdb',
'postgres',
'mysql',
'clickhouse',
@@ -61,7 +50,6 @@ const SUPPORTED_TEST_DRIVERS = [
'metabase',
'looker',
'notion',
- 'gdrive',
'dbt',
'metricflow',
'lookml',
@@ -86,12 +74,6 @@ async function testNativeConnection(
}
const result = await connector.testConnection();
if (!result.success) {
- // Re-throw the driver's original error so connection_test telemetry records
- // its real class (e.g. ConnectionError) and code (e.g. ELOGIN) instead of
- // collapsing every native failure to a generic Error with no code.
- if (result.cause instanceof Error) {
- throw result.cause;
- }
throw new Error(result.error ?? 'connection test failed');
}
return { driver: connector.driver };
@@ -145,7 +127,7 @@ async function createDefaultLookerClient(
connectionId: string,
): Promise {
const factory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project));
- return factory.createLookerClient(connectionId);
+ return (await factory.createClient(connectionId)) as unknown as LookerTestPort;
}
async function testLookerConnection(
@@ -193,34 +175,6 @@ async function testNotionConnection(
return { bot: describeNotionBot(bot) };
}
-async function createDefaultGdriveClient(
- project: KtxLocalProject,
- connectionId: string,
-): Promise {
- const connection = project.config.connections[connectionId];
- if (!connection) {
- throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
- }
- const parsed = parseGdriveConnectionConfig(connection);
- const keyText = await resolveGdriveServiceAccountKey(parsed.service_account_key_ref);
- const key = gdriveServiceAccountKeySchema.parse(JSON.parse(keyText));
- return createGoogleDocsClients(key).drive;
-}
-
-async function testGdriveConnection(
- project: KtxLocalProject,
- connectionId: string,
- createClient: (project: KtxLocalProject, connectionId: string) => Promise,
-): Promise<{ docs: number }> {
- const connection = project.config.connections[connectionId];
- if (!connection) {
- throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
- }
- const parsed = parseGdriveConnectionConfig(connection);
- const client = await createClient(project, connectionId);
- return { docs: await verifyGdriveFolderAndCountDocs(client, parsed.folder_id) };
-}
-
interface GitConnectionFields {
repoUrl: string;
authToken: string | null;
@@ -309,15 +263,6 @@ async function testConnectionByDriver(
return { driver, detailKey: 'Bot', detailValue: result.bot };
}
- if (driver === 'gdrive') {
- const result = await testGdriveConnection(
- project,
- connectionId,
- deps.createGdriveClient ?? createDefaultGdriveClient,
- );
- return { driver, detailKey: 'Docs', detailValue: String(result.docs) };
- }
-
if (driver === 'dbt' || driver === 'metricflow' || driver === 'lookml') {
const result = await testGitRepoConnection(
project,
@@ -373,21 +318,6 @@ async function emitConnectionTest(input: {
...(errorDetail ? { errorDetail } : {}),
},
});
- if (input.error) {
- await reportException({
- error: input.error,
- context: { source: 'connection test', handled: true, fatal: false },
- projectDir: input.project.projectDir,
- io: input.io,
- redactionSecrets: await collectTelemetryRedactionSecrets({
- project: input.project,
- connectionId: input.connectionId,
- includeLlm: false,
- includeEmbeddings: false,
- env: process.env,
- }),
- });
- }
}
function visualWidth(text: string): number {
@@ -495,23 +425,15 @@ export async function runKtxConnection(
io.stdout.write('No connections configured. Run `ktx setup` to add one.\n');
return 0;
}
- const federated = federatedConnectionListing(project.config.connections, args.projectDir);
- const idCandidates = [...entries.map(([id]) => id), ...(federated ? [federated.id] : [])];
- const driverLengths = [
+ const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
+ const driverWidth = Math.max(
+ 'DRIVER'.length,
...entries.map(([, c]) => (c.driver ?? 'unknown').length),
- ...(federated ? [federated.driver.length] : []),
- ];
- const idWidth = Math.max('ID'.length, ...idCandidates.map((id) => id.length));
- const driverWidth = Math.max('DRIVER'.length, ...driverLengths);
+ );
io.stdout.write(`${'ID'.padEnd(idWidth)} ${'DRIVER'.padEnd(driverWidth)}\n`);
for (const [id, connection] of entries) {
io.stdout.write(`${id.padEnd(idWidth)} ${(connection.driver ?? 'unknown').padEnd(driverWidth)}\n`);
}
- if (federated) {
- io.stdout.write(`${federated.id.padEnd(idWidth)} ${federated.driver.padEnd(driverWidth)}\n`);
- io.stdout.write(` federates: ${federated.members.join(', ')}\n`);
- io.stdout.write(` ${federated.hint}\n`);
- }
return 0;
}
diff --git a/packages/cli/src/connectors/athena/connector.ts b/packages/cli/src/connectors/athena/connector.ts
deleted file mode 100644
index 00b53cc0..00000000
--- a/packages/cli/src/connectors/athena/connector.ts
+++ /dev/null
@@ -1,555 +0,0 @@
-import { AthenaClient, StartQueryExecutionCommand, GetQueryExecutionCommand, GetQueryResultsCommand } from '@aws-sdk/client-athena';
-import { GlueClient, GetDatabasesCommand, GetTablesCommand } from '@aws-sdk/client-glue';
-import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
-import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
-import {
- connectorTestFailure,
- createKtxConnectorCapabilities,
- type KtxConnectorTestResult,
- type KtxColumnSampleInput,
- type KtxColumnSampleResult,
- type KtxColumnStatsInput,
- type KtxColumnStatsResult,
- type KtxQueryResult,
- type KtxReadOnlyQueryInput,
- type KtxScanConnector,
- type KtxScanContext,
- type KtxScanInput,
- type KtxSchemaColumn,
- type KtxSchemaSnapshot,
- type KtxSchemaTable,
- type KtxTableListEntry,
- type KtxTableRef,
- type KtxTableSampleInput,
- type KtxTableSampleResult,
-} from '../../context/scan/types.js';
-import { scopedTableNames } from '../../context/scan/table-ref.js';
-import { resolveStringReference } from '../shared/string-reference.js';
-
-export interface KtxAthenaConnectionConfig {
- driver?: string;
- region?: string;
- s3_staging_dir?: string;
- workgroup?: string;
- catalog?: string;
- database?: string;
- databases?: string[];
- [key: string]: unknown;
-}
-
-export interface KtxAthenaResolvedConnectionConfig {
- region: string;
- s3StagingDir: string;
- workgroup: string;
- catalog: string;
- database: string | undefined;
- databases: string[];
-}
-
-interface KtxAthenaQueryExecutionStatus {
- State?: string;
- StateChangeReason?: string;
-}
-
-interface KtxAthenaQueryExecution {
- Status?: KtxAthenaQueryExecutionStatus;
-}
-
-interface KtxAthenaColumnInfo {
- Name?: string;
- Type?: string;
-}
-
-interface KtxAthenaDatum {
- VarCharValue?: string;
-}
-
-interface KtxAthenaRow {
- Data?: KtxAthenaDatum[];
-}
-
-interface KtxAthenaResultSet {
- Rows?: KtxAthenaRow[];
- ResultSetMetadata?: { ColumnInfo?: KtxAthenaColumnInfo[] };
-}
-
-/** @internal */
-export interface KtxAthenaClient {
- startQueryExecution(input: {
- QueryString: string;
- ResultConfiguration: { OutputLocation: string };
- WorkGroup: string;
- QueryExecutionContext?: { Database?: string; Catalog?: string };
- }): Promise<{ QueryExecutionId?: string }>;
- getQueryExecution(input: { QueryExecutionId: string }): Promise<{ QueryExecution?: KtxAthenaQueryExecution }>;
- getQueryResults(input: { QueryExecutionId: string; NextToken?: string }): Promise<{
- ResultSet?: KtxAthenaResultSet;
- NextToken?: string;
- }>;
-}
-
-interface KtxGlueColumnDef {
- Name?: string;
- Type?: string;
- Comment?: string;
-}
-
-interface KtxGlueStorageDescriptor {
- Columns?: KtxGlueColumnDef[];
-}
-
-/** @internal */
-export interface KtxGlueTable {
- Name?: string;
- TableType?: string;
- StorageDescriptor?: KtxGlueStorageDescriptor;
- PartitionKeys?: KtxGlueColumnDef[];
- Description?: string;
- Parameters?: Record;
-}
-
-/** @internal */
-export interface KtxGlueClient {
- getDatabases(input: { CatalogId?: string; NextToken?: string }): Promise<{
- DatabaseList?: Array<{ Name?: string }>;
- NextToken?: string;
- }>;
- getTables(input: { DatabaseName: string; CatalogId?: string; NextToken?: string }): Promise<{
- TableList?: KtxGlueTable[];
- NextToken?: string;
- }>;
-}
-
-export interface KtxAthenaClientFactory {
- createAthenaClient(region: string): KtxAthenaClient;
- createGlueClient(region: string): KtxGlueClient;
-}
-
-class DefaultAthenaClientFactory implements KtxAthenaClientFactory {
- createAthenaClient(region: string): KtxAthenaClient {
- const client = new AthenaClient({ region });
- return {
- startQueryExecution: async (input) => {
- const result = await client.send(
- new StartQueryExecutionCommand({
- QueryString: input.QueryString,
- ResultConfiguration: { OutputLocation: input.ResultConfiguration.OutputLocation },
- WorkGroup: input.WorkGroup,
- QueryExecutionContext: input.QueryExecutionContext,
- }),
- );
- return { QueryExecutionId: result.QueryExecutionId };
- },
- getQueryExecution: async (input) => {
- const result = await client.send(new GetQueryExecutionCommand({ QueryExecutionId: input.QueryExecutionId }));
- return {
- QueryExecution: result.QueryExecution
- ? {
- Status: {
- State: result.QueryExecution.Status?.State,
- StateChangeReason: result.QueryExecution.Status?.StateChangeReason,
- },
- }
- : undefined,
- };
- },
- getQueryResults: async (input) => {
- const result = await client.send(
- new GetQueryResultsCommand({ QueryExecutionId: input.QueryExecutionId, NextToken: input.NextToken }),
- );
- return {
- ResultSet: result.ResultSet as KtxAthenaResultSet | undefined,
- NextToken: result.NextToken,
- };
- },
- };
- }
-
- createGlueClient(region: string): KtxGlueClient {
- const client = new GlueClient({ region });
- return {
- getDatabases: async (input) => {
- const result = await client.send(new GetDatabasesCommand({ CatalogId: input.CatalogId, NextToken: input.NextToken }));
- return {
- DatabaseList: result.DatabaseList?.map((db) => ({ Name: db.Name })),
- NextToken: result.NextToken,
- };
- },
- getTables: async (input) => {
- const result = await client.send(
- new GetTablesCommand({ DatabaseName: input.DatabaseName, CatalogId: input.CatalogId, NextToken: input.NextToken }),
- );
- return {
- TableList: result.TableList as KtxGlueTable[] | undefined,
- NextToken: result.NextToken,
- };
- },
- };
- }
-}
-
-function stringConfigValue(
- connection: KtxAthenaConnectionConfig | undefined,
- key: keyof KtxAthenaConnectionConfig,
- env: NodeJS.ProcessEnv,
-): string | undefined {
- const value = connection?.[key];
- if (typeof value !== 'string' || value.trim().length === 0) return undefined;
- // Resolve before checking emptiness: an unset `env:` reference resolves to '',
- // which must become undefined so `?? default` applies instead of keeping ''.
- const resolved = resolveStringReference(value.trim(), env).trim();
- return resolved.length > 0 ? resolved : undefined;
-}
-
-function configuredAthenaDatabases(connection: KtxAthenaConnectionConfig): string[] {
- if (!Array.isArray(connection.databases)) return [];
- const selected = connection.databases
- .filter((database): database is string => typeof database === 'string' && database.trim().length > 0)
- .map((database) => database.trim());
- return [...new Set(selected)];
-}
-
-export function isKtxAthenaConnectionConfig(
- connection: unknown,
-): connection is KtxAthenaConnectionConfig {
- return (
- typeof connection === 'object' &&
- connection !== null &&
- String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'athena'
- );
-}
-
-/** @internal */
-export function athenaConnectionConfigFromConfig(input: {
- connectionId: string;
- connection: KtxAthenaConnectionConfig | undefined;
- env?: NodeJS.ProcessEnv;
-}): KtxAthenaResolvedConnectionConfig {
- const inputDriver = input.connection?.driver ?? 'unknown';
- if (!isKtxAthenaConnectionConfig(input.connection)) {
- throw new Error(`Native Athena connector cannot run driver "${String(inputDriver)}"`);
- }
- const env = input.env ?? process.env;
- const region = stringConfigValue(input.connection, 'region', env);
- if (!region) {
- throw new Error(`Native Athena connector requires connections.${input.connectionId}.region`);
- }
- const s3StagingDir = stringConfigValue(input.connection, 's3_staging_dir', env);
- if (!s3StagingDir) {
- throw new Error(`Native Athena connector requires connections.${input.connectionId}.s3_staging_dir`);
- }
- return {
- region,
- s3StagingDir,
- workgroup: stringConfigValue(input.connection, 'workgroup', env) ?? 'primary',
- catalog: stringConfigValue(input.connection, 'catalog', env) ?? 'AwsDataCatalog',
- database: stringConfigValue(input.connection, 'database', env),
- databases: configuredAthenaDatabases(input.connection),
- };
-}
-
-function glueTableKind(tableType: string | undefined): 'table' | 'view' {
- const t = String(tableType ?? '').toUpperCase();
- if (t === 'VIRTUAL_VIEW') return 'view';
- return 'table';
-}
-
-const POLL_INTERVAL_MS = 250;
-const QUERY_TIMEOUT_MS = 5 * 60 * 1000;
-
-export interface KtxAthenaScanConnectorOptions {
- connectionId: string;
- connection: KtxAthenaConnectionConfig | undefined;
- clientFactory?: KtxAthenaClientFactory;
- env?: NodeJS.ProcessEnv;
- now?: () => Date;
-}
-
-export class KtxAthenaScanConnector implements KtxScanConnector {
- readonly id: string;
- readonly driver = 'athena' as const;
- readonly capabilities = createKtxConnectorCapabilities({
- tableSampling: true,
- columnSampling: true,
- columnStats: false,
- readOnlySql: true,
- nestedAnalysis: false,
- formalForeignKeys: false,
- estimatedRowCounts: false,
- });
-
- private readonly connectionId: string;
- private readonly resolved: KtxAthenaResolvedConnectionConfig;
- private readonly clientFactory: KtxAthenaClientFactory;
- private readonly now: () => Date;
- private readonly dialect = getSqlDialectForDriver('athena');
- private athenaClient: KtxAthenaClient | null = null;
- private glueClient: KtxGlueClient | null = null;
-
- constructor(options: KtxAthenaScanConnectorOptions) {
- this.connectionId = options.connectionId;
- this.resolved = athenaConnectionConfigFromConfig({
- connectionId: options.connectionId,
- connection: options.connection,
- env: options.env,
- });
- this.clientFactory = options.clientFactory ?? new DefaultAthenaClientFactory();
- this.now = options.now ?? (() => new Date());
- this.id = `athena:${options.connectionId}`;
- }
-
- async testConnection(): Promise {
- try {
- await this.listDatabasesPaginated({ maxResults: 1 });
- return { success: true };
- } catch (error) {
- return connectorTestFailure(error);
- }
- }
-
- async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- // Honor the configured `databases` scope (written by `ktx setup`); fall back
- // to every Glue database only when the scope is unset.
- const databases =
- this.resolved.databases.length > 0 ? this.resolved.databases : await this.listDatabasesPaginated({});
- const tables: KtxSchemaTable[] = [];
- for (const database of databases) {
- const scopedNames = input.tableScope
- ? scopedTableNames(input.tableScope, { catalog: this.resolved.catalog, db: database })
- : null;
- tables.push(...(await this.introspectDatabase(database, scopedNames)));
- }
- return {
- connectionId: this.connectionId,
- driver: 'athena',
- extractedAt: this.now().toISOString(),
- scope: { catalogs: [this.resolved.catalog], datasets: databases },
- metadata: {
- catalog: this.resolved.catalog,
- databases,
- table_count: tables.length,
- total_columns: tables.reduce((sum, t) => sum + t.columns.length, 0),
- },
- tables,
- warnings: [],
- };
- }
-
- async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const result = await this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns));
- return { headers: result.headers, headerTypes: result.headerTypes, rows: result.rows, totalRows: result.totalRows };
- }
-
- async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const result = await this.query(
- this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit),
- );
- return {
- values: result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]),
- nullCount: null,
- distinctCount: null,
- };
- }
-
- async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise {
- return null;
- }
-
- async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows);
- const result = await this.query(limitedSql);
- return { ...result, rowCount: result.rows.length };
- }
-
- async listSchemas(): Promise {
- return this.listDatabasesPaginated({});
- }
-
- async listTables(databases?: string[]): Promise {
- const targetDatabases = databases && databases.length > 0 ? databases : await this.listDatabasesPaginated({});
- const entries: KtxTableListEntry[] = [];
- for (const database of targetDatabases) {
- const glueTables = await this.listGlueTablesPaginated(database);
- for (const t of glueTables) {
- if (!t.Name) continue;
- entries.push({
- catalog: this.resolved.catalog,
- schema: database,
- name: t.Name,
- kind: glueTableKind(t.TableType),
- });
- }
- }
- return entries;
- }
-
- async cleanup(): Promise {
- this.athenaClient = null;
- this.glueClient = null;
- }
-
- qTableName(table: Pick & Partial>): string {
- return this.dialect.formatTableName(table);
- }
-
- private getAthenaClient(): KtxAthenaClient {
- if (!this.athenaClient) {
- this.athenaClient = this.clientFactory.createAthenaClient(this.resolved.region);
- }
- return this.athenaClient;
- }
-
- private getGlueClient(): KtxGlueClient {
- if (!this.glueClient) {
- this.glueClient = this.clientFactory.createGlueClient(this.resolved.region);
- }
- return this.glueClient;
- }
-
- private async listDatabasesPaginated(opts: { maxResults?: number }): Promise {
- const names: string[] = [];
- let nextToken: string | undefined;
- do {
- const result = await this.getGlueClient().getDatabases({ NextToken: nextToken });
- for (const db of result.DatabaseList ?? []) {
- if (db.Name) names.push(db.Name);
- if (opts.maxResults && names.length >= opts.maxResults) return names;
- }
- nextToken = result.NextToken;
- } while (nextToken);
- return names;
- }
-
- private async listGlueTablesPaginated(database: string): Promise {
- const tables: KtxGlueTable[] = [];
- let nextToken: string | undefined;
- do {
- const result = await this.getGlueClient().getTables({ DatabaseName: database, NextToken: nextToken });
- tables.push(...(result.TableList ?? []));
- nextToken = result.NextToken;
- } while (nextToken);
- return tables;
- }
-
- private async introspectDatabase(database: string, scopedNames: readonly string[] | null): Promise {
- if (scopedNames && scopedNames.length === 0) return [];
- const glueTables = await this.listGlueTablesPaginated(database);
- const scopeSet = scopedNames ? new Set(scopedNames) : null;
- return glueTables
- .filter((t): t is KtxGlueTable & { Name: string } => Boolean(t.Name) && (!scopeSet || scopeSet.has(t.Name!)))
- .map((t) => ({
- catalog: this.resolved.catalog,
- db: database,
- name: t.Name,
- kind: glueTableKind(t.TableType),
- comment: t.Description ?? null,
- estimatedRows: null,
- columns: this.toSchemaColumns(t),
- foreignKeys: [],
- }));
- }
-
- private toSchemaColumns(table: KtxGlueTable): KtxSchemaColumn[] {
- const columns = [...(table.StorageDescriptor?.Columns ?? []), ...(table.PartitionKeys ?? [])];
- return columns
- .filter((col): col is KtxGlueColumnDef & { Name: string } => Boolean(col.Name))
- .map((col) => {
- const nativeType = String(col.Type ?? 'string').toLowerCase();
- return {
- name: col.Name,
- nativeType,
- normalizedType: this.dialect.mapDataType(nativeType),
- dimensionType: this.dialect.mapToDimensionType(nativeType),
- nullable: true,
- primaryKey: false,
- comment: col.Comment ?? null,
- };
- });
- }
-
- private async query(sql: string): Promise {
- const athena = this.getAthenaClient();
- const { QueryExecutionId } = await athena.startQueryExecution({
- QueryString: sql,
- ResultConfiguration: { OutputLocation: this.resolved.s3StagingDir },
- WorkGroup: this.resolved.workgroup,
- ...(this.resolved.database || this.resolved.catalog
- ? {
- QueryExecutionContext: {
- ...(this.resolved.database ? { Database: this.resolved.database } : {}),
- ...(this.resolved.catalog ? { Catalog: this.resolved.catalog } : {}),
- },
- }
- : {}),
- });
-
- if (!QueryExecutionId) {
- throw new Error('Athena did not return a QueryExecutionId');
- }
-
- await this.waitForQueryCompletion(athena, QueryExecutionId);
-
- const rows: unknown[][] = [];
- let headers: string[] = [];
- let headerTypes: string[] = [];
- let nextToken: string | undefined;
- let firstPage = true;
-
- do {
- const result = await athena.getQueryResults({ QueryExecutionId, NextToken: nextToken });
- const resultSet = result.ResultSet;
-
- if (firstPage) {
- const columnInfo = resultSet?.ResultSetMetadata?.ColumnInfo ?? [];
- headers = columnInfo.map((col) => col.Name ?? '');
- headerTypes = columnInfo.map((col) => String(col.Type ?? 'varchar').toUpperCase());
- firstPage = false;
- }
-
- const pageRows = resultSet?.Rows ?? [];
- // Athena includes the header row as the first row of the first page — skip it.
- const dataRows = nextToken === undefined ? pageRows.slice(1) : pageRows;
- for (const row of dataRows) {
- rows.push((row.Data ?? []).map((d) => d.VarCharValue ?? null));
- }
-
- nextToken = result.NextToken;
- } while (nextToken);
-
- return {
- headers,
- headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
- rows,
- totalRows: rows.length,
- rowCount: rows.length,
- };
- }
-
- private async waitForQueryCompletion(athena: KtxAthenaClient, queryExecutionId: string): Promise {
- const terminalStates = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED']);
- const deadline = this.now().getTime() + QUERY_TIMEOUT_MS;
- for (;;) {
- const { QueryExecution } = await athena.getQueryExecution({ QueryExecutionId: queryExecutionId });
- const state = QueryExecution?.Status?.State ?? '';
- if (state === 'SUCCEEDED') return;
- if (terminalStates.has(state)) {
- const reason = QueryExecution?.Status?.StateChangeReason ?? state;
- throw new Error(`Athena query ${state}: ${reason}`);
- }
- if (this.now().getTime() >= deadline) {
- throw new Error(`Athena query ${queryExecutionId} timed out after ${QUERY_TIMEOUT_MS / 1000}s`);
- }
- await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
- }
- }
-
- private assertConnection(connectionId: string): void {
- if (connectionId !== this.connectionId) {
- throw new Error(`Athena connector ${this.connectionId} cannot scan connection ${connectionId}`);
- }
- }
-}
diff --git a/packages/cli/src/connectors/athena/dialect.ts b/packages/cli/src/connectors/athena/dialect.ts
deleted file mode 100644
index 81349804..00000000
--- a/packages/cli/src/connectors/athena/dialect.ts
+++ /dev/null
@@ -1,175 +0,0 @@
-import type { KtxSqlDialect } from '../../context/connections/dialects.js';
-import {
- columnDisplayPartCount,
- formatDialectDisplayRef,
- formatDialectTableName,
- parseDialectDisplayRef,
-} from '../../context/connections/dialect-helpers.js';
-import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
-
-type AthenaTableNameRef = Pick & Partial>;
-
-/** @internal */
-export class KtxAthenaDialect implements KtxSqlDialect {
- readonly type = 'athena' as const;
-
- private readonly dimensionTypeMappings: Record = {
- timestamp: 'time',
- date: 'time',
- bigint: 'number',
- int: 'number',
- integer: 'number',
- tinyint: 'number',
- smallint: 'number',
- double: 'number',
- float: 'number',
- real: 'number',
- boolean: 'boolean',
- };
-
- quoteIdentifier(identifier: string): string {
- return `"${identifier.replace(/"/g, '""')}"`;
- }
-
- formatTableName(table: AthenaTableNameRef): string {
- return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi');
- }
-
- formatDisplayRef(table: AthenaTableNameRef): string {
- return formatDialectDisplayRef(table, 'ansi');
- }
-
- parseDisplayRef(display: string): KtxTableRef | null {
- return parseDialectDisplayRef(display, 'ansi');
- }
-
- columnDisplayTablePartCount(): 1 | 2 | 3 {
- return columnDisplayPartCount('ansi');
- }
-
- mapDataType(nativeType: string): string {
- const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim();
- const typeMap: Record = {
- string: 'VARCHAR',
- varchar: 'VARCHAR',
- char: 'CHAR',
- binary: 'VARBINARY',
- bigint: 'BIGINT',
- int: 'INTEGER',
- integer: 'INTEGER',
- tinyint: 'TINYINT',
- smallint: 'SMALLINT',
- double: 'DOUBLE',
- float: 'FLOAT',
- real: 'REAL',
- decimal: 'DECIMAL',
- boolean: 'BOOLEAN',
- timestamp: 'TIMESTAMP',
- date: 'DATE',
- array: 'ARRAY',
- map: 'MAP',
- struct: 'STRUCT',
- uniontype: 'UNION',
- };
- return typeMap[base] ?? nativeType.toUpperCase();
- }
-
- mapToDimensionType(nativeType: string): KtxSchemaDimensionType {
- const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim();
- const mapped = this.dimensionTypeMappings[base];
- if (mapped) return mapped;
- if (base.includes('timestamp') || base.includes('date')) return 'time';
- if (base.includes('int') || base.includes('float') || base.includes('double') || base.includes('decimal') || base.includes('real')) return 'number';
- if (base.includes('bool')) return 'boolean';
- return 'string';
- }
-
- generateSampleQuery(tableName: string, limit: number, columns?: string[]): string {
- const columnList =
- columns && columns.length > 0 ? columns.map((c) => this.quoteIdentifier(c)).join(', ') : '*';
- return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`;
- }
-
- generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string {
- const quoted = this.quoteIdentifier(columnName);
- return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL LIMIT ${limit}`;
- }
-
- generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
- return `
- SELECT approx_distinct(${columnName}) AS cardinality
- FROM (
- SELECT ${columnName}
- FROM ${tableName}
- WHERE ${columnName} IS NOT NULL
- LIMIT ${sampleSize}
- )
- `;
- }
-
- generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
- return `
- SELECT approx_distinct(${columnName}) AS cardinality
- FROM (
- SELECT ${columnName}
- FROM ${tableName}
- WHERE ${columnName} IS NOT NULL
- ORDER BY rand()
- LIMIT ${sampleSize}
- )
- `;
- }
-
- generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string {
- return `
- SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val
- FROM ${tableName}
- WHERE ${columnName} IS NOT NULL
- ORDER BY val
- LIMIT ${limit}
- `;
- }
-
- generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null {
- return null;
- }
-
- getNullCountExpression(column: string): string {
- return `COUNT_IF(${column} IS NULL)`;
- }
-
- getDistinctCountExpression(column: string): string {
- return `approx_distinct(${column})`;
- }
-
- textLengthExpression(columnSql: string): string {
- return `LENGTH(CAST(${columnSql} AS VARCHAR))`;
- }
-
- castToText(columnSql: string): string {
- return `CAST(${columnSql} AS VARCHAR)`;
- }
-
- getSampleValueAggregation(innerSql: string): string {
- return `(SELECT array_join(array_agg(CAST(value AS VARCHAR)), '\u001f') FROM (${innerSql}) AS relationship_profile_values)`;
- }
-
- getLimitOffsetClause(limit: number, offset?: number): string {
- const safeLimit = Math.max(1, Math.floor(limit));
- const safeOffset = offset !== undefined ? Math.floor(offset) : 0;
- return safeOffset > 0 ? `OFFSET ${safeOffset} LIMIT ${safeLimit}` : `LIMIT ${safeLimit}`;
- }
-
- getTopClause(_limit: number): string {
- return '';
- }
-
- getTableSampleClause(_samplePct: number): string {
- return '';
- }
-
- getRandomSampleFilter(samplePct: number): string {
- if (samplePct <= 0 || samplePct >= 1) return '';
- return `rand() < ${samplePct}`;
- }
-}
diff --git a/packages/cli/src/connectors/athena/live-database-introspection.ts b/packages/cli/src/connectors/athena/live-database-introspection.ts
deleted file mode 100644
index 815285ba..00000000
--- a/packages/cli/src/connectors/athena/live-database-introspection.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-import type {
- LiveDatabaseIntrospectionOptions,
- LiveDatabaseIntrospectionPort,
-} from '../../context/ingest/adapters/live-database/types.js';
-import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
-import {
- KtxAthenaScanConnector,
- type KtxAthenaClientFactory,
- type KtxAthenaConnectionConfig,
-} from './connector.js';
-
-interface CreateAthenaLiveDatabaseIntrospectionOptions {
- connections: Record;
- clientFactory?: KtxAthenaClientFactory;
- now?: () => Date;
-}
-
-export function createAthenaLiveDatabaseIntrospection(
- options: CreateAthenaLiveDatabaseIntrospectionOptions,
-): LiveDatabaseIntrospectionPort {
- return {
- async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) {
- const connection = options.connections[connectionId] as KtxAthenaConnectionConfig | undefined;
- const connector = new KtxAthenaScanConnector({
- connectionId,
- connection,
- clientFactory: options.clientFactory,
- now: options.now,
- });
- try {
- return await connector.introspect(
- {
- connectionId,
- driver: 'athena',
- ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
- },
- { runId: `athena-${connectionId}` },
- );
- } finally {
- await connector.cleanup();
- }
- },
- };
-}
diff --git a/packages/cli/src/connectors/bigquery/connector.ts b/packages/cli/src/connectors/bigquery/connector.ts
index 69a166d5..edebe284 100644
--- a/packages/cli/src/connectors/bigquery/connector.ts
+++ b/packages/cli/src/connectors/bigquery/connector.ts
@@ -1,19 +1,11 @@
import { BigQuery, type TableField } from '@google-cloud/bigquery';
-import {
- normalizeBigQueryDatasetId,
- normalizeBigQueryProjectId,
- normalizeBigQueryRegion,
-} from '../../context/connections/bigquery-identifiers.js';
-import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
-import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
+import { normalizeBigQueryProjectId, normalizeBigQueryRegion } from '../../context/connections/bigquery-identifiers.js';
+import { getDialectForDriver } from '../../context/connections/dialects.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
-import { tryIntrospectObject } from '../../context/scan/object-introspection.js';
import { scopedTableNames } from '../../context/scan/table-ref.js';
import {
- connectorTestFailure,
createKtxConnectorCapabilities,
- type KtxConnectorTestResult,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
@@ -32,7 +24,9 @@ import {
type KtxTableSampleInput,
type KtxTableSampleResult,
} from '../../context/scan/types.js';
-import { resolveStringReference } from '../shared/string-reference.js';
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
export interface KtxBigQueryConnectionConfig {
driver?: string;
@@ -41,25 +35,14 @@ export interface KtxBigQueryConnectionConfig {
credentials_json?: string;
location?: string;
max_bytes_billed?: number | string;
- query_timeout_ms?: number;
+ job_timeout_ms?: number;
[key: string]: unknown;
}
-/**
- * A dataset to introspect, paired with the project that hosts it. `project`
- * defaults to the billing project (`credentials.project_id`) when an entry has
- * no `project.` prefix; a fully-qualified `project.dataset` entry resolves to
- * its own host project. Jobs always bill in `credentials.project_id`.
- */
-export interface BigQueryDatasetRef {
- project: string;
- dataset: string;
-}
-
export interface KtxBigQueryResolvedConnectionConfig {
projectId: string;
credentials: Record;
- datasetIds: BigQueryDatasetRef[];
+ datasetIds: string[];
location?: string;
}
@@ -112,7 +95,7 @@ export interface KtxBigQueryDataset {
export interface KtxBigQueryClient {
getDatasets(input?: { maxResults?: number }): Promise<[Array<{ id?: string }>, ...unknown[]]>;
- dataset(datasetId: string, projectId: string): KtxBigQueryDataset;
+ dataset(datasetId: string): KtxBigQueryDataset;
createQueryJob(input: {
query: string;
location?: string;
@@ -133,6 +116,7 @@ export interface KtxBigQueryScanConnectorOptions {
env?: NodeJS.ProcessEnv;
now?: () => Date;
maxBytesBilled?: number | string;
+ queryTimeoutMs?: number;
}
class DefaultBigQueryClientFactory implements KtxBigQueryClientFactory {
@@ -140,8 +124,8 @@ class DefaultBigQueryClientFactory implements KtxBigQueryClientFactory {
const client = new BigQuery(input);
return {
getDatasets: (options) => client.getDatasets(options) as Promise<[Array<{ id?: string }>, ...unknown[]]>,
- dataset: (datasetId, projectId) => {
- const dataset = client.dataset(datasetId, { projectId });
+ dataset: (datasetId) => {
+ const dataset = client.dataset(datasetId);
return {
get: () => dataset.get() as Promise,
getTables: () => dataset.getTables() as Promise<[KtxBigQueryTableRef[], ...unknown[]]>,
@@ -152,6 +136,18 @@ class DefaultBigQueryClientFactory implements KtxBigQueryClientFactory {
}
}
+function resolveStringReference(value: string, env: NodeJS.ProcessEnv): string {
+ if (value.startsWith('env:')) {
+ return env[value.slice('env:'.length)] ?? '';
+ }
+ if (value.startsWith('file:')) {
+ const rawPath = value.slice('file:'.length);
+ const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
+ return readFileSync(path, 'utf-8').trim();
+ }
+ return value;
+}
+
function stringConfigValue(
connection: KtxBigQueryConnectionConfig | undefined,
key: keyof KtxBigQueryConnectionConfig,
@@ -161,48 +157,14 @@ function stringConfigValue(
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
}
-/**
- * Parse one `dataset_ids` / `dataset_id` entry into a canonical
- * {@link BigQueryDatasetRef}. A `project.dataset` prefix selects the host
- * project; a bare entry defaults to `defaultProject` (the billing project).
- * More than one dot, or an empty segment, is a config error naming the
- * connection — never a silent mis-introspection at scan time.
- */
-function parseBigQueryDatasetEntry(entry: string, defaultProject: string, connectionId: string): BigQueryDatasetRef {
- const context = `connections.${connectionId}.dataset_ids entry "${entry}"`;
- const parts = entry.split('.');
- if (parts.length === 1) {
- return { project: defaultProject, dataset: normalizeBigQueryDatasetId(parts[0]!, context) };
+function datasetIds(connection: KtxBigQueryConnectionConfig, env: NodeJS.ProcessEnv): string[] {
+ if (Array.isArray(connection.dataset_ids) && connection.dataset_ids.length > 0) {
+ return connection.dataset_ids
+ .filter((dataset) => dataset.trim().length > 0)
+ .map((dataset) => resolveStringReference(dataset, env));
}
- if (parts.length === 2) {
- const [project, dataset] = parts;
- if (!project || !dataset) {
- throw new Error(`Invalid BigQuery dataset entry for ${context}: empty project or dataset segment`);
- }
- return {
- project: normalizeBigQueryProjectId(project, context),
- dataset: normalizeBigQueryDatasetId(dataset, context),
- };
- }
- throw new Error(
- `Invalid BigQuery dataset entry for ${context}: expected "dataset" or "project.dataset", got more than one "."`,
- );
-}
-
-function resolveDatasetRefs(
- connection: KtxBigQueryConnectionConfig,
- env: NodeJS.ProcessEnv,
- defaultProject: string,
- connectionId: string,
-): BigQueryDatasetRef[] {
- const rawEntries =
- Array.isArray(connection.dataset_ids) && connection.dataset_ids.length > 0
- ? connection.dataset_ids.map((dataset) => resolveStringReference(dataset, env))
- : [stringConfigValue(connection, 'dataset_id', env)].filter((value): value is string => Boolean(value));
- return rawEntries
- .map((entry) => entry.trim())
- .filter((entry) => entry.length > 0)
- .map((entry) => parseBigQueryDatasetEntry(entry, defaultProject, connectionId));
+ const datasetId = stringConfigValue(connection, 'dataset_id', env);
+ return datasetId ? [datasetId] : [];
}
function bigQueryMaxBytesBilledFromConnection(
@@ -219,25 +181,12 @@ function bigQueryMaxBytesBilledFromConnection(
return undefined;
}
-// jobTimeoutMs cancels the job with a "Job timed out" message (or a timeout
-// reason in the errors array) once the deadline elapses.
-function isBigQueryTimeoutError(error: unknown): boolean {
- if (!error || typeof error !== 'object') {
- return false;
+function bigQueryJobTimeoutMsFromConnection(connection: KtxBigQueryConnectionConfig | undefined): number | undefined {
+ const value = connection?.job_timeout_ms;
+ if (typeof value !== 'number') {
+ return undefined;
}
- const topMessage = (error as { message?: unknown }).message;
- if (typeof topMessage === 'string' && /timed out|timeout/i.test(topMessage)) {
- return true;
- }
- const errors = (error as { errors?: unknown }).errors;
- return (
- Array.isArray(errors) &&
- errors.some((entry) => {
- const reason = (entry as { reason?: unknown })?.reason;
- const message = (entry as { message?: unknown })?.message;
- return reason === 'timeout' || (typeof message === 'string' && /timed out|timeout/i.test(message));
- })
- );
+ return Number.isInteger(value) && value > 0 ? value : undefined;
}
function tableKind(metadataType: string | undefined): KtxSchemaTable['kind'] {
@@ -330,7 +279,7 @@ export function bigQueryConnectionConfigFromConfig(input: {
if (!projectId) {
throw new Error(`Native BigQuery connector requires credentials_json.project_id for connections.${input.connectionId}`);
}
- const resolvedDatasetIds = resolveDatasetRefs(input.connection, env, projectId, input.connectionId);
+ const resolvedDatasetIds = datasetIds(input.connection, env);
const location = stringConfigValue(input.connection, 'location', env);
return { projectId, credentials, datasetIds: resolvedDatasetIds, ...(location ? { location } : {}) };
}
@@ -353,8 +302,8 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
private readonly clientFactory: KtxBigQueryClientFactory;
private readonly now: () => Date;
private readonly maxBytesBilled?: number | string;
- private readonly deadlineMs: number;
- private readonly dialect = getSqlDialectForDriver('bigquery');
+ private readonly queryTimeoutMs?: number;
+ private readonly dialect = getDialectForDriver('bigquery');
private client: KtxBigQueryClient | null = null;
constructor(options: KtxBigQueryScanConnectorOptions) {
@@ -367,43 +316,42 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
this.clientFactory = options.clientFactory ?? new DefaultBigQueryClientFactory();
this.now = options.now ?? (() => new Date());
this.maxBytesBilled = options.maxBytesBilled ?? bigQueryMaxBytesBilledFromConnection(options.connection);
- this.deadlineMs = resolveQueryDeadlineMs(options.connection);
+ this.queryTimeoutMs = options.queryTimeoutMs ?? bigQueryJobTimeoutMsFromConnection(options.connection);
this.id = `bigquery:${options.connectionId}`;
}
- async testConnection(): Promise {
+ async testConnection(): Promise<{ success: boolean; error?: string }> {
try {
const client = this.getClient();
await client.getDatasets({ maxResults: 1 });
- for (const ref of this.resolved.datasetIds) {
- await client.dataset(ref.dataset, ref.project).get();
+ for (const datasetId of this.resolved.datasetIds) {
+ await client.dataset(datasetId).get();
}
return { success: true };
} catch (error) {
- return connectorTestFailure(error);
+ return { success: false, error: error instanceof Error ? error.message : String(error) };
}
}
async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise {
this.assertConnection(input.connectionId);
const tables: KtxSchemaTable[] = [];
- const datasetRefs = this.requireDatasetIdsForScan();
+ const datasetIds = this.requireDatasetIdsForScan();
const snapshotWarnings: KtxScanWarning[] = [];
- for (const ref of datasetRefs) {
+ for (const datasetId of datasetIds) {
const scopedNames = input.tableScope
- ? scopedTableNames(input.tableScope, { catalog: ref.project, db: ref.dataset })
+ ? scopedTableNames(input.tableScope, { catalog: this.resolved.projectId, db: datasetId })
: null;
- tables.push(...(await this.introspectDataset(ref, scopedNames, snapshotWarnings)));
+ tables.push(...(await this.introspectDataset(datasetId, scopedNames, snapshotWarnings)));
}
- const datasetLabels = datasetRefs.map((ref) => this.qualifiedDatasetLabel(ref));
return {
connectionId: this.connectionId,
driver: 'bigquery',
extractedAt: this.now().toISOString(),
- scope: { catalogs: [...new Set(datasetRefs.map((ref) => ref.project))], datasets: datasetLabels },
+ scope: { catalogs: [this.resolved.projectId], datasets: datasetIds },
metadata: {
project_id: this.resolved.projectId,
- datasets: datasetLabels,
+ datasets: datasetIds,
table_count: tables.length,
total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
},
@@ -464,14 +412,11 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
return { values: valueRows.filter((row) => row.val !== null).map((row) => String(row.val)), cardinality };
}
- async getTableRowCount(
- tableName: string,
- ref: BigQueryDatasetRef | undefined = this.resolved.datasetIds[0],
- ): Promise {
- if (!ref) {
+ async getTableRowCount(tableName: string, datasetId = this.resolved.datasetIds[0]): Promise {
+ if (!datasetId) {
return 0;
}
- const tables = await this.introspectDataset(ref, null, []);
+ const tables = await this.introspectDataset(datasetId, null, []);
return tables.find((table) => table.name === tableName)?.estimatedRows ?? 0;
}
@@ -489,28 +434,12 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
}
async listTables(datasetIds?: string[]): Promise {
+ const projectId = normalizeBigQueryProjectId(this.resolved.projectId, 'table discovery');
const region = normalizeBigQueryRegion(this.resolved.location ?? 'US', 'table discovery');
- if (!datasetIds || datasetIds.length === 0) {
- return this.listTablesInProject(this.resolved.projectId, region);
- }
- const datasetsByProject = new Map();
- for (const entry of datasetIds) {
- const ref = parseBigQueryDatasetEntry(entry.trim(), this.resolved.projectId, this.connectionId);
- datasetsByProject.set(ref.project, [...(datasetsByProject.get(ref.project) ?? []), ref.dataset]);
- }
- const entries: KtxTableListEntry[] = [];
- for (const [project, datasets] of datasetsByProject) {
- entries.push(...(await this.listTablesInProject(project, region, datasets)));
- }
- return entries;
- }
-
- private async listTablesInProject(project: string, region: string, datasets?: string[]): Promise {
- const projectId = normalizeBigQueryProjectId(project, 'table discovery');
const params: Record = {};
- const filter = datasets && datasets.length > 0 ? 'AND table_schema IN UNNEST(@dataset_ids)' : '';
- if (datasets && datasets.length > 0) {
- params.dataset_ids = datasets;
+ const filter = datasetIds && datasetIds.length > 0 ? 'AND table_schema IN UNNEST(@dataset_ids)' : '';
+ if (datasetIds && datasetIds.length > 0) {
+ params.dataset_ids = datasetIds;
}
const rows = await this.queryRaw<{ table_schema: string; table_name: string; table_type: string }>(
`
@@ -525,7 +454,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
params,
);
return rows.map((row) => ({
- catalog: project,
+ catalog: this.resolved.projectId,
schema: row.table_schema,
name: row.table_name,
kind:
@@ -549,48 +478,34 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
return this.client;
}
- private requireDatasetIdsForScan(): BigQueryDatasetRef[] {
+ private requireDatasetIdsForScan(): string[] {
if (this.resolved.datasetIds.length === 0) {
throw new Error(`Native BigQuery scan requires connections.${this.connectionId}.dataset_ids or dataset_id`);
}
return this.resolved.datasetIds;
}
- // Bare in the billing project, qualified `project.dataset` otherwise, so the
- // snapshot's scope/metadata stay unambiguous when two projects host the same
- // dataset name. The dotless form is the unchanged single-project label.
- private qualifiedDatasetLabel(ref: BigQueryDatasetRef): string {
- return ref.project === this.resolved.projectId ? ref.dataset : `${ref.project}.${ref.dataset}`;
- }
-
private async query(sql: string, params?: Record): Promise {
- try {
- const [job] = await this.getClient().createQueryJob({
- query: sql,
- ...(this.resolved.location ? { location: this.resolved.location } : {}),
- ...(params && Object.keys(params).length > 0 ? { params } : {}),
- ...(this.maxBytesBilled ? { maximumBytesBilled: String(this.maxBytesBilled) } : {}),
- jobTimeoutMs: this.deadlineMs,
- });
- const [rows, , response] = await job.getQueryResults();
- let headers = response?.schema?.fields?.map((field) => field.name || '') ?? [];
- const headerTypes = response?.schema?.fields?.map((field) => String(field.type || 'STRING')) ?? [];
- if (headers.length === 0 && rows.length > 0) {
- headers = Object.keys(rows[0]!);
- }
- return {
- headers,
- headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
- rows: rows.map((row) => headers.map((header) => normalizeValue(row[header]))),
- totalRows: rows.length,
- rowCount: rows.length,
- };
- } catch (error) {
- if (isBigQueryTimeoutError(error)) {
- throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
- }
- throw error;
+ const [job] = await this.getClient().createQueryJob({
+ query: sql,
+ ...(this.resolved.location ? { location: this.resolved.location } : {}),
+ ...(params && Object.keys(params).length > 0 ? { params } : {}),
+ ...(this.maxBytesBilled ? { maximumBytesBilled: String(this.maxBytesBilled) } : {}),
+ ...(this.queryTimeoutMs ? { jobTimeoutMs: this.queryTimeoutMs } : {}),
+ });
+ const [rows, , response] = await job.getQueryResults();
+ let headers = response?.schema?.fields?.map((field) => field.name || '') ?? [];
+ const headerTypes = response?.schema?.fields?.map((field) => String(field.type || 'STRING')) ?? [];
+ if (headers.length === 0 && rows.length > 0) {
+ headers = Object.keys(rows[0]!);
}
+ return {
+ headers,
+ headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
+ rows: rows.map((row) => headers.map((header) => normalizeValue(row[header]))),
+ totalRows: rows.length,
+ rowCount: rows.length,
+ };
}
private async queryRaw>(sql: string, params?: Record): Promise {
@@ -604,18 +519,18 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
}
private async introspectDataset(
- ref: BigQueryDatasetRef,
+ datasetId: string,
scopedNames: readonly string[] | null,
snapshotWarnings: KtxScanWarning[],
): Promise {
if (scopedNames && scopedNames.length === 0) return [];
- const dataset = this.getClient().dataset(ref.dataset, ref.project);
+ const dataset = this.getClient().dataset(datasetId);
const [tableRefs] = await dataset.getTables();
const scopeSet = scopedNames ? new Set(scopedNames) : null;
const filteredTableRefs = scopeSet ? tableRefs.filter((tableRef) => scopeSet.has(tableRef.id ?? '')) : tableRefs;
const primaryKeysResult = await tryConstraintQuery(
- { schema: ref.dataset, kind: 'primary_key', isDeniedError },
- () => this.primaryKeys(ref),
+ { schema: datasetId, kind: 'primary_key', isDeniedError },
+ () => this.primaryKeys(datasetId),
);
const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : new Map>();
if (!primaryKeysResult.ok) {
@@ -624,51 +539,41 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
const tables: KtxSchemaTable[] = [];
for (const tableRef of filteredTableRefs) {
const tableName = tableRef.id || '';
- const outcome = await tryIntrospectObject(
- { object: tableName, catalog: ref.project, db: ref.dataset },
- async () => {
- const [table] = await tableRef.get();
- const fields = table.metadata.schema?.fields ?? [];
- return {
- catalog: ref.project,
- db: ref.dataset,
- name: tableName,
- kind: tableKind(table.metadata.type),
- comment: table.metadata.description || null,
- estimatedRows: firstNumber(table.metadata.numRows) ?? 0,
- columns: fields.map((field) => this.toSchemaColumn(tableName, field, primaryKeys)),
- foreignKeys: [],
- };
- },
- );
- if (outcome.ok) {
- tables.push(outcome.table);
- } else {
- snapshotWarnings.push(outcome.warning);
- }
+ const [table] = await tableRef.get();
+ const fields = table.metadata.schema?.fields ?? [];
+ tables.push({
+ catalog: this.resolved.projectId,
+ db: datasetId,
+ name: tableName,
+ kind: tableKind(table.metadata.type),
+ comment: table.metadata.description || null,
+ estimatedRows: firstNumber(table.metadata.numRows) ?? 0,
+ columns: fields.map((field) => this.toSchemaColumn(tableName, field, primaryKeys)),
+ foreignKeys: [],
+ });
}
return tables;
}
- private async primaryKeys(ref: BigQueryDatasetRef): Promise>> {
+ private async primaryKeys(datasetId: string): Promise>> {
const rows = await this.queryRaw<{ table_name: string; column_name: string }>(
'SELECT tc.table_name, kcu.column_name ' +
'FROM `' +
- ref.project +
+ this.resolved.projectId +
'.' +
- ref.dataset +
+ datasetId +
'.INFORMATION_SCHEMA.TABLE_CONSTRAINTS` tc ' +
'JOIN `' +
- ref.project +
+ this.resolved.projectId +
'.' +
- ref.dataset +
+ datasetId +
'.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` kcu ' +
'ON tc.constraint_name = kcu.constraint_name ' +
'AND tc.table_schema = kcu.table_schema ' +
'AND tc.table_name = kcu.table_name ' +
"WHERE tc.constraint_type = 'PRIMARY KEY' " +
"AND tc.table_schema = '" +
- ref.dataset +
+ datasetId +
"' " +
"AND NOT REGEXP_CONTAINS(kcu.column_name, r'^(stacksync_record_id|sync_primary_key)_') " +
'ORDER BY tc.table_name, kcu.ordinal_position',
diff --git a/packages/cli/src/connectors/bigquery/dialect.ts b/packages/cli/src/connectors/bigquery/dialect.ts
index b63391fe..0e2f883e 100644
--- a/packages/cli/src/connectors/bigquery/dialect.ts
+++ b/packages/cli/src/connectors/bigquery/dialect.ts
@@ -1,4 +1,4 @@
-import type { KtxSqlDialect } from '../../context/connections/dialects.js';
+import type { KtxDialect } from '../../context/connections/dialects.js';
import {
columnDisplayPartCount,
formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
type BigQueryTableNameRef = Pick & Partial>;
/** @internal */
-export class KtxBigQueryDialect implements KtxSqlDialect {
+export class KtxBigQueryDialect implements KtxDialect {
readonly type = 'bigquery' as const;
private readonly typeMappings: Record = {
diff --git a/packages/cli/src/connectors/clickhouse/connector.ts b/packages/cli/src/connectors/clickhouse/connector.ts
index 2f95ea4d..74ef7a77 100644
--- a/packages/cli/src/connectors/clickhouse/connector.ts
+++ b/packages/cli/src/connectors/clickhouse/connector.ts
@@ -1,11 +1,12 @@
import { createClient } from '@clickhouse/client';
-import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
-import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
+import { getDialectForDriver } from '../../context/connections/dialects.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
-import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
+import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
import { scopedTableNames } from '../../context/scan/table-ref.js';
-import { resolveStringReference } from '../shared/string-reference.js';
+import { readFileSync } from 'node:fs';
import { Agent as HttpsAgent } from 'node:https';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
export interface KtxClickHouseConnectionConfig {
driver?: string;
@@ -141,23 +142,21 @@ function stringConfigValue(
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
}
-function maybeNumber(value: unknown): number | undefined {
- return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+function resolveStringReference(value: string, env: NodeJS.ProcessEnv): string {
+ if (value.startsWith('env:')) {
+ const envName = value.slice('env:'.length);
+ return env[envName] ?? '';
+ }
+ if (value.startsWith('file:')) {
+ const rawPath = value.slice('file:'.length);
+ const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
+ return readFileSync(path, 'utf-8').trim();
+ }
+ return value;
}
-// ClickHouse error code 159 = TIMEOUT_EXCEEDED, raised when max_execution_time
-// is hit. The client surfaces it via a numeric/string `code` or a "Code: 159"
-// message prefix depending on transport.
-function isClickHouseTimeoutError(error: unknown): boolean {
- if (!error || typeof error !== 'object') {
- return false;
- }
- const code = (error as { code?: unknown }).code;
- if (code === 159 || code === '159') {
- return true;
- }
- const message = (error as { message?: unknown }).message;
- return typeof message === 'string' && (/\bCode:\s*159\b/.test(message) || message.includes('TIMEOUT_EXCEEDED'));
+function maybeNumber(value: unknown): number | undefined {
+ return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
function parseClickHouseUrl(url: string): Partial {
@@ -300,8 +299,7 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
private readonly clientFactory: KtxClickHouseClientFactory;
private readonly endpointResolver?: KtxClickHouseEndpointResolver;
private readonly now: () => Date;
- private readonly deadlineMs: number;
- private readonly dialect = getSqlDialectForDriver('clickhouse');
+ private readonly dialect = getDialectForDriver('clickhouse');
private client: KtxClickHouseClient | null = null;
private resolvedEndpoint: KtxClickHouseResolvedEndpoint | null = null;
@@ -316,16 +314,15 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
this.clientFactory = options.clientFactory ?? new DefaultClickHouseClientFactory();
this.endpointResolver = options.endpointResolver;
this.now = options.now ?? (() => new Date());
- this.deadlineMs = resolveQueryDeadlineMs(this.connection);
this.id = `clickhouse:${options.connectionId}`;
}
- async testConnection(): Promise {
+ async testConnection(): Promise<{ success: boolean; error?: string }> {
try {
await this.query('SELECT 1');
return { success: true };
} catch (error) {
- return connectorTestFailure(error);
+ return { success: false, error: error instanceof Error ? error.message : String(error) };
}
}
@@ -602,13 +599,9 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
username: config.username,
password: config.password ?? '',
database: config.database,
- // The server aborts at max_execution_time (seconds); request_timeout must
- // outlast it so the HTTP client receives the code-159 error instead of
- // giving up first and leaving the query running.
- request_timeout: this.deadlineMs + 5_000,
+ request_timeout: 30_000,
clickhouse_settings: {
output_format_json_quote_64bit_integers: 1,
- max_execution_time: Math.ceil(this.deadlineMs / 1000),
},
...(isProxied && config.ssl
? {
@@ -635,31 +628,24 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
private async query(sql: string, params?: Record): Promise> {
const client = await this.clientForQuery();
- try {
- const resultSet = await client.query({
- query: assertReadOnlySql(sql),
- format: 'JSONCompact',
- ...(params ? { query_params: params } : {}),
- });
- const response = (await resultSet.json()) as ClickHouseCompactResponse;
- const meta = response.meta ?? [];
- return {
- headers: meta.map((field) => field.name),
- headerTypes: meta.map((field) => field.type),
- rows: response.data ?? [],
- totalRows: response.rows ?? response.data?.length ?? 0,
- };
- } catch (error) {
- if (isClickHouseTimeoutError(error)) {
- throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
- }
- throw error;
- }
+ const resultSet = await client.query({
+ query: assertReadOnlySql(sql),
+ format: 'JSONCompact',
+ ...(params ? { query_params: params } : {}),
+ });
+ const response = (await resultSet.json()) as ClickHouseCompactResponse;
+ const meta = response.meta ?? [];
+ return {
+ headers: meta.map((field) => field.name),
+ headerTypes: meta.map((field) => field.type),
+ rows: response.data ?? [],
+ totalRows: response.rows ?? response.data?.length ?? 0,
+ };
}
private assertConnection(connectionId: string): void {
if (connectionId !== this.connectionId) {
- throw new Error(`ktx ClickHouse connector ${this.id} cannot serve connection ${connectionId}`);
+ throw new Error(`KTX ClickHouse connector ${this.id} cannot serve connection ${connectionId}`);
}
}
}
diff --git a/packages/cli/src/connectors/clickhouse/dialect.ts b/packages/cli/src/connectors/clickhouse/dialect.ts
index 8f3f5bb1..9e470cae 100644
--- a/packages/cli/src/connectors/clickhouse/dialect.ts
+++ b/packages/cli/src/connectors/clickhouse/dialect.ts
@@ -1,4 +1,4 @@
-import type { KtxSqlDialect } from '../../context/connections/dialects.js';
+import type { KtxDialect } from '../../context/connections/dialects.js';
import {
columnDisplayPartCount,
formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
type ClickHouseTableNameRef = Pick & Partial>;
/** @internal */
-export class KtxClickHouseDialect implements KtxSqlDialect {
+export class KtxClickHouseDialect implements KtxDialect {
readonly type = 'clickhouse' as const;
private readonly typeMappings: Record = {
diff --git a/packages/cli/src/connectors/duckdb/connector.ts b/packages/cli/src/connectors/duckdb/connector.ts
deleted file mode 100644
index 5b904c14..00000000
--- a/packages/cli/src/connectors/duckdb/connector.ts
+++ /dev/null
@@ -1,395 +0,0 @@
-import { DuckDBInstance, type DuckDBConnection } from '@duckdb/node-api';
-import { existsSync, statSync } from 'node:fs';
-import { isAbsolute, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { resolveStringReference } from '../shared/string-reference.js';
-import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
-import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
-import { normalizeQueryRows } from '../../context/connections/query-executor.js';
-import { toJsonSafeRows } from '../shared/duckdb-json-safe.js';
-import {
- connectorTestFailure,
- createKtxConnectorCapabilities,
- type KtxColumnSampleInput,
- type KtxColumnSampleResult,
- type KtxColumnStatsInput,
- type KtxColumnStatsResult,
- type KtxConnectorTestResult,
- type KtxQueryResult,
- type KtxReadOnlyQueryInput,
- type KtxScanConnector,
- type KtxScanContext,
- type KtxScanInput,
- type KtxSchemaForeignKey,
- type KtxSchemaSnapshot,
- type KtxSchemaTable,
- type KtxTableListEntry,
- type KtxTableRef,
- type KtxTableSampleInput,
- type KtxTableSampleResult,
-} from '../../context/scan/types.js';
-import { scopedTableNames } from '../../context/scan/table-ref.js';
-
-const MAIN_SCHEMA = 'main';
-
-export interface KtxDuckDbConnectionConfig {
- driver?: string;
- path?: string;
- url?: string;
- [key: string]: unknown;
-}
-
-/** @internal */
-export interface DuckDbDatabasePathInput {
- connectionId: string;
- projectDir?: string;
- connection: KtxDuckDbConnectionConfig | undefined;
-}
-
-export interface KtxDuckDbScanConnectorOptions extends DuckDbDatabasePathInput {
- now?: () => Date;
-}
-
-export interface KtxDuckDbColumnDistinctValuesOptions {
- maxCardinality: number;
- limit: number;
- sampleSize?: number;
-}
-
-export interface KtxDuckDbColumnDistinctValuesResult {
- values: string[] | null;
- cardinality: number;
-}
-
-interface InfoSchemaTableRow {
- table_name: string;
- table_type: string;
-}
-
-interface InfoSchemaColumnRow {
- column_name: string;
- data_type: string;
- is_nullable: string;
-}
-
-// `path` may be an env:/file: reference; `url` resolves env: only, since file:
-// on a url is a native URI form (handled by duckDbPathFromUrl), not a file read.
-function stringConfigValue(
- connection: KtxDuckDbConnectionConfig | undefined,
- key: 'path' | 'url',
-): string | undefined {
- const value = connection?.[key];
- if (typeof value !== 'string' || value.trim().length === 0) {
- return undefined;
- }
- const trimmed = value.trim();
- if (key === 'url') {
- return trimmed.startsWith('env:') ? (process.env[trimmed.slice('env:'.length)] ?? '') : trimmed;
- }
- return resolveStringReference(trimmed, process.env);
-}
-
-function duckDbPathFromUrl(url: string): string {
- if (url.startsWith('file:')) {
- return fileURLToPath(url);
- }
- if (url.startsWith('duckdb:')) {
- const parsed = new URL(url);
- return decodeURIComponent(parsed.pathname);
- }
- return url;
-}
-
-export function isKtxDuckDbConnectionConfig(
- connection: KtxDuckDbConnectionConfig | undefined,
-): connection is KtxDuckDbConnectionConfig {
- return String(connection?.driver ?? '').toLowerCase() === 'duckdb';
-}
-
-/** @internal */
-export function duckDbDatabasePathFromConfig(input: DuckDbDatabasePathInput): string {
- const inputDriver = input.connection?.driver ?? 'unknown';
- if (!isKtxDuckDbConnectionConfig(input.connection)) {
- throw new Error(`Native DuckDB connector cannot run driver "${inputDriver}"`);
- }
- const configuredPath =
- stringConfigValue(input.connection, 'path') ?? duckDbPathFromUrl(stringConfigValue(input.connection, 'url') ?? '');
- if (!configuredPath) {
- throw new Error(`Native DuckDB connector requires connections.${input.connectionId}.path or url`);
- }
- return isAbsolute(configuredPath) ? configuredPath : resolve(input.projectDir ?? process.cwd(), configuredPath);
-}
-
-export class KtxDuckDbScanConnector implements KtxScanConnector {
- readonly id: string;
- readonly driver = 'duckdb' as const;
- readonly capabilities = createKtxConnectorCapabilities({
- tableSampling: true,
- columnSampling: true,
- columnStats: false,
- readOnlySql: true,
- nestedAnalysis: false,
- formalForeignKeys: true,
- estimatedRowCounts: true,
- });
-
- private readonly connectionId: string;
- private readonly dbPath: string;
- private readonly now: () => Date;
- private readonly dialect = getSqlDialectForDriver('duckdb');
- private instance: DuckDBInstance | null = null;
- private connection: DuckDBConnection | null = null;
-
- constructor(options: KtxDuckDbScanConnectorOptions) {
- this.connectionId = options.connectionId;
- this.dbPath = duckDbDatabasePathFromConfig(options);
- this.now = options.now ?? (() => new Date());
- this.id = `duckdb:${options.connectionId}`;
- }
-
- async testConnection(): Promise {
- try {
- if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
- return { success: false, error: `File not found: ${this.dbPath}` };
- }
- await this.query('SELECT 1');
- return { success: true };
- } catch (error) {
- return connectorTestFailure(error);
- }
- }
-
- async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: null, db: null }) : null;
- const tableRows = await this.readTableRows(scopedNames);
- const tables: KtxSchemaTable[] = [];
- for (const row of tableRows) {
- tables.push(await this.readTable(row));
- }
- return {
- connectionId: this.connectionId,
- driver: 'duckdb' as const,
- extractedAt: this.now().toISOString(),
- scope: {},
- metadata: {
- file_path: this.dbPath,
- table_count: tables.length,
- total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
- },
- tables,
- };
- }
-
- async listSchemas(): Promise {
- return [MAIN_SCHEMA];
- }
-
- async listTables(_schemas?: string[]): Promise {
- const rows = await this.readTableRows(null);
- return rows.map((row) => ({
- catalog: null,
- schema: MAIN_SCHEMA,
- name: row.table_name,
- kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const),
- }));
- }
-
- async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const result = await this.query(
- this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns),
- );
- return { headers: result.headers, rows: result.rows, totalRows: result.totalRows };
- }
-
- async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const result = await this.query(
- this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit),
- );
- const values = result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]);
- return { values, nullCount: null, distinctCount: null };
- }
-
- async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise {
- return null;
- }
-
- async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const result = await this.query(limitSqlForExecution(input.sql, input.maxRows));
- return { ...result, rowCount: result.rows.length };
- }
-
- async getColumnDistinctValues(
- table: KtxTableRef,
- columnName: string,
- options: KtxDuckDbColumnDistinctValuesOptions,
- ): Promise {
- const sampleSize = options.sampleSize ?? 10000;
- const tableName = this.qTableName(table);
- const quotedColumn = this.dialect.quoteIdentifier(columnName);
- const cardinalityResult = await this.query(
- this.dialect.generateCardinalitySampleQuery(tableName, quotedColumn, sampleSize),
- );
- if (cardinalityResult.rows.length === 0) {
- return null;
- }
- const cardinality = Number(cardinalityResult.rows[0][0]);
- if (Number.isNaN(cardinality)) {
- return null;
- }
- if (cardinality === 0) {
- return { values: [], cardinality: 0 };
- }
- if (cardinality > options.maxCardinality) {
- return { values: null, cardinality };
- }
- const valuesResult = await this.query(
- this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit),
- );
- return {
- values: valuesResult.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => String(row[0])),
- cardinality,
- };
- }
-
- async getTableRowCount(tableName: string): Promise {
- const result = await this.query(`SELECT COUNT(*) AS count FROM ${this.dialect.quoteIdentifier(tableName)}`);
- return Number(result.rows[0]?.[0] ?? 0);
- }
-
- qTableName(table: Pick): string {
- return this.dialect.formatTableName(table);
- }
-
- quoteIdentifier(identifier: string): string {
- return this.dialect.quoteIdentifier(identifier);
- }
-
- async cleanup(): Promise {
- this.connection?.closeSync();
- this.instance?.closeSync();
- this.connection = null;
- this.instance = null;
- }
-
- private async db(): Promise {
- if (!this.connection) {
- // DuckDBInstance.create() creates the file if missing, so this pre-check
- // enforces the never-create rule. Do not remove it.
- if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
- throw new Error(`File not found: ${this.dbPath}`);
- }
- this.instance = await DuckDBInstance.create(this.dbPath, { access_mode: 'read_only' });
- this.connection = await this.instance.connect();
- }
- return this.connection;
- }
-
- private async query(sql: string): Promise> {
- const connection = await this.db();
- const reader = await connection.runAndReadAll(assertReadOnlySql(sql));
- const rows = toJsonSafeRows(normalizeQueryRows(reader.getRows()));
- return {
- headers: reader.columnNames(),
- rows,
- totalRows: rows.length,
- };
- }
-
- private async readTableRows(scopedNames: string[] | null): Promise {
- if (scopedNames && scopedNames.length === 0) {
- return [];
- }
- const scopeClause = scopedNames
- ? `AND table_name IN (${scopedNames.map((name) => `'${name.replaceAll("'", "''")}'`).join(', ')})`
- : '';
- const result = await this.query(
- `SELECT table_name, table_type
- FROM information_schema.tables
- WHERE table_schema = '${MAIN_SCHEMA}' ${scopeClause}
- ORDER BY table_name`,
- );
- return result.rows.map((row) => ({ table_name: String(row[0]), table_type: String(row[1]) }));
- }
-
- private async readTable(table: InfoSchemaTableRow): Promise {
- const columnsResult = await this.query(
- `SELECT column_name, data_type, is_nullable
- FROM information_schema.columns
- WHERE table_schema = '${MAIN_SCHEMA}' AND table_name = '${table.table_name.replaceAll("'", "''")}'
- ORDER BY ordinal_position`,
- );
- const columns = columnsResult.rows.map((row) => ({
- column_name: String(row[0]),
- data_type: String(row[1]),
- is_nullable: String(row[2]),
- }));
- const primaryKeys = await this.readPrimaryKeyColumns(table.table_name);
- const isView = table.table_type === 'VIEW';
- const estimatedRows = isView ? null : await this.getTableRowCount(table.table_name);
- return {
- catalog: null,
- db: null,
- name: table.table_name,
- kind: isView ? 'view' : 'table',
- comment: null,
- estimatedRows,
- columns: columns.map((column) => ({
- name: column.column_name,
- nativeType: column.data_type,
- normalizedType: this.dialect.mapDataType(column.data_type),
- dimensionType: this.dialect.mapToDimensionType(column.data_type),
- nullable: column.is_nullable === 'YES' && !primaryKeys.has(column.column_name),
- primaryKey: primaryKeys.has(column.column_name),
- comment: null,
- })),
- foreignKeys: await this.readForeignKeys(table.table_name),
- };
- }
-
- private async readPrimaryKeyColumns(tableName: string): Promise> {
- const result = await this.query(
- `SELECT kcu.column_name
- FROM information_schema.table_constraints tc
- JOIN information_schema.key_column_usage kcu
- ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
- WHERE tc.table_schema = '${MAIN_SCHEMA}'
- AND tc.table_name = '${tableName.replaceAll("'", "''")}'
- AND tc.constraint_type = 'PRIMARY KEY'`,
- );
- return new Set(result.rows.map((row) => String(row[0])));
- }
-
- private async readForeignKeys(tableName: string): Promise {
- // information_schema.constraint_column_usage in DuckDB returns the constrained
- // columns (source), not the referenced columns. Use duckdb_constraints() which
- // exposes constraint_column_names and referenced_column_names directly.
- const result = await this.query(
- `SELECT unnest(constraint_column_names) AS from_column,
- referenced_table,
- unnest(referenced_column_names) AS to_column,
- constraint_name
- FROM duckdb_constraints()
- WHERE schema_name = '${MAIN_SCHEMA}'
- AND table_name = '${tableName.replaceAll("'", "''")}'
- AND constraint_type = 'FOREIGN KEY'`,
- );
- return result.rows.map((row) => ({
- fromColumn: String(row[0]),
- toCatalog: null,
- toDb: null,
- toTable: String(row[1]),
- toColumn: String(row[2]),
- constraintName: row[3] === null ? null : String(row[3]),
- }));
- }
-
- private assertConnection(connectionId: string): void {
- if (connectionId !== this.connectionId) {
- throw new Error(`ktx DuckDB connector ${this.id} cannot serve connection ${connectionId}`);
- }
- }
-}
diff --git a/packages/cli/src/connectors/duckdb/dialect.ts b/packages/cli/src/connectors/duckdb/dialect.ts
deleted file mode 100644
index 9fef67dd..00000000
--- a/packages/cli/src/connectors/duckdb/dialect.ts
+++ /dev/null
@@ -1,192 +0,0 @@
-import type { KtxSqlDialect } from '../../context/connections/dialects.js';
-import {
- columnDisplayPartCount,
- formatDialectDisplayRef,
- formatDialectTableName,
- limitOffsetClause,
- parseDialectDisplayRef,
-} from '../../context/connections/dialect-helpers.js';
-import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
-
-type DuckDbTableNameRef = Pick & Partial>;
-
-/** @internal */
-export class KtxDuckDbDialect implements KtxSqlDialect {
- readonly type = 'duckdb' as const;
-
- private readonly typeMappings: Record = {
- TIMESTAMP: 'time',
- 'TIMESTAMP WITH TIME ZONE': 'time',
- TIMESTAMPTZ: 'time',
- DATE: 'time',
- TIME: 'time',
- BIGINT: 'number',
- INTEGER: 'number',
- SMALLINT: 'number',
- TINYINT: 'number',
- HUGEINT: 'number',
- UBIGINT: 'number',
- UINTEGER: 'number',
- DECIMAL: 'number',
- NUMERIC: 'number',
- REAL: 'number',
- FLOAT: 'number',
- DOUBLE: 'number',
- VARCHAR: 'string',
- CHAR: 'string',
- TEXT: 'string',
- BLOB: 'string',
- UUID: 'string',
- BOOLEAN: 'boolean',
- BOOL: 'boolean',
- };
-
- quoteIdentifier(identifier: string): string {
- return `"${identifier.replace(/"/g, '""')}"`;
- }
-
- // v1 introspects the `main` schema only and sets db=null on every table, so
- // refs are single-namespace like SQLite — use the matching display shape.
- formatTableName(table: DuckDbTableNameRef): string {
- return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'sqlite');
- }
-
- formatDisplayRef(table: DuckDbTableNameRef): string {
- return formatDialectDisplayRef(table, 'sqlite');
- }
-
- parseDisplayRef(display: string): KtxTableRef | null {
- return parseDialectDisplayRef(display, 'sqlite');
- }
-
- columnDisplayTablePartCount(): 1 | 2 | 3 {
- return columnDisplayPartCount('sqlite');
- }
-
- mapDataType(nativeType: string): string {
- return nativeType;
- }
-
- mapToDimensionType(nativeType: string): KtxSchemaDimensionType {
- if (!nativeType) {
- return 'string';
- }
- let normalized = nativeType.toUpperCase().trim();
- if (normalized.includes('(')) {
- normalized = normalized.split('(')[0].trim();
- }
- if (this.typeMappings[normalized]) {
- return this.typeMappings[normalized];
- }
- if (normalized.includes('TIME') || normalized.includes('DATE')) {
- return 'time';
- }
- if (
- normalized.includes('INT') ||
- normalized.includes('DEC') ||
- normalized.includes('NUM') ||
- normalized.includes('REAL') ||
- normalized.includes('FLOAT') ||
- normalized.includes('DOUBLE')
- ) {
- return 'number';
- }
- if (normalized.includes('BOOL')) {
- return 'boolean';
- }
- return 'string';
- }
-
- generateSampleQuery(tableName: string, limit: number, columns?: string[]): string {
- const columnList =
- columns && columns.length > 0 ? columns.map((column) => this.quoteIdentifier(column)).join(', ') : '*';
- return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`;
- }
-
- generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string {
- const quoted = this.quoteIdentifier(columnName);
- return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL AND TRIM(CAST(${quoted} AS VARCHAR)) != '' LIMIT ${limit}`;
- }
-
- getRandomSampleFilter(samplePct: number): string {
- if (samplePct <= 0 || samplePct >= 1) {
- return '';
- }
- return `RANDOM() < ${samplePct}`;
- }
-
- getTableSampleClause(samplePct: number): string {
- if (samplePct <= 0 || samplePct >= 1) {
- return '';
- }
- return `USING SAMPLE ${Math.round(samplePct * 100)} PERCENT (bernoulli)`;
- }
-
- getLimitOffsetClause(limit: number, offset?: number): string {
- return limitOffsetClause(limit, offset);
- }
-
- getTopClause(_limit: number): string {
- return '';
- }
-
- getNullCountExpression(column: string): string {
- return `SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END)`;
- }
-
- getDistinctCountExpression(column: string): string {
- return `COUNT(DISTINCT ${column})`;
- }
-
- textLengthExpression(columnSql: string): string {
- return `LENGTH(CAST(${columnSql} AS VARCHAR))`;
- }
-
- castToText(columnSql: string): string {
- return `CAST(${columnSql} AS VARCHAR)`;
- }
-
- getSampleValueAggregation(innerSql: string): string {
- return `(SELECT STRING_AGG(CAST(value AS VARCHAR), chr(31)) FROM (${innerSql}) AS relationship_profile_values)`;
- }
-
- generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
- return `
- WITH sampled AS (
- SELECT ${columnName} AS val
- FROM ${tableName}
- WHERE ${columnName} IS NOT NULL
- LIMIT ${sampleSize}
- )
- SELECT COUNT(DISTINCT val) AS cardinality
- FROM sampled
- `;
- }
-
- generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string {
- return `
- SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val
- FROM ${tableName}
- WHERE ${columnName} IS NOT NULL
- ORDER BY val
- LIMIT ${limit}
- `;
- }
-
- generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null {
- return null;
- }
-
- generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
- return `
- WITH sampled AS (
- SELECT ${columnName} AS val
- FROM ${tableName}
- WHERE ${columnName} IS NOT NULL
- USING SAMPLE ${sampleSize} ROWS
- )
- SELECT COUNT(DISTINCT val) AS cardinality
- FROM sampled
- `;
- }
-}
diff --git a/packages/cli/src/connectors/duckdb/federated-attach.ts b/packages/cli/src/connectors/duckdb/federated-attach.ts
deleted file mode 100644
index c0b9cda0..00000000
--- a/packages/cli/src/connectors/duckdb/federated-attach.ts
+++ /dev/null
@@ -1,97 +0,0 @@
-import { sqliteDatabasePathFromConfig, type KtxSqliteConnectionConfig } from '../sqlite/connector.js';
-import { postgresPoolConfigFromConfig, type KtxPostgresConnectionConfig } from '../postgres/connector.js';
-import {
- mysqlConnectionPoolConfigFromConfig,
- type KtxMysqlConnectionConfig,
-} from '../mysql/connector.js';
-import { duckDbDatabasePathFromConfig, type KtxDuckDbConnectionConfig } from '../../connectors/duckdb/connector.js';
-import type { FederatedMember } from '../../context/connections/federation.js';
-
-function kvKeyword(value: string): string {
- // libpq/DuckDB key-value values quote with single quotes and backslash-escape.
- return /[\s'\\]/.test(value) ? `'${value.replaceAll('\\', '\\\\').replaceAll("'", "\\'")}'` : value;
-}
-
-function withRequiredSslMode(connectionString: string): string {
- // DuckDB passes this libpq URL straight to the server, so an ssl:true member
- // must carry sslmode in the URL itself; keep a stronger mode the URL already pins.
- const url = new URL(connectionString);
- if (url.searchParams.has('sslmode')) {
- return connectionString;
- }
- url.searchParams.set('sslmode', 'require');
- return url.toString();
-}
-
-function postgresAttachString(member: FederatedMember, env: NodeJS.ProcessEnv): string {
- const cfg = postgresPoolConfigFromConfig({
- connectionId: member.connectionId,
- connection: member.connection as KtxPostgresConnectionConfig,
- env,
- });
- if (cfg.connectionString) {
- return cfg.ssl ? withRequiredSslMode(cfg.connectionString) : cfg.connectionString;
- }
- const parts: string[] = [];
- if (cfg.host) parts.push(`host=${kvKeyword(cfg.host)}`);
- if (cfg.port) parts.push(`port=${cfg.port}`);
- if (cfg.database) parts.push(`dbname=${kvKeyword(cfg.database)}`);
- if (cfg.user) parts.push(`user=${kvKeyword(cfg.user)}`);
- if (cfg.password) parts.push(`password=${kvKeyword(cfg.password)}`);
- if (cfg.ssl) {
- parts.push('sslmode=require');
- }
- if (cfg.options) {
- parts.push(`options=${kvKeyword(cfg.options)}`);
- }
- return parts.join(' ');
-}
-
-function mysqlAttachString(member: FederatedMember, env: NodeJS.ProcessEnv): string {
- const cfg = mysqlConnectionPoolConfigFromConfig({
- connectionId: member.connectionId,
- connection: member.connection as KtxMysqlConnectionConfig,
- env,
- });
- const parts: string[] = [
- `host=${kvKeyword(cfg.host)}`,
- `port=${cfg.port}`,
- `database=${kvKeyword(cfg.database)}`,
- `user=${kvKeyword(cfg.user)}`,
- ];
- if (cfg.password) {
- parts.push(`password=${kvKeyword(cfg.password)}`);
- }
- if (cfg.ssl) {
- parts.push('ssl_mode=REQUIRED');
- }
- return parts.join(' ');
-}
-
-/**
- * Resolves a federated member's ktx.yaml config into the connection target
- * DuckDB's ATTACH wants for that driver, reusing each connector's canonical
- * resolver so federation and standalone scans agree on config interpretation.
- */
-export function federatedAttachTarget(member: FederatedMember, env: NodeJS.ProcessEnv): string {
- switch (member.driver.toLowerCase()) {
- case 'duckdb':
- return duckDbDatabasePathFromConfig({
- connectionId: member.connectionId,
- projectDir: member.projectDir,
- connection: member.connection as KtxDuckDbConnectionConfig,
- });
- case 'sqlite':
- return sqliteDatabasePathFromConfig({
- connectionId: member.connectionId,
- projectDir: member.projectDir,
- connection: member.connection as KtxSqliteConnectionConfig,
- });
- case 'postgres':
- return postgresAttachString(member, env);
- case 'mysql':
- return mysqlAttachString(member, env);
- default:
- throw new Error(`Driver "${member.driver}" cannot be attached by DuckDB federation.`);
- }
-}
diff --git a/packages/cli/src/connectors/duckdb/federated-executor.ts b/packages/cli/src/connectors/duckdb/federated-executor.ts
deleted file mode 100644
index 58894cc4..00000000
--- a/packages/cli/src/connectors/duckdb/federated-executor.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import { DuckDBInstance } from '@duckdb/node-api';
-import { federatedAttachTarget } from './federated-attach.js';
-import type {
- KtxSqlQueryExecutionInput,
- KtxSqlQueryExecutionResult,
-} from '../../context/connections/query-executor.js';
-import { normalizeQueryRows } from '../../context/connections/query-executor.js';
-import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
-import { attachTypeForDriver, type FederatedMember } from '../../context/connections/federation.js';
-import { toJsonSafeRows } from '../shared/duckdb-json-safe.js';
-
-function quoteDuckdbIdentifier(id: string): string {
- return `"${id.replaceAll('"', '""')}"`;
-}
-
-/** @internal */
-export function buildAttachStatements(members: FederatedMember[], env: NodeJS.ProcessEnv): string[] {
- const attachments = members.map((member) => ({
- type: attachTypeForDriver(member.driver),
- url: federatedAttachTarget(member, env),
- alias: member.connectionId,
- }));
-
- const loadTypes = [...new Set(attachments.map((a) => a.type).filter((type): type is string => type !== null))];
- const loadStatements = loadTypes.map((type) => `INSTALL ${type}; LOAD ${type};`);
- const attachStatements = attachments.map(({ type, url, alias }) => {
- const options = type === null ? 'READ_ONLY' : `TYPE ${type}, READ_ONLY`;
- return `ATTACH '${url.replaceAll("'", "''")}' AS ${quoteDuckdbIdentifier(alias)} (${options});`;
- });
- return [...loadStatements, ...attachStatements];
-}
-
-export async function executeFederatedQuery(
- members: FederatedMember[],
- input: KtxSqlQueryExecutionInput,
- env: NodeJS.ProcessEnv = process.env,
-): Promise {
- const sql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows);
- const attachStatements = buildAttachStatements(members, env);
-
- const instance = await DuckDBInstance.create(':memory:');
- try {
- const connection = await instance.connect();
- try {
- for (const statement of attachStatements) {
- await connection.run(statement);
- }
- const reader = await connection.runAndReadAll(sql);
- const rows = toJsonSafeRows(normalizeQueryRows(reader.getRows()));
- const headers = reader.columnNames();
- return {
- headers,
- rows,
- totalRows: rows.length,
- command: 'SELECT',
- rowCount: rows.length,
- };
- } finally {
- connection.closeSync();
- }
- } finally {
- instance.closeSync();
- }
-}
diff --git a/packages/cli/src/connectors/duckdb/live-database-introspection.ts b/packages/cli/src/connectors/duckdb/live-database-introspection.ts
deleted file mode 100644
index 4edded29..00000000
--- a/packages/cli/src/connectors/duckdb/live-database-introspection.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import type {
- LiveDatabaseIntrospectionOptions,
- LiveDatabaseIntrospectionPort,
-} from '../../context/ingest/adapters/live-database/types.js';
-import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
-import { KtxDuckDbScanConnector, type KtxDuckDbConnectionConfig } from './connector.js';
-
-export interface CreateDuckDbLiveDatabaseIntrospectionOptions {
- projectDir?: string;
- connections: Record;
- now?: () => Date;
-}
-
-export function createDuckDbLiveDatabaseIntrospection(
- options: CreateDuckDbLiveDatabaseIntrospectionOptions,
-): LiveDatabaseIntrospectionPort {
- return {
- async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) {
- const connection = options.connections[connectionId] as KtxDuckDbConnectionConfig | undefined;
- const connector = new KtxDuckDbScanConnector({
- connectionId,
- connection,
- projectDir: options.projectDir,
- now: options.now,
- });
- try {
- return await connector.introspect(
- {
- connectionId,
- driver: 'duckdb',
- ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
- },
- { runId: `duckdb-${connectionId}` },
- );
- } finally {
- await connector.cleanup();
- }
- },
- };
-}
diff --git a/packages/cli/src/connectors/mongodb/connector.ts b/packages/cli/src/connectors/mongodb/connector.ts
deleted file mode 100644
index f46d6038..00000000
--- a/packages/cli/src/connectors/mongodb/connector.ts
+++ /dev/null
@@ -1,413 +0,0 @@
-import { MongoClient } from 'mongodb';
-import { resolveKtxConfigReference } from '../../context/core/config-reference.js';
-import {
- connectorTestFailure,
- createKtxConnectorCapabilities,
- type KtxColumnSampleInput,
- type KtxColumnSampleResult,
- type KtxConnectorTestResult,
- type KtxScanConnector,
- type KtxScanContext,
- type KtxScanInput,
- type KtxSchemaSnapshot,
- type KtxSchemaTable,
- type KtxTableListEntry,
- type KtxTableRef,
- type KtxTableSampleInput,
- type KtxTableSampleResult,
-} from '../../context/scan/types.js';
-import { scopedTableNames } from '../../context/scan/table-ref.js';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
-import { inferKtxMongoCollectionColumns, type KtxMongoDocument, MONGO_ID_FIELD } from './schema-inference.js';
-
-const DEFAULT_SAMPLE_SIZE = 1000;
-const SAMPLE_MAX_TIME_MS = 30_000;
-
-export interface KtxMongoDbConnectionConfig {
- driver?: string;
- url?: string;
- database?: string;
- databases?: string[];
- enabled_tables?: string[];
- sample_size?: number;
- order_by?: string;
- [key: string]: unknown;
-}
-
-export interface KtxMongoListedCollection {
- name: string;
- type?: string;
-}
-
-interface KtxMongoFindOptions {
- sort: Record;
- limit: number;
- projection?: Record;
-}
-
-/** Driver-agnostic seam over the `mongodb` client so the connector is unit-testable without a server. */
-export interface KtxMongoClient {
- listCollections(databaseName: string): Promise;
- estimatedDocumentCount(databaseName: string, collectionName: string): Promise;
- find(databaseName: string, collectionName: string, options: KtxMongoFindOptions): Promise;
- ping(databaseName: string): Promise;
- close(): Promise;
-}
-
-export interface KtxMongoClientFactory {
- create(url: string): KtxMongoClient;
-}
-
-export interface KtxMongoDbScanConnectorOptions {
- connectionId: string;
- connection: KtxMongoDbConnectionConfig | undefined;
- clientFactory?: KtxMongoClientFactory;
- env?: NodeJS.ProcessEnv;
- now?: () => Date;
-}
-
-class DefaultMongoClient implements KtxMongoClient {
- private readonly client: MongoClient;
- private connected = false;
-
- constructor(url: string) {
- this.client = new MongoClient(url);
- }
-
- private async connectedClient(): Promise {
- if (!this.connected) {
- await this.client.connect();
- this.connected = true;
- }
- return this.client;
- }
-
- async listCollections(databaseName: string): Promise {
- const client = await this.connectedClient();
- const collections = await client.db(databaseName).listCollections({}, { nameOnly: false }).toArray();
- return collections.map((collection) => ({ name: collection.name, type: collection.type }));
- }
-
- async estimatedDocumentCount(databaseName: string, collectionName: string): Promise {
- const client = await this.connectedClient();
- return client.db(databaseName).collection(collectionName).estimatedDocumentCount();
- }
-
- async find(
- databaseName: string,
- collectionName: string,
- options: KtxMongoFindOptions,
- ): Promise {
- const client = await this.connectedClient();
- return client
- .db(databaseName)
- .collection(collectionName)
- .find({}, { sort: options.sort, limit: options.limit, maxTimeMS: SAMPLE_MAX_TIME_MS, ...(options.projection ? { projection: options.projection } : {}) })
- .toArray() as Promise;
- }
-
- async ping(databaseName: string): Promise {
- const client = await this.connectedClient();
- await client.db(databaseName).command({ ping: 1 });
- }
-
- async close(): Promise {
- if (this.connected) {
- await this.client.close();
- this.connected = false;
- }
- }
-}
-
-class DefaultMongoClientFactory implements KtxMongoClientFactory {
- create(url: string): KtxMongoClient {
- return new DefaultMongoClient(url);
- }
-}
-
-export function isKtxMongoDbConnectionConfig(
- connection: KtxMongoDbConnectionConfig | undefined,
-): connection is KtxMongoDbConnectionConfig {
- return String(connection?.driver ?? '').toLowerCase() === 'mongodb';
-}
-
-function databaseFromUrl(url: string): string | undefined {
- try {
- const path = new URL(url).pathname.replace(/^\/+/, '');
- const database = path.split('/')[0];
- return database && database.length > 0 ? decodeURIComponent(database) : undefined;
- } catch {
- return undefined;
- }
-}
-
-function configuredDatabases(connection: KtxMongoDbConnectionConfig, fallback: string | undefined): string[] {
- if (Array.isArray(connection.databases)) {
- const selected = connection.databases
- .filter((database): database is string => typeof database === 'string' && database.trim().length > 0)
- .map((database) => database.trim());
- if (selected.length > 0) {
- return [...new Set(selected)];
- }
- }
- const single = typeof connection.database === 'string' && connection.database.trim().length > 0
- ? connection.database.trim()
- : fallback;
- return single ? [single] : [];
-}
-
-function positiveInteger(value: unknown, fallback: number): number {
- return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : fallback;
-}
-
-function normalizeSampleValue(value: unknown): unknown {
- if (value === null || value === undefined) {
- return null;
- }
- if (value instanceof Date) {
- return value.toISOString();
- }
- if (typeof value === 'object') {
- const bsontype = (value as { _bsontype?: unknown })._bsontype;
- return typeof bsontype === 'string' ? String(value) : JSON.stringify(value);
- }
- return value;
-}
-
-function unionDocumentKeys(documents: readonly KtxMongoDocument[]): string[] {
- const keys: string[] = [];
- const seen = new Set();
- for (const document of documents) {
- for (const key of Object.keys(document)) {
- if (!seen.has(key)) {
- seen.add(key);
- keys.push(key);
- }
- }
- }
- return keys;
-}
-
-export class KtxMongoDbScanConnector implements KtxScanConnector {
- readonly id: string;
- readonly driver = 'mongodb' as const;
- readonly capabilities = createKtxConnectorCapabilities({
- tableSampling: true,
- columnSampling: true,
- columnStats: false,
- readOnlySql: false,
- nestedAnalysis: true,
- formalForeignKeys: false,
- estimatedRowCounts: true,
- });
-
- private readonly connectionId: string;
- private readonly connection: KtxMongoDbConnectionConfig;
- private readonly url: string;
- private readonly databases: string[];
- private readonly sampleSize: number;
- private readonly orderBy: string;
- private readonly enabledTables: ReadonlySet | null;
- private readonly clientFactory: KtxMongoClientFactory;
- private readonly now: () => Date;
- private readonly dialect = getDialectForDriver('mongodb');
- private client: KtxMongoClient | null = null;
-
- constructor(options: KtxMongoDbScanConnectorOptions) {
- const connection = options.connection ?? {};
- const inputDriver = connection.driver ?? 'unknown';
- if (!isKtxMongoDbConnectionConfig(connection)) {
- throw new Error(`Native MongoDB connector cannot run driver "${inputDriver}"`);
- }
- const env = options.env ?? process.env;
- const url = resolveKtxConfigReference(
- typeof connection.url === 'string' ? connection.url.trim() : undefined,
- env,
- );
- if (!url) {
- throw new Error(`Native MongoDB connector requires connections.${options.connectionId}.url`);
- }
- const databases = configuredDatabases(connection, databaseFromUrl(url));
- if (databases.length === 0) {
- throw new Error(
- `Native MongoDB connector requires connections.${options.connectionId}.databases (or a database in the URL)`,
- );
- }
- const enabledTables = Array.isArray(connection.enabled_tables)
- ? new Set(
- connection.enabled_tables
- .filter((table): table is string => typeof table === 'string' && table.trim().length > 0)
- .map((table) => table.trim()),
- )
- : null;
-
- this.connectionId = options.connectionId;
- this.connection = connection;
- this.url = url;
- this.databases = databases;
- this.sampleSize = positiveInteger(connection.sample_size, DEFAULT_SAMPLE_SIZE);
- this.orderBy = typeof connection.order_by === 'string' && connection.order_by.trim().length > 0
- ? connection.order_by.trim()
- : MONGO_ID_FIELD;
- this.enabledTables = enabledTables && enabledTables.size > 0 ? enabledTables : null;
- this.clientFactory = options.clientFactory ?? new DefaultMongoClientFactory();
- this.now = options.now ?? (() => new Date());
- this.id = `mongodb:${options.connectionId}`;
- }
-
- async testConnection(): Promise {
- try {
- await this.clientForQuery().ping(this.databases[0]!);
- return { success: true };
- } catch (error) {
- return connectorTestFailure(error);
- }
- }
-
- async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const client = this.clientForQuery();
- const tables: KtxSchemaTable[] = [];
-
- for (const database of this.databases) {
- const scopedNames = input.tableScope
- ? new Set(scopedTableNames(input.tableScope, { catalog: null, db: database }))
- : null;
- const collections = await client.listCollections(database);
- for (const collection of collections) {
- if (collection.name.startsWith('system.')) {
- continue;
- }
- if (scopedNames && !scopedNames.has(collection.name)) {
- continue;
- }
- if (this.enabledTables && !this.enabledTables.has(`${database}.${collection.name}`)) {
- continue;
- }
- tables.push(await this.introspectCollection(client, database, collection));
- }
- }
-
- return {
- connectionId: this.connectionId,
- driver: 'mongodb',
- extractedAt: this.now().toISOString(),
- scope: { schemas: this.databases },
- metadata: {
- databases: this.databases,
- sample_size: this.sampleSize,
- order_by: this.orderBy,
- table_count: tables.length,
- total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
- },
- tables,
- };
- }
-
- private async introspectCollection(
- client: KtxMongoClient,
- database: string,
- collection: KtxMongoListedCollection,
- ): Promise {
- const isView = collection.type === 'view';
- // estimatedDocumentCount issues a count command, which MongoDB rejects on a
- // view (CommandNotSupportedOnView); only count real collections.
- const estimatedRows = isView ? null : await client.estimatedDocumentCount(database, collection.name);
- const documents = await client.find(database, collection.name, {
- sort: { [this.orderBy]: -1 },
- limit: this.sampleSize,
- });
- return {
- catalog: null,
- db: database,
- name: collection.name,
- kind: isView ? 'view' : 'table',
- comment: null,
- estimatedRows,
- columns: inferKtxMongoCollectionColumns(documents, this.dialect),
- foreignKeys: [],
- };
- }
-
- async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const { database, collection } = this.resolveTableRef(input.table);
- const documents = await this.clientForQuery().find(database, collection, {
- sort: { [this.orderBy]: -1 },
- limit: input.limit,
- });
- const headers = input.columns && input.columns.length > 0 ? input.columns : unionDocumentKeys(documents);
- const rows = documents.map((document) => headers.map((header) => normalizeSampleValue(document[header])));
- return { headers, rows, totalRows: documents.length };
- }
-
- async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise {
- this.assertConnection(input.connectionId);
- const { database, collection } = this.resolveTableRef(input.table);
- const documents = await this.clientForQuery().find(database, collection, {
- sort: { [this.orderBy]: -1 },
- limit: input.limit,
- projection: { [input.column]: 1 },
- });
- const values: unknown[] = [];
- let nullCount = 0;
- for (const document of documents) {
- const value = document[input.column];
- if (value === null || value === undefined) {
- nullCount += 1;
- continue;
- }
- values.push(normalizeSampleValue(value));
- }
- return { values, nullCount, distinctCount: null };
- }
-
- async listSchemas(): Promise {
- return [...this.databases];
- }
-
- async listTables(schemas?: string[]): Promise {
- const client = this.clientForQuery();
- const databases = schemas && schemas.length > 0 ? schemas : this.databases;
- const entries: KtxTableListEntry[] = [];
- for (const database of databases) {
- const collections = await client.listCollections(database);
- for (const collection of collections) {
- if (collection.name.startsWith('system.')) {
- continue;
- }
- entries.push({
- catalog: null,
- schema: database,
- name: collection.name,
- kind: collection.type === 'view' ? 'view' : 'table',
- });
- }
- }
- return entries;
- }
-
- async cleanup(): Promise {
- if (this.client) {
- await this.client.close();
- this.client = null;
- }
- }
-
- private resolveTableRef(table: KtxTableRef): { database: string; collection: string } {
- return { database: table.db ?? this.databases[0]!, collection: table.name };
- }
-
- private clientForQuery(): KtxMongoClient {
- if (!this.client) {
- this.client = this.clientFactory.create(this.url);
- }
- return this.client;
- }
-
- private assertConnection(connectionId: string): void {
- if (connectionId !== this.connectionId) {
- throw new Error(`ktx MongoDB connector ${this.id} cannot serve connection ${connectionId}`);
- }
- }
-}
diff --git a/packages/cli/src/connectors/mongodb/dialect.ts b/packages/cli/src/connectors/mongodb/dialect.ts
deleted file mode 100644
index 3bda71b7..00000000
--- a/packages/cli/src/connectors/mongodb/dialect.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
-import {
- columnDisplayPartCount,
- formatDialectDisplayRef,
- parseDialectDisplayRef,
-} from '../../context/connections/dialect-helpers.js';
-import type { KtxDialectTableRef } from '../../context/connections/dialect-helpers.js';
-import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
-
-/**
- * Display/type-mapping half of the dialect contract for MongoDB. Collections map
- * to `db.collection` display refs (ansi two-part shape). MongoDB is a non-SQL
- * source, so it implements {@link KtxDialect} only — never {@link KtxSqlDialect}.
- */
-export class KtxMongoDbDialect implements KtxDialect {
- readonly type = 'mongodb' as const;
-
- private readonly typeMappings: Record = {
- objectid: 'string',
- string: 'string',
- uuid: 'string',
- binary: 'string',
- regex: 'string',
- int: 'number',
- long: 'number',
- double: 'number',
- decimal: 'number',
- bool: 'boolean',
- date: 'time',
- timestamp: 'time',
- object: 'string',
- array: 'string',
- json: 'string',
- null: 'string',
- mixed: 'string',
- };
-
- formatDisplayRef(table: KtxDialectTableRef): string {
- return formatDialectDisplayRef(table, 'ansi');
- }
-
- parseDisplayRef(display: string): KtxTableRef | null {
- return parseDialectDisplayRef(display, 'ansi');
- }
-
- columnDisplayTablePartCount(): 1 | 2 | 3 {
- return columnDisplayPartCount('ansi');
- }
-
- mapDataType(nativeType: string): string {
- const normalized = nativeType.toLowerCase().trim();
- if (normalized === 'object' || normalized === 'array') {
- return 'json';
- }
- return normalized || 'mixed';
- }
-
- mapToDimensionType(nativeType: string): KtxSchemaDimensionType {
- if (!nativeType) {
- return 'string';
- }
- return this.typeMappings[nativeType.toLowerCase().trim()] ?? 'string';
- }
-}
diff --git a/packages/cli/src/connectors/mongodb/live-database-introspection.ts b/packages/cli/src/connectors/mongodb/live-database-introspection.ts
deleted file mode 100644
index 24cf3ff2..00000000
--- a/packages/cli/src/connectors/mongodb/live-database-introspection.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-import type {
- LiveDatabaseIntrospectionOptions,
- LiveDatabaseIntrospectionPort,
-} from '../../context/ingest/adapters/live-database/types.js';
-import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
-import {
- KtxMongoDbScanConnector,
- type KtxMongoClientFactory,
- type KtxMongoDbConnectionConfig,
-} from './connector.js';
-
-interface CreateMongoDbLiveDatabaseIntrospectionOptions {
- connections: Record
- 
+ 
- 
-