fix(ci): repair slow sl test and sync uv.lock to 0.16.0 (#345)

* test(sl): assert predefined_measures_only in slow sl query test

The slow-only sl query test asserted the outgoing compute.query() request
with an exact nested query object, so the predefined_measures_only flag
added by the semantic-layer-only query policy (#334) broke the match and
turned the Slow TypeScript tests CI job red on main.

* chore: sync uv.lock ktx-sl/ktx-daemon to 0.16.0

The 0.16.0 release bumped the pyproject versions but left uv.lock pinning
the editable ktx-sl/ktx-daemon packages at 0.15.0.

.github/workflows/star-history.yml

@@ -1,72 +0,0 @@
-name: Refresh star history chart
-
-on:
-  schedule:
-    # Twice daily at 06:00 and 18:00 UTC.
-    - cron: "0 6,18 * * *"
-  workflow_dispatch:
-
-permissions:
-  contents: write
-
-env:
-  DO_NOT_TRACK: "1"
-  KTX_TELEMETRY_DISABLED: "1"
-  NEXT_TELEMETRY_DISABLED: "1"
-
-concurrency:
-  group: star-history-refresh
-  cancel-in-progress: true
-
-jobs:
-  refresh:
-    name: Regenerate assets/star-history.svg
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          # RELEASE_PAT can push to the protected main branch; the default
-          # GITHUB_TOKEN is rejected by the branch-protection hook (GH006).
-          token: ${{ secrets.RELEASE_PAT }}
-
-      - name: Fetch fresh star-history SVG
-        run: |
-          set -euo pipefail
-          # cachebust forces star-history to regenerate instead of serving its
-          # own server-side cache; --location follows the slug-normalizing 301.
-          url="https://api.star-history.com/svg?repos=Kaelio/ktx&type=Date&cachebust=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          curl --fail --location --silent --show-error \
-            --retry 3 --retry-delay 5 --max-time 60 \
-            -o assets/star-history.svg.new "$url"
-          # Guard against error pages / truncated responses before overwriting.
-          if ! grep -q "</svg>" assets/star-history.svg.new; then
-            echo "Downloaded file is not a valid SVG; aborting." >&2
-            exit 1
-          fi
-          if [ "$(wc -c < assets/star-history.svg.new)" -lt 1000 ]; then
-            echo "Downloaded SVG is suspiciously small; aborting." >&2
-            exit 1
-          fi
-          # The star-history API returns the SVG without a trailing newline,
-          # which end-of-file-fixer rewrites whenever pre-commit runs
-          # --all-files on a PR. Because the refresh commit below uses [skip ci],
-          # the hook never runs against it here, so an un-normalized file
-          # silently breaks the pre-commit check on every open PR. Normalize to
-          # exactly one trailing newline before committing.
-          printf '%s\n' "$(cat assets/star-history.svg.new)" > assets/star-history.svg
-          rm -f assets/star-history.svg.new
-
-      - name: Commit if changed
-        run: |
-          set -euo pipefail
-          if git diff --quiet -- assets/star-history.svg; then
-            echo "Star-history chart unchanged; nothing to commit."
-            exit 0
-          fi
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add assets/star-history.svg
-          # [skip ci] keeps this housekeeping commit from triggering KTX CI.
-          git commit -m "chore: refresh star history chart [skip ci]"
-          git push

README.md

@@ -103,8 +103,9 @@ upkeep and don't absorb the rest of your company's knowledge.
 - You don't have a SQL warehouse - **ktx** sits on top of one
 - You only need one ad-hoc query - `psql` or a notebook will do
 
-Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and
-SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, and Notion.
+Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server,
+SQLite, DuckDB, Amazon Athena, and MongoDB. Integrates with dbt, MetricFlow,
+LookML, Looker, Metabase, Sigma, Notion, and Google Drive.
 
 ## Quick Start
 

docs-site/components/product-mechanics.tsx

@@ -68,14 +68,14 @@ const EDGE_STROKE = "#94a3b8";
 const sourceData: SourceNodeData[] = [
   {
     title: "Databases",
-    body: "Schemas, columns, keys, row counts, and query history.",
-    items: ["PostgreSQL", "Snowflake", "BigQuery", "SQLite"],
+    body: "Schemas, keys, row counts, query history.",
+    items: ["PostgreSQL", "Snowflake", "BigQuery", "Athena", "MongoDB", "& more"],
     accent: "#3b82f6",
   },
   {
     title: "BI tools",
     body: "Dashboards, questions, explores, usage, and trusted examples.",
-    items: ["Metabase", "Looker"],
+    items: ["Metabase", "Looker", "Sigma"],
     accent: "#f97316",
   },
   {
@@ -87,7 +87,7 @@ const sourceData: SourceNodeData[] = [
   {
     title: "Docs and notes",
     body: "Policies, caveats, team definitions, and analyst context.",
-    items: ["Notion", "Any text"],
+    items: ["Notion", "Google Drive", "Any text"],
     accent: "#10b981",
   },
 ];

docs-site/components/theme-toggle.tsx

@@ -1,14 +1,24 @@
 "use client";
 
-import { useEffect, useState, type ComponentProps, type SVGProps } from "react";
+import {
+  useEffect,
+  useRef,
+  useState,
+  type ComponentProps,
+  type KeyboardEvent,
+  type SVGProps,
+} from "react";
 import { useTheme } from "fumadocs-ui/provider/base";
 
 /**
- * Two-icon theme switcher (light / dark), each icon selecting its own theme —
- * unlike fumadocs' default "light-dark" switcher, which is a single blind
- * toggle that flips on any click. Dropped into the sidebar footer pill via
- * `slots.themeSwitch`, so fumadocs passes the container className (left
- * divider, `ms-auto`, rounded inner buttons); we merge it onto our own base.
+ * Three-icon theme switcher (light / system / dark) rendered as a radio group —
+ * each icon selects its own theme, unlike fumadocs' default "light-dark"
+ * switcher, which is a single blind toggle that flips on any click. Reads
+ * `theme`, not `resolvedTheme`, so the "system" option can show as selected
+ * (resolvedTheme collapses system to light/dark). Dropped into the sidebar
+ * footer pill via `slots.themeSwitch`, so fumadocs passes the container
+ * className (left divider, `ms-auto`, rounded inner buttons); we merge it onto
+ * our own base.
  *
  * Icons are inlined (the project doesn't depend on `lucide-react` directly);
  * `useTheme` is re-exported by fumadocs so we avoid a bare `next-themes` import.
@@ -38,6 +48,25 @@ function SunIcon(props: SVGProps<SVGSVGElement>) {
   );
 }
 
+function MonitorIcon(props: SVGProps<SVGSVGElement>) {
+  return (
+    <svg
+      viewBox="0 0 24 24"
+      fill="none"
+      stroke="currentColor"
+      strokeWidth={2}
+      strokeLinecap="round"
+      strokeLinejoin="round"
+      aria-hidden="true"
+      {...props}
+    >
+      <rect x="3" y="4" width="18" height="12" rx="2" />
+      <path d="M8 20h8" />
+      <path d="M12 16v4" />
+    </svg>
+  );
+}
+
 function MoonIcon(props: SVGProps<SVGSVGElement>) {
   return (
     <svg
@@ -57,6 +86,7 @@ function MoonIcon(props: SVGProps<SVGSVGElement>) {
 
 const OPTIONS = [
   ["light", SunIcon],
+  ["system", MonitorIcon],
   ["dark", MoonIcon],
 ] as const;
 
@@ -65,23 +95,53 @@ function cx(...classes: (string | false | undefined)[]): string {
 }
 
 export function ThemeToggle({ className, ...props }: ComponentProps<"div">) {
-  const { setTheme, resolvedTheme } = useTheme();
+  const { theme, setTheme } = useTheme();
   const [mounted, setMounted] = useState(false);
   useEffect(() => setMounted(true), []);
-  const active = mounted ? resolvedTheme : null;
+  const active = mounted ? theme : null;
+
+  const buttonsRef = useRef<(HTMLButtonElement | null)[]>([]);
+
+  // Pre-mount nothing is selected, so keep the first control tabbable.
+  const selectedIndex = OPTIONS.findIndex(([key]) => key === active);
+  const rovingIndex = selectedIndex === -1 ? 0 : selectedIndex;
+
+  // Radio-group keyboard model: arrows move focus and pick that theme.
+  function onKeyDown(event: KeyboardEvent<HTMLButtonElement>, index: number) {
+    const delta =
+      event.key === "ArrowRight" || event.key === "ArrowDown"
+        ? 1
+        : event.key === "ArrowLeft" || event.key === "ArrowUp"
+          ? -1
+          : 0;
+    if (delta === 0) return;
+    event.preventDefault();
+    const next = (index + delta + OPTIONS.length) % OPTIONS.length;
+    setTheme(OPTIONS[next][0]);
+    buttonsRef.current[next]?.focus();
+  }
 
   return (
     <div
       className={cx("inline-flex items-center overflow-hidden border", className)}
       data-theme-toggle=""
+      role="radiogroup"
+      aria-label="Theme"
       {...props}
     >
-      {OPTIONS.map(([key, Icon]) => (
+      {OPTIONS.map(([key, Icon], index) => (
         <button
           key={key}
+          ref={(el) => {
+            buttonsRef.current[index] = el;
+          }}
           type="button"
+          role="radio"
           aria-label={key}
+          aria-checked={active === key}
+          tabIndex={index === rovingIndex ? 0 : -1}
           onClick={() => setTheme(key)}
+          onKeyDown={(event) => onKeyDown(event, index)}
           className={cx(
             "size-6.5 p-1.5 transition-colors",
             active === key

docs-site/content/docs/cli-reference/ktx-ingest.mdx

@@ -8,7 +8,7 @@ can also capture free-form text into **ktx** memory. Database connections build
 enriched context — schema plus AI-generated descriptions, embeddings, and
 relationship evidence — and require a configured model and embeddings.
 Context-source connections ingest metadata from tools such as dbt, Looker,
-Metabase, MetricFlow, LookML, and Notion. Pass `--text` or `--file` to capture
+Metabase, MetricFlow, LookML, Notion, and Sigma. Pass `--text` or `--file` to capture
 inline text or text files into memory instead.
 
 ## Command signature
@@ -34,8 +34,10 @@ connection is selected.
 | `--query-history` | Include database query-history usage patterns | Stored connection default |
 | `--no-query-history` | Skip database query-history usage patterns for this run | Stored connection default |
 | `--query-history-window-days <days>` | BigQuery/Snowflake query-history lookback window for this run | Stored connection default |
+| `--stages <list>` | Comma-separated enrichment stages to (re)run: `descriptions`, `embeddings`, `relationships` | All three |
 | `--text <content>` | Capture inline text into **ktx** memory; repeatable | `[]` |
 | `--file <path>` | Capture a text file into **ktx** memory; use `-` for stdin; repeatable | `[]` |
+| `--verbatim` | Store each `--text`/`--file` document body unchanged as a `GLOBAL` wiki page; the LLM derives metadata only | `false` |
 | `--connection-id <connectionId>` | **ktx** connection id to tag captured text/file notes | - |
 | `--user-id <id>` | Memory user id for text/file capture attribution | `local-cli` |
 | `--fail-fast` | Stop after the first failed text/file item | `false` |
@@ -63,6 +65,65 @@ use `--no-input` to fail fast with install guidance.
 `--text` and `--file` cannot be combined with a positional `connectionId` or
 `--all`; pass `--connection-id <id>` instead to tag captured notes.
 
+### Verbatim ingest
+
+By default, captured text is routed through the memory agent, which decides what
+to persist and may rewrite, condense, split, or re-title it. For *authoritative*
+documents — metric definitions, formula specs, runbooks, compliance text — that
+paraphrasing is a defect. Add `--verbatim` to store each `--text`/`--file`
+document body **unchanged** as a `GLOBAL` wiki page:
+
+- The stored body is the input document, written by code; the LLM never edits it.
+  It is used only to derive page metadata (`summary`, `tags`, `sl_refs`), and even
+  that is skipped for fields the document's own frontmatter already sets.
+- The page key is deterministic: a `--file` derives it from the filename, inline
+  `--text` from the document's leading Markdown heading (inline text without a
+  heading is rejected — pass it as `--file` instead).
+- Ingest is idempotent. Re-running the same document is a safe no-op; a different
+  body at the same key fails loudly rather than overwriting.
+- `--verbatim` works with `llm.provider.backend: none` — the only ingest path that
+  does. With no backend the `summary` is derived from the heading or first
+  sentence and `tags`/`sl_refs` are left empty; the full body is still stored.
+- Existing frontmatter passes through untouched (including fields **ktx** does not
+  model, such as `effective_date` or `version`); generated metadata only fills
+  absent fields. `--connection-id <id>` scopes the page to that connection by
+  setting its `connections` frontmatter.
+
+### Selecting enrichment stages
+
+Database enrichment runs three stages: `descriptions` (one LLM call per table),
+`embeddings` (vectors over the schema and descriptions), and `relationships`
+(join detection, optionally LLM-proposed). Each stage is cached on a **per-stage
+hash of only its own inputs**, so changing one stage's inputs invalidates only
+that stage. Switching the description LLM re-runs only `descriptions`; upgrading
+the embeddings model re-runs only `embeddings`; turning on
+`scan.relationships.llmProposals` re-runs only `relationships`. The expensive
+per-table descriptions are never thrown away because an unrelated setting moved.
+
+`--stages <list>` re-runs a chosen subset on an already-ingested connection. A
+named stage is **force-recomputed** (it bypasses the completed-stage cache),
+while unselected stages are left exactly as they are on disk:
+
+- `ktx ingest warehouse --stages embeddings` — re-embed on a new model, keeping
+  descriptions and joins.
+- `ktx ingest --all --stages relationships --no-query-history` — backfill joins
+  across every database after enabling `llmProposals`, without re-paying for
+  descriptions.
+- `ktx ingest warehouse --stages descriptions` — re-run thin descriptions (for
+  example after raising `KTX_ENRICH_LLM_TIMEOUT_MS`). When nothing the
+  descriptions depend on changed, the per-table resume record means only the
+  tables that previously failed are re-sent to the LLM.
+
+Stage names are validated: an unknown or empty name (`--stages foo`, `--stages
+descriptions,foo`, `--stages ""`) is a hard parse error. Naming all three
+(`--stages descriptions,embeddings,relationships`) forces a full enrichment
+recompute, which is **not** the same as omitting the flag (omitting resumes
+whatever is already done). After a selective run, **ktx** warns
+(`enrichment_stage_stale`) when an unselected stage's inputs no longer match what
+it was last built from — for example, re-running `descriptions` flags
+`embeddings` as stale until you re-run `--stages embeddings`. The warning is
+informational; **ktx** never silently cascades the extra work.
+
 ## Examples
 
 ```bash
@@ -77,6 +138,11 @@ ktx ingest warehouse --query-history
 # Set the lookback window for BigQuery or Snowflake query history
 ktx ingest warehouse --query-history-window-days 30
 
+# Re-embed one connection on a new embeddings model (descriptions/joins untouched)
+ktx ingest warehouse --stages embeddings
+# Backfill LLM-proposed joins across every database without re-describing
+ktx ingest --all --stages relationships --no-query-history
+
 # Build a context-source connection
 ktx ingest notion
 
@@ -91,6 +157,12 @@ ktx ingest --file docs/revenue-notes.md --connection-id warehouse
 
 # Capture one stdin item
 printf "Refunds are excluded from net revenue." | ktx ingest --file -
+
+# Store an authoritative document verbatim (body preserved exactly)
+ktx ingest --file docs/rfm-bucket-definitions.md --verbatim
+
+# Store it verbatim and scope it to one connection
+ktx ingest --file docs/haversine-formula.md --verbatim --connection-id warehouse
 ```
 
 ## Output
@@ -108,6 +180,21 @@ notion         skipped          skipped        done           done
 Use `--json` when a script or agent needs the selected plan and per-target
 results.
 
+## Final validation pruning
+
+At the end of a context-source ingest, **ktx** validates the composed semantic
+layer and wiki before saving it. If the final validation finds dangling
+references, **ktx** removes the reference instead of failing accepted work. This
+can remove joins that point at missing semantic sources, wiki `refs`, wiki
+`sl_refs`, and inline wiki body references. If a generated semantic source is
+invalid, **ktx** drops that source from the final save.
+
+The stored ingest report records these changes as `finalGatePrunedReferences`
+and `finalGateDroppedSources`. The trace emits `final_gate_reference_pruned`,
+`final_gate_source_dropped`, `final_gate_prune_committed`, and
+`final_gate_prune_finished` events when pruning runs. If validation still fails
+after pruning, the ingest fails and the report keeps the final validation error.
+
 ## Inspect context-source ingest traces
 
 Context-source ingest writes persistent JSONL traces for postmortem debugging.
@@ -191,3 +278,7 @@ according to `ingest.rateLimit`.
 | Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command |
 | Context-source options were ignored | Query-history flags were supplied for a context-source connection | Omit database-only flags when ingesting context-source connections |
 | Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures |
+| `--verbatim requires --text or --file` | `--verbatim` was passed without a document to store | Add `--text` or `--file`, or drop `--verbatim` |
+| Inline verbatim text needs a leading heading | `--text --verbatim` content has no `# Heading` to derive a stable key | Add a leading Markdown heading, or pass the content as `--file <path>` |
+| A different page already exists at key | A verbatim re-run targeted an existing key with a different body | Use a distinct document name/key, or remove the existing page first |
+| Connection scope conflict | Frontmatter `connections` disagrees with `--connection-id` | Remove one so the intended scope is unambiguous |

docs-site/content/docs/cli-reference/ktx-setup.mdx

@@ -29,10 +29,18 @@ below.
 | `--agents` | Install agent configuration and rules only | `false` |
 | `--target <target>` | Agent target: `claude-code`, `claude-desktop`, `codex`, `cursor`, `opencode`, or `universal` | - |
 | `--global` | Install agent integration into the global target scope for `claude-code` or `codex` | `false` |
-| `--install-dir <path>` | Directory to install project-scoped agent config into. Defaults to the ktx project directory; resolved against the current directory and created if missing. Use it to install `.claude/`, `.mcp.json`, and rules where you open your agent (e.g. `--install-dir .`). Mutually exclusive with `--global` and `--local` | ktx project dir |
+| `--install-dir <path>` | Install project-scoped agent configuration | ktx project dir |
 | `--yes` | Accept project creation and runtime install defaults where setup asks for confirmation | `false` |
 | `--no-input` | Disable interactive terminal input | - |
 
+> **`--install-dir <path>`**
+>
+> Installs project-scoped agent configuration into the specified directory.
+> The path is resolved against the current directory and created if it doesn't
+> exist. Use it to install `.claude/`, `.mcp.json`, and rules where you open
+> your agent (for example, `--install-dir .`). This option is mutually exclusive
+> with `--global` and `--local`.
+
 Use the global `--project-dir <path>` option when setup should target a
 specific directory.
 
@@ -112,9 +120,9 @@ runtime features are missing.
 
 | Flag | Description |
 |------|-------------|
-| `--database <driver>` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` |
+| `--database <driver>` | Database driver to configure; repeatable. Choices: `sqlite`, `duckdb`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` |
 | `--database-connection-id <id>` | Existing selected connection id; repeatable. With `--database` or `--database-url`, connection id for the new connection. |
-| `--database-url <url>` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite path |
+| `--database-url <url>` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite or DuckDB path |
 | `--database-schema <schema>` | Database schema or dataset to include; repeatable |
 | `--skip-databases` | Leave database setup incomplete |
 
@@ -126,6 +134,13 @@ incomplete.
 MySQL, and SQL Server; `schema_names` for Snowflake; `dataset_ids` for
 BigQuery; and `databases` for ClickHouse.
 
+A BigQuery `--database-schema` value may be qualified as `project.dataset` to
+scan a dataset hosted in another project (such as
+`bigquery-public-data.austin_311`); a bare value stays in the credentials'
+project. Setup does not discover foreign-project datasets, so supply qualified
+entries explicitly. See
+[Primary sources → BigQuery](/docs/integrations/primary-sources#cross-project-datasets).
+
 With `--no-input`, scope for a scope-bearing driver (PostgreSQL, MySQL,
 ClickHouse, SQL Server, BigQuery, Snowflake) must come from `--database-schema`
 or from existing connection config in `ktx.yaml` (for example
@@ -178,7 +193,7 @@ sources. This is equivalent to passing `--skip-sources` in scripted setup.
 
 | Flag | Description |
 |------|-------------|
-| `--source <type>` | Context-source connector type: `dbt`, `metricflow`, `metabase`, `looker`, `lookml`, or `notion` |
+| `--source <type>` | Context-source connector type: `dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`, or `sigma` |
 | `--source-connection-id <id>` | Connection id for context-source setup |
 | `--source-path <path>` | Local source path for dbt, MetricFlow, or LookML |
 | `--source-git-url <url>` | Git URL for dbt, MetricFlow, or LookML |
@@ -263,6 +278,13 @@ ktx setup \
   --notion-crawl-mode selected_roots \
   --notion-root-page-id abc123def456
 
+# Add a Sigma source
+ktx setup \
+  --source sigma \
+  --source-connection-id sigma-main \
+  --source-client-id your-client-id \
+  --source-client-secret-ref env:SIGMA_CLIENT_SECRET
+
 # Install project-scoped agent integration for Codex
 ktx setup --agents --target codex
 ```

docs-site/content/docs/cli-reference/ktx-wiki.mdx

@@ -28,10 +28,17 @@ Edit the Markdown files under `wiki/` directly, or ingest source content with
 | Flag | Description | Default |
 |------|-------------|---------|
 | `--user-id <id>` | Local user id | `local` |
+| `-c, --connection <id>` | Scope results to one connection: unscoped pages plus pages tagged with that connection | - |
 | `--limit <number>` | Maximum search results (search mode only) | - |
 | `--output <mode>` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` |
 | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` |
 
+`-c, --connection <id>` takes a connection id from the `connections` map in
+`ktx.yaml` (an unknown id is rejected). It narrows both list and search to
+pages that are not tied to any connection plus pages tagged with that
+connection, so an agent working against one database sees only the wiki
+knowledge relevant to it.
+
 `ktx wiki <query>` uses hybrid search when `storage.search` is `sqlite-fts5`.
 **ktx** combines lexical SQLite FTS5 matches, token matches, and semantic matches
 from wiki page embeddings stored in `.ktx/db.sqlite`. If embeddings are not
@@ -50,6 +57,12 @@ ktx wiki --json
 # Search wiki pages
 ktx wiki "monthly recurring revenue"
 
+# List pages scoped to one connection (unscoped + connection-tagged)
+ktx wiki --connection warehouse
+
+# Search within one connection's scope
+ktx wiki "monthly recurring revenue" -c warehouse
+
 # Search wiki pages as JSON
 ktx wiki "monthly recurring revenue" --json --limit 10
 

docs-site/content/docs/concepts/cross-database-federation.mdx

@@ -1,6 +1,6 @@
 ---
 title: Cross-database federation
-description: How ktx federates postgres, mysql, and sqlite connections so a single read-only SQL query can join across them without copying data.
+description: How ktx federates postgres, mysql, sqlite, and duckdb connections so a single read-only SQL query can join across them without copying data.
 ---
 
 Cross-database federation lets a single read-only SQL query join tables that
@@ -20,13 +20,14 @@ block to add. With zero or one compatible connection the behavior is unchanged.
 
 ## Which connections participate
 
-The v1 federation engine supports three drivers:
+The v1 federation engine supports four drivers:
 
 | Driver | Participates in federation |
 |--------|---------------------------|
 | `postgres` | Yes |
 | `mysql` | Yes |
 | `sqlite` | Yes |
+| `duckdb` | Yes |
 | `snowflake` | No — standalone connection |
 | `bigquery` | No — standalone connection |
 | `clickhouse` | No — standalone connection |
@@ -38,7 +39,7 @@ queried independently; they do not appear as federation members.
 ## How it activates
 
 **ktx** inspects the connections in `ktx.yaml` at startup. When it finds two or
-more connections whose driver is `postgres`, `mysql`, or `sqlite`, it
+more connections whose driver is `postgres`, `mysql`, `sqlite`, or `duckdb`, it
 instantiates the DuckDB federation engine and attaches each one read-only.
 There is no `federation:` key, no opt-in flag, and no connection-level setting
 to enable. The engine is derived entirely from what is already declared.
@@ -60,9 +61,10 @@ Two attach-compatible connections are present, so federation is active.
 ## Table naming in federated queries
 
 Inside a federated query, postgres and mysql tables use a three-part name:
-`connectionId.schema.table`. SQLite tables, which have no schema layer in
-DuckDB, use the two-part form `connectionId.table`. In both cases the
-connection's `id` field in `ktx.yaml` becomes the catalog name inside DuckDB.
+`connectionId.schema.table`. SQLite and DuckDB tables use the two-part form
+`connectionId.table`, since ktx addresses both as single-namespace members. In
+both cases the connection's `id` field in `ktx.yaml` becomes the catalog name
+inside DuckDB.
 
 If a connection `id` is not a bare SQL identifier — for example it contains a
 hyphen, like `books-db` — double-quote it in the query the same way DuckDB
@@ -131,12 +133,17 @@ ktx sql -c _ktx_federated \
 Table names follow the rules from
 [Table naming in federated queries](#table-naming-in-federated-queries):
 three-part `connectionId.schema.table` for postgres and mysql, two-part
-`connectionId.table` for sqlite. The `_ktx_federated` id is virtual — it is
-never written to `ktx.yaml` and only exists when two or more attach-compatible
+`connectionId.table` for sqlite and duckdb. The `_ktx_federated` id is virtual —
+it is never written to `ktx.yaml` and only exists when two or more attach-compatible
 connections are declared. It surfaces in `ktx connection` and in the agent's
 connection list so the id is discoverable. Querying a single member database
 directly with its own connection id (`ktx sql -c pg_books ...`) is unchanged.
 
+If any member connection sets
+[`query_policy: semantic-layer-only`](/docs/configuration/ktx-yaml#query-policy),
+raw SQL against `_ktx_federated` is rejected as a whole: a federated query can
+touch any member's tables, so one restricted member restricts the federation.
+
 ## Federated queries are read-only
 
 DuckDB attaches every member database with read-only access. Federated queries
@@ -149,6 +156,6 @@ database through the federation engine.
   them in a source's `joins:` block and automatic discovery of cross-database
   relationships are not available yet. Intra-database relationship discovery for
   each member connection is unchanged.
-- **postgres, mysql, and sqlite only.** Other drivers (snowflake, bigquery,
-  clickhouse, sqlserver) do not participate in federation in this version. They
-  remain usable as standalone connections.
+- **postgres, mysql, sqlite, and duckdb only.** Other drivers (snowflake,
+  bigquery, clickhouse, sqlserver) do not participate in federation in this
+  version. They remain usable as standalone connections.

docs-site/content/docs/configuration/ktx-yaml.mdx

@@ -109,6 +109,7 @@ context-source drivers share the map.
 | `postgres` | Warehouse | `driver` | `url`, `enabled_tables`, `historicSql`, `context.queryHistory` |
 | `mysql` | Warehouse | `driver` | `url`, `enabled_tables` |
 | `sqlite` | Warehouse | `driver` | `url` or `path`, `enabled_tables` |
+| `duckdb` | Warehouse | `driver` | `url` or `path`, `enabled_tables` |
 | `sqlserver` | Warehouse | `driver` | `url`, `enabled_tables` |
 | `bigquery` | Warehouse | `driver` | `credentials_json`, `dataset_ids`, `enabled_tables`, `historicSql` |
 | `snowflake` | Warehouse | `driver` | `schema_names`, `enabled_tables`, `historicSql` |
@@ -119,13 +120,16 @@ context-source drivers share the map.
 | `dbt` | Context source | `driver`, one of `source_dir` or `repo_url` | `branch`, `path`, `profiles_path`, `target`, `project_name` |
 | `metricflow` | Context source | `driver`, `metricflow.repoUrl` | `metricflow.branch`, `metricflow.path`, `metricflow.auth_token_ref` |
 | `notion` | Context source | `driver`, `auth_token_ref` | `crawl_mode`, `root_*_ids`, `max_*_per_run` |
+| `sigma` | Context source | `driver`, `client_id`, `client_secret_ref` | `api_url` |
 
 ### Warehouse drivers
 
 Warehouse connections are open objects: the listed fields are validated, and
 any other field is preserved and passed through to the connector. Use
-`enabled_tables` to scope ingest to a specific list of
-`schema.table` names - useful for smoke tests.
+`enabled_tables` to scope ingest to a specific list of objects - useful for
+smoke tests. Each entry accepts a `catalog.db.name`, `db.name`, or bare `name`
+qualifier. ktx restricts the scan to the listed objects and fails with a clear
+error (naming the available objects) if none match.
 
 ```yaml
 connections:
@@ -137,6 +141,18 @@ connections:
       - public.customers
 ```
 
+For SQLite, which exposes a single `main` schema, the qualified `main.<name>`
+and the bare `<name>` forms select the same object:
+
+```yaml
+connections:
+  local-db:
+    driver: sqlite
+    path: ./warehouse.db
+    enabled_tables:
+      - customers # equivalent to main.customers
+```
+
 Connector-specific scope fields let setup and scan use the same warehouse
 boundary:
 
@@ -158,6 +174,12 @@ connections:
     dataset_ids: [analytics, mart]
 ```
 
+A BigQuery `dataset_ids` / `dataset_id` entry may be written `project.dataset`
+to introspect a dataset hosted in another project (for example
+`bigquery-public-data.austin_311`); jobs still bill to the `project_id` in
+`credentials_json`. A bare `dataset` keeps using your own project. See
+[Primary sources → BigQuery](/docs/integrations/primary-sources#cross-project-datasets).
+
 For Postgres, MySQL, SQL Server, and Snowflake connections, set
 `maxConnections` when scan or ingest work needs to stay below the target's
 connection cap. Postgres, MySQL, and SQL Server default to `10`; Snowflake
@@ -195,6 +217,41 @@ connections:
   observed in-scope query history. The block uses `mode: exclude` and remains
   hand-editable.
 
+### Query policy
+
+Set `query_policy: semantic-layer-only` on a warehouse connection to stop
+agents from authoring SQL against it. The default, `read-only-sql`, allows
+parser-validated read-only SQL through `ktx sql` and the `sql_execution` MCP
+tool alongside semantic-layer queries.
+
+```yaml
+connections:
+  warehouse:
+    driver: snowflake
+    query_policy: semantic-layer-only
+```
+
+With `semantic-layer-only`:
+
+- `ktx sql` and the `sql_execution` MCP tool reject the connection with a
+  clear error. When every SQL connection in the project is restricted, the
+  `sql_execution` tool is not registered at all.
+- Raw SQL against the federated connection (`_ktx_federated`) is rejected
+  when any member connection is restricted.
+- Semantic-layer queries (`ktx sl query`, the `sl_query` tool) accept only
+  measures predefined in the semantic-layer sources. Composed aggregate
+  expressions such as `sum(orders.amount)` are rejected wherever they appear,
+  including inside `filters` (a `HAVING`-style clause may only compare a
+  predefined measure by name, e.g. `orders.revenue > 100`). Grouping by
+  declared dimensions, filtering on columns, and segments remain available.
+- `connection_list` marks the connection as restricted so agents route to
+  `sl_query` instead of burning a failed call.
+
+The policy governs agent-facing query authorship, not data access: **ktx**'s
+own scan, ingest, and semantic-layer-generated SQL still run, and context
+tools such as `entity_details` and `dictionary_search` still expose schema
+metadata and sampled values.
+
 ### Metabase
 
 ```yaml
@@ -325,6 +382,31 @@ connections:
 | `max_knowledge_creates_per_run` | Max new wiki pages created per run (0-25). |
 | `max_knowledge_updates_per_run` | Max existing wiki pages updated per run (0-100). |
 
+### Sigma
+
+```yaml
+connections:
+  sigma-main:
+    driver: sigma
+    api_url: https://api.sigmacomputing.com
+    client_id: "<your-client-id>"
+    client_secret_ref: env:SIGMA_CLIENT_SECRET
+    workbookFilter:
+      includeArchived: false
+      includeExplorations: false
+      updatedSince: "2026-01-01T00:00:00Z"
+```
+
+| Field | Purpose |
+|-------|---------|
+| `api_url` | Sigma API base URL. Defaults to `https://api.sigmacomputing.com` (GCP US). Override for AWS US (`https://aws-api.sigmacomputing.com`) or other regions. |
+| `client_id` | Sigma OAuth client ID. Required. |
+| `client_secret` / `client_secret_ref` | Literal secret or reference. Prefer the `_ref`. |
+| `connectionMappings` | Maps Sigma internal connection UUIDs to **ktx** warehouse connection IDs. Enables `sl_validate` for projected semantic-layer sources. |
+| `workbookFilter.includeArchived` | Include archived workbooks during ingest. Default: `false`. |
+| `workbookFilter.includeExplorations` | Include exploration workbooks during ingest. Default: `false`. |
+| `workbookFilter.updatedSince` | ISO 8601 date string. Only workbooks updated on or after this date are fetched. Useful for limiting ingest scope at large scale. |
+
 ## `setup`
 
 Captured by the setup wizard. The only field **ktx** still reads is
@@ -554,6 +636,7 @@ scan:
     profileConcurrency: 4
     validationConcurrency: 4
     validationBudget: all
+    detectionBudgetMs: 600000
 ```
 
 ### Enrichment
@@ -582,6 +665,7 @@ the manifest.
 | `relationships.profileConcurrency` | `int > 0` | `4` | Parallel relationship-profile queries against the database. For pooled connectors, effective database concurrency is also bounded by the connection's `maxConnections`. |
 | `relationships.validationConcurrency` | `int > 0` | `4` | Parallel relationship validation queries against the database. |
 | `relationships.validationBudget` | `all` \| `int ≥ 0` | runtime default | Cap on validation queries per scan. `all` means unlimited. |
+| `relationships.detectionBudgetMs` | `int > 0` | `600000` | Wall-clock budget (ms) for the whole relationship-detection stage, checked at table-profile, candidate-validation, and composite-probe boundaries. On exhaustion the stage stops scheduling new work and writes the joins found so far, marked partial; descriptions and embeddings are already durable. Sits above the per-query deadline. Raise it to trigger a fresher, fuller run. |
 
 ## `agent`
 

docs-site/content/docs/getting-started/quickstart.mdx

@@ -218,7 +218,8 @@ The wizard walks you through everything **ktx** needs in one pass:
 3. **Embeddings** - picks an embeddings backend. Choose OpenAI for hosted
    embeddings or `sentence-transformers` to run locally without an API key.
 4. **Database** - adds at least one primary connection. Supported drivers:
-   SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, and Snowflake.
+   PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, and
+   DuckDB.
 5. **Context sources** - optionally adds dbt, MetricFlow, LookML, Looker,
    Metabase, or Notion. You can skip and add them later.
 6. **Build** - offers to run the first ingest so semantic sources and wiki

docs-site/content/docs/guides/building-context.mdx

@@ -102,6 +102,7 @@ Supported source types:
 | `looker` | Looker API | Explores, looks, dashboards, and model metadata |
 | `metabase` | Metabase API | Questions, dashboards, table metadata, and mappings |
 | `notion` | Notion API | Wiki pages and business knowledge |
+| `sigma` | Sigma API | Data model specs, pages, element metadata, and workbook metadata |
 
 Context-source ingest writes semantic source YAML and wiki Markdown, reconciling
 with local edits.

docs-site/content/docs/guides/writing-context.mdx

@@ -321,6 +321,23 @@ Useful frontmatter:
 5. Add `sl_refs` for relevant semantic sources.
 6. Search again with a user-like phrase.
 
+### Ingest an authoritative document verbatim
+
+When the document is already the source of truth — a metric-definition sheet, a
+formula spec, a runbook, compliance text — you want **ktx** to index and surface
+it, not re-author it. Instead of hand-copying the file into `wiki/global/`, ingest
+it verbatim:
+
+```bash
+ktx ingest --file docs/rfm-bucket-definitions.md --verbatim
+```
+
+The body is stored byte-for-byte (the LLM only derives `summary`, `tags`, and
+`sl_refs` for the absent frontmatter fields), the page key is derived from the
+filename, and re-running is a safe no-op. Existing frontmatter — including fields
+**ktx** does not model, like `effective_date` — passes through unchanged. See
+[`ktx ingest`](/docs/cli-reference/ktx-ingest) for the full flag reference.
+
 ## Review context changes
 
 Before accepting agent-written context:

docs-site/content/docs/integrations/context-sources.mdx

@@ -1,6 +1,6 @@
 ---
 title: Context Sources
-description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, and Notion.
+description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, Notion, Sigma, and Google Drive.
 ---
 
 Context sources feed your existing analytics tooling into **ktx**. During ingestion, **ktx** extracts metadata from each source and uses a reconciliation agent to reconcile it with your existing semantic layer and knowledge base - preserving accepted edits rather than overwriting.
@@ -27,7 +27,7 @@ LookML uses top-level `repoUrl`, and MetricFlow uses nested
 
 | Field | Required | Description |
 |-------|----------|-------------|
-| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` |
+| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, `notion`, `sigma`, or `gdrive` |
 | `source_dir` | For local file sources | Absolute or project-relative source directory |
 | `repo_url` | For Git-hosted dbt sources | Git repository URL |
 | `repoUrl` | For Git-hosted LookML sources | Git repository URL |
@@ -376,6 +376,167 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in
 - Incremental sync cursors are stored in `.ktx/db.sqlite`; don't add
   `last_successful_cursor` to `ktx.yaml`
 
+---
+
+## Sigma
+
+Ingests data model definitions and workbook metadata from a Sigma workspace as semantic context. Uses the Sigma REST API to fetch data model specs and workbook summaries.
+
+### What it provides
+
+- Data model names, folder paths, and ownership metadata
+- Page and element definitions within each data model
+- Column identifiers and data types where available
+- Workbook names, paths, descriptions, and version metadata
+
+### Connection config
+
+```yaml title="ktx.yaml"
+connections:
+  sigma-main:
+    driver: sigma
+    api_url: https://api.sigmacomputing.com   # Omit for GCP US (default)
+    client_id: "<your-client-id>"
+    client_secret_ref: env:SIGMA_CLIENT_SECRET
+```
+
+For the AWS US region, override `api_url`:
+
+```yaml title="ktx.yaml"
+connections:
+  sigma-main:
+    driver: sigma
+    api_url: https://aws-api.sigmacomputing.com
+    client_id: "<your-client-id>"
+    client_secret_ref: env:SIGMA_CLIENT_SECRET
+```
+
+### Authentication
+
+| Method | Config |
+|--------|--------|
+| OAuth client credentials | `client_id` + `client_secret_ref: env:SIGMA_CLIENT_SECRET` |
+
+Generate a client in Sigma: **Administration → Developer Access → Add New Client**.
+
+### What gets ingested
+
+- Active data model specs, organized by folder into work units
+- Workbook metadata (name, path, description, version) — archived and exploration workbooks excluded by default
+- Models backed by CSV uploads or unsupported connector subtypes are listed in the manifest but skipped during spec fetch (a Sigma API limitation)
+
+### Warehouse connection mapping
+
+`connectionMappings` is optional. Without it, **ktx** produces wiki knowledge only — no semantic-layer sources are written and warehouse validation is skipped. To get semantic-layer output and enable `sl_validate`, map each Sigma internal connection UUID to a **ktx** warehouse connection ID:
+
+```yaml title="ktx.yaml"
+connections:
+  sigma-main:
+    driver: sigma
+    client_id: "<your-client-id>"
+    client_secret_ref: env:SIGMA_CLIENT_SECRET
+    connectionMappings:
+      "<sigma-internal-uuid>": snowflake-prod   # data models using this connection get SL sources
+```
+
+Find the Sigma connection UUID in **Administration → Connections** or from the `source.connectionId` field in a fetched data model spec. Data model elements whose `connectionId` has no mapping are ingested as wiki-only.
+
+### Workbook filter
+
+At large scale, you can limit which workbooks are fetched during ingest using `workbookFilter`:
+
+```yaml title="ktx.yaml"
+connections:
+  sigma-main:
+    driver: sigma
+    client_id: "<your-client-id>"
+    client_secret_ref: env:SIGMA_CLIENT_SECRET
+    workbookFilter:
+      includeArchived: false       # default
+      includeExplorations: false   # default
+      updatedSince: "2026-01-01T00:00:00Z"   # only recently updated workbooks
+```
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `includeArchived` | `false` | Include archived workbooks |
+| `includeExplorations` | `false` | Include exploration workbooks |
+| `updatedSince` | — | ISO 8601 date; only workbooks updated on or after this date are fetched |
+
+### Notes
+
+- `connectionMappings` is optional for wiki-only ingest; it is required to generate semantic-layer sources and run warehouse validation
+- Context ingest (`ktx ingest sigma-main`) fetches from the Sigma API directly
+- Ingest is incremental: items whose `updatedAt` timestamp is unchanged since the last run are skipped
+- Models backed by CSV uploads or unsupported connector subtypes cannot have their spec exported; these are skipped with a warning (a Sigma API limitation)
+- Joins are not projected from Sigma data models in this release; `joins: []` is always written by the projection step. Lookup relationships visible in data model specs are captured as wiki knowledge instead.
+
+---
+
+## Google Drive
+
+Ingests Google Docs from a shared Google Drive folder as wiki-ready knowledge content. This v1 implementation is knowledge-only and ingests Google Docs MIME types only.
+
+### What it provides
+
+- Wiki pages synthesized from Google Docs content
+- Folder-scoped knowledge ingestion from a specific Drive folder
+- Markdown normalization for headings, lists, paragraphs, links, common inline formatting, and Google Docs tables
+
+### Connection config
+
+```yaml title="ktx.yaml"
+connections:
+  company-docs:
+    driver: gdrive
+    service_account_key_ref: file:/absolute/path/to/google-service-account.json
+    folder_id: your-google-drive-folder-id
+    recursive: false
+```
+
+### Authentication
+
+| Method | Config |
+|--------|--------|
+| Service account JSON key file | `service_account_key_ref: file:/absolute/path/to/key.json` |
+
+### Google Cloud setup
+
+1. Create a Google Cloud project.
+2. Enable the Google Drive API.
+3. Enable the Google Docs API.
+4. Create a service account.
+5. Download the service account JSON key.
+6. Share the target Drive folder with the service account email.
+7. Reference the key in `ktx.yaml` with `service_account_key_ref`.
+
+### Required scopes
+
+- `https://www.googleapis.com/auth/drive.readonly`
+- `https://www.googleapis.com/auth/documents.readonly`
+
+### Configuration options
+
+| Field | Description | Default |
+|-------|-------------|---------|
+| `service_account_key_ref` | File reference to the service account JSON key | - |
+| `folder_id` | Google Drive folder ID to ingest | - |
+| `recursive` | Traverse subfolders under `folder_id` | `false` |
+
+### What gets ingested
+
+- Google Docs documents only
+- Wiki-oriented knowledge content
+- One work unit per staged Google Doc
+
+### Notes
+
+- `gdrive` is knowledge-only in v1; it does not produce semantic layer sources
+- `ktx setup` supports Google Drive configuration, including the service-account key ref, folder id, and recursive crawl flag
+- `ktx connection test <connectionId>` supports `gdrive`: it verifies that `folder_id` resolves to a folder the service account can read, then reports the number of Google Docs visible in it. A wrong or unshared `folder_id` fails the test instead of reporting zero docs
+- Only Google Docs are ingested in v1; other file types (Sheets, Slides, PDFs) in the folder are skipped and recorded in the staged manifest
+- The service account must be granted access to the target folder explicitly
+
 ## Common errors
 
 | Error or symptom | Likely cause | Recovery |

docs-site/content/docs/integrations/primary-sources.mdx

@@ -1,6 +1,6 @@
 ---
 title: Primary Sources
-description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, or SQLite.
+description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, MongoDB, or Amazon Athena.
 ---
 
 **ktx** connects to your data warehouse or database to build schema context,
@@ -26,15 +26,23 @@ Agents should prefer environment or file references over literal secrets.
 
 | Field | Required | Applies to | Description |
 |-------|----------|------------|-------------|
-| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, or `sqlite` |
+| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, `mongodb`, or `athena` |
 | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` |
 | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values |
 | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan |
+| `databases` | No | ClickHouse, MongoDB, Athena | List of databases to scan |
+| `sample_size`, `order_by` | No | MongoDB | Schema-inference sampling controls (recent documents, sort field) |
 | `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it |
-| `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference |
+| `path` | Yes for path-style SQLite/DuckDB | SQLite, DuckDB | Local SQLite or DuckDB database path or `env:NAME` reference |
 | `max_bytes_billed` | No | BigQuery | Maximum bytes billed per query job |
-| `job_timeout_ms` | No | BigQuery | BigQuery query job timeout in milliseconds |
+| `query_timeout_ms` | No | all warehouses | Maximum execution time for a single read-only query, in milliseconds (default 30000). A query exceeding it is cancelled server-side (or, for SQLite, by terminating the off-process executor) and returns a `query exceeded Ns` error so the agent can revise. |
 | `project_id` | No | BigQuery | Optional local descriptor and mapping metadata; not used for BigQuery authentication |
+| `region` | Yes | Athena | AWS region where the Athena workgroup and Glue catalog reside (e.g. `us-east-1`) |
+| `s3_staging_dir` | Yes | Athena | S3 URI for Athena query result storage (e.g. `s3://my-bucket/athena-results/`) |
+| `workgroup` | No | Athena | Athena workgroup name (default `primary`) |
+| `catalog` | No | Athena | Glue Data Catalog name (default `AwsDataCatalog`) |
+| `database` | No | Athena | Default Glue database name passed as the query execution context |
+| `databases` | No | Athena | Glue databases to include in schema scans; written by `ktx setup` and read by `ktx ingest` |
 
 ## PostgreSQL
 
@@ -218,6 +226,37 @@ BigQuery dataset scope is stored in `connections.<id>.dataset_ids`. Interactive
 setup discovers datasets from credentials plus location, then writes the chosen
 dataset ids as the scan scope.
 
+### Cross-project datasets
+
+To introspect a dataset hosted in a **different project** than the one your
+credentials bill to — for example Google's `bigquery-public-data`, a partner's
+shared project, or an organization's central data project — qualify the entry
+as `project.dataset`:
+
+```yaml title="ktx.yaml"
+connections:
+  public-bq:
+    driver: bigquery
+    credentials_json: file:~/.config/gcloud/bq-service-account.json
+    location: US
+    dataset_ids:
+      - bigquery-public-data.austin_311
+      - bigquery-public-data.census_bureau_usa
+      - analytics
+```
+
+**ktx** introspects each dataset in its host project while every query job still
+bills to the `project_id` inside your `credentials_json`. A bare `dataset` entry
+(no prefix) is scanned in your own project, exactly as before. A single
+connection may mix datasets from several projects, and two projects may host
+datasets with the same name without colliding.
+
+Interactive setup does not enumerate datasets in projects your credentials don't
+own, so hand-write `project.dataset` entries for foreign datasets. The wizard's
+table picker also only lists datasets in your connection's `location` region;
+this affects table selection only — ingest and `discover_data` introspect a
+cross-project dataset regardless of region.
+
 ### Authentication
 
 | Method | Config |
@@ -267,7 +306,77 @@ staged artifact shape as Postgres and Snowflake.
 - Parameter binding uses named `@param` syntax
 - Arrays flattened to comma-separated strings in results
 - Location specified at query execution time
-- Supports `max_bytes_billed` and `job_timeout_ms` limits from `ktx.yaml`
+- Supports the `max_bytes_billed` limit from `ktx.yaml`; the shared `query_timeout_ms` field maps to the query job's `jobTimeoutMs`
+
+---
+
+## Amazon Athena
+
+Connects to Amazon Athena using the AWS Glue Data Catalog for schema introspection and the Athena query API for read-only SQL execution. Authentication uses the standard AWS credential chain — no credentials are embedded in `ktx.yaml`.
+
+### Connection config
+
+```yaml title="ktx.yaml"
+connections:
+  my-athena:
+    driver: athena
+    region: us-east-1
+    s3_staging_dir: s3://my-bucket/athena-results/
+```
+
+With optional fields:
+
+```yaml title="ktx.yaml"
+connections:
+  my-athena:
+    driver: athena
+    region: us-east-1
+    s3_staging_dir: env:ATHENA_S3_STAGING_DIR
+    workgroup: analytics
+    catalog: AwsDataCatalog
+    database: my_default_database
+    databases:
+      - analytics
+      - raw
+```
+
+`ktx setup` writes the `databases` array when you select Glue databases during setup. `ktx scan` reads it to limit introspection to those databases.
+
+### Authentication
+
+**ktx** uses the AWS SDK default credential chain — no credentials appear in `ktx.yaml`. The chain resolves credentials in this order:
+
+| Method | How to configure |
+|--------|-----------------|
+| Environment variables | Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN` |
+| Shared credentials file | Configure `~/.aws/credentials` with a `[default]` or named profile; set `AWS_PROFILE` to select a non-default profile |
+| IAM instance profile | Attach an IAM role to the EC2 instance or ECS task — no local configuration needed |
+| IAM roles for service accounts (EKS) | Annotate the pod's service account with the IAM role ARN |
+
+The IAM principal must have `athena:StartQueryExecution`, `athena:GetQueryExecution`, `athena:GetQueryResults`, `glue:GetDatabases`, and `glue:GetTables` permissions, plus read access to the S3 results bucket.
+
+### Features
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Tables & views | Yes | Via AWS Glue Data Catalog |
+| Primary keys | No | Glue does not expose constraint metadata |
+| Foreign keys | No | Not available in Glue/Athena |
+| Row count estimates | No | Glue table statistics are often stale |
+| Column statistics | No | - |
+| Query history | No | - |
+| Table sampling | Yes | `SELECT ... LIMIT n` |
+
+### Dialect notes
+
+- SQL dialect is Presto/Trino; identifiers are quoted with double-quotes
+- Table names use three-part format: `"catalog"."database"."table"` (e.g. `"AwsDataCatalog"."analytics"."orders"`)
+- Partition columns (`PartitionKeys` in Glue) are included after regular columns in the schema and are fully queryable
+- Athena does not support `TABLESAMPLE`; random sampling uses `ORDER BY rand()`
+- Query execution is asynchronous: **ktx** starts the query, polls until completion, then fetches results from S3
+- Results are stored in `s3_staging_dir`; the IAM principal needs write access to that bucket
+- Use `workgroup` to apply per-workgroup cost controls and result configuration
+- The connector always uses your account's default Glue Data Catalog; cross-account catalog access (`CatalogId` pointing to another account) is not supported
 
 ---
 
@@ -510,12 +619,141 @@ No authentication required - SQLite is file-based. The file must be readable by
 - Foreign key enforcement requires explicit `PRAGMA foreign_keys = ON`
 - Database file must exist before `ktx connection test` or ingest runs
 
+---
+
+## DuckDB
+
+File-based connector using the DuckDB Node.js API. Ideal for local analytics, embedded warehouses, and cross-database federation.
+
+### Connection config
+
+```yaml title="ktx.yaml"
+connections:
+  warehouse:
+    driver: duckdb
+    path: data/warehouse.duckdb
+```
+
+`path` is resolved relative to the project directory. The `.duckdb` file must already exist — **ktx** never creates a missing database file.
+
+### Authentication
+
+No authentication required — DuckDB is file-based. The `.duckdb` file must be readable by the process running **ktx**.
+
+### Features
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Tables & views | Yes | Via `information_schema` on the `main` schema |
+| Primary keys | Yes | Via `information_schema.table_constraints` |
+| Foreign keys | Yes | Via DuckDB's `duckdb_constraints()` catalog function |
+| Row count estimates | Yes | Exact count via `SELECT COUNT(*)` |
+| Column statistics | No | - |
+| Query history | No | - |
+| Table sampling | Yes | - |
+| Nested analysis | No | - |
+
+### Dialect notes
+
+- Introspection scans the `main` schema only
+- Execution is read-only; **ktx** opens the file without write access
+- Parameter binding uses positional `?` placeholders
+- Uses `LIMIT X OFFSET Y` for pagination
+- Database file must exist before `ktx connection test` or ingest runs
+
+### Cross-database federation
+
+When a project declares two or more attach-compatible connections — any combination of `postgres`, `mysql`, `sqlite`, and `duckdb` — **ktx** derives a cross-database federation connection. That connection can ATTACH a native `.duckdb` file, allowing semantic queries to join across sources without manually copying data.
+
+---
+
+## MongoDB
+
+Connects to MongoDB as a primary context source. **ktx** treats each collection
+as a table and each inferred top-level field as a column. MongoDB is a non-SQL
+source: `ktx sql` and semantic-layer metric compilation do not apply to a MongoDB
+connection, but its collections still flow through `ktx ingest`, descriptions, and
+relationship discovery.
+
+### Connection config
+
+```yaml title="ktx.yaml"
+connections:
+  mongo-prod:
+    driver: mongodb
+    url: env:MONGO_URL
+    databases: [app]
+    enabled_tables: [app.users, app.orders] # optional collection allowlist
+    sample_size: 1000
+    # order_by: createdAt                    # only when _id is not an ObjectId
+```
+
+Standard `mongodb://` and `mongodb+srv://` connection strings are supported,
+including TLS and MongoDB Atlas — pass the full connection string (with its
+query parameters) as `url`. The `databases` list selects which databases to
+introspect; if omitted, **ktx** uses the database in the URL path. `ktx setup`
+also offers MongoDB and stores the selected databases under
+`connections.<id>.databases`.
+
+### Authentication
+
+| Method | Config |
+|--------|--------|
+| Connection string | `url: env:MONGO_URL` or `url: file:/path/to/secret` |
+| Atlas / TLS | Use a `mongodb+srv://` URL with the credentials and TLS options Atlas provides |
+
+### Schema inference
+
+MongoDB has no fixed schema, so **ktx** infers one by sampling the most recent
+`sample_size` documents per collection (default 1000), sorted by `_id`
+descending. Because an ObjectId embeds its creation time, this captures the
+collection's current shape with zero configuration. When `_id` is not an
+ObjectId (custom string or UUID keys), set `order_by` to a timestamp field such
+as `createdAt` so "most recent" is well-defined. A custom `order_by` field
+should be indexed — an unindexed sort hits MongoDB's in-memory sort limit and
+fails on large collections (`_id`, the default, is always indexed).
+
+For each top-level field, **ktx** unions the BSON types seen and derives
+nullability from how often the field is present:
+
+- Scalar BSON types map to `string`, `number`, `time`, or `boolean`
+- A field seen with more than one type is recorded as `mixed` and treated as a string
+- Sub-documents and arrays become a single opaque `json` column (no dotted-path
+  columns); their sampled values are stringified, not faithfully serialized
+- `_id` is the primary key
+
+### Features
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Collections (as tables) | Yes | Via `listCollections`; `system.*` collections are excluded |
+| Primary keys | Yes | `_id` |
+| Foreign keys | No | MongoDB has no formal foreign keys |
+| Row count estimates | Yes | Via `estimatedDocumentCount` |
+| Column statistics | No | - |
+| Query history | No | - |
+| Table sampling | Yes | Reads the most recent documents |
+| Nested analysis | Yes | Sub-documents and arrays modeled as opaque `json` |
+| Read-only SQL (`ktx sql`) | No | MongoDB is not a SQL source |
+
+### Dialect notes
+
+- Strictly read-only: the connector only issues `find`, `listCollections`,
+  `estimatedDocumentCount`, and read aggregations
+- Sampling rides the `_id` index and uses a server-side time limit so large
+  collections do not stall a run; a custom `order_by` must be indexed for the
+  same guarantee
+- `sample_size` trades inference coverage for speed; raise it for collections
+  with highly variable documents
+
 ## Common errors
 
 | Error or symptom | Likely cause | Recovery |
 |------------------|--------------|----------|
 | Connection URL appears in git diff | A literal credential URL was written to `ktx.yaml` | Replace it with `env:NAME` or `file:/path/to/secret` and rotate exposed credentials |
-| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions |
+| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions. For Athena, confirm the IAM principal has `glue:GetDatabases` and `glue:GetTables` permissions |
 | Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest <connectionId> --query-history` or `ktx setup` |
 | Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on schema-level context without column statistics |
 | Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test <id>` and check the `ktx sl query` flags |
+| Athena query fails with `ACCESS_DENIED` | IAM principal lacks `athena:StartQueryExecution` or S3 write access to `s3_staging_dir` | Attach a policy granting Athena query permissions and `s3:PutObject` on the staging bucket |
+| Athena ingest finds databases but no tables | IAM principal has `glue:GetDatabases` but not `glue:GetTables` | Grant `glue:GetTables` on the relevant Glue catalog resources |

examples/README.md

@@ -23,6 +23,14 @@ at the Orbit-style no-declared-constraint relationship fixture and verifies that
 relationship enrichment writes nine accepted joins without requiring a local
 warehouse credential.
 
+## mongodb
+
+`mongodb/` is a manual container-backed example for the MongoDB connector. It
+seeds a representative dataset (nested documents, arrays, a mixed-type field, a
+nullable field, and a view), then exercises the connector as a fast no-LLM
+introspection smoke (`scripts/smoke.sh`) and documents a full keyless
+`ktx ingest` run. Works with Docker Compose or `podman compose`.
+
 ## postgres-historic
 
 `postgres-historic/` is a manual Docker-backed smoke for Postgres

examples/mongodb/README.md

@@ -0,0 +1,127 @@
+# MongoDB Connector Example
+
+A manual, self-contained example for the **ktx** MongoDB connector. It starts a
+local MongoDB, seeds a representative dataset, and exercises the connector both
+as a fast no-LLM introspection smoke and as a full `ktx ingest` run.
+
+MongoDB is a **context-only** primary source: collections become tables and
+inferred top-level fields become columns, but `ktx sql` and semantic-layer
+metric compilation do not apply. See
+[`docs-site/content/docs/integrations/primary-sources.mdx`](../../docs-site/content/docs/integrations/primary-sources.mdx).
+
+## Prerequisites
+
+- Docker with Compose v2, or Podman with `podman compose`
+- Node and pnpm matching the **ktx** workspace
+- The built CLI: `pnpm --filter @kaelio/ktx run build`
+- For the full ingest only: `uv` on `PATH` and a usable local Claude Code
+  session (the keyless `claude-code` LLM backend)
+
+## What the seed contains
+
+[`init/seed.js`](init/seed.js) creates the `app` database with:
+
+- `users` — `_id` (ObjectId), scalar fields, a nested `address`, an array
+  `tags`, a `Decimal128` `balance`, a `ref` field that holds more than one type
+  (inferred `mixed`), and an `age` field absent from one document (nullable)
+- `orders` — an ObjectId `user_id` reference for relationship discovery
+- `active_users` — a **view** (to confirm introspection never runs a count
+  command on a view)
+
+MongoDB applies the script once on first container start. Apply it by hand with:
+
+```bash
+mongosh "mongodb://localhost:27117" < examples/mongodb/init/seed.js
+```
+
+## Smoke (no LLM credentials)
+
+From the **ktx** repository root:
+
+```bash
+examples/mongodb/scripts/smoke.sh
+```
+
+It starts MongoDB on `127.0.0.1:27117`, seeds it, and asserts the connector's
+inferred schema (collections → tables, nested → `json`, `mixed`, nullability,
+`_id` primary key, and a view introspected with `estimatedRows: null`). This
+drives the same entry point `ktx ingest`'s "database schema" stage uses, without
+needing an LLM or embeddings.
+
+Podman:
+
+```bash
+KTX_MONGODB_COMPOSE="podman compose" examples/mongodb/scripts/smoke.sh
+```
+
+Set `KTX_MONGODB_KEEP=1` to leave the container running after the script exits.
+
+## Full `ktx ingest`
+
+The public database-ingest path requires a configured model and embeddings.
+This runs entirely locally with the keyless `claude-code` LLM backend and the
+**ktx**-managed `sentence-transformers` embedding daemon — no API keys.
+
+Start MongoDB and create a project:
+
+```bash
+docker compose -f examples/mongodb/docker-compose.yml up -d --wait   # or: podman compose
+node packages/cli/dist/bin.js admin init /tmp/ktx-mongodb-example
+```
+
+Add the connection and a keyless enrichment stack to
+`/tmp/ktx-mongodb-example/ktx.yaml`:
+
+```yaml
+connections:
+  mongo-prod:
+    driver: mongodb
+    url: mongodb://localhost:27117/app
+    databases:
+      - app
+llm:
+  provider:
+    backend: claude-code
+  models:
+    default: sonnet
+scan:
+  enrichment:
+    mode: llm
+    embeddings:
+      backend: sentence-transformers
+      model: all-MiniLM-L6-v2
+      dimensions: 384
+      sentenceTransformers:
+        base_url: ""
+```
+
+Test the connection and ingest:
+
+```bash
+node packages/cli/dist/bin.js connection test mongo-prod --project-dir /tmp/ktx-mongodb-example
+node packages/cli/dist/bin.js ingest mongo-prod --project-dir /tmp/ktx-mongodb-example --yes --plain
+```
+
+The first ingest starts the **ktx** embedding daemon and downloads the
+`all-MiniLM-L6-v2` model. Expected final state: `Database schema: done`.
+
+Inspect the result:
+
+- `raw-sources/mongo-prod/live-database/<run>/tables/*.json` — one per
+  collection, including the `active_users` view with `estimatedRows: null`
+- `raw-sources/mongo-prod/live-database/<run>/enrichment/relationships.json` —
+  inferred relationships sit in `review` (a non-SQL source has no read-only SQL
+  coverage validation), with `accepted: []`
+- `semantic-layer/mongo-prod/_schema/app.yaml` — the schema with per-column AI
+  descriptions
+
+`ktx sql -c mongo-prod "SELECT 1"` is refused by the read-only SQL capability
+gate, and `ktx sl query -c mongo-prod ...` is refused because MongoDB is not a
+SQL source.
+
+## Cleanup
+
+```bash
+docker compose -f examples/mongodb/docker-compose.yml down -v   # or: podman compose
+rm -rf /tmp/ktx-mongodb-example
+```

examples/mongodb/docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  mongodb:
+    image: mongo:7
+    ports:
+      # Non-default host port so the example does not clash with a local MongoDB.
+      - "27117:27017"
+    healthcheck:
+      test: ["CMD-SHELL", "mongosh --quiet --eval \"db.runCommand({ ping: 1 }).ok\" | grep -q 1"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    volumes:
+      # MongoDB runs *.js here once, on first start, against an empty data dir.
+      - ./init:/docker-entrypoint-initdb.d:ro

examples/mongodb/init/seed.js

@@ -0,0 +1,64 @@
+// Seed a representative MongoDB dataset for the ktx connector example.
+//
+// MongoDB runs this once on first container start (it is mounted into
+// /docker-entrypoint-initdb.d). It can also be applied by hand:
+//   mongosh "mongodb://localhost:27117" < examples/mongodb/init/seed.js
+//
+// The shapes here exercise the connector's schema inference end to end:
+// scalar BSON types, a nested sub-document, an array, Decimal128, dates, a
+// field with more than one type (-> "mixed"), an absent field (-> nullable),
+// an ObjectId reference for relationship discovery, and a view (to confirm
+// introspection never runs a count command on a view).
+const app = db.getSiblingDB('app');
+
+app.users.drop();
+app.orders.drop();
+
+app.users.insertMany([
+  {
+    email: 'ada@example.com',
+    age: 31,
+    active: true,
+    created: new Date('2026-01-04T10:00:00Z'),
+    balance: NumberDecimal('120.50'),
+    address: { city: 'NY', zip: '10001' },
+    tags: ['admin', 'early-access'],
+    ref: 'abc',
+  },
+  {
+    email: 'grace@example.com',
+    active: false,
+    created: new Date('2026-02-11T08:30:00Z'),
+    balance: NumberDecimal('0.00'),
+    address: { city: 'SF', zip: '94016' },
+    tags: [],
+    ref: 42, // a second type for this field -> inferred "mixed"
+    // age intentionally absent -> inferred nullable
+  },
+  {
+    email: 'linus@example.com',
+    age: 27,
+    active: true,
+    created: new Date('2026-03-01T12:00:00Z'),
+    balance: NumberDecimal('9.99'),
+    address: { city: 'Austin', zip: '73301' },
+    tags: ['beta'],
+    ref: null,
+  },
+]);
+
+const userIds = app.users.find({}, { _id: 1 }).toArray().map((u) => u._id);
+
+app.orders.insertMany([
+  { user_id: userIds[0], total: 120.5, status: 'paid', placed: new Date('2026-03-02T09:00:00Z') },
+  { user_id: userIds[0], total: 9.99, status: 'pending', placed: new Date('2026-03-05T14:00:00Z') },
+  { user_id: userIds[1], total: 50.25, status: 'paid', placed: new Date('2026-03-06T16:00:00Z') },
+]);
+
+// A view, to confirm introspection does not issue a count command on it
+// (MongoDB rejects count on a view with CommandNotSupportedOnView).
+app.createView('active_users', 'users', [{ $match: { active: true } }]);
+
+print('users: ' + app.users.countDocuments());
+print('orders: ' + app.orders.countDocuments());
+print('collections: ' + app.getCollectionNames().join(', '));

examples/mongodb/scripts/introspect-smoke.mjs

@@ -0,0 +1,53 @@
+// Deterministic, no-LLM smoke for the MongoDB connector. Drives the same
+// introspection entry point ktx ingest's "database schema" stage uses, against
+// the seeded example database, and asserts the inferred schema.
+//
+// Usage: node introspect-smoke.mjs [mongoUrl]
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve } from 'node:path';
+
+const here = dirname(fileURLToPath(import.meta.url));
+const ktxRoot = resolve(here, '../../..');
+const connectorUrl = `file://${resolve(
+  ktxRoot,
+  'packages/cli/dist/connectors/mongodb/live-database-introspection.js',
+)}`;
+
+const mongoUrl = process.argv[2] ?? 'mongodb://localhost:27117/app';
+
+const { createMongoDbLiveDatabaseIntrospection } = await import(connectorUrl);
+
+function assert(condition, message) {
+  if (!condition) {
+    throw new Error(`assertion failed: ${message}`);
+  }
+}
+
+const port = createMongoDbLiveDatabaseIntrospection({
+  connections: { 'mongo-example': { driver: 'mongodb', url: mongoUrl, databases: ['app'] } },
+});
+
+const snapshot = await port.extractSchema('mongo-example');
+const tables = new Map(snapshot.tables.map((table) => [table.name, table]));
+
+assert(snapshot.driver === 'mongodb', 'snapshot driver is mongodb');
+assert(['orders', 'users'].every((name) => tables.has(name)), 'users and orders collections introspected');
+
+const users = tables.get('users');
+const columns = new Map(users.columns.map((column) => [column.name, column]));
+assert(columns.get('_id')?.primaryKey === true && columns.get('_id')?.nullable === false, '_id is the non-null primary key');
+assert(columns.get('age')?.nullable === true, 'age is nullable (absent in one document)');
+assert(columns.get('email')?.nullable === false, 'email is non-nullable (present in every document)');
+assert(columns.get('address')?.normalizedType === 'json', 'nested address maps to opaque json');
+assert(columns.get('tags')?.normalizedType === 'json', 'array tags maps to opaque json');
+assert(columns.get('ref')?.nativeType === 'mixed', 'ref with two types is inferred as mixed');
+
+const view = tables.get('active_users');
+assert(view?.kind === 'view', 'active_users is a view');
+assert(view?.estimatedRows === null, 'a view is introspected without a count (estimatedRows null)');
+
+console.log(`OK: introspected ${snapshot.tables.length} collections from ${mongoUrl}`);
+for (const table of snapshot.tables) {
+  console.log(`  - ${table.db}.${table.name} (${table.kind}, ${table.columns.length} columns)`);
+}
+process.exit(0);

examples/mongodb/scripts/smoke.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Manual smoke for the MongoDB connector: start MongoDB, seed it, and assert the
+# connector's schema introspection (the deterministic, no-LLM half of ktx ingest's
+# "database schema" stage). The full enrichment ingest is documented in README.md.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXAMPLE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+KTX_ROOT="$(cd "$EXAMPLE_DIR/../.." && pwd)"
+COMPOSE_FILE="$EXAMPLE_DIR/docker-compose.yml"
+CONNECTOR="$KTX_ROOT/packages/cli/dist/connectors/mongodb/live-database-introspection.js"
+MONGO_URL="${KTX_MONGODB_URL:-mongodb://localhost:27117/app}"
+
+# Compose engine: docker by default, override for podman:
+#   KTX_MONGODB_COMPOSE="podman compose" examples/mongodb/scripts/smoke.sh
+COMPOSE="${KTX_MONGODB_COMPOSE:-docker compose}"
+
+cleanup() {
+  if [[ "${KTX_MONGODB_KEEP:-0}" != "1" ]]; then
+    $COMPOSE -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT
+
+if [[ ! -f "$CONNECTOR" ]]; then
+  echo "Build the CLI first: pnpm --filter @kaelio/ktx run build" >&2
+  exit 1
+fi
+
+echo "Starting MongoDB and seeding (${COMPOSE})…"
+$COMPOSE -f "$COMPOSE_FILE" up -d --wait
+
+echo "Asserting connector introspection against ${MONGO_URL}…"
+node "$SCRIPT_DIR/introspect-smoke.mjs" "$MONGO_URL"
+
+echo "Smoke passed."

knip.json

@@ -17,7 +17,9 @@
         "test/**/*.test-utils.ts",
         "test/**/acceptance-fixtures.ts",
         "src/context/scan/relationship-benchmarks.ts!",
-        "src/context/scan/relationship-benchmark-report.ts!"
+        "src/context/scan/relationship-benchmark-report.ts!",
+        "src/connectors/sqlite/read-query-child.ts!",
+        "src/context/llm/subprocess-generate-object-child.ts!"
       ]
     },
     "docs-site": {
@@ -38,7 +40,8 @@
     "conventional-changelog-conventionalcommits"
   ],
   "ignore": [
-    ".context/**"
+    ".context/**",
+    "examples/**"
   ],
   "ignoreBinaries": [
     "uv",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ktx-workspace",
-  "version": "0.13.0",
+  "version": "0.16.0",
   "description": "Workspace root for ktx packages",
   "private": true,
   "type": "module",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kaelio/ktx",
-  "version": "0.13.0",
+  "version": "0.16.0",
   "description": "Standalone ktx context layer for data agents",
   "author": {
     "name": "Kaelio",
@@ -51,12 +51,15 @@
     "@ai-sdk/devtools": "0.0.18",
     "@ai-sdk/google-vertex": "^4.0.134",
     "@anthropic-ai/claude-agent-sdk": "0.3.146",
+    "@aws-sdk/client-athena": "^3.1068.0",
+    "@aws-sdk/client-glue": "^3.1068.0",
     "@clack/core": "1.3.1",
     "@clack/prompts": "1.4.0",
     "@clickhouse/client": "^1.18.5",
     "@commander-js/extra-typings": "14.0.0",
     "@duckdb/node-api": "1.5.3-r.3",
     "@google-cloud/bigquery": "^8.3.1",
+    "google-auth-library": "10.6.2",
     "@looker/sdk": "^26.8.0",
     "@looker/sdk-node": "^26.8.0",
     "@looker/sdk-rtl": "^21.6.5",
@@ -71,11 +74,14 @@
     "ink": "^7.0.3",
     "lookml-parser": "7.1.0",
     "minimatch": "^10.2.5",
+    "mongodb": "^6.12.0",
     "mssql": "^12.5.4",
     "mysql2": "^3.22.3",
     "openai": "^6.38.0",
     "p-limit": "^7.3.0",
     "pg": "^8.21.0",
+    "pino": "^10.3.1",
+    "pino-pretty": "^13.1.3",
     "posthog-node": "^5.34.9",
     "react": "^19.2.6",
     "semver": "^7.8.1",

packages/cli/scripts/copy-runtime-assets.mjs

@@ -7,10 +7,17 @@ const promptsSource = join(packageRoot, 'src', 'prompts');
 const promptsTarget = join(packageRoot, 'dist', 'prompts');
 const skillsSource = join(packageRoot, 'src', 'skills');
 const skillsTarget = join(packageRoot, 'dist', 'skills');
+// Per-dialect SQL notes are markdown served by the sql_dialect_notes MCP tool;
+// tsc does not emit non-.ts files, so copy them next to their compiled module.
+const dialectNotesSource = join(packageRoot, 'src', 'context', 'sql-analysis', 'dialects');
+const dialectNotesTarget = join(packageRoot, 'dist', 'context', 'sql-analysis', 'dialects');
 
 await rm(promptsTarget, { recursive: true, force: true });
 await rm(skillsTarget, { recursive: true, force: true });
+await rm(dialectNotesTarget, { recursive: true, force: true });
 await mkdir(dirname(promptsTarget), { recursive: true });
 await mkdir(dirname(skillsTarget), { recursive: true });
+await mkdir(dirname(dialectNotesTarget), { recursive: true });
 await cp(promptsSource, promptsTarget, { recursive: true });
 await cp(skillsSource, skillsTarget, { recursive: true });
+await cp(dialectNotesSource, dialectNotesTarget, { recursive: true });

packages/cli/src/cli-program.ts

@@ -133,7 +133,7 @@ export function parseBooleanStringOption(value: string): boolean {
 }
 
 export function parseSafeConnectionIdOption(value: string): string {
-  if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
+  if (!/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/.test(value)) {
     throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
   }
   return value;

packages/cli/src/commands/ingest-commands.ts

@@ -1,10 +1,12 @@
-import { type Command, Option } from '@commander-js/extra-typings';
+import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
 import {
   collectOption,
   type KtxCliCommandContext,
   parsePositiveIntegerOption,
   resolveCommandProjectDir,
 } from '../cli-program.js';
+import { KTX_SCAN_ENRICHMENT_STAGES } from '../context/scan/enrichment-state.js';
+import type { KtxScanEnrichmentStage } from '../context/scan/types.js';
 import type { KtxCliDeps, KtxCliIo } from '../index.js';
 import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
 import type { KtxPublicIngestArgs } from '../public-ingest.js';
@@ -14,6 +16,36 @@ import { resolveConnectionSelection } from './connection-selection.js';
 
 profileMark('module:commands/ingest-commands');
 
+/**
+ * Parses `--stages` into an ordered, de-duplicated subset of the canonical
+ * enrichment-stage registry. An unknown or empty name is a hard parse error so
+ * a typo never silently degrades to "run everything."
+ *
+ * @internal
+ */
+export function parseEnrichmentStagesOption(value: string): KtxScanEnrichmentStage[] {
+  const names = value
+    .split(',')
+    .map((name) => name.trim())
+    .filter((name) => name.length > 0);
+  if (names.length === 0) {
+    throw new InvalidArgumentError(
+      `must be a non-empty comma-separated list of stages (${KTX_SCAN_ENRICHMENT_STAGES.join(', ')})`,
+    );
+  }
+  const valid = new Set<string>(KTX_SCAN_ENRICHMENT_STAGES);
+  const selected = new Set<KtxScanEnrichmentStage>();
+  for (const name of names) {
+    if (!valid.has(name)) {
+      throw new InvalidArgumentError(
+        `unknown stage "${name}"; valid stages are ${KTX_SCAN_ENRICHMENT_STAGES.join(', ')}`,
+      );
+    }
+    selected.add(name as KtxScanEnrichmentStage);
+  }
+  return KTX_SCAN_ENRICHMENT_STAGES.filter((stage) => selected.has(stage));
+}
+
 interface IngestCommandOptions {
   runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise<number>;
 }
@@ -32,8 +64,18 @@ export function registerIngestCommands(
     .addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory'))
     .addOption(new Option('--no-query-history', 'Skip database query-history usage patterns'))
     .option('--query-history-window-days <days>', 'Query-history lookback window for this run', parsePositiveIntegerOption)
+    .option(
+      '--stages <stages>',
+      'Comma-separated enrichment stages to (re)run (descriptions,embeddings,relationships); omit to run all',
+      parseEnrichmentStagesOption,
+    )
     .option('--text <content>', 'Capture inline text into ktx memory; repeatable', collectOption, [])
     .option('--file <path>', 'Capture a text file into ktx memory; use - for stdin; repeatable', collectOption, [])
+    .option(
+      '--verbatim',
+      'Store each --text/--file document body unchanged as a GLOBAL wiki page; the LLM derives only metadata',
+      false,
+    )
     .option('--connection-id <connectionId>', 'ktx connection id to tag captured text/file notes')
     .option('--user-id <id>', 'Memory user id for text/file capture attribution', 'local-cli')
     .option('--fail-fast', 'Stop after the first failed text/file item', false)
@@ -47,6 +89,14 @@ export function registerIngestCommands(
     const projectDir = resolveCommandProjectDir(command);
     const hasTextCapture = options.text.length > 0 || options.file.length > 0;
 
+    if (options.verbatim === true && !hasTextCapture) {
+      command.error('error: --verbatim requires --text or --file');
+    }
+
+    if (options.stages !== undefined && hasTextCapture) {
+      command.error('error: --stages applies to database ingest only; it cannot be combined with --text or --file');
+    }
+
     if (hasTextCapture) {
       if (connectionId !== undefined) {
         command.error(
@@ -66,6 +116,7 @@ export function registerIngestCommands(
             userId: options.userId,
             json: options.json === true,
             failFast: options.failFast === true,
+            ...(options.verbatim === true ? { verbatim: true } : {}),
           },
           context.io,
           context.deps,
@@ -87,6 +138,7 @@ export function registerIngestCommands(
       inputMode: options.input === false ? 'disabled' : 'auto',
       queryHistory,
       ...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
+      ...(options.stages ? { stages: options.stages } : {}),
       cliVersion: context.packageInfo.version,
       runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
     };

packages/cli/src/commands/knowledge-commands.ts

@@ -27,6 +27,7 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
     .usage('[options] [query...]')
     .argument('[query...]', 'Search query; omit to list all pages')
     .option('--user-id <id>', 'Local user id', 'local')
+    .option('-c, --connection <id>', 'Scope results to one connection (unscoped pages plus pages tagged with it)')
     .option('--limit <number>', 'Maximum search results (search mode only)', parsePositiveIntegerOption)
     .addOption(
       new Option('--output <mode>', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
@@ -46,6 +47,7 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
         query: string[],
         options: {
           userId: string;
+          connection?: string;
           limit?: number;
           output?: 'pretty' | 'plain' | 'json';
           json?: boolean;
@@ -57,6 +59,7 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
             command: 'list',
             projectDir: resolveCommandProjectDir(command),
             userId: options.userId,
+            ...(options.connection !== undefined ? { connectionId: options.connection } : {}),
             output: options.output,
             json: options.json,
             cliVersion: context.packageInfo.version,
@@ -68,6 +71,7 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
           projectDir: resolveCommandProjectDir(command),
           query: query.join(' '),
           userId: options.userId,
+          ...(options.connection !== undefined ? { connectionId: options.connection } : {}),
           output: options.output,
           json: options.json,
           ...(isDebugEnabled(command) ? { debug: true } : {}),

packages/cli/src/commands/setup-commands.ts

@@ -38,6 +38,7 @@ function llmBackend(value: string): KtxSetupLlmBackend {
 function databaseDriver(value: string): KtxSetupDatabaseDriver {
   if (
     value === 'sqlite' ||
+    value === 'duckdb' ||
     value === 'postgres' ||
     value === 'mysql' ||
     value === 'clickhouse' ||
@@ -57,7 +58,9 @@ function sourceType(value: string): KtxSetupSourceType {
     value === 'metabase' ||
     value === 'looker' ||
     value === 'lookml' ||
-    value === 'notion'
+    value === 'notion' ||
+    value === 'sigma' ||
+    value === 'gdrive'
   ) {
     return value;
   }
@@ -132,6 +135,9 @@ function shouldShowSetupEntryMenu(
     metabaseDatabaseId?: number;
     notionCrawlMode?: string;
     notionRootPageId?: string[];
+    gdriveServiceAccountKeyRef?: string;
+    gdriveFolderId?: string;
+    gdriveRecursive?: boolean;
     skipSources?: boolean;
   },
   command: Command,
@@ -197,6 +203,9 @@ function shouldShowSetupEntryMenu(
     'sourceTarget',
     'metabaseDatabaseId',
     'notionCrawlMode',
+    'gdriveServiceAccountKeyRef',
+    'gdriveFolderId',
+    'gdriveRecursive',
     'skipSources',
   ].some((optionName) => optionWasSpecified(command, optionName));
 }
@@ -337,6 +346,12 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
         .default([] as string[])
         .hideHelp(),
     )
+    .addOption(
+      new Option('--gdrive-service-account-key-ref <ref>', 'file: reference to a Google service account JSON key')
+        .hideHelp(),
+    )
+    .addOption(new Option('--gdrive-folder-id <id>', 'Google Drive folder id to ingest').hideHelp())
+    .addOption(new Option('--gdrive-recursive', 'Recursively traverse Google Drive subfolders').hideHelp().default(false))
     .addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false))
     .showHelpAfterError();
 
@@ -486,6 +501,11 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
       ...(options.metabaseDatabaseId !== undefined ? { metabaseDatabaseId: options.metabaseDatabaseId } : {}),
       ...(options.notionCrawlMode ? { notionCrawlMode: options.notionCrawlMode } : {}),
       ...(options.notionRootPageId.length > 0 ? { notionRootPageIds: options.notionRootPageId } : {}),
+      ...(options.gdriveServiceAccountKeyRef
+        ? { gdriveServiceAccountKeyRef: options.gdriveServiceAccountKeyRef }
+        : {}),
+      ...(options.gdriveFolderId ? { gdriveFolderId: options.gdriveFolderId } : {}),
+      ...(options.gdriveRecursive ? { gdriveRecursive: true } : {}),
       runInitialSourceIngest: false,
       skipSources: options.skipSources === true,
       showEntryMenu: shouldShowSetupEntryMenu(options, command),

packages/cli/src/connection-drivers.ts

@@ -1,14 +1,21 @@
 import type { KtxProjectConnectionConfig } from './context/project/config.js';
 
-const KTX_DATABASE_DRIVER_IDS = new Set([
+/** @internal Canonical SQL-warehouse driver ids; the dialect-notes coverage test derives its required coverage from this set. */
+export const KTX_DATABASE_DRIVER_IDS = [
   'sqlite',
+  'duckdb',
   'postgres',
   'mysql',
   'clickhouse',
   'sqlserver',
   'bigquery',
   'snowflake',
-]);
+  'athena',
+] as const;
+
+// mongodb is a database driver but has no SQL dialect, so it sits outside the
+// dialect-notes coverage set above.
+const databaseDriverIds = new Set<string>([...KTX_DATABASE_DRIVER_IDS, 'mongodb']);
 
 export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string {
   return String(connection.driver ?? '')
@@ -17,5 +24,5 @@ export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig
 }
 
 export function isDatabaseDriver(driver: string): boolean {
-  return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
+  return databaseDriverIds.has(driver.trim().toLowerCase());
 }

packages/cli/src/connection.ts

@@ -3,8 +3,11 @@ import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/
 import type { LookerClient } from './context/ingest/adapters/looker/client.js';
 import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
 import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js';
+import { parseGdriveConnectionConfig, resolveGdriveServiceAccountKey } from './context/connections/gdrive-config.js';
 import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js';
 import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js';
+import { createGoogleDocsClients, verifyGdriveFolderAndCountDocs } from './context/ingest/adapters/gdrive/gdrive-client.js';
+import { gdriveServiceAccountKeySchema } from './context/ingest/adapters/gdrive/types.js';
 import { testRepoConnection } from './context/ingest/repo-fetch.js';
 import { federatedConnectionListing } from './context/connections/federation.js';
 import { getDriverRegistration } from './context/connections/drivers.js';
@@ -31,6 +34,10 @@ export type KtxConnectionArgs =
 type MetabaseTestPort = Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>;
 type LookerTestPort = Pick<LookerClient, 'testConnection'>;
 type NotionTestPort = Pick<NotionClient, 'retrieveBotUser'>;
+type GdriveTestPort = Pick<
+  ReturnType<typeof createGoogleDocsClients>['drive'],
+  'listFiles' | 'getFile'
+>;
 type TestRepoConnection = typeof testRepoConnection;
 
 export interface KtxConnectionDeps {
@@ -38,11 +45,13 @@ export interface KtxConnectionDeps {
   createMetabaseClient?: (project: KtxLocalProject, connectionId: string) => Promise<MetabaseTestPort>;
   createLookerClient?: (project: KtxLocalProject, connectionId: string) => Promise<LookerTestPort>;
   createNotionClient?: (project: KtxLocalProject, connectionId: string) => Promise<NotionTestPort>;
+  createGdriveClient?: (project: KtxLocalProject, connectionId: string) => Promise<GdriveTestPort>;
   testRepoConnection?: TestRepoConnection;
 }
 
 const SUPPORTED_TEST_DRIVERS = [
   'sqlite',
+  'duckdb',
   'postgres',
   'mysql',
   'clickhouse',
@@ -52,6 +61,7 @@ const SUPPORTED_TEST_DRIVERS = [
   'metabase',
   'looker',
   'notion',
+  'gdrive',
   'dbt',
   'metricflow',
   'lookml',
@@ -183,6 +193,34 @@ async function testNotionConnection(
   return { bot: describeNotionBot(bot) };
 }
 
+async function createDefaultGdriveClient(
+  project: KtxLocalProject,
+  connectionId: string,
+): Promise<GdriveTestPort> {
+  const connection = project.config.connections[connectionId];
+  if (!connection) {
+    throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
+  }
+  const parsed = parseGdriveConnectionConfig(connection);
+  const keyText = await resolveGdriveServiceAccountKey(parsed.service_account_key_ref);
+  const key = gdriveServiceAccountKeySchema.parse(JSON.parse(keyText));
+  return createGoogleDocsClients(key).drive;
+}
+
+async function testGdriveConnection(
+  project: KtxLocalProject,
+  connectionId: string,
+  createClient: (project: KtxLocalProject, connectionId: string) => Promise<GdriveTestPort>,
+): Promise<{ docs: number }> {
+  const connection = project.config.connections[connectionId];
+  if (!connection) {
+    throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
+  }
+  const parsed = parseGdriveConnectionConfig(connection);
+  const client = await createClient(project, connectionId);
+  return { docs: await verifyGdriveFolderAndCountDocs(client, parsed.folder_id) };
+}
+
 interface GitConnectionFields {
   repoUrl: string;
   authToken: string | null;
@@ -271,6 +309,15 @@ async function testConnectionByDriver(
     return { driver, detailKey: 'Bot', detailValue: result.bot };
   }
 
+  if (driver === 'gdrive') {
+    const result = await testGdriveConnection(
+      project,
+      connectionId,
+      deps.createGdriveClient ?? createDefaultGdriveClient,
+    );
+    return { driver, detailKey: 'Docs', detailValue: String(result.docs) };
+  }
+
   if (driver === 'dbt' || driver === 'metricflow' || driver === 'lookml') {
     const result = await testGitRepoConnection(
       project,

packages/cli/src/connectors/athena/connector.ts

@@ -0,0 +1,555 @@
+import { AthenaClient, StartQueryExecutionCommand, GetQueryExecutionCommand, GetQueryResultsCommand } from '@aws-sdk/client-athena';
+import { GlueClient, GetDatabasesCommand, GetTablesCommand } from '@aws-sdk/client-glue';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
+import {
+  connectorTestFailure,
+  createKtxConnectorCapabilities,
+  type KtxConnectorTestResult,
+  type KtxColumnSampleInput,
+  type KtxColumnSampleResult,
+  type KtxColumnStatsInput,
+  type KtxColumnStatsResult,
+  type KtxQueryResult,
+  type KtxReadOnlyQueryInput,
+  type KtxScanConnector,
+  type KtxScanContext,
+  type KtxScanInput,
+  type KtxSchemaColumn,
+  type KtxSchemaSnapshot,
+  type KtxSchemaTable,
+  type KtxTableListEntry,
+  type KtxTableRef,
+  type KtxTableSampleInput,
+  type KtxTableSampleResult,
+} from '../../context/scan/types.js';
+import { scopedTableNames } from '../../context/scan/table-ref.js';
+import { resolveStringReference } from '../shared/string-reference.js';
+
+export interface KtxAthenaConnectionConfig {
+  driver?: string;
+  region?: string;
+  s3_staging_dir?: string;
+  workgroup?: string;
+  catalog?: string;
+  database?: string;
+  databases?: string[];
+  [key: string]: unknown;
+}
+
+export interface KtxAthenaResolvedConnectionConfig {
+  region: string;
+  s3StagingDir: string;
+  workgroup: string;
+  catalog: string;
+  database: string | undefined;
+  databases: string[];
+}
+
+interface KtxAthenaQueryExecutionStatus {
+  State?: string;
+  StateChangeReason?: string;
+}
+
+interface KtxAthenaQueryExecution {
+  Status?: KtxAthenaQueryExecutionStatus;
+}
+
+interface KtxAthenaColumnInfo {
+  Name?: string;
+  Type?: string;
+}
+
+interface KtxAthenaDatum {
+  VarCharValue?: string;
+}
+
+interface KtxAthenaRow {
+  Data?: KtxAthenaDatum[];
+}
+
+interface KtxAthenaResultSet {
+  Rows?: KtxAthenaRow[];
+  ResultSetMetadata?: { ColumnInfo?: KtxAthenaColumnInfo[] };
+}
+
+/** @internal */
+export interface KtxAthenaClient {
+  startQueryExecution(input: {
+    QueryString: string;
+    ResultConfiguration: { OutputLocation: string };
+    WorkGroup: string;
+    QueryExecutionContext?: { Database?: string; Catalog?: string };
+  }): Promise<{ QueryExecutionId?: string }>;
+  getQueryExecution(input: { QueryExecutionId: string }): Promise<{ QueryExecution?: KtxAthenaQueryExecution }>;
+  getQueryResults(input: { QueryExecutionId: string; NextToken?: string }): Promise<{
+    ResultSet?: KtxAthenaResultSet;
+    NextToken?: string;
+  }>;
+}
+
+interface KtxGlueColumnDef {
+  Name?: string;
+  Type?: string;
+  Comment?: string;
+}
+
+interface KtxGlueStorageDescriptor {
+  Columns?: KtxGlueColumnDef[];
+}
+
+/** @internal */
+export interface KtxGlueTable {
+  Name?: string;
+  TableType?: string;
+  StorageDescriptor?: KtxGlueStorageDescriptor;
+  PartitionKeys?: KtxGlueColumnDef[];
+  Description?: string;
+  Parameters?: Record<string, string>;
+}
+
+/** @internal */
+export interface KtxGlueClient {
+  getDatabases(input: { CatalogId?: string; NextToken?: string }): Promise<{
+    DatabaseList?: Array<{ Name?: string }>;
+    NextToken?: string;
+  }>;
+  getTables(input: { DatabaseName: string; CatalogId?: string; NextToken?: string }): Promise<{
+    TableList?: KtxGlueTable[];
+    NextToken?: string;
+  }>;
+}
+
+export interface KtxAthenaClientFactory {
+  createAthenaClient(region: string): KtxAthenaClient;
+  createGlueClient(region: string): KtxGlueClient;
+}
+
+class DefaultAthenaClientFactory implements KtxAthenaClientFactory {
+  createAthenaClient(region: string): KtxAthenaClient {
+    const client = new AthenaClient({ region });
+    return {
+      startQueryExecution: async (input) => {
+        const result = await client.send(
+          new StartQueryExecutionCommand({
+            QueryString: input.QueryString,
+            ResultConfiguration: { OutputLocation: input.ResultConfiguration.OutputLocation },
+            WorkGroup: input.WorkGroup,
+            QueryExecutionContext: input.QueryExecutionContext,
+          }),
+        );
+        return { QueryExecutionId: result.QueryExecutionId };
+      },
+      getQueryExecution: async (input) => {
+        const result = await client.send(new GetQueryExecutionCommand({ QueryExecutionId: input.QueryExecutionId }));
+        return {
+          QueryExecution: result.QueryExecution
+            ? {
+                Status: {
+                  State: result.QueryExecution.Status?.State,
+                  StateChangeReason: result.QueryExecution.Status?.StateChangeReason,
+                },
+              }
+            : undefined,
+        };
+      },
+      getQueryResults: async (input) => {
+        const result = await client.send(
+          new GetQueryResultsCommand({ QueryExecutionId: input.QueryExecutionId, NextToken: input.NextToken }),
+        );
+        return {
+          ResultSet: result.ResultSet as KtxAthenaResultSet | undefined,
+          NextToken: result.NextToken,
+        };
+      },
+    };
+  }
+
+  createGlueClient(region: string): KtxGlueClient {
+    const client = new GlueClient({ region });
+    return {
+      getDatabases: async (input) => {
+        const result = await client.send(new GetDatabasesCommand({ CatalogId: input.CatalogId, NextToken: input.NextToken }));
+        return {
+          DatabaseList: result.DatabaseList?.map((db) => ({ Name: db.Name })),
+          NextToken: result.NextToken,
+        };
+      },
+      getTables: async (input) => {
+        const result = await client.send(
+          new GetTablesCommand({ DatabaseName: input.DatabaseName, CatalogId: input.CatalogId, NextToken: input.NextToken }),
+        );
+        return {
+          TableList: result.TableList as KtxGlueTable[] | undefined,
+          NextToken: result.NextToken,
+        };
+      },
+    };
+  }
+}
+
+function stringConfigValue(
+  connection: KtxAthenaConnectionConfig | undefined,
+  key: keyof KtxAthenaConnectionConfig,
+  env: NodeJS.ProcessEnv,
+): string | undefined {
+  const value = connection?.[key];
+  if (typeof value !== 'string' || value.trim().length === 0) return undefined;
+  // Resolve before checking emptiness: an unset `env:` reference resolves to '',
+  // which must become undefined so `?? default` applies instead of keeping ''.
+  const resolved = resolveStringReference(value.trim(), env).trim();
+  return resolved.length > 0 ? resolved : undefined;
+}
+
+function configuredAthenaDatabases(connection: KtxAthenaConnectionConfig): string[] {
+  if (!Array.isArray(connection.databases)) return [];
+  const selected = connection.databases
+    .filter((database): database is string => typeof database === 'string' && database.trim().length > 0)
+    .map((database) => database.trim());
+  return [...new Set(selected)];
+}
+
+export function isKtxAthenaConnectionConfig(
+  connection: unknown,
+): connection is KtxAthenaConnectionConfig {
+  return (
+    typeof connection === 'object' &&
+    connection !== null &&
+    String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'athena'
+  );
+}
+
+/** @internal */
+export function athenaConnectionConfigFromConfig(input: {
+  connectionId: string;
+  connection: KtxAthenaConnectionConfig | undefined;
+  env?: NodeJS.ProcessEnv;
+}): KtxAthenaResolvedConnectionConfig {
+  const inputDriver = input.connection?.driver ?? 'unknown';
+  if (!isKtxAthenaConnectionConfig(input.connection)) {
+    throw new Error(`Native Athena connector cannot run driver "${String(inputDriver)}"`);
+  }
+  const env = input.env ?? process.env;
+  const region = stringConfigValue(input.connection, 'region', env);
+  if (!region) {
+    throw new Error(`Native Athena connector requires connections.${input.connectionId}.region`);
+  }
+  const s3StagingDir = stringConfigValue(input.connection, 's3_staging_dir', env);
+  if (!s3StagingDir) {
+    throw new Error(`Native Athena connector requires connections.${input.connectionId}.s3_staging_dir`);
+  }
+  return {
+    region,
+    s3StagingDir,
+    workgroup: stringConfigValue(input.connection, 'workgroup', env) ?? 'primary',
+    catalog: stringConfigValue(input.connection, 'catalog', env) ?? 'AwsDataCatalog',
+    database: stringConfigValue(input.connection, 'database', env),
+    databases: configuredAthenaDatabases(input.connection),
+  };
+}
+
+function glueTableKind(tableType: string | undefined): 'table' | 'view' {
+  const t = String(tableType ?? '').toUpperCase();
+  if (t === 'VIRTUAL_VIEW') return 'view';
+  return 'table';
+}
+
+const POLL_INTERVAL_MS = 250;
+const QUERY_TIMEOUT_MS = 5 * 60 * 1000;
+
+export interface KtxAthenaScanConnectorOptions {
+  connectionId: string;
+  connection: KtxAthenaConnectionConfig | undefined;
+  clientFactory?: KtxAthenaClientFactory;
+  env?: NodeJS.ProcessEnv;
+  now?: () => Date;
+}
+
+export class KtxAthenaScanConnector implements KtxScanConnector {
+  readonly id: string;
+  readonly driver = 'athena' as const;
+  readonly capabilities = createKtxConnectorCapabilities({
+    tableSampling: true,
+    columnSampling: true,
+    columnStats: false,
+    readOnlySql: true,
+    nestedAnalysis: false,
+    formalForeignKeys: false,
+    estimatedRowCounts: false,
+  });
+
+  private readonly connectionId: string;
+  private readonly resolved: KtxAthenaResolvedConnectionConfig;
+  private readonly clientFactory: KtxAthenaClientFactory;
+  private readonly now: () => Date;
+  private readonly dialect = getSqlDialectForDriver('athena');
+  private athenaClient: KtxAthenaClient | null = null;
+  private glueClient: KtxGlueClient | null = null;
+
+  constructor(options: KtxAthenaScanConnectorOptions) {
+    this.connectionId = options.connectionId;
+    this.resolved = athenaConnectionConfigFromConfig({
+      connectionId: options.connectionId,
+      connection: options.connection,
+      env: options.env,
+    });
+    this.clientFactory = options.clientFactory ?? new DefaultAthenaClientFactory();
+    this.now = options.now ?? (() => new Date());
+    this.id = `athena:${options.connectionId}`;
+  }
+
+  async testConnection(): Promise<KtxConnectorTestResult> {
+    try {
+      await this.listDatabasesPaginated({ maxResults: 1 });
+      return { success: true };
+    } catch (error) {
+      return connectorTestFailure(error);
+    }
+  }
+
+  async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
+    this.assertConnection(input.connectionId);
+    // Honor the configured `databases` scope (written by `ktx setup`); fall back
+    // to every Glue database only when the scope is unset.
+    const databases =
+      this.resolved.databases.length > 0 ? this.resolved.databases : await this.listDatabasesPaginated({});
+    const tables: KtxSchemaTable[] = [];
+    for (const database of databases) {
+      const scopedNames = input.tableScope
+        ? scopedTableNames(input.tableScope, { catalog: this.resolved.catalog, db: database })
+        : null;
+      tables.push(...(await this.introspectDatabase(database, scopedNames)));
+    }
+    return {
+      connectionId: this.connectionId,
+      driver: 'athena',
+      extractedAt: this.now().toISOString(),
+      scope: { catalogs: [this.resolved.catalog], datasets: databases },
+      metadata: {
+        catalog: this.resolved.catalog,
+        databases,
+        table_count: tables.length,
+        total_columns: tables.reduce((sum, t) => sum + t.columns.length, 0),
+      },
+      tables,
+      warnings: [],
+    };
+  }
+
+  async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxTableSampleResult & { headerTypes?: string[] }> {
+    this.assertConnection(input.connectionId);
+    const result = await this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns));
+    return { headers: result.headers, headerTypes: result.headerTypes, rows: result.rows, totalRows: result.totalRows };
+  }
+
+  async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult> {
+    this.assertConnection(input.connectionId);
+    const result = await this.query(
+      this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit),
+    );
+    return {
+      values: result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]),
+      nullCount: null,
+      distinctCount: null,
+    };
+  }
+
+  async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise<KtxColumnStatsResult | null> {
+    return null;
+  }
+
+  async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
+    this.assertConnection(input.connectionId);
+    const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows);
+    const result = await this.query(limitedSql);
+    return { ...result, rowCount: result.rows.length };
+  }
+
+  async listSchemas(): Promise<string[]> {
+    return this.listDatabasesPaginated({});
+  }
+
+  async listTables(databases?: string[]): Promise<KtxTableListEntry[]> {
+    const targetDatabases = databases && databases.length > 0 ? databases : await this.listDatabasesPaginated({});
+    const entries: KtxTableListEntry[] = [];
+    for (const database of targetDatabases) {
+      const glueTables = await this.listGlueTablesPaginated(database);
+      for (const t of glueTables) {
+        if (!t.Name) continue;
+        entries.push({
+          catalog: this.resolved.catalog,
+          schema: database,
+          name: t.Name,
+          kind: glueTableKind(t.TableType),
+        });
+      }
+    }
+    return entries;
+  }
+
+  async cleanup(): Promise<void> {
+    this.athenaClient = null;
+    this.glueClient = null;
+  }
+
+  qTableName(table: Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>): string {
+    return this.dialect.formatTableName(table);
+  }
+
+  private getAthenaClient(): KtxAthenaClient {
+    if (!this.athenaClient) {
+      this.athenaClient = this.clientFactory.createAthenaClient(this.resolved.region);
+    }
+    return this.athenaClient;
+  }
+
+  private getGlueClient(): KtxGlueClient {
+    if (!this.glueClient) {
+      this.glueClient = this.clientFactory.createGlueClient(this.resolved.region);
+    }
+    return this.glueClient;
+  }
+
+  private async listDatabasesPaginated(opts: { maxResults?: number }): Promise<string[]> {
+    const names: string[] = [];
+    let nextToken: string | undefined;
+    do {
+      const result = await this.getGlueClient().getDatabases({ NextToken: nextToken });
+      for (const db of result.DatabaseList ?? []) {
+        if (db.Name) names.push(db.Name);
+        if (opts.maxResults && names.length >= opts.maxResults) return names;
+      }
+      nextToken = result.NextToken;
+    } while (nextToken);
+    return names;
+  }
+
+  private async listGlueTablesPaginated(database: string): Promise<KtxGlueTable[]> {
+    const tables: KtxGlueTable[] = [];
+    let nextToken: string | undefined;
+    do {
+      const result = await this.getGlueClient().getTables({ DatabaseName: database, NextToken: nextToken });
+      tables.push(...(result.TableList ?? []));
+      nextToken = result.NextToken;
+    } while (nextToken);
+    return tables;
+  }
+
+  private async introspectDatabase(database: string, scopedNames: readonly string[] | null): Promise<KtxSchemaTable[]> {
+    if (scopedNames && scopedNames.length === 0) return [];
+    const glueTables = await this.listGlueTablesPaginated(database);
+    const scopeSet = scopedNames ? new Set(scopedNames) : null;
+    return glueTables
+      .filter((t): t is KtxGlueTable & { Name: string } => Boolean(t.Name) && (!scopeSet || scopeSet.has(t.Name!)))
+      .map((t) => ({
+        catalog: this.resolved.catalog,
+        db: database,
+        name: t.Name,
+        kind: glueTableKind(t.TableType),
+        comment: t.Description ?? null,
+        estimatedRows: null,
+        columns: this.toSchemaColumns(t),
+        foreignKeys: [],
+      }));
+  }
+
+  private toSchemaColumns(table: KtxGlueTable): KtxSchemaColumn[] {
+    const columns = [...(table.StorageDescriptor?.Columns ?? []), ...(table.PartitionKeys ?? [])];
+    return columns
+      .filter((col): col is KtxGlueColumnDef & { Name: string } => Boolean(col.Name))
+      .map((col) => {
+        const nativeType = String(col.Type ?? 'string').toLowerCase();
+        return {
+          name: col.Name,
+          nativeType,
+          normalizedType: this.dialect.mapDataType(nativeType),
+          dimensionType: this.dialect.mapToDimensionType(nativeType),
+          nullable: true,
+          primaryKey: false,
+          comment: col.Comment ?? null,
+        };
+      });
+  }
+
+  private async query(sql: string): Promise<KtxQueryResult> {
+    const athena = this.getAthenaClient();
+    const { QueryExecutionId } = await athena.startQueryExecution({
+      QueryString: sql,
+      ResultConfiguration: { OutputLocation: this.resolved.s3StagingDir },
+      WorkGroup: this.resolved.workgroup,
+      ...(this.resolved.database || this.resolved.catalog
+        ? {
+            QueryExecutionContext: {
+              ...(this.resolved.database ? { Database: this.resolved.database } : {}),
+              ...(this.resolved.catalog ? { Catalog: this.resolved.catalog } : {}),
+            },
+          }
+        : {}),
+    });
+
+    if (!QueryExecutionId) {
+      throw new Error('Athena did not return a QueryExecutionId');
+    }
+
+    await this.waitForQueryCompletion(athena, QueryExecutionId);
+
+    const rows: unknown[][] = [];
+    let headers: string[] = [];
+    let headerTypes: string[] = [];
+    let nextToken: string | undefined;
+    let firstPage = true;
+
+    do {
+      const result = await athena.getQueryResults({ QueryExecutionId, NextToken: nextToken });
+      const resultSet = result.ResultSet;
+
+      if (firstPage) {
+        const columnInfo = resultSet?.ResultSetMetadata?.ColumnInfo ?? [];
+        headers = columnInfo.map((col) => col.Name ?? '');
+        headerTypes = columnInfo.map((col) => String(col.Type ?? 'varchar').toUpperCase());
+        firstPage = false;
+      }
+
+      const pageRows = resultSet?.Rows ?? [];
+      // Athena includes the header row as the first row of the first page — skip it.
+      const dataRows = nextToken === undefined ? pageRows.slice(1) : pageRows;
+      for (const row of dataRows) {
+        rows.push((row.Data ?? []).map((d) => d.VarCharValue ?? null));
+      }
+
+      nextToken = result.NextToken;
+    } while (nextToken);
+
+    return {
+      headers,
+      headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
+      rows,
+      totalRows: rows.length,
+      rowCount: rows.length,
+    };
+  }
+
+  private async waitForQueryCompletion(athena: KtxAthenaClient, queryExecutionId: string): Promise<void> {
+    const terminalStates = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED']);
+    const deadline = this.now().getTime() + QUERY_TIMEOUT_MS;
+    for (;;) {
+      const { QueryExecution } = await athena.getQueryExecution({ QueryExecutionId: queryExecutionId });
+      const state = QueryExecution?.Status?.State ?? '';
+      if (state === 'SUCCEEDED') return;
+      if (terminalStates.has(state)) {
+        const reason = QueryExecution?.Status?.StateChangeReason ?? state;
+        throw new Error(`Athena query ${state}: ${reason}`);
+      }
+      if (this.now().getTime() >= deadline) {
+        throw new Error(`Athena query ${queryExecutionId} timed out after ${QUERY_TIMEOUT_MS / 1000}s`);
+      }
+      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+    }
+  }
+
+  private assertConnection(connectionId: string): void {
+    if (connectionId !== this.connectionId) {
+      throw new Error(`Athena connector ${this.connectionId} cannot scan connection ${connectionId}`);
+    }
+  }
+}

packages/cli/src/connectors/athena/dialect.ts

@@ -0,0 +1,175 @@
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
+import {
+  columnDisplayPartCount,
+  formatDialectDisplayRef,
+  formatDialectTableName,
+  parseDialectDisplayRef,
+} from '../../context/connections/dialect-helpers.js';
+import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
+
+type AthenaTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
+
+/** @internal */
+export class KtxAthenaDialect implements KtxSqlDialect {
+  readonly type = 'athena' as const;
+
+  private readonly dimensionTypeMappings: Record<string, KtxSchemaDimensionType> = {
+    timestamp: 'time',
+    date: 'time',
+    bigint: 'number',
+    int: 'number',
+    integer: 'number',
+    tinyint: 'number',
+    smallint: 'number',
+    double: 'number',
+    float: 'number',
+    real: 'number',
+    boolean: 'boolean',
+  };
+
+  quoteIdentifier(identifier: string): string {
+    return `"${identifier.replace(/"/g, '""')}"`;
+  }
+
+  formatTableName(table: AthenaTableNameRef): string {
+    return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi');
+  }
+
+  formatDisplayRef(table: AthenaTableNameRef): string {
+    return formatDialectDisplayRef(table, 'ansi');
+  }
+
+  parseDisplayRef(display: string): KtxTableRef | null {
+    return parseDialectDisplayRef(display, 'ansi');
+  }
+
+  columnDisplayTablePartCount(): 1 | 2 | 3 {
+    return columnDisplayPartCount('ansi');
+  }
+
+  mapDataType(nativeType: string): string {
+    const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim();
+    const typeMap: Record<string, string> = {
+      string: 'VARCHAR',
+      varchar: 'VARCHAR',
+      char: 'CHAR',
+      binary: 'VARBINARY',
+      bigint: 'BIGINT',
+      int: 'INTEGER',
+      integer: 'INTEGER',
+      tinyint: 'TINYINT',
+      smallint: 'SMALLINT',
+      double: 'DOUBLE',
+      float: 'FLOAT',
+      real: 'REAL',
+      decimal: 'DECIMAL',
+      boolean: 'BOOLEAN',
+      timestamp: 'TIMESTAMP',
+      date: 'DATE',
+      array: 'ARRAY',
+      map: 'MAP',
+      struct: 'STRUCT',
+      uniontype: 'UNION',
+    };
+    return typeMap[base] ?? nativeType.toUpperCase();
+  }
+
+  mapToDimensionType(nativeType: string): KtxSchemaDimensionType {
+    const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim();
+    const mapped = this.dimensionTypeMappings[base];
+    if (mapped) return mapped;
+    if (base.includes('timestamp') || base.includes('date')) return 'time';
+    if (base.includes('int') || base.includes('float') || base.includes('double') || base.includes('decimal') || base.includes('real')) return 'number';
+    if (base.includes('bool')) return 'boolean';
+    return 'string';
+  }
+
+  generateSampleQuery(tableName: string, limit: number, columns?: string[]): string {
+    const columnList =
+      columns && columns.length > 0 ? columns.map((c) => this.quoteIdentifier(c)).join(', ') : '*';
+    return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`;
+  }
+
+  generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string {
+    const quoted = this.quoteIdentifier(columnName);
+    return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL LIMIT ${limit}`;
+  }
+
+  generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
+    return `
+      SELECT approx_distinct(${columnName}) AS cardinality
+      FROM (
+        SELECT ${columnName}
+        FROM ${tableName}
+        WHERE ${columnName} IS NOT NULL
+        LIMIT ${sampleSize}
+      )
+    `;
+  }
+
+  generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
+    return `
+      SELECT approx_distinct(${columnName}) AS cardinality
+      FROM (
+        SELECT ${columnName}
+        FROM ${tableName}
+        WHERE ${columnName} IS NOT NULL
+        ORDER BY rand()
+        LIMIT ${sampleSize}
+      )
+    `;
+  }
+
+  generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string {
+    return `
+      SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val
+      FROM ${tableName}
+      WHERE ${columnName} IS NOT NULL
+      ORDER BY val
+      LIMIT ${limit}
+    `;
+  }
+
+  generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null {
+    return null;
+  }
+
+  getNullCountExpression(column: string): string {
+    return `COUNT_IF(${column} IS NULL)`;
+  }
+
+  getDistinctCountExpression(column: string): string {
+    return `approx_distinct(${column})`;
+  }
+
+  textLengthExpression(columnSql: string): string {
+    return `LENGTH(CAST(${columnSql} AS VARCHAR))`;
+  }
+
+  castToText(columnSql: string): string {
+    return `CAST(${columnSql} AS VARCHAR)`;
+  }
+
+  getSampleValueAggregation(innerSql: string): string {
+    return `(SELECT array_join(array_agg(CAST(value AS VARCHAR)), '\u001f') FROM (${innerSql}) AS relationship_profile_values)`;
+  }
+
+  getLimitOffsetClause(limit: number, offset?: number): string {
+    const safeLimit = Math.max(1, Math.floor(limit));
+    const safeOffset = offset !== undefined ? Math.floor(offset) : 0;
+    return safeOffset > 0 ? `OFFSET ${safeOffset} LIMIT ${safeLimit}` : `LIMIT ${safeLimit}`;
+  }
+
+  getTopClause(_limit: number): string {
+    return '';
+  }
+
+  getTableSampleClause(_samplePct: number): string {
+    return '';
+  }
+
+  getRandomSampleFilter(samplePct: number): string {
+    if (samplePct <= 0 || samplePct >= 1) return '';
+    return `rand() < ${samplePct}`;
+  }
+}

packages/cli/src/connectors/athena/live-database-introspection.ts

@@ -0,0 +1,44 @@
+import type {
+  LiveDatabaseIntrospectionOptions,
+  LiveDatabaseIntrospectionPort,
+} from '../../context/ingest/adapters/live-database/types.js';
+import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
+import {
+  KtxAthenaScanConnector,
+  type KtxAthenaClientFactory,
+  type KtxAthenaConnectionConfig,
+} from './connector.js';
+
+interface CreateAthenaLiveDatabaseIntrospectionOptions {
+  connections: Record<string, KtxProjectConnectionConfig>;
+  clientFactory?: KtxAthenaClientFactory;
+  now?: () => Date;
+}
+
+export function createAthenaLiveDatabaseIntrospection(
+  options: CreateAthenaLiveDatabaseIntrospectionOptions,
+): LiveDatabaseIntrospectionPort {
+  return {
+    async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) {
+      const connection = options.connections[connectionId] as KtxAthenaConnectionConfig | undefined;
+      const connector = new KtxAthenaScanConnector({
+        connectionId,
+        connection,
+        clientFactory: options.clientFactory,
+        now: options.now,
+      });
+      try {
+        return await connector.introspect(
+          {
+            connectionId,
+            driver: 'athena',
+            ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
+          },
+          { runId: `athena-${connectionId}` },
+        );
+      } finally {
+        await connector.cleanup();
+      }
+    },
+  };
+}

packages/cli/src/connectors/bigquery/connector.ts

@@ -1,8 +1,14 @@
 import { BigQuery, type TableField } from '@google-cloud/bigquery';
-import { normalizeBigQueryProjectId, normalizeBigQueryRegion } from '../../context/connections/bigquery-identifiers.js';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import {
+  normalizeBigQueryDatasetId,
+  normalizeBigQueryProjectId,
+  normalizeBigQueryRegion,
+} from '../../context/connections/bigquery-identifiers.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
+import { tryIntrospectObject } from '../../context/scan/object-introspection.js';
 import { scopedTableNames } from '../../context/scan/table-ref.js';
 import {
   connectorTestFailure,
@@ -35,14 +41,25 @@ export interface KtxBigQueryConnectionConfig {
   credentials_json?: string;
   location?: string;
   max_bytes_billed?: number | string;
-  job_timeout_ms?: number;
+  query_timeout_ms?: number;
   [key: string]: unknown;
 }
 
+/**
+ * A dataset to introspect, paired with the project that hosts it. `project`
+ * defaults to the billing project (`credentials.project_id`) when an entry has
+ * no `project.` prefix; a fully-qualified `project.dataset` entry resolves to
+ * its own host project. Jobs always bill in `credentials.project_id`.
+ */
+export interface BigQueryDatasetRef {
+  project: string;
+  dataset: string;
+}
+
 export interface KtxBigQueryResolvedConnectionConfig {
   projectId: string;
   credentials: Record<string, unknown>;
-  datasetIds: string[];
+  datasetIds: BigQueryDatasetRef[];
   location?: string;
 }
 
@@ -95,7 +112,7 @@ export interface KtxBigQueryDataset {
 
 export interface KtxBigQueryClient {
   getDatasets(input?: { maxResults?: number }): Promise<[Array<{ id?: string }>, ...unknown[]]>;
-  dataset(datasetId: string): KtxBigQueryDataset;
+  dataset(datasetId: string, projectId: string): KtxBigQueryDataset;
   createQueryJob(input: {
     query: string;
     location?: string;
@@ -116,7 +133,6 @@ export interface KtxBigQueryScanConnectorOptions {
   env?: NodeJS.ProcessEnv;
   now?: () => Date;
   maxBytesBilled?: number | string;
-  queryTimeoutMs?: number;
 }
 
 class DefaultBigQueryClientFactory implements KtxBigQueryClientFactory {
@@ -124,8 +140,8 @@ class DefaultBigQueryClientFactory implements KtxBigQueryClientFactory {
     const client = new BigQuery(input);
     return {
       getDatasets: (options) => client.getDatasets(options) as Promise<[Array<{ id?: string }>, ...unknown[]]>,
-      dataset: (datasetId) => {
-        const dataset = client.dataset(datasetId);
+      dataset: (datasetId, projectId) => {
+        const dataset = client.dataset(datasetId, { projectId });
         return {
           get: () => dataset.get() as Promise<unknown>,
           getTables: () => dataset.getTables() as Promise<[KtxBigQueryTableRef[], ...unknown[]]>,
@@ -145,14 +161,48 @@ function stringConfigValue(
   return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
 }
 
-function datasetIds(connection: KtxBigQueryConnectionConfig, env: NodeJS.ProcessEnv): string[] {
-  if (Array.isArray(connection.dataset_ids) && connection.dataset_ids.length > 0) {
-    return connection.dataset_ids
-      .filter((dataset) => dataset.trim().length > 0)
-      .map((dataset) => resolveStringReference(dataset, env));
+/**
+ * Parse one `dataset_ids` / `dataset_id` entry into a canonical
+ * {@link BigQueryDatasetRef}. A `project.dataset` prefix selects the host
+ * project; a bare entry defaults to `defaultProject` (the billing project).
+ * More than one dot, or an empty segment, is a config error naming the
+ * connection — never a silent mis-introspection at scan time.
+ */
+function parseBigQueryDatasetEntry(entry: string, defaultProject: string, connectionId: string): BigQueryDatasetRef {
+  const context = `connections.${connectionId}.dataset_ids entry "${entry}"`;
+  const parts = entry.split('.');
+  if (parts.length === 1) {
+    return { project: defaultProject, dataset: normalizeBigQueryDatasetId(parts[0]!, context) };
   }
-  const datasetId = stringConfigValue(connection, 'dataset_id', env);
-  return datasetId ? [datasetId] : [];
+  if (parts.length === 2) {
+    const [project, dataset] = parts;
+    if (!project || !dataset) {
+      throw new Error(`Invalid BigQuery dataset entry for ${context}: empty project or dataset segment`);
+    }
+    return {
+      project: normalizeBigQueryProjectId(project, context),
+      dataset: normalizeBigQueryDatasetId(dataset, context),
+    };
+  }
+  throw new Error(
+    `Invalid BigQuery dataset entry for ${context}: expected "dataset" or "project.dataset", got more than one "."`,
+  );
+}
+
+function resolveDatasetRefs(
+  connection: KtxBigQueryConnectionConfig,
+  env: NodeJS.ProcessEnv,
+  defaultProject: string,
+  connectionId: string,
+): BigQueryDatasetRef[] {
+  const rawEntries =
+    Array.isArray(connection.dataset_ids) && connection.dataset_ids.length > 0
+      ? connection.dataset_ids.map((dataset) => resolveStringReference(dataset, env))
+      : [stringConfigValue(connection, 'dataset_id', env)].filter((value): value is string => Boolean(value));
+  return rawEntries
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0)
+    .map((entry) => parseBigQueryDatasetEntry(entry, defaultProject, connectionId));
 }
 
 function bigQueryMaxBytesBilledFromConnection(
@@ -169,12 +219,25 @@ function bigQueryMaxBytesBilledFromConnection(
   return undefined;
 }
 
-function bigQueryJobTimeoutMsFromConnection(connection: KtxBigQueryConnectionConfig | undefined): number | undefined {
-  const value = connection?.job_timeout_ms;
-  if (typeof value !== 'number') {
-    return undefined;
+// jobTimeoutMs cancels the job with a "Job timed out" message (or a timeout
+// reason in the errors array) once the deadline elapses.
+function isBigQueryTimeoutError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') {
+    return false;
   }
-  return Number.isInteger(value) && value > 0 ? value : undefined;
+  const topMessage = (error as { message?: unknown }).message;
+  if (typeof topMessage === 'string' && /timed out|timeout/i.test(topMessage)) {
+    return true;
+  }
+  const errors = (error as { errors?: unknown }).errors;
+  return (
+    Array.isArray(errors) &&
+    errors.some((entry) => {
+      const reason = (entry as { reason?: unknown })?.reason;
+      const message = (entry as { message?: unknown })?.message;
+      return reason === 'timeout' || (typeof message === 'string' && /timed out|timeout/i.test(message));
+    })
+  );
 }
 
 function tableKind(metadataType: string | undefined): KtxSchemaTable['kind'] {
@@ -267,7 +330,7 @@ export function bigQueryConnectionConfigFromConfig(input: {
   if (!projectId) {
     throw new Error(`Native BigQuery connector requires credentials_json.project_id for connections.${input.connectionId}`);
   }
-  const resolvedDatasetIds = datasetIds(input.connection, env);
+  const resolvedDatasetIds = resolveDatasetRefs(input.connection, env, projectId, input.connectionId);
   const location = stringConfigValue(input.connection, 'location', env);
   return { projectId, credentials, datasetIds: resolvedDatasetIds, ...(location ? { location } : {}) };
 }
@@ -290,8 +353,8 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
   private readonly clientFactory: KtxBigQueryClientFactory;
   private readonly now: () => Date;
   private readonly maxBytesBilled?: number | string;
-  private readonly queryTimeoutMs?: number;
-  private readonly dialect = getDialectForDriver('bigquery');
+  private readonly deadlineMs: number;
+  private readonly dialect = getSqlDialectForDriver('bigquery');
   private client: KtxBigQueryClient | null = null;
 
   constructor(options: KtxBigQueryScanConnectorOptions) {
@@ -304,7 +367,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
     this.clientFactory = options.clientFactory ?? new DefaultBigQueryClientFactory();
     this.now = options.now ?? (() => new Date());
     this.maxBytesBilled = options.maxBytesBilled ?? bigQueryMaxBytesBilledFromConnection(options.connection);
-    this.queryTimeoutMs = options.queryTimeoutMs ?? bigQueryJobTimeoutMsFromConnection(options.connection);
+    this.deadlineMs = resolveQueryDeadlineMs(options.connection);
     this.id = `bigquery:${options.connectionId}`;
   }
 
@@ -312,8 +375,8 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
     try {
       const client = this.getClient();
       await client.getDatasets({ maxResults: 1 });
-      for (const datasetId of this.resolved.datasetIds) {
-        await client.dataset(datasetId).get();
+      for (const ref of this.resolved.datasetIds) {
+        await client.dataset(ref.dataset, ref.project).get();
       }
       return { success: true };
     } catch (error) {
@@ -324,22 +387,23 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
   async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
     this.assertConnection(input.connectionId);
     const tables: KtxSchemaTable[] = [];
-    const datasetIds = this.requireDatasetIdsForScan();
+    const datasetRefs = this.requireDatasetIdsForScan();
     const snapshotWarnings: KtxScanWarning[] = [];
-    for (const datasetId of datasetIds) {
+    for (const ref of datasetRefs) {
       const scopedNames = input.tableScope
-        ? scopedTableNames(input.tableScope, { catalog: this.resolved.projectId, db: datasetId })
+        ? scopedTableNames(input.tableScope, { catalog: ref.project, db: ref.dataset })
         : null;
-      tables.push(...(await this.introspectDataset(datasetId, scopedNames, snapshotWarnings)));
+      tables.push(...(await this.introspectDataset(ref, scopedNames, snapshotWarnings)));
     }
+    const datasetLabels = datasetRefs.map((ref) => this.qualifiedDatasetLabel(ref));
     return {
       connectionId: this.connectionId,
       driver: 'bigquery',
       extractedAt: this.now().toISOString(),
-      scope: { catalogs: [this.resolved.projectId], datasets: datasetIds },
+      scope: { catalogs: [...new Set(datasetRefs.map((ref) => ref.project))], datasets: datasetLabels },
       metadata: {
         project_id: this.resolved.projectId,
-        datasets: datasetIds,
+        datasets: datasetLabels,
         table_count: tables.length,
         total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
       },
@@ -400,11 +464,14 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
     return { values: valueRows.filter((row) => row.val !== null).map((row) => String(row.val)), cardinality };
   }
 
-  async getTableRowCount(tableName: string, datasetId = this.resolved.datasetIds[0]): Promise<number> {
-    if (!datasetId) {
+  async getTableRowCount(
+    tableName: string,
+    ref: BigQueryDatasetRef | undefined = this.resolved.datasetIds[0],
+  ): Promise<number> {
+    if (!ref) {
       return 0;
     }
-    const tables = await this.introspectDataset(datasetId, null, []);
+    const tables = await this.introspectDataset(ref, null, []);
     return tables.find((table) => table.name === tableName)?.estimatedRows ?? 0;
   }
 
@@ -422,12 +489,28 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
   }
 
   async listTables(datasetIds?: string[]): Promise<KtxTableListEntry[]> {
-    const projectId = normalizeBigQueryProjectId(this.resolved.projectId, 'table discovery');
     const region = normalizeBigQueryRegion(this.resolved.location ?? 'US', 'table discovery');
+    if (!datasetIds || datasetIds.length === 0) {
+      return this.listTablesInProject(this.resolved.projectId, region);
+    }
+    const datasetsByProject = new Map<string, string[]>();
+    for (const entry of datasetIds) {
+      const ref = parseBigQueryDatasetEntry(entry.trim(), this.resolved.projectId, this.connectionId);
+      datasetsByProject.set(ref.project, [...(datasetsByProject.get(ref.project) ?? []), ref.dataset]);
+    }
+    const entries: KtxTableListEntry[] = [];
+    for (const [project, datasets] of datasetsByProject) {
+      entries.push(...(await this.listTablesInProject(project, region, datasets)));
+    }
+    return entries;
+  }
+
+  private async listTablesInProject(project: string, region: string, datasets?: string[]): Promise<KtxTableListEntry[]> {
+    const projectId = normalizeBigQueryProjectId(project, 'table discovery');
     const params: Record<string, unknown> = {};
-    const filter = datasetIds && datasetIds.length > 0 ? 'AND table_schema IN UNNEST(@dataset_ids)' : '';
-    if (datasetIds && datasetIds.length > 0) {
-      params.dataset_ids = datasetIds;
+    const filter = datasets && datasets.length > 0 ? 'AND table_schema IN UNNEST(@dataset_ids)' : '';
+    if (datasets && datasets.length > 0) {
+      params.dataset_ids = datasets;
     }
     const rows = await this.queryRaw<{ table_schema: string; table_name: string; table_type: string }>(
       `
@@ -442,7 +525,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
       params,
     );
     return rows.map((row) => ({
-      catalog: this.resolved.projectId,
+      catalog: project,
       schema: row.table_schema,
       name: row.table_name,
       kind:
@@ -466,34 +549,48 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
     return this.client;
   }
 
-  private requireDatasetIdsForScan(): string[] {
+  private requireDatasetIdsForScan(): BigQueryDatasetRef[] {
     if (this.resolved.datasetIds.length === 0) {
       throw new Error(`Native BigQuery scan requires connections.${this.connectionId}.dataset_ids or dataset_id`);
     }
     return this.resolved.datasetIds;
   }
 
+  // Bare in the billing project, qualified `project.dataset` otherwise, so the
+  // snapshot's scope/metadata stay unambiguous when two projects host the same
+  // dataset name. The dotless form is the unchanged single-project label.
+  private qualifiedDatasetLabel(ref: BigQueryDatasetRef): string {
+    return ref.project === this.resolved.projectId ? ref.dataset : `${ref.project}.${ref.dataset}`;
+  }
+
   private async query(sql: string, params?: Record<string, unknown>): Promise<KtxQueryResult> {
-    const [job] = await this.getClient().createQueryJob({
-      query: sql,
-      ...(this.resolved.location ? { location: this.resolved.location } : {}),
-      ...(params && Object.keys(params).length > 0 ? { params } : {}),
-      ...(this.maxBytesBilled ? { maximumBytesBilled: String(this.maxBytesBilled) } : {}),
-      ...(this.queryTimeoutMs ? { jobTimeoutMs: this.queryTimeoutMs } : {}),
-    });
-    const [rows, , response] = await job.getQueryResults();
-    let headers = response?.schema?.fields?.map((field) => field.name || '') ?? [];
-    const headerTypes = response?.schema?.fields?.map((field) => String(field.type || 'STRING')) ?? [];
-    if (headers.length === 0 && rows.length > 0) {
-      headers = Object.keys(rows[0]!);
+    try {
+      const [job] = await this.getClient().createQueryJob({
+        query: sql,
+        ...(this.resolved.location ? { location: this.resolved.location } : {}),
+        ...(params && Object.keys(params).length > 0 ? { params } : {}),
+        ...(this.maxBytesBilled ? { maximumBytesBilled: String(this.maxBytesBilled) } : {}),
+        jobTimeoutMs: this.deadlineMs,
+      });
+      const [rows, , response] = await job.getQueryResults();
+      let headers = response?.schema?.fields?.map((field) => field.name || '') ?? [];
+      const headerTypes = response?.schema?.fields?.map((field) => String(field.type || 'STRING')) ?? [];
+      if (headers.length === 0 && rows.length > 0) {
+        headers = Object.keys(rows[0]!);
+      }
+      return {
+        headers,
+        headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
+        rows: rows.map((row) => headers.map((header) => normalizeValue(row[header]))),
+        totalRows: rows.length,
+        rowCount: rows.length,
+      };
+    } catch (error) {
+      if (isBigQueryTimeoutError(error)) {
+        throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
+      }
+      throw error;
     }
-    return {
-      headers,
-      headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
-      rows: rows.map((row) => headers.map((header) => normalizeValue(row[header]))),
-      totalRows: rows.length,
-      rowCount: rows.length,
-    };
   }
 
   private async queryRaw<T extends Record<string, unknown>>(sql: string, params?: Record<string, unknown>): Promise<T[]> {
@@ -507,18 +604,18 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
   }
 
   private async introspectDataset(
-    datasetId: string,
+    ref: BigQueryDatasetRef,
     scopedNames: readonly string[] | null,
     snapshotWarnings: KtxScanWarning[],
   ): Promise<KtxSchemaTable[]> {
     if (scopedNames && scopedNames.length === 0) return [];
-    const dataset = this.getClient().dataset(datasetId);
+    const dataset = this.getClient().dataset(ref.dataset, ref.project);
     const [tableRefs] = await dataset.getTables();
     const scopeSet = scopedNames ? new Set(scopedNames) : null;
     const filteredTableRefs = scopeSet ? tableRefs.filter((tableRef) => scopeSet.has(tableRef.id ?? '')) : tableRefs;
     const primaryKeysResult = await tryConstraintQuery(
-      { schema: datasetId, kind: 'primary_key', isDeniedError },
-      () => this.primaryKeys(datasetId),
+      { schema: ref.dataset, kind: 'primary_key', isDeniedError },
+      () => this.primaryKeys(ref),
     );
     const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : new Map<string, Set<string>>();
     if (!primaryKeysResult.ok) {
@@ -527,41 +624,51 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
     const tables: KtxSchemaTable[] = [];
     for (const tableRef of filteredTableRefs) {
       const tableName = tableRef.id || '';
-      const [table] = await tableRef.get();
-      const fields = table.metadata.schema?.fields ?? [];
-      tables.push({
-        catalog: this.resolved.projectId,
-        db: datasetId,
-        name: tableName,
-        kind: tableKind(table.metadata.type),
-        comment: table.metadata.description || null,
-        estimatedRows: firstNumber(table.metadata.numRows) ?? 0,
-        columns: fields.map((field) => this.toSchemaColumn(tableName, field, primaryKeys)),
-        foreignKeys: [],
-      });
+      const outcome = await tryIntrospectObject<KtxSchemaTable>(
+        { object: tableName, catalog: ref.project, db: ref.dataset },
+        async () => {
+          const [table] = await tableRef.get();
+          const fields = table.metadata.schema?.fields ?? [];
+          return {
+            catalog: ref.project,
+            db: ref.dataset,
+            name: tableName,
+            kind: tableKind(table.metadata.type),
+            comment: table.metadata.description || null,
+            estimatedRows: firstNumber(table.metadata.numRows) ?? 0,
+            columns: fields.map((field) => this.toSchemaColumn(tableName, field, primaryKeys)),
+            foreignKeys: [],
+          };
+        },
+      );
+      if (outcome.ok) {
+        tables.push(outcome.table);
+      } else {
+        snapshotWarnings.push(outcome.warning);
+      }
     }
     return tables;
   }
 
-  private async primaryKeys(datasetId: string): Promise<Map<string, Set<string>>> {
+  private async primaryKeys(ref: BigQueryDatasetRef): Promise<Map<string, Set<string>>> {
     const rows = await this.queryRaw<{ table_name: string; column_name: string }>(
       'SELECT tc.table_name, kcu.column_name ' +
         'FROM `' +
-        this.resolved.projectId +
+        ref.project +
         '.' +
-        datasetId +
+        ref.dataset +
         '.INFORMATION_SCHEMA.TABLE_CONSTRAINTS` tc ' +
         'JOIN `' +
-        this.resolved.projectId +
+        ref.project +
         '.' +
-        datasetId +
+        ref.dataset +
         '.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` kcu ' +
         'ON tc.constraint_name = kcu.constraint_name ' +
         'AND tc.table_schema = kcu.table_schema ' +
         'AND tc.table_name = kcu.table_name ' +
         "WHERE tc.constraint_type = 'PRIMARY KEY' " +
         "AND tc.table_schema = '" +
-        datasetId +
+        ref.dataset +
         "' " +
         "AND NOT REGEXP_CONTAINS(kcu.column_name, r'^(stacksync_record_id|sync_primary_key)_') " +
         'ORDER BY tc.table_name, kcu.ordinal_position',

packages/cli/src/connectors/bigquery/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type BigQueryTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxBigQueryDialect implements KtxDialect {
+export class KtxBigQueryDialect implements KtxSqlDialect {
   readonly type = 'bigquery' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/connectors/clickhouse/connector.ts

@@ -1,5 +1,6 @@
 import { createClient } from '@clickhouse/client';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
 import { scopedTableNames } from '../../context/scan/table-ref.js';
@@ -144,6 +145,21 @@ function maybeNumber(value: unknown): number | undefined {
   return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
 }
 
+// ClickHouse error code 159 = TIMEOUT_EXCEEDED, raised when max_execution_time
+// is hit. The client surfaces it via a numeric/string `code` or a "Code: 159"
+// message prefix depending on transport.
+function isClickHouseTimeoutError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') {
+    return false;
+  }
+  const code = (error as { code?: unknown }).code;
+  if (code === 159 || code === '159') {
+    return true;
+  }
+  const message = (error as { message?: unknown }).message;
+  return typeof message === 'string' && (/\bCode:\s*159\b/.test(message) || message.includes('TIMEOUT_EXCEEDED'));
+}
+
 function parseClickHouseUrl(url: string): Partial<KtxClickHouseConnectionConfig> {
   const parsed = new URL(url);
   return {
@@ -284,7 +300,8 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
   private readonly clientFactory: KtxClickHouseClientFactory;
   private readonly endpointResolver?: KtxClickHouseEndpointResolver;
   private readonly now: () => Date;
-  private readonly dialect = getDialectForDriver('clickhouse');
+  private readonly deadlineMs: number;
+  private readonly dialect = getSqlDialectForDriver('clickhouse');
   private client: KtxClickHouseClient | null = null;
   private resolvedEndpoint: KtxClickHouseResolvedEndpoint | null = null;
 
@@ -299,6 +316,7 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
     this.clientFactory = options.clientFactory ?? new DefaultClickHouseClientFactory();
     this.endpointResolver = options.endpointResolver;
     this.now = options.now ?? (() => new Date());
+    this.deadlineMs = resolveQueryDeadlineMs(this.connection);
     this.id = `clickhouse:${options.connectionId}`;
   }
 
@@ -584,9 +602,13 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
         username: config.username,
         password: config.password ?? '',
         database: config.database,
-        request_timeout: 30_000,
+        // The server aborts at max_execution_time (seconds); request_timeout must
+        // outlast it so the HTTP client receives the code-159 error instead of
+        // giving up first and leaving the query running.
+        request_timeout: this.deadlineMs + 5_000,
         clickhouse_settings: {
           output_format_json_quote_64bit_integers: 1,
+          max_execution_time: Math.ceil(this.deadlineMs / 1000),
         },
         ...(isProxied && config.ssl
           ? {
@@ -613,19 +635,26 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
 
   private async query(sql: string, params?: Record<string, unknown>): Promise<Omit<KtxQueryResult, 'rowCount'>> {
     const client = await this.clientForQuery();
-    const resultSet = await client.query({
-      query: assertReadOnlySql(sql),
-      format: 'JSONCompact',
-      ...(params ? { query_params: params } : {}),
-    });
-    const response = (await resultSet.json()) as ClickHouseCompactResponse;
-    const meta = response.meta ?? [];
-    return {
-      headers: meta.map((field) => field.name),
-      headerTypes: meta.map((field) => field.type),
-      rows: response.data ?? [],
-      totalRows: response.rows ?? response.data?.length ?? 0,
-    };
+    try {
+      const resultSet = await client.query({
+        query: assertReadOnlySql(sql),
+        format: 'JSONCompact',
+        ...(params ? { query_params: params } : {}),
+      });
+      const response = (await resultSet.json()) as ClickHouseCompactResponse;
+      const meta = response.meta ?? [];
+      return {
+        headers: meta.map((field) => field.name),
+        headerTypes: meta.map((field) => field.type),
+        rows: response.data ?? [],
+        totalRows: response.rows ?? response.data?.length ?? 0,
+      };
+    } catch (error) {
+      if (isClickHouseTimeoutError(error)) {
+        throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
+      }
+      throw error;
+    }
   }
 
   private assertConnection(connectionId: string): void {

packages/cli/src/connectors/clickhouse/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type ClickHouseTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxClickHouseDialect implements KtxDialect {
+export class KtxClickHouseDialect implements KtxSqlDialect {
   readonly type = 'clickhouse' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/connectors/duckdb/connector.ts

@@ -0,0 +1,395 @@
+import { DuckDBInstance, type DuckDBConnection } from '@duckdb/node-api';
+import { existsSync, statSync } from 'node:fs';
+import { isAbsolute, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { resolveStringReference } from '../shared/string-reference.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
+import { normalizeQueryRows } from '../../context/connections/query-executor.js';
+import { toJsonSafeRows } from '../shared/duckdb-json-safe.js';
+import {
+  connectorTestFailure,
+  createKtxConnectorCapabilities,
+  type KtxColumnSampleInput,
+  type KtxColumnSampleResult,
+  type KtxColumnStatsInput,
+  type KtxColumnStatsResult,
+  type KtxConnectorTestResult,
+  type KtxQueryResult,
+  type KtxReadOnlyQueryInput,
+  type KtxScanConnector,
+  type KtxScanContext,
+  type KtxScanInput,
+  type KtxSchemaForeignKey,
+  type KtxSchemaSnapshot,
+  type KtxSchemaTable,
+  type KtxTableListEntry,
+  type KtxTableRef,
+  type KtxTableSampleInput,
+  type KtxTableSampleResult,
+} from '../../context/scan/types.js';
+import { scopedTableNames } from '../../context/scan/table-ref.js';
+
+const MAIN_SCHEMA = 'main';
+
+export interface KtxDuckDbConnectionConfig {
+  driver?: string;
+  path?: string;
+  url?: string;
+  [key: string]: unknown;
+}
+
+/** @internal */
+export interface DuckDbDatabasePathInput {
+  connectionId: string;
+  projectDir?: string;
+  connection: KtxDuckDbConnectionConfig | undefined;
+}
+
+export interface KtxDuckDbScanConnectorOptions extends DuckDbDatabasePathInput {
+  now?: () => Date;
+}
+
+export interface KtxDuckDbColumnDistinctValuesOptions {
+  maxCardinality: number;
+  limit: number;
+  sampleSize?: number;
+}
+
+export interface KtxDuckDbColumnDistinctValuesResult {
+  values: string[] | null;
+  cardinality: number;
+}
+
+interface InfoSchemaTableRow {
+  table_name: string;
+  table_type: string;
+}
+
+interface InfoSchemaColumnRow {
+  column_name: string;
+  data_type: string;
+  is_nullable: string;
+}
+
+// `path` may be an env:/file: reference; `url` resolves env: only, since file:
+// on a url is a native URI form (handled by duckDbPathFromUrl), not a file read.
+function stringConfigValue(
+  connection: KtxDuckDbConnectionConfig | undefined,
+  key: 'path' | 'url',
+): string | undefined {
+  const value = connection?.[key];
+  if (typeof value !== 'string' || value.trim().length === 0) {
+    return undefined;
+  }
+  const trimmed = value.trim();
+  if (key === 'url') {
+    return trimmed.startsWith('env:') ? (process.env[trimmed.slice('env:'.length)] ?? '') : trimmed;
+  }
+  return resolveStringReference(trimmed, process.env);
+}
+
+function duckDbPathFromUrl(url: string): string {
+  if (url.startsWith('file:')) {
+    return fileURLToPath(url);
+  }
+  if (url.startsWith('duckdb:')) {
+    const parsed = new URL(url);
+    return decodeURIComponent(parsed.pathname);
+  }
+  return url;
+}
+
+export function isKtxDuckDbConnectionConfig(
+  connection: KtxDuckDbConnectionConfig | undefined,
+): connection is KtxDuckDbConnectionConfig {
+  return String(connection?.driver ?? '').toLowerCase() === 'duckdb';
+}
+
+/** @internal */
+export function duckDbDatabasePathFromConfig(input: DuckDbDatabasePathInput): string {
+  const inputDriver = input.connection?.driver ?? 'unknown';
+  if (!isKtxDuckDbConnectionConfig(input.connection)) {
+    throw new Error(`Native DuckDB connector cannot run driver "${inputDriver}"`);
+  }
+  const configuredPath =
+    stringConfigValue(input.connection, 'path') ?? duckDbPathFromUrl(stringConfigValue(input.connection, 'url') ?? '');
+  if (!configuredPath) {
+    throw new Error(`Native DuckDB connector requires connections.${input.connectionId}.path or url`);
+  }
+  return isAbsolute(configuredPath) ? configuredPath : resolve(input.projectDir ?? process.cwd(), configuredPath);
+}
+
+export class KtxDuckDbScanConnector implements KtxScanConnector {
+  readonly id: string;
+  readonly driver = 'duckdb' as const;
+  readonly capabilities = createKtxConnectorCapabilities({
+    tableSampling: true,
+    columnSampling: true,
+    columnStats: false,
+    readOnlySql: true,
+    nestedAnalysis: false,
+    formalForeignKeys: true,
+    estimatedRowCounts: true,
+  });
+
+  private readonly connectionId: string;
+  private readonly dbPath: string;
+  private readonly now: () => Date;
+  private readonly dialect = getSqlDialectForDriver('duckdb');
+  private instance: DuckDBInstance | null = null;
+  private connection: DuckDBConnection | null = null;
+
+  constructor(options: KtxDuckDbScanConnectorOptions) {
+    this.connectionId = options.connectionId;
+    this.dbPath = duckDbDatabasePathFromConfig(options);
+    this.now = options.now ?? (() => new Date());
+    this.id = `duckdb:${options.connectionId}`;
+  }
+
+  async testConnection(): Promise<KtxConnectorTestResult> {
+    try {
+      if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
+        return { success: false, error: `File not found: ${this.dbPath}` };
+      }
+      await this.query('SELECT 1');
+      return { success: true };
+    } catch (error) {
+      return connectorTestFailure(error);
+    }
+  }
+
+  async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
+    this.assertConnection(input.connectionId);
+    const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: null, db: null }) : null;
+    const tableRows = await this.readTableRows(scopedNames);
+    const tables: KtxSchemaTable[] = [];
+    for (const row of tableRows) {
+      tables.push(await this.readTable(row));
+    }
+    return {
+      connectionId: this.connectionId,
+      driver: 'duckdb' as const,
+      extractedAt: this.now().toISOString(),
+      scope: {},
+      metadata: {
+        file_path: this.dbPath,
+        table_count: tables.length,
+        total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
+      },
+      tables,
+    };
+  }
+
+  async listSchemas(): Promise<string[]> {
+    return [MAIN_SCHEMA];
+  }
+
+  async listTables(_schemas?: string[]): Promise<KtxTableListEntry[]> {
+    const rows = await this.readTableRows(null);
+    return rows.map((row) => ({
+      catalog: null,
+      schema: MAIN_SCHEMA,
+      name: row.table_name,
+      kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const),
+    }));
+  }
+
+  async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxTableSampleResult> {
+    this.assertConnection(input.connectionId);
+    const result = await this.query(
+      this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns),
+    );
+    return { headers: result.headers, rows: result.rows, totalRows: result.totalRows };
+  }
+
+  async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult> {
+    this.assertConnection(input.connectionId);
+    const result = await this.query(
+      this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit),
+    );
+    const values = result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]);
+    return { values, nullCount: null, distinctCount: null };
+  }
+
+  async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise<KtxColumnStatsResult | null> {
+    return null;
+  }
+
+  async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
+    this.assertConnection(input.connectionId);
+    const result = await this.query(limitSqlForExecution(input.sql, input.maxRows));
+    return { ...result, rowCount: result.rows.length };
+  }
+
+  async getColumnDistinctValues(
+    table: KtxTableRef,
+    columnName: string,
+    options: KtxDuckDbColumnDistinctValuesOptions,
+  ): Promise<KtxDuckDbColumnDistinctValuesResult | null> {
+    const sampleSize = options.sampleSize ?? 10000;
+    const tableName = this.qTableName(table);
+    const quotedColumn = this.dialect.quoteIdentifier(columnName);
+    const cardinalityResult = await this.query(
+      this.dialect.generateCardinalitySampleQuery(tableName, quotedColumn, sampleSize),
+    );
+    if (cardinalityResult.rows.length === 0) {
+      return null;
+    }
+    const cardinality = Number(cardinalityResult.rows[0][0]);
+    if (Number.isNaN(cardinality)) {
+      return null;
+    }
+    if (cardinality === 0) {
+      return { values: [], cardinality: 0 };
+    }
+    if (cardinality > options.maxCardinality) {
+      return { values: null, cardinality };
+    }
+    const valuesResult = await this.query(
+      this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit),
+    );
+    return {
+      values: valuesResult.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => String(row[0])),
+      cardinality,
+    };
+  }
+
+  async getTableRowCount(tableName: string): Promise<number> {
+    const result = await this.query(`SELECT COUNT(*) AS count FROM ${this.dialect.quoteIdentifier(tableName)}`);
+    return Number(result.rows[0]?.[0] ?? 0);
+  }
+
+  qTableName(table: Pick<KtxTableRef, 'name'>): string {
+    return this.dialect.formatTableName(table);
+  }
+
+  quoteIdentifier(identifier: string): string {
+    return this.dialect.quoteIdentifier(identifier);
+  }
+
+  async cleanup(): Promise<void> {
+    this.connection?.closeSync();
+    this.instance?.closeSync();
+    this.connection = null;
+    this.instance = null;
+  }
+
+  private async db(): Promise<DuckDBConnection> {
+    if (!this.connection) {
+      // DuckDBInstance.create() creates the file if missing, so this pre-check
+      // enforces the never-create rule. Do not remove it.
+      if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
+        throw new Error(`File not found: ${this.dbPath}`);
+      }
+      this.instance = await DuckDBInstance.create(this.dbPath, { access_mode: 'read_only' });
+      this.connection = await this.instance.connect();
+    }
+    return this.connection;
+  }
+
+  private async query(sql: string): Promise<Omit<KtxQueryResult, 'rowCount'>> {
+    const connection = await this.db();
+    const reader = await connection.runAndReadAll(assertReadOnlySql(sql));
+    const rows = toJsonSafeRows(normalizeQueryRows(reader.getRows()));
+    return {
+      headers: reader.columnNames(),
+      rows,
+      totalRows: rows.length,
+    };
+  }
+
+  private async readTableRows(scopedNames: string[] | null): Promise<InfoSchemaTableRow[]> {
+    if (scopedNames && scopedNames.length === 0) {
+      return [];
+    }
+    const scopeClause = scopedNames
+      ? `AND table_name IN (${scopedNames.map((name) => `'${name.replaceAll("'", "''")}'`).join(', ')})`
+      : '';
+    const result = await this.query(
+      `SELECT table_name, table_type
+       FROM information_schema.tables
+       WHERE table_schema = '${MAIN_SCHEMA}' ${scopeClause}
+       ORDER BY table_name`,
+    );
+    return result.rows.map((row) => ({ table_name: String(row[0]), table_type: String(row[1]) }));
+  }
+
+  private async readTable(table: InfoSchemaTableRow): Promise<KtxSchemaTable> {
+    const columnsResult = await this.query(
+      `SELECT column_name, data_type, is_nullable
+       FROM information_schema.columns
+       WHERE table_schema = '${MAIN_SCHEMA}' AND table_name = '${table.table_name.replaceAll("'", "''")}'
+       ORDER BY ordinal_position`,
+    );
+    const columns = columnsResult.rows.map<InfoSchemaColumnRow>((row) => ({
+      column_name: String(row[0]),
+      data_type: String(row[1]),
+      is_nullable: String(row[2]),
+    }));
+    const primaryKeys = await this.readPrimaryKeyColumns(table.table_name);
+    const isView = table.table_type === 'VIEW';
+    const estimatedRows = isView ? null : await this.getTableRowCount(table.table_name);
+    return {
+      catalog: null,
+      db: null,
+      name: table.table_name,
+      kind: isView ? 'view' : 'table',
+      comment: null,
+      estimatedRows,
+      columns: columns.map((column) => ({
+        name: column.column_name,
+        nativeType: column.data_type,
+        normalizedType: this.dialect.mapDataType(column.data_type),
+        dimensionType: this.dialect.mapToDimensionType(column.data_type),
+        nullable: column.is_nullable === 'YES' && !primaryKeys.has(column.column_name),
+        primaryKey: primaryKeys.has(column.column_name),
+        comment: null,
+      })),
+      foreignKeys: await this.readForeignKeys(table.table_name),
+    };
+  }
+
+  private async readPrimaryKeyColumns(tableName: string): Promise<Set<string>> {
+    const result = await this.query(
+      `SELECT kcu.column_name
+       FROM information_schema.table_constraints tc
+       JOIN information_schema.key_column_usage kcu
+         ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
+       WHERE tc.table_schema = '${MAIN_SCHEMA}'
+         AND tc.table_name = '${tableName.replaceAll("'", "''")}'
+         AND tc.constraint_type = 'PRIMARY KEY'`,
+    );
+    return new Set(result.rows.map((row) => String(row[0])));
+  }
+
+  private async readForeignKeys(tableName: string): Promise<KtxSchemaForeignKey[]> {
+    // information_schema.constraint_column_usage in DuckDB returns the constrained
+    // columns (source), not the referenced columns. Use duckdb_constraints() which
+    // exposes constraint_column_names and referenced_column_names directly.
+    const result = await this.query(
+      `SELECT unnest(constraint_column_names) AS from_column,
+              referenced_table,
+              unnest(referenced_column_names) AS to_column,
+              constraint_name
+       FROM duckdb_constraints()
+       WHERE schema_name = '${MAIN_SCHEMA}'
+         AND table_name = '${tableName.replaceAll("'", "''")}'
+         AND constraint_type = 'FOREIGN KEY'`,
+    );
+    return result.rows.map((row) => ({
+      fromColumn: String(row[0]),
+      toCatalog: null,
+      toDb: null,
+      toTable: String(row[1]),
+      toColumn: String(row[2]),
+      constraintName: row[3] === null ? null : String(row[3]),
+    }));
+  }
+
+  private assertConnection(connectionId: string): void {
+    if (connectionId !== this.connectionId) {
+      throw new Error(`ktx DuckDB connector ${this.id} cannot serve connection ${connectionId}`);
+    }
+  }
+}

packages/cli/src/connectors/duckdb/dialect.ts

@@ -0,0 +1,192 @@
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
+import {
+  columnDisplayPartCount,
+  formatDialectDisplayRef,
+  formatDialectTableName,
+  limitOffsetClause,
+  parseDialectDisplayRef,
+} from '../../context/connections/dialect-helpers.js';
+import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
+
+type DuckDbTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
+
+/** @internal */
+export class KtxDuckDbDialect implements KtxSqlDialect {
+  readonly type = 'duckdb' as const;
+
+  private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {
+    TIMESTAMP: 'time',
+    'TIMESTAMP WITH TIME ZONE': 'time',
+    TIMESTAMPTZ: 'time',
+    DATE: 'time',
+    TIME: 'time',
+    BIGINT: 'number',
+    INTEGER: 'number',
+    SMALLINT: 'number',
+    TINYINT: 'number',
+    HUGEINT: 'number',
+    UBIGINT: 'number',
+    UINTEGER: 'number',
+    DECIMAL: 'number',
+    NUMERIC: 'number',
+    REAL: 'number',
+    FLOAT: 'number',
+    DOUBLE: 'number',
+    VARCHAR: 'string',
+    CHAR: 'string',
+    TEXT: 'string',
+    BLOB: 'string',
+    UUID: 'string',
+    BOOLEAN: 'boolean',
+    BOOL: 'boolean',
+  };
+
+  quoteIdentifier(identifier: string): string {
+    return `"${identifier.replace(/"/g, '""')}"`;
+  }
+
+  // v1 introspects the `main` schema only and sets db=null on every table, so
+  // refs are single-namespace like SQLite — use the matching display shape.
+  formatTableName(table: DuckDbTableNameRef): string {
+    return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'sqlite');
+  }
+
+  formatDisplayRef(table: DuckDbTableNameRef): string {
+    return formatDialectDisplayRef(table, 'sqlite');
+  }
+
+  parseDisplayRef(display: string): KtxTableRef | null {
+    return parseDialectDisplayRef(display, 'sqlite');
+  }
+
+  columnDisplayTablePartCount(): 1 | 2 | 3 {
+    return columnDisplayPartCount('sqlite');
+  }
+
+  mapDataType(nativeType: string): string {
+    return nativeType;
+  }
+
+  mapToDimensionType(nativeType: string): KtxSchemaDimensionType {
+    if (!nativeType) {
+      return 'string';
+    }
+    let normalized = nativeType.toUpperCase().trim();
+    if (normalized.includes('(')) {
+      normalized = normalized.split('(')[0].trim();
+    }
+    if (this.typeMappings[normalized]) {
+      return this.typeMappings[normalized];
+    }
+    if (normalized.includes('TIME') || normalized.includes('DATE')) {
+      return 'time';
+    }
+    if (
+      normalized.includes('INT') ||
+      normalized.includes('DEC') ||
+      normalized.includes('NUM') ||
+      normalized.includes('REAL') ||
+      normalized.includes('FLOAT') ||
+      normalized.includes('DOUBLE')
+    ) {
+      return 'number';
+    }
+    if (normalized.includes('BOOL')) {
+      return 'boolean';
+    }
+    return 'string';
+  }
+
+  generateSampleQuery(tableName: string, limit: number, columns?: string[]): string {
+    const columnList =
+      columns && columns.length > 0 ? columns.map((column) => this.quoteIdentifier(column)).join(', ') : '*';
+    return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`;
+  }
+
+  generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string {
+    const quoted = this.quoteIdentifier(columnName);
+    return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL AND TRIM(CAST(${quoted} AS VARCHAR)) != '' LIMIT ${limit}`;
+  }
+
+  getRandomSampleFilter(samplePct: number): string {
+    if (samplePct <= 0 || samplePct >= 1) {
+      return '';
+    }
+    return `RANDOM() < ${samplePct}`;
+  }
+
+  getTableSampleClause(samplePct: number): string {
+    if (samplePct <= 0 || samplePct >= 1) {
+      return '';
+    }
+    return `USING SAMPLE ${Math.round(samplePct * 100)} PERCENT (bernoulli)`;
+  }
+
+  getLimitOffsetClause(limit: number, offset?: number): string {
+    return limitOffsetClause(limit, offset);
+  }
+
+  getTopClause(_limit: number): string {
+    return '';
+  }
+
+  getNullCountExpression(column: string): string {
+    return `SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END)`;
+  }
+
+  getDistinctCountExpression(column: string): string {
+    return `COUNT(DISTINCT ${column})`;
+  }
+
+  textLengthExpression(columnSql: string): string {
+    return `LENGTH(CAST(${columnSql} AS VARCHAR))`;
+  }
+
+  castToText(columnSql: string): string {
+    return `CAST(${columnSql} AS VARCHAR)`;
+  }
+
+  getSampleValueAggregation(innerSql: string): string {
+    return `(SELECT STRING_AGG(CAST(value AS VARCHAR), chr(31)) FROM (${innerSql}) AS relationship_profile_values)`;
+  }
+
+  generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
+    return `
+      WITH sampled AS (
+        SELECT ${columnName} AS val
+        FROM ${tableName}
+        WHERE ${columnName} IS NOT NULL
+        LIMIT ${sampleSize}
+      )
+      SELECT COUNT(DISTINCT val) AS cardinality
+      FROM sampled
+    `;
+  }
+
+  generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string {
+    return `
+      SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val
+      FROM ${tableName}
+      WHERE ${columnName} IS NOT NULL
+      ORDER BY val
+      LIMIT ${limit}
+    `;
+  }
+
+  generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null {
+    return null;
+  }
+
+  generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
+    return `
+      WITH sampled AS (
+        SELECT ${columnName} AS val
+        FROM ${tableName}
+        WHERE ${columnName} IS NOT NULL
+        USING SAMPLE ${sampleSize} ROWS
+      )
+      SELECT COUNT(DISTINCT val) AS cardinality
+      FROM sampled
+    `;
+  }
+}

packages/cli/src/connectors/duckdb/federated-attach.ts

@@ -4,6 +4,7 @@ import {
   mysqlConnectionPoolConfigFromConfig,
   type KtxMysqlConnectionConfig,
 } from '../mysql/connector.js';
+import { duckDbDatabasePathFromConfig, type KtxDuckDbConnectionConfig } from '../../connectors/duckdb/connector.js';
 import type { FederatedMember } from '../../context/connections/federation.js';
 
 function kvKeyword(value: string): string {
@@ -74,6 +75,12 @@ function mysqlAttachString(member: FederatedMember, env: NodeJS.ProcessEnv): str
  */
 export function federatedAttachTarget(member: FederatedMember, env: NodeJS.ProcessEnv): string {
   switch (member.driver.toLowerCase()) {
+    case 'duckdb':
+      return duckDbDatabasePathFromConfig({
+        connectionId: member.connectionId,
+        projectDir: member.projectDir,
+        connection: member.connection as KtxDuckDbConnectionConfig,
+      });
     case 'sqlite':
       return sqliteDatabasePathFromConfig({
         connectionId: member.connectionId,

packages/cli/src/connectors/duckdb/federated-executor.ts

@@ -7,25 +7,12 @@ import type {
 import { normalizeQueryRows } from '../../context/connections/query-executor.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { attachTypeForDriver, type FederatedMember } from '../../context/connections/federation.js';
+import { toJsonSafeRows } from '../shared/duckdb-json-safe.js';
 
 function quoteDuckdbIdentifier(id: string): string {
   return `"${id.replaceAll('"', '""')}"`;
 }
 
-const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
-const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
-
-// DuckDB returns integer columns as JS bigint (unserializable by JSON). Values
-// in Number's safe range become Number; larger magnitudes become strings so a
-// BIGINT beyond 2^53 keeps its exact value instead of silently rounding.
-function jsonSafeBigint(value: bigint): number | string {
-  return value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT ? Number(value) : value.toString();
-}
-
-function toJsonSafeRows(rows: unknown[][]): unknown[][] {
-  return rows.map((row) => row.map((cell) => (typeof cell === 'bigint' ? jsonSafeBigint(cell) : cell)));
-}
-
 /** @internal */
 export function buildAttachStatements(members: FederatedMember[], env: NodeJS.ProcessEnv): string[] {
   const attachments = members.map((member) => ({
@@ -34,13 +21,12 @@ export function buildAttachStatements(members: FederatedMember[], env: NodeJS.Pr
     alias: member.connectionId,
   }));
 
-  const loadStatements = [...new Set(attachments.map((a) => a.type))].map(
-    (type) => `INSTALL ${type}; LOAD ${type};`,
-  );
-  const attachStatements = attachments.map(
-    ({ type, url, alias }) =>
-      `ATTACH '${url.replaceAll("'", "''")}' AS ${quoteDuckdbIdentifier(alias)} (TYPE ${type}, READ_ONLY);`,
-  );
+  const loadTypes = [...new Set(attachments.map((a) => a.type).filter((type): type is string => type !== null))];
+  const loadStatements = loadTypes.map((type) => `INSTALL ${type}; LOAD ${type};`);
+  const attachStatements = attachments.map(({ type, url, alias }) => {
+    const options = type === null ? 'READ_ONLY' : `TYPE ${type}, READ_ONLY`;
+    return `ATTACH '${url.replaceAll("'", "''")}' AS ${quoteDuckdbIdentifier(alias)} (${options});`;
+  });
   return [...loadStatements, ...attachStatements];
 }
 

packages/cli/src/connectors/duckdb/live-database-introspection.ts

@@ -0,0 +1,40 @@
+import type {
+  LiveDatabaseIntrospectionOptions,
+  LiveDatabaseIntrospectionPort,
+} from '../../context/ingest/adapters/live-database/types.js';
+import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
+import { KtxDuckDbScanConnector, type KtxDuckDbConnectionConfig } from './connector.js';
+
+export interface CreateDuckDbLiveDatabaseIntrospectionOptions {
+  projectDir?: string;
+  connections: Record<string, KtxProjectConnectionConfig>;
+  now?: () => Date;
+}
+
+export function createDuckDbLiveDatabaseIntrospection(
+  options: CreateDuckDbLiveDatabaseIntrospectionOptions,
+): LiveDatabaseIntrospectionPort {
+  return {
+    async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) {
+      const connection = options.connections[connectionId] as KtxDuckDbConnectionConfig | undefined;
+      const connector = new KtxDuckDbScanConnector({
+        connectionId,
+        connection,
+        projectDir: options.projectDir,
+        now: options.now,
+      });
+      try {
+        return await connector.introspect(
+          {
+            connectionId,
+            driver: 'duckdb',
+            ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
+          },
+          { runId: `duckdb-${connectionId}` },
+        );
+      } finally {
+        await connector.cleanup();
+      }
+    },
+  };
+}

packages/cli/src/connectors/mongodb/connector.ts

@@ -0,0 +1,413 @@
+import { MongoClient } from 'mongodb';
+import { resolveKtxConfigReference } from '../../context/core/config-reference.js';
+import {
+  connectorTestFailure,
+  createKtxConnectorCapabilities,
+  type KtxColumnSampleInput,
+  type KtxColumnSampleResult,
+  type KtxConnectorTestResult,
+  type KtxScanConnector,
+  type KtxScanContext,
+  type KtxScanInput,
+  type KtxSchemaSnapshot,
+  type KtxSchemaTable,
+  type KtxTableListEntry,
+  type KtxTableRef,
+  type KtxTableSampleInput,
+  type KtxTableSampleResult,
+} from '../../context/scan/types.js';
+import { scopedTableNames } from '../../context/scan/table-ref.js';
+import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { inferKtxMongoCollectionColumns, type KtxMongoDocument, MONGO_ID_FIELD } from './schema-inference.js';
+
+const DEFAULT_SAMPLE_SIZE = 1000;
+const SAMPLE_MAX_TIME_MS = 30_000;
+
+export interface KtxMongoDbConnectionConfig {
+  driver?: string;
+  url?: string;
+  database?: string;
+  databases?: string[];
+  enabled_tables?: string[];
+  sample_size?: number;
+  order_by?: string;
+  [key: string]: unknown;
+}
+
+export interface KtxMongoListedCollection {
+  name: string;
+  type?: string;
+}
+
+interface KtxMongoFindOptions {
+  sort: Record<string, 1 | -1>;
+  limit: number;
+  projection?: Record<string, 1>;
+}
+
+/** Driver-agnostic seam over the `mongodb` client so the connector is unit-testable without a server. */
+export interface KtxMongoClient {
+  listCollections(databaseName: string): Promise<KtxMongoListedCollection[]>;
+  estimatedDocumentCount(databaseName: string, collectionName: string): Promise<number>;
+  find(databaseName: string, collectionName: string, options: KtxMongoFindOptions): Promise<KtxMongoDocument[]>;
+  ping(databaseName: string): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface KtxMongoClientFactory {
+  create(url: string): KtxMongoClient;
+}
+
+export interface KtxMongoDbScanConnectorOptions {
+  connectionId: string;
+  connection: KtxMongoDbConnectionConfig | undefined;
+  clientFactory?: KtxMongoClientFactory;
+  env?: NodeJS.ProcessEnv;
+  now?: () => Date;
+}
+
+class DefaultMongoClient implements KtxMongoClient {
+  private readonly client: MongoClient;
+  private connected = false;
+
+  constructor(url: string) {
+    this.client = new MongoClient(url);
+  }
+
+  private async connectedClient(): Promise<MongoClient> {
+    if (!this.connected) {
+      await this.client.connect();
+      this.connected = true;
+    }
+    return this.client;
+  }
+
+  async listCollections(databaseName: string): Promise<KtxMongoListedCollection[]> {
+    const client = await this.connectedClient();
+    const collections = await client.db(databaseName).listCollections({}, { nameOnly: false }).toArray();
+    return collections.map((collection) => ({ name: collection.name, type: collection.type }));
+  }
+
+  async estimatedDocumentCount(databaseName: string, collectionName: string): Promise<number> {
+    const client = await this.connectedClient();
+    return client.db(databaseName).collection(collectionName).estimatedDocumentCount();
+  }
+
+  async find(
+    databaseName: string,
+    collectionName: string,
+    options: KtxMongoFindOptions,
+  ): Promise<KtxMongoDocument[]> {
+    const client = await this.connectedClient();
+    return client
+      .db(databaseName)
+      .collection(collectionName)
+      .find({}, { sort: options.sort, limit: options.limit, maxTimeMS: SAMPLE_MAX_TIME_MS, ...(options.projection ? { projection: options.projection } : {}) })
+      .toArray() as Promise<KtxMongoDocument[]>;
+  }
+
+  async ping(databaseName: string): Promise<void> {
+    const client = await this.connectedClient();
+    await client.db(databaseName).command({ ping: 1 });
+  }
+
+  async close(): Promise<void> {
+    if (this.connected) {
+      await this.client.close();
+      this.connected = false;
+    }
+  }
+}
+
+class DefaultMongoClientFactory implements KtxMongoClientFactory {
+  create(url: string): KtxMongoClient {
+    return new DefaultMongoClient(url);
+  }
+}
+
+export function isKtxMongoDbConnectionConfig(
+  connection: KtxMongoDbConnectionConfig | undefined,
+): connection is KtxMongoDbConnectionConfig {
+  return String(connection?.driver ?? '').toLowerCase() === 'mongodb';
+}
+
+function databaseFromUrl(url: string): string | undefined {
+  try {
+    const path = new URL(url).pathname.replace(/^\/+/, '');
+    const database = path.split('/')[0];
+    return database && database.length > 0 ? decodeURIComponent(database) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function configuredDatabases(connection: KtxMongoDbConnectionConfig, fallback: string | undefined): string[] {
+  if (Array.isArray(connection.databases)) {
+    const selected = connection.databases
+      .filter((database): database is string => typeof database === 'string' && database.trim().length > 0)
+      .map((database) => database.trim());
+    if (selected.length > 0) {
+      return [...new Set(selected)];
+    }
+  }
+  const single = typeof connection.database === 'string' && connection.database.trim().length > 0
+    ? connection.database.trim()
+    : fallback;
+  return single ? [single] : [];
+}
+
+function positiveInteger(value: unknown, fallback: number): number {
+  return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : fallback;
+}
+
+function normalizeSampleValue(value: unknown): unknown {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  if (value instanceof Date) {
+    return value.toISOString();
+  }
+  if (typeof value === 'object') {
+    const bsontype = (value as { _bsontype?: unknown })._bsontype;
+    return typeof bsontype === 'string' ? String(value) : JSON.stringify(value);
+  }
+  return value;
+}
+
+function unionDocumentKeys(documents: readonly KtxMongoDocument[]): string[] {
+  const keys: string[] = [];
+  const seen = new Set<string>();
+  for (const document of documents) {
+    for (const key of Object.keys(document)) {
+      if (!seen.has(key)) {
+        seen.add(key);
+        keys.push(key);
+      }
+    }
+  }
+  return keys;
+}
+
+export class KtxMongoDbScanConnector implements KtxScanConnector {
+  readonly id: string;
+  readonly driver = 'mongodb' as const;
+  readonly capabilities = createKtxConnectorCapabilities({
+    tableSampling: true,
+    columnSampling: true,
+    columnStats: false,
+    readOnlySql: false,
+    nestedAnalysis: true,
+    formalForeignKeys: false,
+    estimatedRowCounts: true,
+  });
+
+  private readonly connectionId: string;
+  private readonly connection: KtxMongoDbConnectionConfig;
+  private readonly url: string;
+  private readonly databases: string[];
+  private readonly sampleSize: number;
+  private readonly orderBy: string;
+  private readonly enabledTables: ReadonlySet<string> | null;
+  private readonly clientFactory: KtxMongoClientFactory;
+  private readonly now: () => Date;
+  private readonly dialect = getDialectForDriver('mongodb');
+  private client: KtxMongoClient | null = null;
+
+  constructor(options: KtxMongoDbScanConnectorOptions) {
+    const connection = options.connection ?? {};
+    const inputDriver = connection.driver ?? 'unknown';
+    if (!isKtxMongoDbConnectionConfig(connection)) {
+      throw new Error(`Native MongoDB connector cannot run driver "${inputDriver}"`);
+    }
+    const env = options.env ?? process.env;
+    const url = resolveKtxConfigReference(
+      typeof connection.url === 'string' ? connection.url.trim() : undefined,
+      env,
+    );
+    if (!url) {
+      throw new Error(`Native MongoDB connector requires connections.${options.connectionId}.url`);
+    }
+    const databases = configuredDatabases(connection, databaseFromUrl(url));
+    if (databases.length === 0) {
+      throw new Error(
+        `Native MongoDB connector requires connections.${options.connectionId}.databases (or a database in the URL)`,
+      );
+    }
+    const enabledTables = Array.isArray(connection.enabled_tables)
+      ? new Set(
+          connection.enabled_tables
+            .filter((table): table is string => typeof table === 'string' && table.trim().length > 0)
+            .map((table) => table.trim()),
+        )
+      : null;
+
+    this.connectionId = options.connectionId;
+    this.connection = connection;
+    this.url = url;
+    this.databases = databases;
+    this.sampleSize = positiveInteger(connection.sample_size, DEFAULT_SAMPLE_SIZE);
+    this.orderBy = typeof connection.order_by === 'string' && connection.order_by.trim().length > 0
+      ? connection.order_by.trim()
+      : MONGO_ID_FIELD;
+    this.enabledTables = enabledTables && enabledTables.size > 0 ? enabledTables : null;
+    this.clientFactory = options.clientFactory ?? new DefaultMongoClientFactory();
+    this.now = options.now ?? (() => new Date());
+    this.id = `mongodb:${options.connectionId}`;
+  }
+
+  async testConnection(): Promise<KtxConnectorTestResult> {
+    try {
+      await this.clientForQuery().ping(this.databases[0]!);
+      return { success: true };
+    } catch (error) {
+      return connectorTestFailure(error);
+    }
+  }
+
+  async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
+    this.assertConnection(input.connectionId);
+    const client = this.clientForQuery();
+    const tables: KtxSchemaTable[] = [];
+
+    for (const database of this.databases) {
+      const scopedNames = input.tableScope
+        ? new Set(scopedTableNames(input.tableScope, { catalog: null, db: database }))
+        : null;
+      const collections = await client.listCollections(database);
+      for (const collection of collections) {
+        if (collection.name.startsWith('system.')) {
+          continue;
+        }
+        if (scopedNames && !scopedNames.has(collection.name)) {
+          continue;
+        }
+        if (this.enabledTables && !this.enabledTables.has(`${database}.${collection.name}`)) {
+          continue;
+        }
+        tables.push(await this.introspectCollection(client, database, collection));
+      }
+    }
+
+    return {
+      connectionId: this.connectionId,
+      driver: 'mongodb',
+      extractedAt: this.now().toISOString(),
+      scope: { schemas: this.databases },
+      metadata: {
+        databases: this.databases,
+        sample_size: this.sampleSize,
+        order_by: this.orderBy,
+        table_count: tables.length,
+        total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
+      },
+      tables,
+    };
+  }
+
+  private async introspectCollection(
+    client: KtxMongoClient,
+    database: string,
+    collection: KtxMongoListedCollection,
+  ): Promise<KtxSchemaTable> {
+    const isView = collection.type === 'view';
+    // estimatedDocumentCount issues a count command, which MongoDB rejects on a
+    // view (CommandNotSupportedOnView); only count real collections.
+    const estimatedRows = isView ? null : await client.estimatedDocumentCount(database, collection.name);
+    const documents = await client.find(database, collection.name, {
+      sort: { [this.orderBy]: -1 },
+      limit: this.sampleSize,
+    });
+    return {
+      catalog: null,
+      db: database,
+      name: collection.name,
+      kind: isView ? 'view' : 'table',
+      comment: null,
+      estimatedRows,
+      columns: inferKtxMongoCollectionColumns(documents, this.dialect),
+      foreignKeys: [],
+    };
+  }
+
+  async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxTableSampleResult> {
+    this.assertConnection(input.connectionId);
+    const { database, collection } = this.resolveTableRef(input.table);
+    const documents = await this.clientForQuery().find(database, collection, {
+      sort: { [this.orderBy]: -1 },
+      limit: input.limit,
+    });
+    const headers = input.columns && input.columns.length > 0 ? input.columns : unionDocumentKeys(documents);
+    const rows = documents.map((document) => headers.map((header) => normalizeSampleValue(document[header])));
+    return { headers, rows, totalRows: documents.length };
+  }
+
+  async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult> {
+    this.assertConnection(input.connectionId);
+    const { database, collection } = this.resolveTableRef(input.table);
+    const documents = await this.clientForQuery().find(database, collection, {
+      sort: { [this.orderBy]: -1 },
+      limit: input.limit,
+      projection: { [input.column]: 1 },
+    });
+    const values: unknown[] = [];
+    let nullCount = 0;
+    for (const document of documents) {
+      const value = document[input.column];
+      if (value === null || value === undefined) {
+        nullCount += 1;
+        continue;
+      }
+      values.push(normalizeSampleValue(value));
+    }
+    return { values, nullCount, distinctCount: null };
+  }
+
+  async listSchemas(): Promise<string[]> {
+    return [...this.databases];
+  }
+
+  async listTables(schemas?: string[]): Promise<KtxTableListEntry[]> {
+    const client = this.clientForQuery();
+    const databases = schemas && schemas.length > 0 ? schemas : this.databases;
+    const entries: KtxTableListEntry[] = [];
+    for (const database of databases) {
+      const collections = await client.listCollections(database);
+      for (const collection of collections) {
+        if (collection.name.startsWith('system.')) {
+          continue;
+        }
+        entries.push({
+          catalog: null,
+          schema: database,
+          name: collection.name,
+          kind: collection.type === 'view' ? 'view' : 'table',
+        });
+      }
+    }
+    return entries;
+  }
+
+  async cleanup(): Promise<void> {
+    if (this.client) {
+      await this.client.close();
+      this.client = null;
+    }
+  }
+
+  private resolveTableRef(table: KtxTableRef): { database: string; collection: string } {
+    return { database: table.db ?? this.databases[0]!, collection: table.name };
+  }
+
+  private clientForQuery(): KtxMongoClient {
+    if (!this.client) {
+      this.client = this.clientFactory.create(this.url);
+    }
+    return this.client;
+  }
+
+  private assertConnection(connectionId: string): void {
+    if (connectionId !== this.connectionId) {
+      throw new Error(`ktx MongoDB connector ${this.id} cannot serve connection ${connectionId}`);
+    }
+  }
+}

packages/cli/src/connectors/mongodb/dialect.ts

@@ -0,0 +1,64 @@
+import type { KtxDialect } from '../../context/connections/dialects.js';
+import {
+  columnDisplayPartCount,
+  formatDialectDisplayRef,
+  parseDialectDisplayRef,
+} from '../../context/connections/dialect-helpers.js';
+import type { KtxDialectTableRef } from '../../context/connections/dialect-helpers.js';
+import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
+
+/**
+ * Display/type-mapping half of the dialect contract for MongoDB. Collections map
+ * to `db.collection` display refs (ansi two-part shape). MongoDB is a non-SQL
+ * source, so it implements {@link KtxDialect} only — never {@link KtxSqlDialect}.
+ */
+export class KtxMongoDbDialect implements KtxDialect {
+  readonly type = 'mongodb' as const;
+
+  private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {
+    objectid: 'string',
+    string: 'string',
+    uuid: 'string',
+    binary: 'string',
+    regex: 'string',
+    int: 'number',
+    long: 'number',
+    double: 'number',
+    decimal: 'number',
+    bool: 'boolean',
+    date: 'time',
+    timestamp: 'time',
+    object: 'string',
+    array: 'string',
+    json: 'string',
+    null: 'string',
+    mixed: 'string',
+  };
+
+  formatDisplayRef(table: KtxDialectTableRef): string {
+    return formatDialectDisplayRef(table, 'ansi');
+  }
+
+  parseDisplayRef(display: string): KtxTableRef | null {
+    return parseDialectDisplayRef(display, 'ansi');
+  }
+
+  columnDisplayTablePartCount(): 1 | 2 | 3 {
+    return columnDisplayPartCount('ansi');
+  }
+
+  mapDataType(nativeType: string): string {
+    const normalized = nativeType.toLowerCase().trim();
+    if (normalized === 'object' || normalized === 'array') {
+      return 'json';
+    }
+    return normalized || 'mixed';
+  }
+
+  mapToDimensionType(nativeType: string): KtxSchemaDimensionType {
+    if (!nativeType) {
+      return 'string';
+    }
+    return this.typeMappings[nativeType.toLowerCase().trim()] ?? 'string';
+  }
+}

packages/cli/src/connectors/mongodb/live-database-introspection.ts

@@ -0,0 +1,44 @@
+import type {
+  LiveDatabaseIntrospectionOptions,
+  LiveDatabaseIntrospectionPort,
+} from '../../context/ingest/adapters/live-database/types.js';
+import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
+import {
+  KtxMongoDbScanConnector,
+  type KtxMongoClientFactory,
+  type KtxMongoDbConnectionConfig,
+} from './connector.js';
+
+interface CreateMongoDbLiveDatabaseIntrospectionOptions {
+  connections: Record<string, KtxProjectConnectionConfig>;
+  clientFactory?: KtxMongoClientFactory;
+  now?: () => Date;
+}
+
+export function createMongoDbLiveDatabaseIntrospection(
+  options: CreateMongoDbLiveDatabaseIntrospectionOptions,
+): LiveDatabaseIntrospectionPort {
+  return {
+    async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) {
+      const connection = options.connections[connectionId] as KtxMongoDbConnectionConfig | undefined;
+      const connector = new KtxMongoDbScanConnector({
+        connectionId,
+        connection,
+        clientFactory: options.clientFactory,
+        now: options.now,
+      });
+      try {
+        return await connector.introspect(
+          {
+            connectionId,
+            driver: 'mongodb',
+            ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
+          },
+          { runId: `mongodb-${connectionId}` },
+        );
+      } finally {
+        await connector.cleanup();
+      }
+    },
+  };
+}

packages/cli/src/connectors/mongodb/schema-inference.ts

@@ -0,0 +1,129 @@
+import type { KtxSchemaColumn } from '../../context/scan/types.js';
+import type { KtxDialect } from '../../context/connections/dialects.js';
+
+export type KtxMongoDocument = Record<string, unknown>;
+
+/** Top-level field name MongoDB guarantees on every document; used as the primary key. */
+export const MONGO_ID_FIELD = '_id';
+
+const BSON_TYPE_NAMES: Record<string, string> = {
+  objectid: 'objectId',
+  int32: 'int',
+  long: 'long',
+  double: 'double',
+  decimal128: 'decimal',
+  binary: 'binary',
+  uuid: 'uuid',
+  timestamp: 'timestamp',
+  bsonregexp: 'regex',
+  bsonsymbol: 'string',
+};
+
+/**
+ * Canonical BSON type name for a sampled value as the `mongodb` driver hydrates
+ * it: BSON wrapper objects expose `_bsontype`; everything else maps from the JS
+ * runtime type. Sub-documents and arrays collapse to opaque `object`/`array`.
+ * @internal
+ */
+export function bsonTypeOf(value: unknown): string {
+  if (value === null || value === undefined) {
+    return 'null';
+  }
+  if (typeof value === 'string') {
+    return 'string';
+  }
+  if (typeof value === 'boolean') {
+    return 'bool';
+  }
+  if (typeof value === 'number') {
+    return Number.isInteger(value) ? 'int' : 'double';
+  }
+  if (typeof value === 'bigint') {
+    return 'long';
+  }
+  if (value instanceof Date) {
+    return 'date';
+  }
+  if (Array.isArray(value)) {
+    return 'array';
+  }
+  if (typeof value === 'object') {
+    const bsontype = (value as { _bsontype?: unknown })._bsontype;
+    if (typeof bsontype === 'string') {
+      return BSON_TYPE_NAMES[bsontype.toLowerCase()] ?? bsontype;
+    }
+    return 'object';
+  }
+  return 'mixed';
+}
+
+interface FieldAccumulator {
+  types: Set<string>;
+  present: number;
+  nullSeen: boolean;
+}
+
+function resolveNativeType(types: ReadonlySet<string>): string {
+  if (types.size === 0) {
+    return 'null';
+  }
+  if (types.size > 1) {
+    return 'mixed';
+  }
+  return [...types][0]!;
+}
+
+/**
+ * Infer a flat column schema from sampled documents. Each top-level field becomes
+ * one column: BSON types are unioned (a field seen with >1 type is `mixed` and
+ * treated as a string), nullability is derived from field presence and observed
+ * nulls, and sub-documents/arrays remain a single opaque `json` column. `_id` is
+ * the non-nullable primary key.
+ */
+export function inferKtxMongoCollectionColumns(
+  documents: readonly KtxMongoDocument[],
+  dialect: KtxDialect,
+): KtxSchemaColumn[] {
+  const total = documents.length;
+  const order: string[] = [];
+  const fields = new Map<string, FieldAccumulator>();
+
+  for (const document of documents) {
+    if (!document || typeof document !== 'object') {
+      continue;
+    }
+    for (const [name, value] of Object.entries(document)) {
+      let accumulator = fields.get(name);
+      if (!accumulator) {
+        accumulator = { types: new Set(), present: 0, nullSeen: false };
+        fields.set(name, accumulator);
+        order.push(name);
+      }
+      accumulator.present += 1;
+      const bsonType = bsonTypeOf(value);
+      if (bsonType === 'null') {
+        accumulator.nullSeen = true;
+      } else {
+        accumulator.types.add(bsonType);
+      }
+    }
+  }
+
+  return order.map((name) => {
+    const accumulator = fields.get(name)!;
+    const nativeType = resolveNativeType(accumulator.types);
+    const isId = name === MONGO_ID_FIELD;
+    const nullable = isId
+      ? false
+      : accumulator.present < total || accumulator.nullSeen || accumulator.types.size === 0;
+    return {
+      name,
+      nativeType,
+      normalizedType: dialect.mapDataType(nativeType),
+      dimensionType: dialect.mapToDimensionType(nativeType),
+      nullable,
+      primaryKey: isId,
+      comment: null,
+    };
+  });
+}

packages/cli/src/connectors/mysql/connector.ts

@@ -1,5 +1,6 @@
 import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/promise';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { resolveStringReference } from '../shared/string-reference.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import {
@@ -282,6 +283,11 @@ function isDeniedError(error: unknown): boolean {
   );
 }
 
+// errno 3024 = ER_QUERY_TIMEOUT, raised when max_execution_time is exceeded.
+function isMysqlTimeoutError(error: unknown): boolean {
+  return Boolean(error) && typeof error === 'object' && (error as { errno?: unknown }).errno === 3024;
+}
+
 function pushConstraintWarnings(
   warnings: KtxScanWarning[],
   schemas: readonly string[],
@@ -391,7 +397,8 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
   private readonly poolFactory: KtxMysqlPoolFactory;
   private readonly endpointResolver?: KtxMysqlEndpointResolver;
   private readonly now: () => Date;
-  private readonly dialect = getDialectForDriver('mysql');
+  private readonly deadlineMs: number;
+  private readonly dialect = getSqlDialectForDriver('mysql');
   private pool: KtxMysqlPool | null = null;
   private resolvedEndpoint: KtxMysqlResolvedEndpoint | null = null;
 
@@ -406,6 +413,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
     this.poolFactory = options.poolFactory ?? new DefaultMysqlPoolFactory();
     this.endpointResolver = options.endpointResolver;
     this.now = options.now ?? (() => new Date());
+    this.deadlineMs = resolveQueryDeadlineMs(this.connection);
     this.id = `mysql:${options.connectionId}`;
   }
 
@@ -763,6 +771,9 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
     const pool = await this.poolForQuery();
     const connection = await pool.getConnection();
     try {
+      // max_execution_time (ms) bounds read-only SELECTs server-side; our path
+      // only runs SELECT/WITH, so the session setting always applies.
+      await connection.query('SET SESSION max_execution_time = ?', [this.deadlineMs]);
       const [rows, fields] = await connection.query(assertReadOnlySql(sql), queryParams(params));
       const headers = fields.map((field) => field.name);
       const headerTypes = fields.map((field) => String(field.type ?? 'unknown'));
@@ -772,6 +783,11 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
         rows: rows.map((row) => headers.map((header) => row[header])),
         totalRows: rows.length,
       };
+    } catch (error) {
+      if (isMysqlTimeoutError(error)) {
+        throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
+      }
+      throw error;
     } finally {
       connection.release();
     }

packages/cli/src/connectors/mysql/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type MysqlTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxMysqlDialect implements KtxDialect {
+export class KtxMysqlDialect implements KtxSqlDialect {
   readonly type = 'mysql' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/connectors/postgres/connector.ts

@@ -1,5 +1,6 @@
 import { resolveStringReference } from '../shared/string-reference.js';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
 import { scopedTableNames } from '../../context/scan/table-ref.js';
@@ -260,6 +261,11 @@ function isDeniedError(error: unknown): boolean {
   return code === '42501' || code === '42P01';
 }
 
+// 57014 = query_canceled, which is how statement_timeout surfaces.
+function isPostgresTimeoutError(error: unknown): boolean {
+  return Boolean(error) && typeof error === 'object' && (error as { code?: unknown }).code === '57014';
+}
+
 function queryRows(result: KtxPostgresQueryResult): unknown[][] {
   const headers = (result.fields ?? []).map((field) => field.name);
   return result.rows.map((row) => headers.map((header) => row[header]));
@@ -384,9 +390,13 @@ export function postgresPoolConfigFromConfig(input: {
       : { host, port: numberValue(merged.port) ?? 5432, database, user, password }),
   };
   const searchPathSchemas = searchPathSchemasFromConnection(merged);
+  // statement_timeout (ms) bounds every query on connections from this pool, so
+  // the server itself aborts a runaway query and frees the connection cleanly.
+  const serverOptions = [`-c statement_timeout=${resolveQueryDeadlineMs(merged)}`];
   if (searchPathSchemas.length > 0) {
-    config.options = `-c search_path=${searchPathSchemas.join(',')}`;
+    serverOptions.unshift(`-c search_path=${searchPathSchemas.join(',')}`);
   }
+  config.options = serverOptions.join(' ');
   if (merged.ssl && sslmode !== 'prefer' && sslmode !== 'disable') {
     config.ssl = { rejectUnauthorized: merged.rejectUnauthorized ?? true };
   }
@@ -412,7 +422,8 @@ export class KtxPostgresScanConnector implements KtxScanConnector {
   private readonly poolFactory: KtxPostgresPoolFactory;
   private readonly endpointResolver?: KtxPostgresEndpointResolver;
   private readonly now: () => Date;
-  private readonly dialect = getDialectForDriver('postgres');
+  private readonly deadlineMs: number;
+  private readonly dialect = getSqlDialectForDriver('postgres');
   private pool: KtxPostgresPool | null = null;
   private lastIdlePoolError: Error | null = null;
   private resolvedEndpoint: KtxPostgresResolvedEndpoint | null = null;
@@ -428,6 +439,7 @@ export class KtxPostgresScanConnector implements KtxScanConnector {
     this.poolFactory = options.poolFactory ?? new DefaultPostgresPoolFactory();
     this.endpointResolver = options.endpointResolver;
     this.now = options.now ?? (() => new Date());
+    this.deadlineMs = resolveQueryDeadlineMs(this.connection);
     this.id = `postgres:${options.connectionId}`;
   }
 
@@ -819,6 +831,11 @@ export class KtxPostgresScanConnector implements KtxScanConnector {
         totalRows: result.rows.length,
         rowCount: result.rows.length,
       };
+    } catch (error) {
+      if (isPostgresTimeoutError(error)) {
+        throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
+      }
+      throw error;
     } finally {
       client.release();
     }

packages/cli/src/connectors/postgres/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type PostgresTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxPostgresDialect implements KtxDialect {
+export class KtxPostgresDialect implements KtxSqlDialect {
   readonly type = 'postgres' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/connectors/shared/duckdb-json-safe.ts

@@ -0,0 +1,14 @@
+const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
+const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
+
+// DuckDB returns integer columns as JS bigint (unserializable by JSON). Values
+// in Number's safe range become Number; larger magnitudes become strings so a
+// BIGINT beyond 2^53 keeps its exact value instead of silently rounding.
+/** @internal */
+export function jsonSafeBigint(value: bigint): number | string {
+  return value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT ? Number(value) : value.toString();
+}
+
+export function toJsonSafeRows(rows: unknown[][]): unknown[][] {
+  return rows.map((row) => row.map((cell) => (typeof cell === 'bigint' ? jsonSafeBigint(cell) : cell)));
+}

packages/cli/src/connectors/snowflake/connector.ts

@@ -1,5 +1,6 @@
 import { createPrivateKey } from 'node:crypto';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { resolveStringReference } from '../shared/string-reference.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
@@ -60,6 +61,7 @@ export interface KtxSnowflakeResolvedConnectionConfig {
   passphrase?: string;
   role?: string;
   maxConnections: number;
+  deadlineMs: number;
 }
 
 export interface KtxSnowflakeRawColumnMetadata {
@@ -181,6 +183,22 @@ function isDeniedError(error: unknown): boolean {
   return false;
 }
 
+// Snowflake cancels with code 604 and a "reached its statement ... timeout"
+// message once STATEMENT_TIMEOUT_IN_SECONDS elapses.
+function isSnowflakeTimeoutError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') {
+    return false;
+  }
+  const code = (error as { code?: unknown }).code;
+  const message = (error as { message?: unknown }).message;
+  return (
+    code === 604 ||
+    code === '604' ||
+    code === '000604' ||
+    (typeof message === 'string' && /reached its (statement|warehouse) .*timeout/i.test(message))
+  );
+}
+
 function normalizeSnowflakeValue(value: unknown, columnType?: string): unknown {
   if (columnType && DATE_TYPES.some((type) => columnType.toUpperCase().includes(type))) {
     if (typeof value === 'number') {
@@ -282,6 +300,7 @@ export function snowflakeConnectionConfigFromConfig(input: {
       connectionId: input.connectionId,
       defaultValue: 4,
     }),
+    deadlineMs: resolveQueryDeadlineMs(input.connection),
   };
   const role = stringConfigValue(input.connection, 'role', env);
   if (role) {
@@ -339,13 +358,23 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver {
 
   async query(sql: string, params?: unknown): Promise<KtxQueryResult> {
     const binds = Array.isArray(params) ? toSnowflakeBinds(params) : undefined;
+    const statementTimeoutSeconds = Math.ceil(this.resolved.deadlineMs / 1000);
     try {
       const pool = await this.getPool();
-      const result = await pool.use(async (connection: snowflake.Connection) =>
-        this.executeSnowflakeQuery(connection, sql, binds),
-      );
+      const result = await pool.use(async (connection: snowflake.Connection) => {
+        // Bound the statement server-side; Snowflake cancels and frees the
+        // warehouse slot when STATEMENT_TIMEOUT_IN_SECONDS is reached.
+        await this.executeSnowflakeQuery(
+          connection,
+          `ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = ${statementTimeoutSeconds}`,
+        );
+        return this.executeSnowflakeQuery(connection, sql, binds);
+      });
       return { ...result, totalRows: result.rows.length, rowCount: result.rows.length };
     } catch (error) {
+      if (isSnowflakeTimeoutError(error)) {
+        throw queryDeadlineExceededError(this.resolved.deadlineMs, { cause: error });
+      }
       const message = error instanceof Error ? error.message : String(error);
       if (/timeout/i.test(message) && /pool|acquire/i.test(message)) {
         throw new Error(
@@ -547,7 +576,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector {
 
   private readonly resolved: KtxSnowflakeResolvedConnectionConfig;
   private readonly driverFactory: KtxSnowflakeDriverFactory;
-  private readonly dialect = getDialectForDriver('snowflake');
+  private readonly dialect = getSqlDialectForDriver('snowflake');
   private readonly now: () => Date;
   private driverInstance: KtxSnowflakeDriver | null = null;
 

packages/cli/src/connectors/snowflake/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type SnowflakeTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxSnowflakeDialect implements KtxDialect {
+export class KtxSnowflakeDialect implements KtxSqlDialect {
   readonly type = 'snowflake' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/connectors/sqlite/connector.ts

@@ -3,19 +3,44 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { isAbsolute, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { fork, type ChildProcess } from 'node:child_process';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { normalizeQueryRows } from '../../context/connections/query-executor.js';
-import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
+import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxScanWarning, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
 import { scopedTableNames } from '../../context/scan/table-ref.js';
+import { tryIntrospectObject } from '../../context/scan/object-introspection.js';
 
 export interface KtxSqliteConnectionConfig {
   driver?: string;
   path?: string;
   url?: string;
+  query_timeout_ms?: number;
   [key: string]: unknown;
 }
 
+// In dist, connector.js and read-query-child.js are siblings; under vitest the
+// compiled .js is absent and Node strips types from the .ts when forking it.
+const readQueryChildUrl = existsSync(fileURLToPath(new URL('./read-query-child.js', import.meta.url)))
+  ? new URL('./read-query-child.js', import.meta.url)
+  : new URL('./read-query-child.ts', import.meta.url);
+
+/** @internal */
+export function forkReadQueryChild(): ChildProcess {
+  // Empty execArgv so the child is a clean Node process (no inherited vitest /
+  // inspector flags); advanced serialization preserves BigInt/Buffer in rows.
+  return fork(readQueryChildUrl, {
+    execArgv: [],
+    serialization: 'advanced',
+    stdio: ['ignore', 'ignore', 'inherit', 'ipc'],
+  });
+}
+
+type ReadQueryChildMessage =
+  | { ok: true; headers: string[]; rows: unknown[]; totalRows: number }
+  | { ok: false; message: string };
+
 /** @internal */
 export interface SqliteDatabasePathInput {
   connectionId: string;
@@ -25,6 +50,8 @@ export interface SqliteDatabasePathInput {
 
 export interface KtxSqliteScanConnectorOptions extends SqliteDatabasePathInput {
   now?: () => Date;
+  /** @internal Test seam: spawn the read-query child so tests can observe its lifecycle. */
+  spawnReadQueryChild?: () => ChildProcess;
 }
 
 export interface KtxSqliteReadOnlyQueryInput extends KtxReadOnlyQueryInput {
@@ -133,13 +160,17 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
   private readonly connectionId: string;
   private readonly dbPath: string;
   private readonly now: () => Date;
-  private readonly dialect = getDialectForDriver('sqlite');
+  private readonly deadlineMs: number;
+  private readonly spawnReadQueryChild: () => ChildProcess;
+  private readonly dialect = getSqlDialectForDriver('sqlite');
   private db: Database.Database | null = null;
 
   constructor(options: KtxSqliteScanConnectorOptions) {
     this.connectionId = options.connectionId;
     this.dbPath = sqliteDatabasePathFromConfig(options);
     this.now = options.now ?? (() => new Date());
+    this.deadlineMs = resolveQueryDeadlineMs(options.connection);
+    this.spawnReadQueryChild = options.spawnReadQueryChild ?? forkReadQueryChild;
     this.id = `sqlite:${options.connectionId}`;
   }
 
@@ -158,17 +189,27 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
   async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
     this.assertConnection(input.connectionId);
     const database = this.database();
-    const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: null, db: null }) : null;
-    const scopeClause = scopedNames ? `AND name IN (${scopedNames.map(() => '?').join(', ')})` : '';
-    const rawTables =
-      scopedNames && scopedNames.length === 0
-        ? []
-        : (database
-            .prepare(
-              `SELECT name, type FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ${scopeClause} ORDER BY name`,
-            )
-            .all(...(scopedNames ?? [])) as SqliteMasterRow[]);
-    const tables = rawTables.map((table) => this.readTable(database, table));
+    const allObjects = database
+      .prepare(
+        `SELECT name, type FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY name`,
+      )
+      .all() as SqliteMasterRow[];
+    const scopedNames = input.tableScope
+      ? new Set(scopedTableNames(input.tableScope, { catalog: null, db: null }))
+      : null;
+    const selectedObjects = scopedNames ? allObjects.filter((object) => scopedNames.has(object.name)) : allObjects;
+
+    const tables: KtxSchemaTable[] = [];
+    const warnings: KtxScanWarning[] = [];
+    for (const object of selectedObjects) {
+      const outcome = await tryIntrospectObject({ object: object.name }, () => this.readTable(database, object));
+      if (outcome.ok) {
+        tables.push(outcome.table);
+      } else {
+        warnings.push(outcome.warning);
+      }
+    }
+
     const fileStats = existsSync(this.dbPath) ? statSync(this.dbPath) : null;
     return {
       connectionId: this.connectionId,
@@ -180,8 +221,12 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
         file_size: fileStats ? fileStats.size : 0,
         table_count: tables.length,
         total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
+        // Carries the full object inventory so a zero-match enabled_tables scope
+        // can report which objects were actually available.
+        ...(scopedNames ? { discovered_object_names: allObjects.map((object) => object.name) } : {}),
       },
       tables,
+      ...(warnings.length > 0 ? { warnings } : {}),
     };
   }
 
@@ -229,12 +274,81 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
     return null;
   }
 
-  async executeReadOnly(input: KtxSqliteReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
+  async executeReadOnly(input: KtxSqliteReadOnlyQueryInput, ctx: KtxScanContext): Promise<KtxQueryResult> {
     this.assertConnection(input.connectionId);
-    const result = this.query(limitSqlForExecution(input.sql, input.maxRows), input.params);
+    // Validate and row-limit on the main thread so invalid SQL fails instantly
+    // without spawning a process and read-only enforcement stays at the boundary.
+    const sql = limitSqlForExecution(input.sql, input.maxRows);
+    const result = await this.runReadQueryOffProcess(sql, input.params, ctx.signal);
     return { ...result, rowCount: result.rows.length };
   }
 
+  // The LLM-SQL path runs off the event loop in a short-lived child process so a
+  // pathological scan cannot freeze the MCP server, and the deadline is enforced
+  // by SIGKILL-ing that process. A synchronous better-sqlite3 scan never yields,
+  // so a worker-thread terminate cannot interrupt it — only the OS reclaiming the
+  // whole process frees the CPU. One short-lived process per call; killed on
+  // completion, deadline, or external abort.
+  private runReadQueryOffProcess(
+    sql: string,
+    params: Record<string, unknown> | unknown[] | undefined,
+    signal: AbortSignal | undefined,
+  ): Promise<Omit<KtxQueryResult, 'rowCount'>> {
+    const deadlineMs = this.deadlineMs;
+    const dbPath = this.dbPath;
+    return new Promise((resolvePromise, rejectPromise) => {
+      const child = this.spawnReadQueryChild();
+      let settled = false;
+      const onDeadline = () => settle(() => rejectPromise(queryDeadlineExceededError(deadlineMs)));
+      const timer = setTimeout(onDeadline, deadlineMs);
+      function settle(finish: () => void): void {
+        if (settled) {
+          return;
+        }
+        settled = true;
+        clearTimeout(timer);
+        signal?.removeEventListener('abort', onDeadline);
+        if (child.exitCode === null && child.signalCode === null) {
+          child.kill('SIGKILL');
+        }
+        finish();
+      }
+      child.on('message', (message: ReadQueryChildMessage) => {
+        if (message.ok) {
+          settle(() =>
+            resolvePromise({
+              headers: message.headers,
+              rows: normalizeQueryRows(message.rows),
+              totalRows: message.totalRows,
+            }),
+          );
+        } else {
+          settle(() => rejectPromise(new Error(message.message)));
+        }
+      });
+      child.on('error', (error) => settle(() => rejectPromise(error)));
+      child.on('exit', (code, processSignal) => {
+        if (!settled) {
+          settle(() =>
+            rejectPromise(
+              new Error(`SQLite read process exited before returning a result (code ${code}, signal ${processSignal}).`),
+            ),
+          );
+        }
+      });
+      if (signal?.aborted) {
+        onDeadline();
+        return;
+      }
+      signal?.addEventListener('abort', onDeadline, { once: true });
+      try {
+        child.send({ dbPath, sql, params });
+      } catch (error) {
+        settle(() => rejectPromise(error instanceof Error ? error : new Error(String(error))));
+      }
+    });
+  }
+
   async getColumnDistinctValues(
     table: KtxTableRef,
     columnName: string,
@@ -310,16 +424,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
     const foreignKeys = database
       .prepare(`PRAGMA foreign_key_list(${this.dialect.quoteIdentifier(table.name)})`)
       .all() as SqliteForeignKeyRow[];
-    const estimatedRows =
-      table.type === 'table'
-        ? Number(
-            (
-              database
-                .prepare(`SELECT COUNT(*) AS count FROM ${this.dialect.quoteIdentifier(table.name)}`)
-                .get() as { count: unknown }
-            ).count,
-          )
-        : null;
+    const estimatedRows = table.type === 'table' ? this.readRowCount(database, table.name) : null;
     return {
       catalog: null,
       db: null,
@@ -340,6 +445,19 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
     };
   }
 
+  // A row-count read is profiling, not structure: a failure here leaves the
+  // object's structure intact rather than skipping the whole object.
+  private readRowCount(database: Database.Database, name: string): number | null {
+    try {
+      const row = database.prepare(`SELECT COUNT(*) AS count FROM ${this.dialect.quoteIdentifier(name)}`).get() as {
+        count: unknown;
+      };
+      return Number(row.count);
+    } catch {
+      return null;
+    }
+  }
+
   private mapForeignKeys(rows: SqliteForeignKeyRow[]): KtxSchemaForeignKey[] {
     return rows
       .sort((a, b) => a.id - b.id || a.seq - b.seq)

packages/cli/src/connectors/sqlite/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type SqliteTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxSqliteDialect implements KtxDialect {
+export class KtxSqliteDialect implements KtxSqlDialect {
   readonly type = 'sqlite' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/connectors/sqlite/read-query-child.ts

@@ -0,0 +1,40 @@
+import Database from 'better-sqlite3';
+
+// Runs on a forked child process (no bundler, no test transform), so it imports
+// only better-sqlite3 and node builtins. The SQL is already read-only-validated
+// and row-limited by the parent; this process just executes it and posts the
+// structured-cloneable raw rows back over IPC. Its only cancellation mechanism
+// is the parent sending SIGKILL: a synchronous better-sqlite3 scan never yields,
+// so neither a worker-thread terminate nor any in-process timer can interrupt
+// it — only the OS reclaiming the whole process can.
+
+interface ReadQueryRequest {
+  dbPath: string;
+  sql: string;
+  params?: Record<string, unknown> | unknown[];
+}
+
+type ReadQueryResponse =
+  | { ok: true; headers: string[]; rows: unknown[]; totalRows: number }
+  | { ok: false; message: string };
+
+process.once('message', (request: ReadQueryRequest) => {
+  let db: Database.Database | undefined;
+  let response: ReadQueryResponse;
+  try {
+    db = new Database(request.dbPath, { readonly: true, fileMustExist: true });
+    const statement = db.prepare(request.sql);
+    const rows = (request.params ? statement.all(request.params) : statement.all()) as unknown[];
+    response = {
+      ok: true,
+      headers: statement.columns().map((column) => column.name),
+      rows,
+      totalRows: rows.length,
+    };
+  } catch (error) {
+    response = { ok: false, message: error instanceof Error ? error.message : String(error) };
+  } finally {
+    db?.close();
+  }
+  process.send?.(response, () => process.exit(0));
+});

packages/cli/src/connectors/sqlserver/connector.ts

@@ -1,5 +1,6 @@
-import { assertReadOnlySql, stripTrailingSqlNoise } from '../../context/connections/read-only-sql.js';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { assertReadOnlySql, hoistLeadingCte, stripTrailingSqlNoise } from '../../context/connections/read-only-sql.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
 import { scopedTableNames } from '../../context/scan/table-ref.js';
 import {
@@ -50,6 +51,8 @@ export interface KtxSqlServerPoolConfig {
   database: string;
   user: string;
   password?: string;
+  // ms; on expiry mssql sends a TDS attention that cancels the query server-side.
+  requestTimeout: number;
   options: { encrypt: true; trustServerCertificate: boolean };
   pool: { max: number; min: number; idleTimeoutMillis: number };
 }
@@ -269,6 +272,11 @@ function isDeniedError(error: unknown): boolean {
   return number === 229 || number === 230 || number === 297;
 }
 
+// mssql raises a RequestError with code 'ETIMEOUT' once requestTimeout elapses.
+function isSqlServerTimeoutError(error: unknown): boolean {
+  return Boolean(error) && typeof error === 'object' && (error as { code?: unknown }).code === 'ETIMEOUT';
+}
+
 function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefined): string {
   const trimmed = stripTrailingSqlNoise(assertReadOnlySql(sqlText));
   if (!maxRows) {
@@ -277,7 +285,8 @@ function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefi
   if (!Number.isInteger(maxRows) || maxRows <= 0) {
     throw new Error('maxRows must be a positive integer.');
   }
-  return `SELECT TOP ${maxRows} * FROM (${trimmed}) AS ktx_query_result`;
+  const { withPrefix, body } = hoistLeadingCte(trimmed);
+  return `${withPrefix}SELECT TOP ${maxRows} * FROM (${body}) AS ktx_query_result`;
 }
 
 export function isKtxSqlServerConnectionConfig(
@@ -327,6 +336,7 @@ export function sqlServerConnectionPoolConfigFromConfig(input: {
     database,
     user,
     password: stringConfigValue(merged, 'password', env),
+    requestTimeout: resolveQueryDeadlineMs(merged),
     options: { encrypt: true, trustServerCertificate: merged.trustServerCertificate ?? true },
     pool: { max: maxConnections, min: 0, idleTimeoutMillis: 30000 },
   };
@@ -352,7 +362,8 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
   private readonly poolFactory: KtxSqlServerPoolFactory;
   private readonly endpointResolver?: KtxSqlServerEndpointResolver;
   private readonly now: () => Date;
-  private readonly dialect = getDialectForDriver('sqlserver');
+  private readonly deadlineMs: number;
+  private readonly dialect = getSqlDialectForDriver('sqlserver');
   private pool: KtxSqlServerPool | null = null;
   private resolvedEndpoint: KtxSqlServerResolvedEndpoint | null = null;
 
@@ -369,6 +380,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
     this.poolFactory = options.poolFactory ?? new DefaultSqlServerPoolFactory();
     this.endpointResolver = options.endpointResolver;
     this.now = options.now ?? (() => new Date());
+    this.deadlineMs = resolveQueryDeadlineMs(this.connection);
     this.id = `sqlserver:${options.connectionId}`;
   }
 
@@ -803,7 +815,15 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
         request.input(key, value);
       }
     }
-    const result = await request.query(assertReadOnlySql(query));
+    let result: KtxSqlServerQueryResult;
+    try {
+      result = await request.query(assertReadOnlySql(query));
+    } catch (error) {
+      if (isSqlServerTimeoutError(error)) {
+        throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
+      }
+      throw error;
+    }
     const recordset = result.recordset ?? [];
     const columnMetadata = recordset.columns ?? {};
     const metadataHeaders = Object.keys(columnMetadata);

packages/cli/src/connectors/sqlserver/dialect.ts

@@ -1,4 +1,4 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import {
   columnDisplayPartCount,
   formatDialectDisplayRef,
@@ -11,7 +11,7 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/typ
 type SqlServerTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 
 /** @internal */
-export class KtxSqlServerDialect implements KtxDialect {
+export class KtxSqlServerDialect implements KtxSqlDialect {
   readonly type = 'sqlserver' as const;
 
   private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {

packages/cli/src/context-build-view.ts

@@ -98,6 +98,7 @@ export interface ContextBuildArgs {
   queryHistory?: Extract<KtxPublicIngestArgs, { command: 'run' }>['queryHistory'];
   queryHistoryWindowDays?: number;
   scanMode?: Extract<KtxPublicIngestArgs, { command: 'run' }>['scanMode'];
+  stages?: Extract<KtxPublicIngestArgs, { command: 'run' }>['stages'];
   detectRelationships?: boolean;
   cliVersion?: string;
   runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
@@ -990,6 +991,7 @@ export async function runContextBuild(
     ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
     ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
     ...(args.scanMode ? { scanMode: args.scanMode } : {}),
+    ...(args.stages ? { stages: args.stages } : {}),
     ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
     ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
     ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),

packages/cli/src/context/cache/content-result-cache.ts

@@ -0,0 +1,64 @@
+import { createHash } from 'node:crypto';
+
+type ContentCacheMetadata = Record<string, unknown>;
+
+export interface ContentResultCacheLookup {
+  namespace: string;
+  scopeKey: string;
+  inputHash: string;
+}
+
+export interface ContentResultCacheCompleted<TOutput = unknown> extends ContentResultCacheLookup {
+  runId: string;
+  status: 'completed';
+  output: TOutput;
+  errorMessage: null;
+  metadata: ContentCacheMetadata;
+  updatedAt: string;
+}
+
+export interface ContentResultCacheFailed extends ContentResultCacheLookup {
+  runId: string;
+  status: 'failed';
+  output: null;
+  errorMessage: string;
+  metadata: ContentCacheMetadata;
+  updatedAt: string;
+}
+
+export type ContentResultCacheRecord<TOutput = unknown> =
+  | ContentResultCacheCompleted<TOutput>
+  | ContentResultCacheFailed;
+
+export interface ContentResultCache {
+  findCompletedResult<TOutput = unknown>(
+    input: ContentResultCacheLookup,
+  ): Promise<ContentResultCacheCompleted<TOutput> | null>;
+  findLatestCompletedResult(input: {
+    namespace: string;
+    scopeKey: string;
+  }): Promise<ContentResultCacheCompleted | null>;
+  saveCompletedResult<TOutput = unknown>(
+    input: Omit<ContentResultCacheCompleted<TOutput>, 'status' | 'errorMessage'>,
+  ): Promise<void>;
+  saveFailedResult(input: Omit<ContentResultCacheFailed, 'status' | 'output'>): Promise<void>;
+  deleteResult(input: ContentResultCacheLookup): Promise<void>;
+  listRunResults(runId: string): Promise<ContentResultCacheRecord[]>;
+}
+
+function stableJson(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `[${value.map(stableJson).join(',')}]`;
+  }
+  if (value && typeof value === 'object') {
+    const entries = Object.entries(value as Record<string, unknown>).sort(([left], [right]) =>
+      left.localeCompare(right),
+    );
+    return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(',')}}`;
+  }
+  return JSON.stringify(value) ?? 'undefined';
+}
+
+export function stableContentHash(value: unknown): string {
+  return createHash('sha256').update(stableJson(value)).digest('hex');
+}

packages/cli/src/context/cache/sqlite-content-result-cache.ts

@@ -0,0 +1,281 @@
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import Database from 'better-sqlite3';
+import type {
+  ContentResultCache,
+  ContentResultCacheCompleted,
+  ContentResultCacheFailed,
+  ContentResultCacheLookup,
+  ContentResultCacheRecord,
+} from './content-result-cache.js';
+
+export interface SqliteContentResultCacheOptions {
+  dbPath: string;
+}
+
+interface ResultRow {
+  run_id: string;
+  namespace: string;
+  scope_key: string;
+  input_hash: string;
+  status: 'completed' | 'failed';
+  output_json: string | null;
+  error_message: string | null;
+  metadata_json: string;
+  updated_at: string;
+}
+
+const RESULTS_TABLE = 'local_content_results';
+const RESULTS_PRIMARY_KEY = ['namespace', 'scope_key', 'input_hash'] as const;
+
+function isSafeRunId(runId: string): boolean {
+  return /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(runId);
+}
+
+function parseResultRow<TOutput = unknown>(row: ResultRow): ContentResultCacheRecord<TOutput> {
+  const base = {
+    runId: row.run_id,
+    namespace: row.namespace,
+    scopeKey: row.scope_key,
+    inputHash: row.input_hash,
+    metadata: JSON.parse(row.metadata_json || '{}') as Record<string, unknown>,
+    updatedAt: row.updated_at,
+  };
+  if (row.status === 'completed') {
+    return {
+      ...base,
+      status: 'completed',
+      output: JSON.parse(row.output_json ?? 'null') as TOutput,
+      errorMessage: null,
+    };
+  }
+  return {
+    ...base,
+    status: 'failed',
+    output: null,
+    errorMessage: row.error_message ?? 'Unknown content result failure',
+  };
+}
+
+export class SqliteContentResultCache implements ContentResultCache {
+  private readonly db: Database.Database;
+
+  constructor(options: SqliteContentResultCacheOptions) {
+    mkdirSync(dirname(options.dbPath), { recursive: true });
+    this.db = new Database(options.dbPath);
+    this.db.pragma('journal_mode = WAL');
+    this.db.exec('DROP TABLE IF EXISTS local_scan_enrichment_stages');
+    this.dropResultsTableIfPrimaryKeyDiffers();
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS local_content_results (
+        run_id TEXT NOT NULL,
+        namespace TEXT NOT NULL,
+        scope_key TEXT NOT NULL,
+        input_hash TEXT NOT NULL,
+        status TEXT NOT NULL,
+        output_json TEXT,
+        error_message TEXT,
+        metadata_json TEXT NOT NULL,
+        updated_at TEXT NOT NULL,
+        PRIMARY KEY (namespace, scope_key, input_hash)
+      );
+
+      CREATE INDEX IF NOT EXISTS local_content_results_lookup_idx
+        ON local_content_results (namespace, scope_key, input_hash, updated_at);
+      CREATE INDEX IF NOT EXISTS local_content_results_run_idx
+        ON local_content_results (run_id, updated_at, namespace);
+    `);
+  }
+
+  private dropResultsTableIfPrimaryKeyDiffers(): void {
+    const columns = this.db.prepare(`PRAGMA table_info(${RESULTS_TABLE})`).all() as Array<{
+      name: string;
+      pk: number;
+    }>;
+    if (columns.length === 0) {
+      return;
+    }
+    const primaryKey = columns
+      .filter((column) => column.pk > 0)
+      .sort((left, right) => left.pk - right.pk)
+      .map((column) => column.name);
+    const matches =
+      primaryKey.length === RESULTS_PRIMARY_KEY.length &&
+      primaryKey.every((name, index) => name === RESULTS_PRIMARY_KEY[index]);
+    if (!matches) {
+      this.db.exec(`DROP TABLE ${RESULTS_TABLE}`);
+    }
+  }
+
+  async findCompletedResult<TOutput = unknown>(
+    input: ContentResultCacheLookup,
+  ): Promise<ContentResultCacheCompleted<TOutput> | null> {
+    const row = this.db
+      .prepare(
+        `
+        SELECT *
+        FROM local_content_results
+        WHERE namespace = ?
+          AND scope_key = ?
+          AND input_hash = ?
+          AND status = 'completed'
+        ORDER BY updated_at DESC
+        LIMIT 1
+      `,
+      )
+      .get(input.namespace, input.scopeKey, input.inputHash) as ResultRow | undefined;
+    if (!row) {
+      return null;
+    }
+    const parsed = parseResultRow<TOutput>(row);
+    return parsed.status === 'completed' ? parsed : null;
+  }
+
+  async findLatestCompletedResult(input: {
+    namespace: string;
+    scopeKey: string;
+  }): Promise<ContentResultCacheCompleted | null> {
+    const row = this.db
+      .prepare(
+        `
+        SELECT *
+        FROM local_content_results
+        WHERE namespace = ?
+          AND scope_key = ?
+          AND status = 'completed'
+        ORDER BY updated_at DESC
+        LIMIT 1
+      `,
+      )
+      .get(input.namespace, input.scopeKey) as ResultRow | undefined;
+    if (!row) {
+      return null;
+    }
+    const parsed = parseResultRow(row);
+    return parsed.status === 'completed' ? parsed : null;
+  }
+
+  async saveCompletedResult<TOutput = unknown>(
+    input: Omit<ContentResultCacheCompleted<TOutput>, 'status' | 'errorMessage'>,
+  ): Promise<void> {
+    this.db
+      .prepare(
+        `
+        INSERT INTO local_content_results (
+          run_id,
+          namespace,
+          scope_key,
+          input_hash,
+          status,
+          output_json,
+          error_message,
+          metadata_json,
+          updated_at
+        )
+        VALUES (
+          @runId,
+          @namespace,
+          @scopeKey,
+          @inputHash,
+          'completed',
+          @outputJson,
+          NULL,
+          @metadataJson,
+          @updatedAt
+        )
+        ON CONFLICT(namespace, scope_key, input_hash) DO UPDATE SET
+          run_id = excluded.run_id,
+          status = excluded.status,
+          output_json = excluded.output_json,
+          error_message = excluded.error_message,
+          metadata_json = excluded.metadata_json,
+          updated_at = excluded.updated_at
+      `,
+      )
+      .run({
+        runId: input.runId,
+        namespace: input.namespace,
+        scopeKey: input.scopeKey,
+        inputHash: input.inputHash,
+        outputJson: JSON.stringify(input.output),
+        metadataJson: JSON.stringify(input.metadata),
+        updatedAt: input.updatedAt,
+      });
+  }
+
+  async saveFailedResult(input: Omit<ContentResultCacheFailed, 'status' | 'output'>): Promise<void> {
+    this.db
+      .prepare(
+        `
+        INSERT INTO local_content_results (
+          run_id,
+          namespace,
+          scope_key,
+          input_hash,
+          status,
+          output_json,
+          error_message,
+          metadata_json,
+          updated_at
+        )
+        VALUES (
+          @runId,
+          @namespace,
+          @scopeKey,
+          @inputHash,
+          'failed',
+          NULL,
+          @errorMessage,
+          @metadataJson,
+          @updatedAt
+        )
+        ON CONFLICT(namespace, scope_key, input_hash) DO UPDATE SET
+          run_id = excluded.run_id,
+          status = excluded.status,
+          output_json = excluded.output_json,
+          error_message = excluded.error_message,
+          metadata_json = excluded.metadata_json,
+          updated_at = excluded.updated_at
+      `,
+      )
+      .run({
+        runId: input.runId,
+        namespace: input.namespace,
+        scopeKey: input.scopeKey,
+        inputHash: input.inputHash,
+        errorMessage: input.errorMessage,
+        metadataJson: JSON.stringify(input.metadata),
+        updatedAt: input.updatedAt,
+      });
+  }
+
+  async deleteResult(input: ContentResultCacheLookup): Promise<void> {
+    this.db
+      .prepare(
+        `
+        DELETE FROM local_content_results
+        WHERE namespace = ?
+          AND scope_key = ?
+          AND input_hash = ?
+      `,
+      )
+      .run(input.namespace, input.scopeKey, input.inputHash);
+  }
+
+  async listRunResults(runId: string): Promise<ContentResultCacheRecord[]> {
+    if (!isSafeRunId(runId)) {
+      return [];
+    }
+    const rows = this.db
+      .prepare(
+        `
+        SELECT *
+        FROM local_content_results
+        WHERE run_id = ?
+        ORDER BY updated_at ASC, namespace ASC
+      `,
+      )
+      .all(runId) as ResultRow[];
+    return rows.map((row) => parseResultRow(row));
+  }
+}

packages/cli/src/context/connections/bigquery-identifiers.ts

@@ -1,4 +1,5 @@
 const BIGQUERY_PROJECT_ID_PATTERN = /^[A-Za-z0-9_-]+$/;
+const BIGQUERY_DATASET_ID_PATTERN = /^[A-Za-z0-9_]+$/;
 const BIGQUERY_REGION_PATTERN = /^[a-z0-9-]+$/;
 
 export function normalizeBigQueryProjectId(value: string, context = 'historic-SQL ingest'): string {
@@ -8,6 +9,13 @@ export function normalizeBigQueryProjectId(value: string, context = 'historic-SQ
   return value;
 }
 
+export function normalizeBigQueryDatasetId(value: string, context = 'historic-SQL ingest'): string {
+  if (!BIGQUERY_DATASET_ID_PATTERN.test(value)) {
+    throw new Error(`Invalid BigQuery dataset id for ${context}: ${value}`);
+  }
+  return value;
+}
+
 export function normalizeBigQueryRegion(value: string, context = 'historic-SQL ingest'): string {
   const normalized = value.trim().toLowerCase().replace(/^region-/, '');
   if (!BIGQUERY_REGION_PATTERN.test(normalized)) {

packages/cli/src/context/connections/configured-connections.ts

@@ -0,0 +1,24 @@
+import type { KtxProjectConnectionConfig } from '../project/config.js';
+
+function listConfiguredConnectionIds(connections: Record<string, KtxProjectConnectionConfig>): string[] {
+  return Object.keys(connections).sort();
+}
+
+/**
+ * Validate a connection id supplied as an explicit command/tool argument against
+ * the canonical `ktx.yaml` connections map. Returns the id when configured;
+ * otherwise throws an error that lists the configured ids so the caller can fix
+ * the typo. Use for explicit arguments only — persisted page frontmatter that
+ * references a since-removed connection must warn, not fail.
+ */
+export function assertConfiguredConnectionId(
+  connections: Record<string, KtxProjectConnectionConfig>,
+  connectionId: string,
+): string {
+  if (Object.hasOwn(connections, connectionId)) {
+    return connectionId;
+  }
+  const ids = listConfiguredConnectionIds(connections);
+  const configured = ids.length > 0 ? ids.join(', ') : '(none configured)';
+  throw new Error(`Unknown connection "${connectionId}". Configured connections: ${configured}.`);
+}

packages/cli/src/context/connections/connection-type-dialect.ts

@@ -0,0 +1,24 @@
+import type { ConnectionType } from './connection-type.js';
+
+const CONNECTION_TYPE_TO_SQLGLOT = {
+  POSTGRESQL: 'postgres',
+  SQLITE: 'sqlite',
+  DUCKDB: 'duckdb',
+  SQLSERVER: 'tsql',
+  BIGQUERY: 'bigquery',
+  SNOWFLAKE: 'snowflake',
+  MYSQL: 'mysql',
+  CLICKHOUSE: 'clickhouse',
+  ATHENA: 'athena',
+  METABASE: null,
+  LOOKER: null,
+  NOTION: null,
+} satisfies Record<ConnectionType, string | null>;
+
+export function dialectForConnectionType(connectionType: string): string {
+  return CONNECTION_TYPE_TO_SQLGLOT[connectionType.toUpperCase() as ConnectionType] ?? 'postgres';
+}
+
+export function warehouseTargetDialect(connectionType: string): string | null {
+  return CONNECTION_TYPE_TO_SQLGLOT[connectionType.toUpperCase() as ConnectionType] ?? null;
+}

packages/cli/src/context/connections/connection-type.ts

@@ -3,25 +3,16 @@ import { z } from 'zod';
 export const connectionTypeSchema = z.enum([
   'POSTGRESQL',
   'SQLITE',
+  'DUCKDB',
   'SQLSERVER',
   'BIGQUERY',
   'SNOWFLAKE',
-  'CENTRALREACH',
-  'EPIC',
-  'CERNER',
   'ATHENA',
-  'QUICKBOOKS',
-  'WORKDAY',
-  'REST',
-  'S3',
-  'SLACK',
   'METABASE',
   'LOOKER',
   'NOTION',
   'MYSQL',
   'CLICKHOUSE',
-  'PLAIN',
-  'BETTERSTACK',
 ]);
 
 export type ConnectionType = z.infer<typeof connectionTypeSchema>;

packages/cli/src/context/connections/dialects.ts

@@ -1,20 +1,40 @@
+import { KtxAthenaDialect } from '../../connectors/athena/dialect.js';
 import { KtxBigQueryDialect } from '../../connectors/bigquery/dialect.js';
 import { KtxClickHouseDialect } from '../../connectors/clickhouse/dialect.js';
+import { KtxDuckDbDialect } from '../../connectors/duckdb/dialect.js';
+import { KtxMongoDbDialect } from '../../connectors/mongodb/dialect.js';
 import { KtxMysqlDialect } from '../../connectors/mysql/dialect.js';
 import { KtxPostgresDialect } from '../../connectors/postgres/dialect.js';
 import { KtxSqliteDialect } from '../../connectors/sqlite/dialect.js';
 import { KtxSnowflakeDialect } from '../../connectors/snowflake/dialect.js';
 import { KtxSqlServerDialect } from '../../connectors/sqlserver/dialect.js';
+import { KtxExpectedError } from '../../errors.js';
 import type { KtxConnectionDriver, KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js';
 import type { KtxDialectTableRef } from './dialect-helpers.js';
 
+/**
+ * Driver-agnostic dialect surface every connection implements, including
+ * non-SQL sources like MongoDB: display/ref formatting and type mapping. The
+ * catalog and entity-details paths resolve this for any snapshot driver, so it
+ * must stay free of SQL generation.
+ */
 export interface KtxDialect {
   readonly type: KtxConnectionDriver;
-  quoteIdentifier(identifier: string): string;
-  formatTableName(table: KtxDialectTableRef): string;
   formatDisplayRef(table: KtxDialectTableRef): string;
   parseDisplayRef(display: string): KtxTableRef | null;
   columnDisplayTablePartCount(): 1 | 2 | 3;
+  mapToDimensionType(nativeType: string): KtxSchemaDimensionType;
+  mapDataType(nativeType: string): string;
+}
+
+/**
+ * SQL query generation, implemented only by SQL warehouse drivers. The relationship
+ * profiling/validation pipeline is the sole caller and is gated on the
+ * `readOnlySql` capability, so these methods are unreachable for a non-SQL source.
+ */
+export interface KtxSqlDialect extends KtxDialect {
+  quoteIdentifier(identifier: string): string;
+  formatTableName(table: KtxDialectTableRef): string;
   getLimitOffsetClause(limit: number, offset?: number): string;
   getTopClause(limit: number): string;
   getRandomSampleFilter(samplePct: number): string;
@@ -30,23 +50,15 @@ export interface KtxDialect {
   getDistinctCountExpression(column: string): string;
   textLengthExpression(columnSql: string): string;
   castToText(columnSql: string): string;
-  mapToDimensionType(nativeType: string): KtxSchemaDimensionType;
-  mapDataType(nativeType: string): string;
 }
 
-const supportedDrivers: KtxConnectionDriver[] = [
-  'bigquery',
-  'clickhouse',
-  'mysql',
-  'postgres',
-  'sqlite',
-  'snowflake',
-  'sqlserver',
-];
+type KtxSqlDriver = Exclude<KtxConnectionDriver, 'mongodb'>;
 
-const dialectFactories: Record<KtxConnectionDriver, () => KtxDialect> = {
+const sqlDialectFactories: Record<KtxSqlDriver, () => KtxSqlDialect> = {
+  athena: () => new KtxAthenaDialect(),
   bigquery: () => new KtxBigQueryDialect(),
   clickhouse: () => new KtxClickHouseDialect(),
+  duckdb: () => new KtxDuckDbDialect(),
   mysql: () => new KtxMysqlDialect(),
   postgres: () => new KtxPostgresDialect(),
   sqlite: () => new KtxSqliteDialect(),
@@ -54,11 +66,54 @@ const dialectFactories: Record<KtxConnectionDriver, () => KtxDialect> = {
   sqlserver: () => new KtxSqlServerDialect(),
 };
 
+const dialectFactories: Record<KtxConnectionDriver, () => KtxDialect> = {
+  ...sqlDialectFactories,
+  mongodb: () => new KtxMongoDbDialect(),
+};
+
+const supportedSqlDrivers = Object.keys(sqlDialectFactories).sort();
+
 export function getDialectForDriver(driver: string): KtxDialect {
   const normalized = driver.toLowerCase().trim();
   const factory = dialectFactories[normalized as KtxConnectionDriver];
   if (factory) {
     return factory();
   }
-  throw new Error(`Unsupported warehouse driver "${driver}". Supported drivers: ${supportedDrivers.join(', ')}`);
+  throw new Error(
+    `Unsupported driver "${driver}". Supported drivers: ${Object.keys(dialectFactories).sort().join(', ')}`,
+  );
+}
+
+export function getSqlDialectForDriver(driver: string): KtxSqlDialect {
+  const normalized = driver.toLowerCase().trim();
+  const factory = sqlDialectFactories[normalized as KtxSqlDriver];
+  if (factory) {
+    return factory();
+  }
+  throw new Error(`Driver "${driver}" has no SQL dialect. SQL drivers: ${supportedSqlDrivers.join(', ')}`);
+}
+
+/**
+ * Whether a driver can generate and execute SQL. Single source of truth for the
+ * SQL/non-SQL boundary: a driver is SQL-queryable iff it has a SQL dialect, so
+ * non-SQL sources (e.g. mongodb) are excluded without a hand-maintained list.
+ */
+export function isSqlQueryableDriver(driver: string | undefined): boolean {
+  const normalized = (driver ?? '').toLowerCase().trim();
+  return Object.prototype.hasOwnProperty.call(sqlDialectFactories, normalized);
+}
+
+/**
+ * Refuse a non-SQL connection (e.g. mongodb) at a read-only-SQL entry point before
+ * any dialect selection or parser/daemon work, so it is never validated as Postgres.
+ * The federated `duckdb` connection has no driver — callers skip this guard for it.
+ */
+export function assertSqlQueryableConnection(connectionId: string, driver: string | undefined): void {
+  if (!isSqlQueryableDriver(driver)) {
+    throw new KtxExpectedError(
+      `Connection '${connectionId}' uses the non-SQL driver '${driver ?? 'unknown'}'. ` +
+        'Read-only SQL (ktx sql, the sql_execution tool) requires a SQL warehouse connection; ' +
+        'MongoDB and other context-only sources are searchable and ingestable, not SQL-queryable.',
+    );
+  }
 }

packages/cli/src/context/connections/drivers.ts

@@ -26,6 +26,23 @@ function invalidConnectionConfig(driver: KtxConnectionDriver): Error {
 
 /** @internal */
 export const driverRegistrations: Record<KtxConnectionDriver, KtxDriverRegistration> = {
+  athena: {
+    driver: 'athena',
+    scopeConfigKey: 'databases',
+    hasHistoricSqlReader: false,
+    load: async () => {
+      const m = await import('../../connectors/athena/connector.js');
+      return {
+        isConnectionConfig: (connection) => m.isKtxAthenaConnectionConfig(connection),
+        createScanConnector: ({ connectionId, connection }) => {
+          if (!m.isKtxAthenaConnectionConfig(connection)) {
+            throw invalidConnectionConfig('athena');
+          }
+          return new m.KtxAthenaScanConnector({ connectionId, connection });
+        },
+      };
+    },
+  },
   bigquery: {
     driver: 'bigquery',
     scopeConfigKey: 'dataset_ids',
@@ -68,6 +85,48 @@ export const driverRegistrations: Record<KtxConnectionDriver, KtxDriverRegistrat
       };
     },
   },
+  duckdb: {
+    driver: 'duckdb',
+    scopeConfigKey: null,
+    hasHistoricSqlReader: false,
+    load: async () => {
+      const m = await import('../../connectors/duckdb/connector.js');
+      return {
+        isConnectionConfig: (connection) => {
+          const typedConnection = connection as Parameters<typeof m.isKtxDuckDbConnectionConfig>[0];
+          return m.isKtxDuckDbConnectionConfig(typedConnection);
+        },
+        createScanConnector: ({ connectionId, connection, projectDir }) => {
+          const typedConnection = connection as Parameters<typeof m.isKtxDuckDbConnectionConfig>[0];
+          if (!m.isKtxDuckDbConnectionConfig(typedConnection)) {
+            throw invalidConnectionConfig('duckdb');
+          }
+          return new m.KtxDuckDbScanConnector({ connectionId, connection: typedConnection, projectDir });
+        },
+      };
+    },
+  },
+  mongodb: {
+    driver: 'mongodb',
+    scopeConfigKey: 'databases',
+    hasHistoricSqlReader: false,
+    load: async () => {
+      const m = await import('../../connectors/mongodb/connector.js');
+      return {
+        isConnectionConfig: (connection) => {
+          const typedConnection = connection as Parameters<typeof m.isKtxMongoDbConnectionConfig>[0];
+          return m.isKtxMongoDbConnectionConfig(typedConnection);
+        },
+        createScanConnector: ({ connectionId, connection }) => {
+          const typedConnection = connection as Parameters<typeof m.isKtxMongoDbConnectionConfig>[0];
+          if (!m.isKtxMongoDbConnectionConfig(typedConnection)) {
+            throw invalidConnectionConfig('mongodb');
+          }
+          return new m.KtxMongoDbScanConnector({ connectionId, connection: typedConnection });
+        },
+      };
+    },
+  },
   mysql: {
     driver: 'mysql',
     scopeConfigKey: 'schemas',

packages/cli/src/context/connections/federation.ts

@@ -4,18 +4,19 @@ import type { KtxProjectConnectionConfig } from '../project/config.js';
 export const FEDERATED_CONNECTION_ID = '_ktx_federated';
 
 /**
- * Drivers DuckDB can ATTACH for federation. The driver name doubles as the
- * DuckDB extension/TYPE name, so this set is the single source of truth for
- * both membership (a driver participates iff it appears here) and attach type.
+ * Drivers DuckDB can ATTACH for federation. Membership is governed by this set;
+ * the attach TYPE is governed by attachTypeForDriver, which returns the driver
+ * name for extension-backed engines and null for a native DuckDB file (attached
+ * with no INSTALL/LOAD and no TYPE).
  */
-const ATTACH_COMPATIBLE_DRIVERS = new Set(['postgres', 'mysql', 'sqlite']);
+const ATTACH_COMPATIBLE_DRIVERS = new Set(['postgres', 'mysql', 'sqlite', 'duckdb']);
 
-export function attachTypeForDriver(driver: string): string {
+export function attachTypeForDriver(driver: string): string | null {
   const normalized = driver.toLowerCase();
   if (!ATTACH_COMPATIBLE_DRIVERS.has(normalized)) {
     throw new Error(`Driver "${driver}" cannot be attached by DuckDB federation.`);
   }
-  return normalized;
+  return normalized === 'duckdb' ? null : normalized;
 }
 
 export interface FederatedMember {

packages/cli/src/context/connections/gdrive-config.ts

@@ -0,0 +1,87 @@
+import { readFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+import type { KtxProjectConnectionConfig } from '../project/config.js';
+import type { GdrivePullConfig } from '../ingest/adapters/gdrive/types.js';
+import { gdrivePullConfigSchema } from '../ingest/adapters/gdrive/types.js';
+
+type RawKtxGdriveConnectionConfig = Extract<KtxProjectConnectionConfig, { driver: 'gdrive' }>;
+
+export type KtxGdriveConnectionConfig = Omit<
+  RawKtxGdriveConnectionConfig,
+  'service_account_key_ref' | 'folder_id' | 'recursive'
+> & {
+  driver: 'gdrive';
+  service_account_key_ref: string;
+  folder_id: string;
+  recursive: boolean;
+};
+
+interface ResolveKeyOptions {
+  readTextFile?: (path: string) => Promise<string>;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function expandHome(path: string): string {
+  return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path;
+}
+
+export function parseGdriveConnectionConfig(raw: unknown): KtxGdriveConnectionConfig {
+  if (!isRecord(raw)) {
+    throw new Error('gdrive connection config must be an object');
+  }
+  if (raw.driver !== 'gdrive') {
+    throw new Error('gdrive connection config requires driver: gdrive');
+  }
+  const keyRef =
+    typeof raw.service_account_key_ref === 'string' && raw.service_account_key_ref.trim().length > 0 // pragma: allowlist secret
+      ? raw.service_account_key_ref.trim()
+      : null;
+  if (!keyRef) {
+    throw new Error('gdrive connection config requires service_account_key_ref');
+  }
+  if (!keyRef.startsWith('file:')) {
+    throw new Error('gdrive service_account_key_ref must use file:/path/to/key.json');
+  }
+  const folderId = typeof raw.folder_id === 'string' && raw.folder_id.trim().length > 0 ? raw.folder_id.trim() : null;
+  if (!folderId) {
+    throw new Error('gdrive connection config requires folder_id');
+  }
+  return {
+    driver: 'gdrive',
+    service_account_key_ref: keyRef,
+    folder_id: folderId,
+    recursive: raw.recursive === true,
+  };
+}
+
+/** @internal */
+export async function resolveGdriveServiceAccountKey(
+  serviceAccountKeyRef: string,
+  options: ResolveKeyOptions = {},
+): Promise<string> {
+  if (!serviceAccountKeyRef.startsWith('file:')) {
+    throw new Error('gdrive service_account_key_ref must use file:/path/to/key.json');
+  }
+  const path = expandHome(serviceAccountKeyRef.slice('file:'.length));
+  const readTextFile = options.readTextFile ?? ((filePath: string) => readFile(filePath, 'utf-8'));
+  const value = (await readTextFile(path)).trim();
+  if (!value) {
+    throw new Error(`gdrive service account key file is empty: ${path}`);
+  }
+  return value;
+}
+
+export async function gdriveConnectionToPullConfig(
+  config: KtxGdriveConnectionConfig,
+  options: ResolveKeyOptions = {},
+): Promise<GdrivePullConfig> {
+  return gdrivePullConfigSchema.parse({
+    serviceAccountKey: await resolveGdriveServiceAccountKey(config.service_account_key_ref, options),
+    folderId: config.folder_id,
+    recursive: config.recursive,
+  });
+}

packages/cli/src/context/connections/local-warehouse-descriptor.ts

@@ -1,5 +1,6 @@
 import type { KtxProjectConnectionConfig } from '../project/config.js';
 import type { ConnectionType } from './connection-type.js';
+import { connectionQueryPolicy } from './query-policy.js';
 
 export interface LocalWarehouseDescriptor {
   id: string;
@@ -18,16 +19,19 @@ export interface LocalConnectionInfo {
   connectionType: string;
   members?: string[];
   hint?: string;
+  queryPolicy?: 'semantic-layer-only';
 }
 
 const DRIVER_TO_CONNECTION_TYPE: Record<string, ConnectionType> = {
   postgres: 'POSTGRESQL',
   sqlite: 'SQLITE',
+  duckdb: 'DUCKDB',
   sqlserver: 'SQLSERVER',
   mysql: 'MYSQL',
   clickhouse: 'CLICKHOUSE',
   snowflake: 'SNOWFLAKE',
   bigquery: 'BIGQUERY',
+  athena: 'ATHENA',
 };
 
 export function localConnectionToWarehouseDescriptor(
@@ -90,10 +94,17 @@ export function localConnectionInfoFromConfig(
   if (!connection) {
     return null;
   }
+  const restricted = connectionQueryPolicy(connection) === 'semantic-layer-only';
   return {
     id,
     name: id,
     connectionType: localConnectionTypeForConfig(id, connection),
+    ...(restricted
+      ? {
+          queryPolicy: 'semantic-layer-only' as const,
+          hint: 'Raw SQL is disabled on this connection; query it with sl_query using predefined measures.',
+        }
+      : {}),
   };
 }
 

packages/cli/src/context/connections/project-sql-executor.ts

@@ -1,8 +1,15 @@
 import { executeFederatedQuery } from '../../connectors/duckdb/federated-executor.js';
+import { KtxExpectedError, KtxQueryError, isNativeProgrammingFault } from '../../errors.js';
+import { sqlAnalysisDialectForDriver } from '../sql-analysis/dialect.js';
+import type { SqlAnalysisPort } from '../sql-analysis/ports.js';
+import { assertSafeConnectionId } from '../sl/source-files.js';
 import type { KtxLocalProject } from '../project/project.js';
 import type { KtxScanConnector, KtxScanContext } from '../scan/types.js';
+import { assertSqlQueryableConnection } from './dialects.js';
 import { deriveFederatedConnection, FEDERATED_CONNECTION_ID } from './federation.js';
+import { assertRawSqlAllowed } from './query-policy.js';
 import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutionResult } from './query-executor.js';
+import { resolveConfiguredConnection } from './resolve-connection.js';
 
 export interface ExecuteProjectReadOnlySqlDeps {
   project: KtxLocalProject;
@@ -56,3 +63,71 @@ export async function executeProjectReadOnlySql(
     await connector?.cleanup?.();
   }
 }
+
+type RawSqlProgressCallback = (event: { progress: number; message: string }) => void | Promise<void>;
+
+export interface ExecuteProjectRawSqlDeps {
+  project: KtxLocalProject;
+  connectionId: string;
+  sql: string;
+  maxRows: number;
+  sqlAnalysis: SqlAnalysisPort;
+  createConnector: (connectionId: string) => Promise<KtxScanConnector> | KtxScanConnector;
+  executeFederated?: typeof executeFederatedQuery;
+  runId: string;
+  onProgress?: RawSqlProgressCallback;
+}
+
+/**
+ * Single guarded path for user-authored (raw) SQL — `ktx sql` and the MCP
+ * sql_execution tool. Enforces the connection's query_policy and the parser
+ * read-only guard before executing; ktx-internal SQL (semantic-layer, ingest)
+ * calls executeProjectReadOnlySql directly and is not subject to query_policy.
+ */
+export async function executeProjectRawSql(deps: ExecuteProjectRawSqlDeps): Promise<KtxSqlQueryExecutionResult> {
+  const { project } = deps;
+  await deps.onProgress?.({ progress: 0, message: 'Validating SQL' });
+
+  const isFederated = deps.connectionId === FEDERATED_CONNECTION_ID;
+  const connectionId = isFederated ? deps.connectionId : assertSafeConnectionId(deps.connectionId);
+  const connection = isFederated ? undefined : resolveConfiguredConnection(project.config, connectionId);
+  if (!isFederated) {
+    assertSqlQueryableConnection(connectionId, connection!.driver);
+  }
+  assertRawSqlAllowed(project.config, project.projectDir, connectionId);
+
+  const dialect = sqlAnalysisDialectForDriver(isFederated ? 'duckdb' : connection!.driver);
+  const validation = await deps.sqlAnalysis.validateReadOnly(deps.sql, dialect);
+  if (!validation.ok) {
+    // A read-only guard rejecting the caller's SQL is an expected outcome, not a
+    // ktx fault: classify it so reportException keeps it out of Error Tracking.
+    throw new KtxQueryError(validation.error ?? 'SQL is not read-only.');
+  }
+
+  await deps.onProgress?.({ progress: 0.3, message: 'Executing' });
+  const result = await executeProjectReadOnlySql({
+    project,
+    input: {
+      connectionId,
+      projectDir: project.projectDir,
+      connection,
+      sql: deps.sql,
+      maxRows: deps.maxRows,
+    },
+    createConnector: deps.createConnector,
+    executeFederated: deps.executeFederated,
+    runId: deps.runId,
+  }).catch((error: unknown) => {
+    // A warehouse/driver rejection (e.g. the caller's SQL failed to compile) is a
+    // surfaced operational outcome, not a ktx fault: mark it expected while
+    // preserving the warehouse's own diagnostics. A native JS error (TypeError,
+    // etc.) signals a bug in connector code — let it propagate unchanged so Error
+    // Tracking still sees it.
+    if (isNativeProgrammingFault(error) || error instanceof KtxExpectedError) {
+      throw error;
+    }
+    throw new KtxQueryError(error instanceof Error ? error.message : String(error), { cause: error });
+  });
+  await deps.onProgress?.({ progress: 1, message: `Fetched ${result.rowCount ?? result.rows.length} rows` });
+  return result;
+}

packages/cli/src/context/connections/query-deadline.ts

@@ -0,0 +1,45 @@
+import { KtxQueryError } from '../../errors.js';
+
+/**
+ * Canonical default bound on read-query execution time. Generous headroom over
+ * any indexed aggregate or normal profiling probe; a pathological nested-loop
+ * scan blows past it immediately. Overridable per-connection via
+ * `query_timeout_ms`. Production reads it through {@link resolveQueryDeadlineMs};
+ * exported for the resolver's own unit tests.
+ * @internal
+ */
+export const DEFAULT_QUERY_TIMEOUT_MS = 30_000;
+
+interface QueryTimeoutConnectionConfig {
+  query_timeout_ms?: unknown;
+  [key: string]: unknown;
+}
+
+/**
+ * Single source of truth for the read-query deadline: the per-connection
+ * `query_timeout_ms` override (milliseconds) when present, else the default.
+ * Every connector resolves through here so the default and override precedence
+ * live in exactly one place. A malformed override (zero, negative, non-integer,
+ * non-number) is a config error — surfaced here even though `ktx.yaml`
+ * validation also rejects it, so programmatically-built connectors cannot
+ * silently run unbounded.
+ */
+export function resolveQueryDeadlineMs(connection: QueryTimeoutConnectionConfig | undefined): number {
+  const raw = connection?.query_timeout_ms;
+  if (raw === undefined || raw === null) {
+    return DEFAULT_QUERY_TIMEOUT_MS;
+  }
+  if (typeof raw !== 'number' || !Number.isInteger(raw) || raw <= 0) {
+    throw new Error(`query_timeout_ms must be a positive integer in milliseconds, received ${JSON.stringify(raw)}.`);
+  }
+  return raw;
+}
+
+/**
+ * The canonical, driver-independent timeout error an agent sees regardless of
+ * which connector enforced the deadline. Reads in whole seconds. Remote
+ * connectors pass the driver's own timeout error as `cause`.
+ */
+export function queryDeadlineExceededError(deadlineMs: number, options?: ErrorOptions): KtxQueryError {
+  return new KtxQueryError(`query exceeded ${Math.round(deadlineMs / 1000)}s`, options);
+}

packages/cli/src/context/connections/query-policy.ts

@@ -0,0 +1,61 @@
+import { KtxQueryError } from '../../errors.js';
+import type { KtxProjectConfig, KtxProjectConnectionConfig } from '../project/config.js';
+import { deriveFederatedConnection, FEDERATED_CONNECTION_ID } from './federation.js';
+import { isSqlQueryableDriver } from './dialects.js';
+
+export type KtxConnectionQueryPolicy = 'read-only-sql' | 'semantic-layer-only';
+
+export function connectionQueryPolicy(
+  connection: KtxProjectConnectionConfig | undefined,
+): KtxConnectionQueryPolicy {
+  return connection !== undefined && connection.query_policy === 'semantic-layer-only'
+    ? 'semantic-layer-only'
+    : 'read-only-sql';
+}
+
+/** Member ids whose policy blocks raw SQL through the federated connection. */
+export function restrictedFederatedMemberIds(config: KtxProjectConfig, projectDir: string): string[] {
+  const descriptor = deriveFederatedConnection(config.connections, projectDir);
+  if (!descriptor) {
+    return [];
+  }
+  return descriptor.members
+    .filter((member) => connectionQueryPolicy(member.connection) === 'semantic-layer-only')
+    .map((member) => member.connectionId);
+}
+
+export function assertRawSqlAllowed(config: KtxProjectConfig, projectDir: string, connectionId: string): void {
+  if (connectionId === FEDERATED_CONNECTION_ID) {
+    const restricted = restrictedFederatedMemberIds(config, projectDir);
+    if (restricted.length > 0) {
+      throw new KtxQueryError(
+        `Federated SQL execution is disabled: member connection(s) ${restricted
+          .map((id) => `"${id}"`)
+          .join(', ')} are restricted to semantic-layer queries (query_policy: semantic-layer-only in ktx.yaml).`,
+      );
+    }
+    return;
+  }
+  if (connectionQueryPolicy(config.connections[connectionId]) === 'semantic-layer-only') {
+    throw new KtxQueryError(
+      `Connection "${connectionId}" is restricted to semantic-layer queries (query_policy: semantic-layer-only in ktx.yaml); ` +
+        'raw SQL execution is disabled. Query it through the semantic layer with predefined measures instead ' +
+        '(the sl_query tool or `ktx sl query`).',
+    );
+  }
+}
+
+/**
+ * False only when the project has SQL-queryable connections and every one of
+ * them is restricted — then no raw-SQL surface can succeed and the
+ * sql_execution tool should not be offered at all.
+ */
+export function projectAllowsRawSql(config: KtxProjectConfig): boolean {
+  const sqlConnections = Object.values(config.connections).filter((connection) =>
+    isSqlQueryableDriver(connection.driver),
+  );
+  if (sqlConnections.length === 0) {
+    return true;
+  }
+  return sqlConnections.some((connection) => connectionQueryPolicy(connection) === 'read-only-sql');
+}

packages/cli/src/context/connections/read-only-sql.ts

@@ -97,6 +97,161 @@ export function assertReadOnlySql(sql: string): string {
   return trimmed;
 }
 
+function isSqlIdentifierPart(char: string | undefined): boolean {
+  return char !== undefined && /[A-Za-z0-9_$]/.test(char);
+}
+
+function keywordAt(sql: string, index: number, keyword: string): boolean {
+  if (sql.slice(index, index + keyword.length).toLowerCase() !== keyword.toLowerCase()) {
+    return false;
+  }
+  return !isSqlIdentifierPart(sql[index - 1]) && !isSqlIdentifierPart(sql[index + keyword.length]);
+}
+
+function skipWhitespaceAndComments(sql: string, index: number): number {
+  let current = index;
+  while (current < sql.length) {
+    while (/\s/.test(sql[current] ?? '')) {
+      current += 1;
+    }
+    if (sql.startsWith('--', current) || sql.startsWith('/*', current)) {
+      current = skipQuotedOrComment(sql, current);
+      continue;
+    }
+    return current;
+  }
+  return current;
+}
+
+function skipBracketIdentifier(sql: string, index: number): number {
+  let current = index + 1;
+  while (current < sql.length) {
+    if (sql[current] === ']') {
+      if (sql[current + 1] === ']') {
+        current += 2;
+        continue;
+      }
+      return current + 1;
+    }
+    current += 1;
+  }
+  return -1;
+}
+
+function skipBacktickIdentifier(sql: string, index: number): number {
+  let current = index + 1;
+  while (current < sql.length) {
+    if (sql[current] === '`') {
+      if (sql[current + 1] === '`') {
+        current += 2;
+        continue;
+      }
+      return current + 1;
+    }
+    current += 1;
+  }
+  return -1;
+}
+
+function skipIdentifier(sql: string, index: number): number {
+  if (sql[index] === '"') {
+    const skipped = skipQuotedOrComment(sql, index);
+    return skipped > index ? skipped : -1;
+  }
+  if (sql[index] === '[') {
+    return skipBracketIdentifier(sql, index);
+  }
+  if (sql[index] === '`') {
+    return skipBacktickIdentifier(sql, index);
+  }
+  let current = index;
+  while (isSqlIdentifierPart(sql[current])) {
+    current += 1;
+  }
+  return current > index ? current : -1;
+}
+
+function skipBalancedParentheses(sql: string, index: number): number {
+  if (sql[index] !== '(') {
+    return -1;
+  }
+
+  let current = index;
+  let depth = 0;
+  while (current < sql.length) {
+    const skipped = skipQuotedOrComment(sql, current);
+    if (skipped > current) {
+      current = skipped;
+      continue;
+    }
+
+    if (sql[current] === '(') {
+      depth += 1;
+    } else if (sql[current] === ')') {
+      depth -= 1;
+      if (depth === 0) {
+        return current + 1;
+      }
+    }
+    current += 1;
+  }
+
+  return -1;
+}
+
+/** @internal */
+export function hoistLeadingCte(sql: string): { withPrefix: string; body: string } {
+  const trimmed = sql.trim();
+  if (!keywordAt(trimmed, 0, 'with')) {
+    return { withPrefix: '', body: sql };
+  }
+
+  let current = skipWhitespaceAndComments(trimmed, 4);
+  if (keywordAt(trimmed, current, 'recursive')) {
+    current = skipWhitespaceAndComments(trimmed, current + 'recursive'.length);
+  }
+
+  while (current < trimmed.length) {
+    current = skipIdentifier(trimmed, current);
+    if (current < 0) {
+      return { withPrefix: '', body: trimmed };
+    }
+
+    current = skipWhitespaceAndComments(trimmed, current);
+    if (trimmed[current] === '(') {
+      current = skipBalancedParentheses(trimmed, current);
+      if (current < 0) {
+        return { withPrefix: '', body: trimmed };
+      }
+      current = skipWhitespaceAndComments(trimmed, current);
+    }
+
+    if (!keywordAt(trimmed, current, 'as')) {
+      return { withPrefix: '', body: trimmed };
+    }
+
+    current = skipWhitespaceAndComments(trimmed, current + 2);
+    current = skipBalancedParentheses(trimmed, current);
+    if (current < 0) {
+      return { withPrefix: '', body: trimmed };
+    }
+
+    current = skipWhitespaceAndComments(trimmed, current);
+    if (trimmed[current] === ',') {
+      current = skipWhitespaceAndComments(trimmed, current + 1);
+      continue;
+    }
+
+    const body = trimmed.slice(current).trimStart();
+    if (!body) {
+      return { withPrefix: '', body: trimmed };
+    }
+    return { withPrefix: `${trimmed.slice(0, current).trimEnd()} `, body };
+  }
+
+  return { withPrefix: '', body: trimmed };
+}
+
 // `assertReadOnlySql` deliberately keeps trailing semicolons, comments, and
 // whitespace (e.g. `select 1; -- done`) — harmless for direct single-statement
 // execution. A row-limit subquery wrapper needs a bare expression instead: a
@@ -137,5 +292,6 @@ export function limitSqlForExecution(sql: string, maxRows: number | undefined):
   if (!Number.isInteger(maxRows) || maxRows <= 0) {
     throw new KtxQueryError('maxRows must be a positive integer.');
   }
-  return `select * from (${trimmed}) as ktx_query_result limit ${maxRows}`;
+  const { withPrefix, body } = hoistLeadingCte(trimmed);
+  return `${withPrefix}select * from (${body}) as ktx_query_result limit ${maxRows}`;
 }

packages/cli/src/context/core/config-reference.ts

@@ -1,6 +1,7 @@
 import { readFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { resolve } from 'node:path';
+import { KtxExpectedError } from '../../errors.js';
 
 /** @internal */
 export function resolveKtxHomePath(path: string): string {
@@ -28,7 +29,15 @@ export function resolveKtxConfigReference(value: string | undefined, env: NodeJS
 
   if (value.startsWith('file:')) {
     const filePath = resolveKtxHomePath(value.slice('file:'.length).trim());
-    const fileValue = readFileSync(filePath, 'utf8').trim();
+    let fileValue: string;
+    try {
+      fileValue = readFileSync(filePath, 'utf8').trim();
+    } catch (error) {
+      // A `file:` secret whose target is missing or unreadable is a config
+      // problem (secrets/ is gitignored, so absent after a clone), not a ktx
+      // fault — classify it so it stays out of Error Tracking.
+      throw new KtxExpectedError(`Could not read the secret file referenced in ktx.yaml: ${filePath}`, { cause: error });
+    }
     return fileValue.length > 0 ? fileValue : undefined;
   }
 

packages/cli/src/context/daemon/semantic-layer-compute.ts

@@ -4,6 +4,44 @@ import { URL } from 'node:url';
 import { spawn } from 'node:child_process';
 import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js';
 
+// The daemon signals "the caller sent a well-formed request I refused as
+// invalid" with a distinct subprocess exit code (INPUT_REJECTED_EXIT_CODE in
+// python/ktx-daemon/__main__.py) or HTTP 400. Kept in sync across the boundary.
+const DAEMON_INPUT_REJECTED_EXIT_CODE = 3;
+const DAEMON_INPUT_REJECTED_HTTP_STATUS = 400;
+
+/**
+ * A ktx-daemon compute call failed. `inputRejected` is true when the daemon
+ * refused the request as invalid (a caller-driven outcome) rather than faulting;
+ * callers can promote those to an expected KtxQueryError while letting genuine
+ * daemon crashes reach Error Tracking. `detail` is the daemon's own message,
+ * unwrapped for surfacing to the caller.
+ */
+export class KtxDaemonComputeError extends Error {
+  readonly inputRejected: boolean;
+  readonly detail: string;
+
+  constructor(message: string, options: { inputRejected: boolean; detail: string; cause?: unknown }) {
+    super(message, options.cause !== undefined ? { cause: options.cause } : undefined);
+    this.name = 'KtxDaemonComputeError';
+    this.inputRejected = options.inputRejected;
+    this.detail = options.detail;
+  }
+}
+
+/** The daemon reports HTTP failures as `{ "detail": "…" }`; fall back to the raw body. */
+function daemonHttpErrorDetail(raw: string): string {
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    if (parsed && typeof parsed === 'object' && typeof (parsed as { detail?: unknown }).detail === 'string') {
+      return (parsed as { detail: string }).detail;
+    }
+  } catch {
+    // Non-JSON body (e.g. a proxy error page) — surface it verbatim.
+  }
+  return raw;
+}
+
 interface KtxSemanticLayerComputeQueryResult {
   sql: string;
   dialect: string;
@@ -128,7 +166,13 @@ function runProcessJson(
         const stdoutText = Buffer.concat(stdout).toString('utf8').trim();
         const stderrText = Buffer.concat(stderr).toString('utf8').trim();
         if (code !== 0) {
-          reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
+          const detail = stderrText || `exit code ${code}`;
+          reject(
+            new KtxDaemonComputeError(`ktx-daemon ${subcommand} failed: ${detail}`, {
+              inputRejected: code === DAEMON_INPUT_REJECTED_EXIT_CODE,
+              detail,
+            }),
+          );
           return;
         }
         try {
@@ -168,7 +212,12 @@ function postJson(baseUrl: string): KtxDaemonHttpJsonRunner {
             const text = Buffer.concat(chunks).toString('utf8');
             const statusCode = response.statusCode ?? 0;
             if (statusCode < 200 || statusCode >= 300) {
-              reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
+              reject(
+                new KtxDaemonComputeError(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`, {
+                  inputRejected: statusCode === DAEMON_INPUT_REJECTED_HTTP_STATUS,
+                  detail: daemonHttpErrorDetail(text),
+                }),
+              );
               return;
             }
             try {

packages/cli/src/context/ingest/adapters/gdrive/chunk.ts

@@ -0,0 +1,85 @@
+import { createHash } from 'node:crypto';
+import { readdir, readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../types.js';
+import { gdriveManifestSchema, gdriveMetadataSchema } from './types.js';
+
+const GDRIVE_RECONCILE_GUIDANCE =
+  'Synthesize durable wiki knowledge from this Google Doc. Preserve product definitions, process documentation, and operating rules as wiki pages. Do not create semantic-layer sources from gdrive content in v1.';
+
+function normalizeRawPath(path: string): string {
+  return path.replace(/\\/g, '/');
+}
+
+async function walk(root: string): Promise<string[]> {
+  const entries = await readdir(root, { withFileTypes: true, recursive: true });
+  return entries
+    .filter((entry) => entry.isFile())
+    .map((entry) => normalizeRawPath(relative(root, join(entry.parentPath, entry.name))))
+    .sort();
+}
+
+function safeUnitKey(path: string): string {
+  return `gdrive-${path.replace(/^docs\//, '').replace(/\/page\.md$/, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`;
+}
+
+async function readManifest(stagedDir: string) {
+  try {
+    return gdriveManifestSchema.parse(JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8')));
+  } catch (error) {
+    throw new Error(`Invalid gdrive manifest: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+export async function chunkGdriveStagedDir(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {
+  const files = await walk(stagedDir);
+  const manifest = await readManifest(stagedDir);
+  const touched = diffSet
+    ? new Set([...diffSet.added, ...diffSet.modified].map((path) => normalizeRawPath(path)))
+    : null;
+  const workUnits: WorkUnit[] = [];
+
+  for (const pagePath of files.filter((path) => path.endsWith('/page.md'))) {
+    const metadataPath = pagePath.replace(/\/page\.md$/, '/metadata.json');
+    const primary = [metadataPath, pagePath].filter((path) => files.includes(path));
+    if (touched && !primary.some((path) => touched.has(path))) {
+      continue;
+    }
+    const metadata = gdriveMetadataSchema.parse(JSON.parse(await readFile(join(stagedDir, metadataPath), 'utf-8')));
+    const rawFiles = touched ? primary.filter((path) => touched.has(path)).sort() : primary.sort();
+    const dependencyPaths = ['manifest.json'].filter((path) => !rawFiles.includes(path));
+    const excluded = new Set([...rawFiles, ...dependencyPaths]);
+    const peerFileIndex = files.filter((path) => !excluded.has(path)).sort();
+    workUnits.push({
+      unitKey: safeUnitKey(pagePath),
+      displayLabel: metadata.path,
+      rawFiles,
+      dependencyPaths,
+      peerFileIndex,
+      notes: GDRIVE_RECONCILE_GUIDANCE,
+    });
+  }
+
+  return {
+    workUnits,
+    eviction:
+      diffSet && diffSet.deleted.length > 0
+        ? { deletedRawPaths: diffSet.deleted.map((path) => normalizeRawPath(path)).sort() }
+        : undefined,
+    reconcileNotes: ['Google Drive docs are knowledge-only in v1; keep output in wiki pages unless later follow-up work expands scope.'],
+    contextReport: { capped: false, warnings: manifest.warnings },
+  };
+}
+
+export async function describeGdriveScope(stagedDir: string): Promise<ScopeDescriptor> {
+  const manifest = await readManifest(stagedDir);
+  const scopeKey = JSON.stringify({
+    folderId: manifest.folderId,
+    recursive: manifest.recursive,
+  });
+  const fingerprint = createHash('sha256').update(scopeKey).digest('hex');
+  return {
+    fingerprint,
+    isPathInScope: (rawPath) => rawPath === 'manifest.json' || rawPath.startsWith('docs/'),
+  };
+}

packages/cli/src/context/ingest/adapters/gdrive/detect.ts

@@ -0,0 +1,20 @@
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export async function detectGdriveStagedDir(stagedDir: string): Promise<boolean> {
+  try {
+    const manifest = JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8')) as { source?: unknown };
+    if (manifest.source === 'gdrive') {
+      return true;
+    }
+  } catch {
+    // Fall through to structural detection.
+  }
+
+  try {
+    const entries = await readdir(stagedDir, { withFileTypes: true, recursive: true });
+    return entries.some((entry) => entry.isFile() && entry.name === 'page.md');
+  } catch {
+    return false;
+  }
+}

packages/cli/src/context/ingest/adapters/gdrive/fetch.ts

@@ -0,0 +1,132 @@
+import { createHash } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { createGoogleDocsClients, driveFolderChildrenQuery } from './gdrive-client.js';
+import { normalizeGoogleDocToMarkdown } from './normalize.js';
+import type { GdriveFileRecord, GdriveManifest, GdrivePullConfig } from './types.js';
+import { GDRIVE_DOC_MIME_TYPE, GDRIVE_FOLDER_MIME_TYPE, GDRIVE_SOURCE_KEY } from './types.js';
+
+async function writeJson(path: string, value: unknown): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, 'utf-8');
+}
+
+async function writeText(path: string, value: string): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, value.endsWith('\n') ? value : `${value}\n`, 'utf-8');
+}
+
+function slugifySegment(value: string): string {
+  const normalized = value
+    .normalize('NFKD')
+    .replace(/[^\x00-\x7F]/g, '')
+    .replace(/[^a-zA-Z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .toLowerCase();
+  return normalized || 'untitled';
+}
+
+function compactSegment(value: string, maxLength = 24): string {
+  const slug = slugifySegment(value);
+  return slug.length > maxLength ? slug.slice(0, maxLength).replace(/-+$/g, '') || 'untitled' : slug;
+}
+
+function shortHash(value: string, length = 10): string {
+  return createHash('sha1').update(value).digest('hex').slice(0, length);
+}
+
+function gdriveDocDirName(title: string, fileId: string): string {
+  return `${compactSegment(title)}-${shortHash(fileId)}`;
+}
+
+interface GdriveDocRecord {
+  file: GdriveFileRecord;
+  drivePath: string[];
+  folderId: string;
+}
+
+interface GdriveSkippedFile {
+  externalId: string;
+  reason: string;
+}
+
+interface ListFolderResult {
+  docs: GdriveDocRecord[];
+  skipped: GdriveSkippedFile[];
+}
+
+async function listFolderFiles(
+  drive: ReturnType<typeof createGoogleDocsClients>['drive'],
+  folderId: string,
+  recursive: boolean,
+  parents: string[] = [],
+): Promise<ListFolderResult> {
+  const q = driveFolderChildrenQuery(folderId);
+  const docs: GdriveDocRecord[] = [];
+  const skipped: GdriveSkippedFile[] = [];
+  let pageToken: string | undefined;
+  do {
+    const page = await drive.listFiles({ q, pageToken });
+    for (const file of page.files) {
+      if (file.mimeType === GDRIVE_FOLDER_MIME_TYPE) {
+        if (recursive) {
+          const nested = await listFolderFiles(drive, file.id, true, [...parents, file.name]);
+          docs.push(...nested.docs);
+          skipped.push(...nested.skipped);
+        }
+        continue;
+      }
+      if (file.mimeType !== GDRIVE_DOC_MIME_TYPE) {
+        skipped.push({ externalId: file.id, reason: `unsupported mime type: ${file.mimeType}` });
+        continue;
+      }
+      docs.push({ file, drivePath: parents, folderId });
+    }
+    pageToken = page.nextPageToken ?? undefined;
+  } while (pageToken);
+  return { docs, skipped };
+}
+
+export async function fetchGdriveSnapshot(params: {
+  key: unknown;
+  config: GdrivePullConfig;
+  stagedDir: string;
+}): Promise<GdriveManifest> {
+  await mkdir(params.stagedDir, { recursive: true });
+  const clients = createGoogleDocsClients(params.key);
+  const { docs, skipped } = await listFolderFiles(clients.drive, params.config.folderId, params.config.recursive);
+
+  for (const { file, drivePath, folderId } of docs) {
+    const document = await clients.docs.getDocument(file.id);
+    const title = (document.title?.trim() || file.name).trim();
+    const relDir = join('docs', ...drivePath.map((segment) => compactSegment(segment)), gdriveDocDirName(title, file.id));
+    const markdownBody = normalizeGoogleDocToMarkdown(document);
+    const pageMarkdown = [`# ${title}`, markdownBody].filter(Boolean).join('\n\n');
+    await writeJson(join(params.stagedDir, relDir, 'metadata.json'), {
+      id: file.id,
+      title,
+      path: [...drivePath, title].join(' / ') || title,
+      url: file.webViewLink,
+      mimeType: file.mimeType,
+      folderId,
+      drivePath,
+      modifiedTime: file.modifiedTime,
+    });
+    await writeText(join(params.stagedDir, relDir, 'page.md'), pageMarkdown);
+  }
+
+  const manifest: GdriveManifest = {
+    source: GDRIVE_SOURCE_KEY,
+    folderId: params.config.folderId,
+    recursive: params.config.recursive,
+    fetchedAt: new Date().toISOString(),
+    fileCount: docs.length,
+    skipped,
+    warnings:
+      skipped.length > 0
+        ? [`Skipped ${skipped.length} non-Google-Doc file(s); only Google Docs are ingested in v1.`]
+        : [],
+  };
+  await writeJson(join(params.stagedDir, 'manifest.json'), manifest);
+  return manifest;
+}

packages/cli/src/context/ingest/adapters/gdrive/gdrive-client.ts

@@ -0,0 +1,188 @@
+import { JWT } from 'google-auth-library';
+import type { GdriveFileRecord, GdriveServiceAccountKey, GoogleDocsDocument } from './types.js';
+import { GDRIVE_DOC_MIME_TYPE, GDRIVE_FOLDER_MIME_TYPE, GDRIVE_SCOPES, gdriveServiceAccountKeySchema } from './types.js';
+
+const GOOGLE_DRIVE_BASE_URL = 'https://www.googleapis.com/drive/v3';
+const GOOGLE_DOCS_BASE_URL = 'https://docs.googleapis.com/v1';
+const GOOGLE_FILE_FIELDS = 'id,name,mimeType,parents,webViewLink,modifiedTime';
+
+const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
+const MAX_REQUEST_ATTEMPTS = 4;
+
+interface GoogleApiFile {
+  id?: string;
+  name?: string;
+  mimeType?: string;
+  parents?: string[];
+  webViewLink?: string;
+  modifiedTime?: string;
+}
+
+interface GoogleApiListResponse {
+  files?: GoogleApiFile[];
+  nextPageToken?: string;
+}
+
+export interface GoogleDriveClient {
+  listFiles(args: { q: string; pageToken?: string }): Promise<{ files: GdriveFileRecord[]; nextPageToken: string | null }>;
+  getFile(fileId: string): Promise<GdriveFileRecord | null>;
+}
+
+export interface GoogleDocsClients {
+  drive: GoogleDriveClient;
+  docs: {
+    getDocument(documentId: string): Promise<GoogleDocsDocument>;
+  };
+}
+
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function retryDelayMs(attempt: number, retryAfterHeader: string | null): number {
+  const retryAfterSeconds = retryAfterHeader ? Number.parseInt(retryAfterHeader, 10) : Number.NaN;
+  if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0) {
+    return Math.min(retryAfterSeconds * 1000, 30_000);
+  }
+  return Math.min(500 * 2 ** attempt, 8_000);
+}
+
+/** @internal Retries transient Google API responses (429/5xx) honoring Retry-After. */
+export async function fetchWithGoogleRetry(
+  doFetch: () => Promise<Response>,
+  options: { maxAttempts?: number; sleep?: (ms: number) => Promise<void> } = {},
+): Promise<Response> {
+  const maxAttempts = options.maxAttempts ?? MAX_REQUEST_ATTEMPTS;
+  const sleep = options.sleep ?? defaultSleep;
+  let response = await doFetch();
+  for (let attempt = 1; attempt < maxAttempts && !response.ok && RETRYABLE_STATUSES.has(response.status); attempt += 1) {
+    await sleep(retryDelayMs(attempt - 1, response.headers.get('retry-after')));
+    response = await doFetch();
+  }
+  return response;
+}
+
+async function parseGoogleResponse<T>(response: Response): Promise<T> {
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(`Google API request failed (${response.status}): ${body || response.statusText}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function authorizedFetch(client: JWT, url: string): Promise<Response> {
+  return fetchWithGoogleRetry(async () => {
+    const headers = await client.getRequestHeaders(url);
+    return fetch(url, { headers });
+  });
+}
+
+function isGoogleApiFileRecord(file: GoogleApiFile): file is GoogleApiFile & {
+  id: string;
+  name: string;
+  mimeType: string;
+} {
+  return typeof file.id === 'string' && typeof file.name === 'string' && typeof file.mimeType === 'string';
+}
+
+function toFileRecord(file: GoogleApiFile & { id: string; name: string; mimeType: string }): GdriveFileRecord {
+  return {
+    id: file.id,
+    name: file.name,
+    mimeType: file.mimeType,
+    parents: Array.isArray(file.parents) ? file.parents.filter((parent): parent is string => typeof parent === 'string') : [],
+    webViewLink: typeof file.webViewLink === 'string' ? file.webViewLink : null,
+    modifiedTime: typeof file.modifiedTime === 'string' ? file.modifiedTime : null,
+  };
+}
+
+function escapeDriveQueryValue(value: string): string {
+  return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+}
+
+/** Builds the Drive query for the non-trashed direct children of a folder, escaping the folder id. */
+export function driveFolderChildrenQuery(folderId: string): string {
+  return `'${escapeDriveQueryValue(folderId)}' in parents and trashed = false`;
+}
+
+/**
+ * Confirms `folderId` resolves to a folder the service account can read, then counts the
+ * Google Docs directly inside it. Throws a caller-facing error when the id is missing or not a folder.
+ */
+export async function verifyGdriveFolderAndCountDocs(
+  drive: GoogleDriveClient,
+  folderId: string,
+): Promise<number> {
+  const folder = await drive.getFile(folderId);
+  if (!folder) {
+    throw new Error(
+      `Google Drive folder "${folderId}" is not accessible. Share it with the service account email and verify folder_id.`,
+    );
+  }
+  if (folder.mimeType !== GDRIVE_FOLDER_MIME_TYPE) {
+    throw new Error(`Google Drive id "${folderId}" is not a folder (mimeType: ${folder.mimeType}).`);
+  }
+  const q = driveFolderChildrenQuery(folderId);
+  let docs = 0;
+  let pageToken: string | undefined;
+  do {
+    const page = await drive.listFiles({ q, pageToken });
+    docs += page.files.filter((file) => file.mimeType === GDRIVE_DOC_MIME_TYPE).length;
+    pageToken = page.nextPageToken ?? undefined;
+  } while (pageToken);
+  return docs;
+}
+
+export function createGoogleDocsClients(rawKey: unknown): GoogleDocsClients {
+  const key = gdriveServiceAccountKeySchema.parse(rawKey) satisfies GdriveServiceAccountKey;
+  const client = new JWT({
+    email: key.client_email,
+    key: key.private_key,
+    scopes: [...GDRIVE_SCOPES],
+  });
+
+  return {
+    drive: {
+      async listFiles(args) {
+        const params = new URLSearchParams({
+          q: args.q,
+          supportsAllDrives: 'true',
+          includeItemsFromAllDrives: 'true',
+          pageSize: '1000',
+          fields: `nextPageToken,files(${GOOGLE_FILE_FIELDS})`,
+        });
+        if (args.pageToken) {
+          params.set('pageToken', args.pageToken);
+        }
+        const response = await authorizedFetch(client, `${GOOGLE_DRIVE_BASE_URL}/files?${params.toString()}`);
+        const parsed = await parseGoogleResponse<GoogleApiListResponse>(response);
+        return {
+          files: (parsed.files ?? []).filter(isGoogleApiFileRecord).map(toFileRecord),
+          nextPageToken: typeof parsed.nextPageToken === 'string' ? parsed.nextPageToken : null,
+        };
+      },
+      async getFile(fileId: string) {
+        const params = new URLSearchParams({ supportsAllDrives: 'true', fields: GOOGLE_FILE_FIELDS });
+        const response = await authorizedFetch(
+          client,
+          `${GOOGLE_DRIVE_BASE_URL}/files/${encodeURIComponent(fileId)}?${params.toString()}`,
+        );
+        if (response.status === 404) {
+          return null;
+        }
+        const file = await parseGoogleResponse<GoogleApiFile>(response);
+        return isGoogleApiFileRecord(file) ? toFileRecord(file) : null;
+      },
+    },
+    docs: {
+      async getDocument(documentId: string) {
+        const params = new URLSearchParams({
+          includeTabsContent: 'true',
+          suggestionsViewMode: 'PREVIEW_WITHOUT_SUGGESTIONS',
+        });
+        const response = await authorizedFetch(client, `${GOOGLE_DOCS_BASE_URL}/documents/${documentId}?${params.toString()}`);
+        return await parseGoogleResponse<GoogleDocsDocument>(response);
+      },
+    },
+  };
+}

packages/cli/src/context/ingest/adapters/gdrive/gdrive.adapter.ts

@@ -0,0 +1,33 @@
+import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js';
+import { chunkGdriveStagedDir, describeGdriveScope } from './chunk.js';
+import { detectGdriveStagedDir } from './detect.js';
+import { fetchGdriveSnapshot } from './fetch.js';
+import { gdrivePullConfigSchema } from './types.js';
+
+export class GdriveSourceAdapter implements SourceAdapter {
+  readonly source = 'gdrive';
+  readonly skillNames = ['gdrive_synthesize'];
+  readonly reconcileSkillNames: string[] = [];
+  readonly evidenceIndexing = 'documents' as const;
+
+  detect(stagedDir: string): Promise<boolean> {
+    return detectGdriveStagedDir(stagedDir);
+  }
+
+  async fetch(pullConfig: unknown, stagedDir: string, _ctx: FetchContext): Promise<void> {
+    const config = gdrivePullConfigSchema.parse(pullConfig);
+    await fetchGdriveSnapshot({
+      key: JSON.parse(config.serviceAccountKey),
+      config,
+      stagedDir,
+    });
+  }
+
+  chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {
+    return chunkGdriveStagedDir(stagedDir, diffSet);
+  }
+
+  describeScope(stagedDir: string): Promise<ScopeDescriptor> {
+    return describeGdriveScope(stagedDir);
+  }
+}

packages/cli/src/context/ingest/adapters/gdrive/normalize.ts

@@ -0,0 +1,323 @@
+import type {
+  GoogleDocsDocument,
+  GoogleDocsDocumentStyle,
+  GoogleDocsHeaderFooter,
+  GoogleDocsLinkTarget,
+  GoogleDocsList,
+  GoogleDocsParagraph,
+  GoogleDocsParagraphElement,
+  GoogleDocsStructuralElement,
+  GoogleDocsTab,
+  GoogleDocsTable,
+  GoogleDocsTableCell,
+} from './types.js';
+
+function escapeMarkdownText(value: string): string {
+  return value.replace(/([*_~`])/g, '\\$1');
+}
+
+function normalizeInternalLinkTarget(prefix: 'heading' | 'bookmark', target: GoogleDocsLinkTarget | string | undefined): string | null {
+  const id = typeof target === 'string' ? target : target?.id;
+  if (!id?.trim()) {
+    return null;
+  }
+  return `#${prefix}-${id.trim()}`;
+}
+
+function resolveLinkHref(element: GoogleDocsParagraphElement): string | null {
+  const link = element.textRun?.textStyle?.link;
+  const href = link?.url?.trim();
+  if (href) {
+    return href;
+  }
+  return (
+    normalizeInternalLinkTarget('heading', link?.heading) ??
+    normalizeInternalLinkTarget('heading', link?.headingId) ??
+    normalizeInternalLinkTarget('bookmark', link?.bookmark) ??
+    normalizeInternalLinkTarget('bookmark', link?.bookmarkId) ??
+    null
+  );
+}
+
+function normalizeTextRun(element: GoogleDocsParagraphElement): string {
+  const content = element.textRun?.content ?? '';
+  const style = element.textRun?.textStyle;
+  let text = escapeMarkdownText(content.replace(/\r/g, ''));
+  if (!text && element.inlineObjectElement) {
+    return '[Embedded object]';
+  }
+  if (!text && element.pageBreak) {
+    return '\n---\n';
+  }
+  if (!text) {
+    return '';
+  }
+  const href = resolveLinkHref(element);
+  const isCode = style?.weightedFontFamily?.fontFamily === 'Courier New';
+  if (isCode) {
+    text = `\`${text.replace(/`/g, '\\`')}\``;
+  }
+  if (style?.bold) {
+    text = `**${text}**`;
+  }
+  if (style?.italic) {
+    text = `*${text}*`;
+  }
+  if (style?.underline) {
+    text = `<u>${text}</u>`;
+  }
+  if (style?.strikethrough) {
+    text = `~~${text}~~`;
+  }
+  if (href) {
+    text = `[${text}](${href.replace(/\)/g, '\\)')})`;
+  }
+  if (style?.baselineOffset === 'SUPERSCRIPT') {
+    text = `<sup>${text}</sup>`;
+  } else if (style?.baselineOffset === 'SUBSCRIPT') {
+    text = `<sub>${text}</sub>`;
+  }
+  return text;
+}
+
+function paragraphText(paragraph: GoogleDocsParagraph | undefined): string {
+  return (paragraph?.elements ?? [])
+    .map((element) => normalizeTextRun(element))
+    .join('')
+    .replace(/\n/g, '')
+    .trim();
+}
+
+function headingPrefix(namedStyleType: string | undefined): string | null {
+  if (namedStyleType === 'TITLE') {
+    return '#';
+  }
+  if (namedStyleType === 'SUBTITLE') {
+    return '##';
+  }
+  if (!namedStyleType?.startsWith('HEADING_')) {
+    return null;
+  }
+  const level = Number.parseInt(namedStyleType.slice('HEADING_'.length), 10);
+  if (Number.isNaN(level) || level < 1) {
+    return null;
+  }
+  return '#'.repeat(Math.min(level, 6));
+}
+
+function isOrderedListLevel(level: { glyphType?: string; glyphSymbol?: string } | undefined): boolean {
+  const glyphType = level?.glyphType?.toUpperCase();
+  if (glyphType) {
+    return (
+      glyphType.includes('NUMBER') ||
+      glyphType.includes('DECIMAL') ||
+      glyphType.includes('ALPHA') ||
+      glyphType.includes('ROMAN') ||
+      glyphType.includes('LATIN')
+    );
+  }
+  const glyphSymbol = level?.glyphSymbol?.trim();
+  return glyphSymbol === '%0.' || glyphSymbol === '%0)' || glyphSymbol === '1.' || glyphSymbol === '1)';
+}
+
+function listPrefix(paragraph: GoogleDocsParagraph, lists: Record<string, GoogleDocsList> | undefined): string | null {
+  if (!paragraph.bullet) {
+    return null;
+  }
+  const level = Math.max(paragraph.bullet.nestingLevel ?? 0, 0);
+  const indent = '  '.repeat(level);
+  const listDefinition = paragraph.bullet.listId ? lists?.[paragraph.bullet.listId] : undefined;
+  const listLevel = listDefinition?.listProperties?.nestingLevels?.[level];
+  return `${indent}${isOrderedListLevel(listLevel) ? '1. ' : '- '}`;
+}
+
+function paragraphToMarkdown(
+  paragraph: GoogleDocsParagraph | undefined,
+  lists: Record<string, GoogleDocsList> | undefined,
+): string | null {
+  const text = paragraphText(paragraph);
+  if (!text) {
+    return null;
+  }
+  const prefix = paragraph ? listPrefix(paragraph, lists) : null;
+  if (prefix) {
+    return `${prefix}${text}`;
+  }
+  const heading = headingPrefix(paragraph?.paragraphStyle?.namedStyleType);
+  if (heading) {
+    const headingLine = `${heading} ${text}`;
+    const headingId = paragraph?.paragraphStyle?.headingId?.trim();
+    return headingId ? `<a id="heading-${headingId}"></a>\n${headingLine}` : headingLine;
+  }
+  return text;
+}
+
+function normalizeTableCell(
+  cell: GoogleDocsTableCell | undefined,
+  lists: Record<string, GoogleDocsList> | undefined,
+): string {
+  const blocks = normalizeStructuralElements(cell?.content ?? [], lists);
+  return blocks
+    .map((block) => block.replace(/\n/g, ' <br> '))
+    .join(' / ')
+    .replace(/\|/g, '\\|')
+    .trim();
+}
+
+function markdownTableDivider(columnCount: number): string {
+  return `| ${Array.from({ length: columnCount }, () => '---').join(' | ')} |`;
+}
+
+function normalizeTable(table: GoogleDocsTable | undefined, lists: Record<string, GoogleDocsList> | undefined): string[] {
+  const rows = table?.tableRows ?? [];
+  const normalizedRows = rows
+    .map((row) => (row.tableCells ?? []).map((cell) => normalizeTableCell(cell, lists)))
+    .filter((cells) => cells.length > 0);
+  if (normalizedRows.length === 0) {
+    return [];
+  }
+  const columnCount = Math.max(...normalizedRows.map((cells) => cells.length));
+  const paddedRows = normalizedRows.map((cells) =>
+    Array.from({ length: columnCount }, (_, index) => cells[index] ?? ''),
+  );
+  const [header, ...body] = paddedRows;
+  const blocks = [`| ${header.join(' | ')} |`, markdownTableDivider(columnCount)];
+  for (const row of body) {
+    blocks.push(`| ${row.join(' | ')} |`);
+  }
+  return [blocks.join('\n')];
+}
+
+function normalizeStructuralElements(
+  elements: GoogleDocsStructuralElement[],
+  lists: Record<string, GoogleDocsList> | undefined,
+): string[] {
+  const blocks: string[] = [];
+  for (const element of elements) {
+    const line = paragraphToMarkdown(element.paragraph, lists);
+    if (line) {
+      blocks.push(line);
+      continue;
+    }
+    if (element.table) {
+      blocks.push(...normalizeTable(element.table, lists));
+    }
+  }
+  return blocks;
+}
+
+function headerFooterRoleMap(
+  label: 'Headers' | 'Footers',
+  documentStyle: GoogleDocsDocumentStyle | undefined,
+): Map<string, string> {
+  const roleMap = new Map<string, string>();
+  const roleEntries =
+    label === 'Headers'
+      ? [
+          [documentStyle?.defaultHeaderId, 'Default Header'],
+          [documentStyle?.firstPageHeaderId, 'First Page Header'],
+          [documentStyle?.evenPageHeaderId, 'Even Page Header'],
+        ]
+      : [
+          [documentStyle?.defaultFooterId, 'Default Footer'],
+          [documentStyle?.firstPageFooterId, 'First Page Footer'],
+          [documentStyle?.evenPageFooterId, 'Even Page Footer'],
+        ];
+  for (const [id, role] of roleEntries) {
+    const normalizedId = id?.trim();
+    if (!normalizedId || roleMap.has(normalizedId)) {
+      continue;
+    }
+    roleMap.set(normalizedId, role ?? normalizedId);
+  }
+  return roleMap;
+}
+
+function normalizeHeaderFooterMap(
+  label: 'Headers' | 'Footers',
+  entries: Record<string, GoogleDocsHeaderFooter> | undefined,
+  lists: Record<string, GoogleDocsList> | undefined,
+  documentStyle: GoogleDocsDocumentStyle | undefined,
+): string | null {
+  if (!entries) {
+    return null;
+  }
+  const ids = Object.keys(entries).sort();
+  const roles = headerFooterRoleMap(label, documentStyle);
+  const sections: string[] = [];
+  for (const id of ids) {
+    const blocks = normalizeStructuralElements(entries[id]?.content ?? [], lists);
+    if (blocks.length === 0) {
+      continue;
+    }
+    const title = roles.get(id) ?? `${label.slice(0, -1)} ${escapeMarkdownText(id)}`;
+    sections.push(`### ${title}\n\n${blocks.join('\n\n').trim()}`);
+  }
+  if (sections.length === 0) {
+    return null;
+  }
+  return `## ${label}\n\n${sections.join('\n\n').trim()}`;
+}
+
+function joinNonEmptySections(sections: Array<string | null>): string | null {
+  const nonEmpty = sections.filter((section): section is string => Boolean(section?.trim()));
+  if (nonEmpty.length === 0) {
+    return null;
+  }
+  return nonEmpty.join('\n\n').trim();
+}
+
+function flattenGoogleDocsTabs(tabs: GoogleDocsTab[] | undefined): GoogleDocsTab[] {
+  if (!tabs?.length) {
+    return [];
+  }
+  const flattened: GoogleDocsTab[] = [];
+  for (const tab of tabs) {
+    flattened.push(tab);
+    flattened.push(...flattenGoogleDocsTabs(tab.childTabs));
+  }
+  return flattened;
+}
+
+function normalizeTab(tab: GoogleDocsTab, fallbackLists: Record<string, GoogleDocsList> | undefined): string | null {
+  const lists = tab.documentTab?.lists ?? fallbackLists;
+  const headerSection = normalizeHeaderFooterMap(
+    'Headers',
+    tab.documentTab?.headers,
+    lists,
+    tab.documentTab?.documentStyle,
+  );
+  const bodySection = normalizeStructuralElements(tab.documentTab?.body?.content ?? [], lists).join('\n\n').trim();
+  const footerSection = normalizeHeaderFooterMap(
+    'Footers',
+    tab.documentTab?.footers,
+    lists,
+    tab.documentTab?.documentStyle,
+  );
+  const content = joinNonEmptySections([headerSection, bodySection, footerSection]);
+  if (!content) {
+    return null;
+  }
+  const title = tab.tabProperties?.title?.trim();
+  if (!title) {
+    return content;
+  }
+  return [`# ${escapeMarkdownText(title)}`, content].join('\n\n').trim();
+}
+
+export function normalizeGoogleDocToMarkdown(document: GoogleDocsDocument): string {
+  const normalizedTabs = flattenGoogleDocsTabs(document.tabs)
+    .map((tab) => normalizeTab(tab, document.lists))
+    .filter((tab): tab is string => Boolean(tab));
+  if (normalizedTabs.length > 0) {
+    return normalizedTabs.join('\n\n').trim();
+  }
+  const bodySection = normalizeStructuralElements(document.body?.content ?? [], document.lists).join('\n\n').trim();
+  return (
+    joinNonEmptySections([
+      normalizeHeaderFooterMap('Headers', document.headers, document.lists, document.documentStyle),
+      bodySection,
+      normalizeHeaderFooterMap('Footers', document.footers, document.lists, document.documentStyle),
+    ]) ?? ''
+  );
+}

packages/cli/src/context/ingest/adapters/gdrive/types.ts

@@ -0,0 +1,168 @@
+import { z } from 'zod';
+
+const GDRIVE_DOCS_SCOPE = 'https://www.googleapis.com/auth/documents.readonly';
+const GDRIVE_DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly';
+export const GDRIVE_SCOPES = [GDRIVE_DRIVE_SCOPE, GDRIVE_DOCS_SCOPE] as const;
+export const GDRIVE_SOURCE_KEY = 'gdrive';
+export const GDRIVE_DOC_MIME_TYPE = 'application/vnd.google-apps.document';
+export const GDRIVE_FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder';
+
+export const gdrivePullConfigSchema = z.object({
+  serviceAccountKey: z.string().min(1),
+  folderId: z.string().min(1),
+  recursive: z.boolean().default(false),
+});
+export type GdrivePullConfig = z.infer<typeof gdrivePullConfigSchema>;
+
+export const gdriveManifestSchema = z.object({
+  source: z.literal(GDRIVE_SOURCE_KEY),
+  folderId: z.string().min(1),
+  recursive: z.boolean(),
+  fetchedAt: z.string().datetime(),
+  fileCount: z.number().int().nonnegative(),
+  skipped: z.array(z.object({ externalId: z.string(), reason: z.string() })).default([]),
+  warnings: z.array(z.string()).default([]),
+});
+export type GdriveManifest = z.infer<typeof gdriveManifestSchema>;
+
+export const gdriveMetadataSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  path: z.string(),
+  url: z.string().nullable().default(null),
+  mimeType: z.literal(GDRIVE_DOC_MIME_TYPE),
+  folderId: z.string(),
+  drivePath: z.array(z.string()).default([]),
+  modifiedTime: z.string().datetime().nullable().default(null),
+});
+
+export const gdriveServiceAccountKeySchema = z.object({
+  client_email: z.string().email(),
+  private_key: z.string().min(1),
+  project_id: z.string().min(1).optional(),
+});
+export type GdriveServiceAccountKey = z.infer<typeof gdriveServiceAccountKeySchema>;
+
+export interface GdriveFileRecord {
+  id: string;
+  name: string;
+  mimeType: string;
+  parents: string[];
+  webViewLink: string | null;
+  modifiedTime: string | null;
+}
+
+export interface GoogleDocsDocument {
+  documentId?: string;
+  title?: string;
+  body?: {
+    content?: GoogleDocsStructuralElement[];
+  };
+  documentStyle?: GoogleDocsDocumentStyle;
+  lists?: Record<string, GoogleDocsList>;
+  headers?: Record<string, GoogleDocsHeaderFooter>;
+  footers?: Record<string, GoogleDocsHeaderFooter>;
+  tabs?: GoogleDocsTab[];
+}
+
+export interface GoogleDocsList {
+  listProperties?: {
+    nestingLevels?: GoogleDocsListNestingLevel[];
+  };
+}
+
+interface GoogleDocsListNestingLevel {
+  glyphType?: string;
+  glyphSymbol?: string;
+}
+
+export interface GoogleDocsTab {
+  tabProperties?: {
+    tabId?: string;
+    title?: string;
+  };
+  childTabs?: GoogleDocsTab[];
+  documentTab?: {
+    body?: {
+      content?: GoogleDocsStructuralElement[];
+    };
+    documentStyle?: GoogleDocsDocumentStyle;
+    lists?: Record<string, GoogleDocsList>;
+    headers?: Record<string, GoogleDocsHeaderFooter>;
+    footers?: Record<string, GoogleDocsHeaderFooter>;
+  };
+}
+
+export interface GoogleDocsDocumentStyle {
+  defaultHeaderId?: string;
+  defaultFooterId?: string;
+  firstPageHeaderId?: string;
+  firstPageFooterId?: string;
+  evenPageHeaderId?: string;
+  evenPageFooterId?: string;
+}
+
+export interface GoogleDocsHeaderFooter {
+  headerId?: string;
+  footerId?: string;
+  content?: GoogleDocsStructuralElement[];
+}
+
+export interface GoogleDocsStructuralElement {
+  paragraph?: GoogleDocsParagraph;
+  table?: GoogleDocsTable;
+  sectionBreak?: unknown;
+}
+
+export interface GoogleDocsTable {
+  tableRows?: GoogleDocsTableRow[];
+}
+
+interface GoogleDocsTableRow {
+  tableCells?: GoogleDocsTableCell[];
+}
+
+export interface GoogleDocsTableCell {
+  content?: GoogleDocsStructuralElement[];
+}
+
+export interface GoogleDocsParagraph {
+  elements?: GoogleDocsParagraphElement[];
+  bullet?: {
+    listId?: string;
+    nestingLevel?: number;
+  };
+  paragraphStyle?: {
+    namedStyleType?: string;
+    headingId?: string;
+  };
+}
+
+export interface GoogleDocsLinkTarget {
+  id?: string;
+  tabId?: string;
+}
+
+export interface GoogleDocsParagraphElement {
+  textRun?: {
+    content?: string;
+    textStyle?: {
+      bold?: boolean;
+      italic?: boolean;
+      underline?: boolean;
+      strikethrough?: boolean;
+      link?: {
+        url?: string;
+        tabId?: string;
+        headingId?: string;
+        bookmarkId?: string;
+        heading?: GoogleDocsLinkTarget;
+        bookmark?: GoogleDocsLinkTarget;
+      };
+      weightedFontFamily?: { fontFamily?: string };
+      baselineOffset?: 'SUPERSCRIPT' | 'SUBSCRIPT' | string;
+    };
+  };
+  inlineObjectElement?: unknown;
+  pageBreak?: unknown;
+}

packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts

@@ -3,8 +3,9 @@ import { request as httpRequest } from 'node:http';
 import { request as httpsRequest } from 'node:https';
 import { URL } from 'node:url';
 import type { KtxProjectConnectionConfig } from '../../../project/config.js';
+import { isKtxScanWarningCode } from '../../../scan/local-structural-artifacts.js';
 import { tableRefFromKey } from '../../../scan/table-ref.js';
-import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
+import type { KtxScanWarning, KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
 import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
 import type { LiveDatabaseIntrospectionOptions, LiveDatabaseIntrospectionPort } from './types.js';
 
@@ -206,10 +207,32 @@ function mapTable(raw: Record<string, unknown>): KtxSchemaTable {
   };
 }
 
+function mapWarning(raw: Record<string, unknown>): KtxScanWarning | null {
+  const code = optionalString(raw.code);
+  // Drop codes Node cannot render, keeping the daemon and Node warning catalogs
+  // in parity rather than surfacing an unknown code downstream.
+  if (!code || !isKtxScanWarningCode(code)) return null;
+  const table = optionalString(raw.table);
+  const column = optionalString(raw.column);
+  return {
+    code,
+    message: requiredString(raw.message, 'warnings[].message'),
+    recoverable: raw.recoverable !== false,
+    ...(table ? { table } : {}),
+    ...(column ? { column } : {}),
+    ...(raw.metadata && typeof raw.metadata === 'object' && !Array.isArray(raw.metadata)
+      ? { metadata: recordValue(raw.metadata) }
+      : {}),
+  };
+}
+
 function mapDaemonSnapshot(
   raw: Record<string, unknown>,
   input: { connectionId: string; extractedAt: string; schemas: string[] },
 ): KtxSchemaSnapshot {
+  const warnings = recordArray(raw.warnings)
+    .map(mapWarning)
+    .filter((warning): warning is KtxScanWarning => warning !== null);
   return {
     connectionId: requiredString(raw.connection_id, 'connection_id') || input.connectionId,
     driver: 'postgres',
@@ -217,6 +240,7 @@ function mapDaemonSnapshot(
     scope: { schemas: input.schemas },
     metadata: recordValue(raw.metadata),
     tables: recordArray(raw.tables).map(mapTable),
+    ...(warnings.length > 0 ? { warnings } : {}),
   };
 }
 

packages/cli/src/context/ingest/adapters/live-database/fetch-report.ts

@@ -0,0 +1,48 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { SourceFetchReport } from '../../types.js';
+import { LIVE_DATABASE_WARNINGS_FILE } from './stage.js';
+
+const OBJECT_SKIP_CODE = 'object_introspection_failed';
+
+interface RawWarning {
+  code?: unknown;
+  message?: unknown;
+  table?: unknown;
+}
+
+/**
+ * Derives the fetch report from the staged `warnings.json`: objects that failed
+ * introspection become `skipped` entries so the run report, ingest summary, and
+ * `ktx status` can surface them. Returns null when nothing was skipped, keeping
+ * clean ingests free of an empty report.
+ */
+export async function readLiveDatabaseFetchReport(stagedDir: string): Promise<SourceFetchReport | null> {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), 'utf8'));
+  } catch {
+    return null;
+  }
+  const warnings =
+    parsed && typeof parsed === 'object' && Array.isArray((parsed as { warnings?: unknown }).warnings)
+      ? ((parsed as { warnings: RawWarning[] }).warnings)
+      : [];
+
+  const skipped = warnings
+    .filter((warning) => warning.code === OBJECT_SKIP_CODE)
+    .map((warning) => ({
+      rawPath: '',
+      entityType: 'database_object',
+      entityId: typeof warning.table === 'string' ? warning.table : null,
+      severity: 'warning' as const,
+      statusCode: null,
+      message: typeof warning.message === 'string' ? warning.message : 'introspection failed',
+      retryRecommended: false,
+    }));
+
+  if (skipped.length === 0) {
+    return null;
+  }
+  return { status: 'partial', retryRecommended: false, skipped, warnings: [] };
+}

packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.ts

@@ -1,5 +1,7 @@
-import type { ChunkResult, DiffSet, FetchContext, SourceAdapter } from '../../types.js';
+import type { ChunkResult, DiffSet, FetchContext, SourceAdapter, SourceFetchReport } from '../../types.js';
 import { chunkLiveDatabaseStagedDir } from './chunk.js';
+import { readLiveDatabaseFetchReport } from './fetch-report.js';
+import { assertLiveDatabaseScanOutcome } from './scan-outcome.js';
 import { detectLiveDatabaseStagedDir, writeLiveDatabaseSnapshot } from './stage.js';
 import type { LiveDatabaseSourceAdapterDeps } from './types.js';
 
@@ -13,14 +15,20 @@ export class LiveDatabaseSourceAdapter implements SourceAdapter {
     return detectLiveDatabaseStagedDir(stagedDir);
   }
 
+  readFetchReport(stagedDir: string): Promise<SourceFetchReport | null> {
+    return readLiveDatabaseFetchReport(stagedDir);
+  }
+
   async fetch(_pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void> {
     const tableScope = ctx.tableScope;
     const snapshot = await this.deps.introspection.extractSchema(ctx.connectionId, { tableScope });
-    await writeLiveDatabaseSnapshot(stagedDir, {
+    const finalized = {
       ...snapshot,
       connectionId: ctx.connectionId,
       extractedAt: snapshot.extractedAt ?? (this.deps.now ?? (() => new Date()))().toISOString(),
-    });
+    };
+    assertLiveDatabaseScanOutcome({ connectionId: ctx.connectionId, scope: tableScope, snapshot: finalized });
+    await writeLiveDatabaseSnapshot(stagedDir, finalized);
   }
 
   chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {

packages/cli/src/context/ingest/adapters/live-database/manifest.ts

@@ -162,7 +162,8 @@ function getShardKey(connectionType: string, catalog: string | null, db: string
   }
 }
 
-function buildTableRef(name: string, catalog: string | null, db: string | null): string {
+/** @internal */
+export function buildTableRef(name: string, catalog: string | null, db: string | null): string {
   const parts: string[] = [];
   if (catalog) {
     parts.push(catalog);
@@ -273,7 +274,10 @@ export function buildLiveDatabaseManifestShards(
   for (const table of input.tables) {
     const shardKey = getShardKey(input.connectionType, table.catalog, table.db);
     const shard = shards.get(shardKey) ?? { tables: {} };
-    const existingDescriptions = input.existingDescriptions?.get(table.name);
+    // Existing descriptions/usage are keyed by the fully-qualified ref so two
+    // same-named tables in different schemas never share an entry.
+    const fullRef = buildTableRef(table.name, table.catalog, table.db);
+    const existingDescriptions = input.existingDescriptions?.get(fullRef);
 
     const columns: LiveDatabaseManifestColumn[] = table.columns.map((column) => {
       const manifestColumn: LiveDatabaseManifestColumn = {
@@ -297,7 +301,7 @@ export function buildLiveDatabaseManifestShards(
     });
 
     const entry: LiveDatabaseManifestTableEntry = {
-      table: buildTableRef(table.name, table.catalog, table.db),
+      table: fullRef,
       columns,
     };
 
@@ -306,7 +310,7 @@ export function buildLiveDatabaseManifestShards(
       entry.descriptions = tableDescriptions;
     }
 
-    const usage = mergeUsagePreservingExternal(input.existingUsage?.get(table.name), table.usage);
+    const usage = mergeUsagePreservingExternal(input.existingUsage?.get(fullRef), table.usage);
     if (usage) {
       entry.usage = usage;
     }

packages/cli/src/context/ingest/adapters/live-database/scan-outcome.ts

@@ -0,0 +1,55 @@
+import { KtxExpectedError } from '../../../../errors.js';
+import { tableRefFromKey, type KtxTableRefKey } from '../../../scan/table-ref.js';
+import type { KtxSchemaSnapshot } from '../../../scan/types.js';
+
+const OBJECT_SKIP_CODE = 'object_introspection_failed';
+
+function formatScopeEntry(key: KtxTableRefKey): string {
+  const ref = tableRefFromKey(key);
+  return [ref.catalog, ref.db, ref.name].filter((part): part is string => Boolean(part)).join('.');
+}
+
+function discoveredObjectNames(snapshot: KtxSchemaSnapshot): string[] {
+  const raw = (snapshot.metadata as Record<string, unknown>).discovered_object_names;
+  return Array.isArray(raw) ? raw.filter((value): value is string => typeof value === 'string') : [];
+}
+
+/**
+ * Enforces the partial-vs-total outcome rules for a live-database snapshot,
+ * uniformly for every connector. Outcomes follow from object counts, not a
+ * mode: a connection with at least one ingested object succeeds (any broken
+ * objects ride along as warnings); a connection where every introspected object
+ * failed, or a non-empty enabled_tables scope that matched nothing, raises a
+ * clear connection error instead of staging an empty layer that would later
+ * surface as the generic "did not recognize" message. A legitimately empty
+ * database (no scope, no objects) succeeds with an empty layer.
+ */
+export function assertLiveDatabaseScanOutcome(input: {
+  connectionId: string;
+  scope: ReadonlySet<KtxTableRefKey> | undefined;
+  snapshot: KtxSchemaSnapshot;
+}): void {
+  const { connectionId, scope, snapshot } = input;
+  if (snapshot.tables.length > 0) {
+    return;
+  }
+
+  const skipped = (snapshot.warnings ?? []).filter((warning) => warning.code === OBJECT_SKIP_CODE);
+  if (skipped.length > 0) {
+    const detail = skipped.map((warning) => `${warning.table ?? 'object'}: ${warning.message}`).join('; ');
+    throw new KtxExpectedError(
+      `Connection "${connectionId}" produced no semantic layer: all ${skipped.length} introspected ` +
+        `${skipped.length === 1 ? 'object' : 'objects'} failed (${detail}).`,
+    );
+  }
+
+  if (scope && scope.size > 0) {
+    const requested = [...scope].map(formatScopeEntry).sort();
+    const available = discoveredObjectNames(snapshot);
+    const availableClause = available.length > 0 ? ` Available objects: ${available.join(', ')}.` : '';
+    throw new KtxExpectedError(
+      `enabled_tables for connection "${connectionId}" matched no objects ` +
+        `(looked for: ${requested.join(', ')}).${availableClause}`,
+    );
+  }
+}

packages/cli/src/context/ingest/adapters/live-database/stage.ts

@@ -136,13 +136,13 @@ export async function readLiveDatabaseTableFiles(stagedDir: string): Promise<Liv
 }
 
 export async function detectLiveDatabaseStagedDir(stagedDir: string): Promise<boolean> {
+  // A valid live-database staging is identified by its connection.json marker.
+  // An empty table set is a legitimate outcome (an empty database), so the
+  // presence of table files is not required — the total-vs-partial decision is
+  // made earlier by assertLiveDatabaseScanOutcome, before staging.
   try {
     const meta = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_META_FILE), 'utf8')) as unknown;
-    if (!meta || typeof meta !== 'object' || Array.isArray(meta)) {
-      return false;
-    }
-    const files = await readLiveDatabaseTableFiles(stagedDir);
-    return files.length > 0;
+    return Boolean(meta) && typeof meta === 'object' && !Array.isArray(meta);
   } catch {
     return false;
   }

packages/cli/src/context/ingest/adapters/looker/mapping.ts

@@ -1,4 +1,5 @@
 import type { ParsedTargetTable } from '../../parsed-target-table.js';
+import { warehouseTargetDialect } from '../../../connections/connection-type-dialect.js';
 import type { LookerWarehouseConnectionInfo } from './client.js';
 import type { LookerPullConfig, LookerRuntimeCursors, StagedExploreFile, StagedLookmlModelsFile } from './types.js';
 
@@ -11,6 +12,7 @@ const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
   sqlite: 'SQLITE',
   sqlserver: 'SQLSERVER',
   clickhouse: 'CLICKHOUSE',
+  awsathena: 'ATHENA',
 } as const;
 
 /** @internal */
@@ -72,16 +74,6 @@ export interface LookerMappingClient {
   getExplore(modelName: string, exploreName: string): Promise<StagedExploreFile>;
 }
 
-const SQLGLOT_DIALECT_BY_CONNECTION_TYPE: Partial<Record<LookerWarehouseTargetConnectionType, string>> = {
-  BIGQUERY: 'bigquery',
-  SNOWFLAKE: 'snowflake',
-  POSTGRESQL: 'postgres',
-  MYSQL: 'mysql',
-  SQLITE: 'sqlite',
-  SQLSERVER: 'tsql',
-  CLICKHOUSE: 'clickhouse',
-};
-
 export async function discoverLookerConnections(
   client: Pick<LookerMappingClient, 'listLookerConnections'>,
 ): Promise<LookerWarehouseConnectionInfo[]> {
@@ -100,7 +92,7 @@ export function lookerDialectToConnectionType(dialect: string | null): LookerWar
 
 /** @internal */
 export function sqlglotDialectForConnectionType(connectionType: string): string | null {
-  return SQLGLOT_DIALECT_BY_CONNECTION_TYPE[connectionType as LookerWarehouseTargetConnectionType] ?? null;
+  return warehouseTargetDialect(connectionType);
 }
 
 /** @internal */

packages/cli/src/context/ingest/adapters/metabase/mapping.ts

@@ -8,9 +8,9 @@ export const METABASE_ENGINE_TO_CONNECTION_TYPE = {
   snowflake: 'SNOWFLAKE',
   sqlserver: 'SQLSERVER',
   mysql: 'MYSQL',
+  athena: 'ATHENA',
 } as const;
 
-
 export interface DiscoveredMetabaseDatabase {
   id: number;
   name: string;

packages/cli/src/context/ingest/adapters/metabase/types.ts

@@ -3,7 +3,7 @@ import { z } from 'zod';
 const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']);
 export type MetabaseSyncMode = z.infer<typeof metabaseSyncModeSchema>;
 
-const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/);
+const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/);
 
 /**
  * The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's

packages/cli/src/context/ingest/adapters/sigma/chunk.ts

@@ -0,0 +1,148 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import type { ChunkResult, DiffSet, WorkUnit } from '../../types.js';
+import {
+  type SigmaManifest,
+  type StagedDataModelFile,
+  type StagedWorkbookFile,
+  sigmaManifestSchema,
+  stagedDataModelFileSchema,
+  stagedWorkbookFileSchema,
+  STAGED_FILES,
+} from './types.js';
+
+interface LoadedBundle {
+  manifest: SigmaManifest | null;
+  dataModelsByPath: Map<string, StagedDataModelFile>;
+  workbooksByPath: Map<string, StagedWorkbookFile>;
+  allPaths: string[];
+}
+
+async function walkStagedDir(stagedDir: string): Promise<string[]> {
+  let entries;
+  try {
+    entries = await readdir(stagedDir, { withFileTypes: true, recursive: true });
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return [];
+    throw err;
+  }
+  const paths: string[] = [];
+  for (const entry of entries) {
+    if (!entry.isFile()) continue;
+    const abs = join(entry.parentPath, entry.name);
+    paths.push(relative(stagedDir, abs).replace(/\\/g, '/'));
+  }
+  paths.sort();
+  return paths;
+}
+
+async function loadBundle(stagedDir: string): Promise<LoadedBundle> {
+  const allPaths = await walkStagedDir(stagedDir);
+  let manifest: SigmaManifest | null = null;
+  try {
+    const body = await readFile(join(stagedDir, STAGED_FILES.manifest), 'utf-8');
+    manifest = sigmaManifestSchema.parse(JSON.parse(body));
+  } catch {
+    manifest = null;
+  }
+
+  const dataModelsByPath = new Map<string, StagedDataModelFile>();
+  const dmPrefix = `${STAGED_FILES.dataModelsDir}/`;
+  for (const path of allPaths) {
+    if (!path.startsWith(dmPrefix) || !path.endsWith('.json')) continue;
+    try {
+      const body = await readFile(join(stagedDir, path), 'utf-8');
+      const parsed = stagedDataModelFileSchema.parse(JSON.parse(body));
+      dataModelsByPath.set(path, parsed);
+    } catch {
+      // Malformed file — skip.
+    }
+  }
+
+  const workbooksByPath = new Map<string, StagedWorkbookFile>();
+  const wbPrefix = `${STAGED_FILES.workbooksDir}/`;
+  for (const path of allPaths) {
+    if (!path.startsWith(wbPrefix) || !path.endsWith('.json')) continue;
+    try {
+      const body = await readFile(join(stagedDir, path), 'utf-8');
+      const parsed = stagedWorkbookFileSchema.parse(JSON.parse(body));
+      workbooksByPath.set(path, parsed);
+    } catch {
+      // Malformed file — skip.
+    }
+  }
+
+  return { manifest, dataModelsByPath, workbooksByPath, allPaths };
+}
+
+/** Max data models per LLM work unit. Controls parallel processing granularity. */
+const DATA_MODELS_PER_UNIT = 50;
+/** Max workbooks per LLM work unit. Controls incremental re-sync granularity. */
+const WORKBOOKS_PER_UNIT = 2000;
+
+function emitBatches(
+  paths: string[],
+  perUnit: number,
+  unitKeyBase: string,
+  labelBase: string,
+  noun: string,
+  allPaths: string[],
+): WorkUnit[] {
+  const batches = Math.ceil(paths.length / perUnit) || 0;
+  const units: WorkUnit[] = [];
+  for (let i = 0; i < batches; i++) {
+    const batch = paths.slice(i * perUnit, (i + 1) * perUnit);
+    const rawFiles = [...batch].sort();
+    const rawFilesSet = new Set(rawFiles);
+    const suffix = batches > 1 ? `-${i}` : '';
+    units.push({
+      unitKey: `${unitKeyBase}${suffix}`,
+      displayLabel: batches > 1 ? `${labelBase} (${i + 1}/${batches})` : labelBase,
+      rawFiles,
+      peerFileIndex: allPaths.filter((p) => !rawFilesSet.has(p)).sort(),
+      dependencyPaths: [],
+      notes: `${batch.length} ${noun}${batch.length === 1 ? '' : 's'}`,
+    });
+  }
+  return units;
+}
+
+function emitWorkUnits(bundle: LoadedBundle): WorkUnit[] {
+  if (!bundle.manifest) return [];
+  const dmPaths = [...bundle.dataModelsByPath.keys()].sort();
+  const wbPaths = [...bundle.workbooksByPath.keys()].sort();
+  return [
+    ...emitBatches(dmPaths, DATA_MODELS_PER_UNIT, 'sigma-data-models', 'Sigma: data models', 'data model', bundle.allPaths),
+    ...emitBatches(wbPaths, WORKBOOKS_PER_UNIT, 'sigma-workbooks', 'Sigma: workbooks', 'workbook', bundle.allPaths),
+  ];
+}
+
+interface ChunkOptions {
+  diffSet?: DiffSet;
+}
+
+export async function chunkSigmaStagedDir(stagedDir: string, opts: ChunkOptions = {}): Promise<ChunkResult> {
+  const bundle = await loadBundle(stagedDir);
+  if (!bundle.manifest) {
+    return { workUnits: [] };
+  }
+
+  const firstRunUnits = emitWorkUnits(bundle);
+  if (!opts.diffSet) {
+    return { workUnits: firstRunUnits };
+  }
+
+  const touched = new Set([...opts.diffSet.added, ...opts.diffSet.modified]);
+  const kept: WorkUnit[] = [];
+  for (const wu of firstRunUnits) {
+    const anyTouched = wu.rawFiles.some((p) => touched.has(p));
+    if (!anyTouched) continue;
+    const changedFiles = wu.rawFiles.filter((p) => touched.has(p));
+    const unchangedFiles = wu.rawFiles.filter((p) => !touched.has(p));
+    const deps = new Set([...wu.dependencyPaths, ...unchangedFiles]);
+    kept.push({ ...wu, rawFiles: changedFiles.sort(), dependencyPaths: [...deps].sort() });
+  }
+  const eviction =
+    opts.diffSet.deleted.length > 0 ? { deletedRawPaths: [...opts.diffSet.deleted].sort() } : undefined;
+  return { workUnits: kept, eviction };
+}

packages/cli/src/context/ingest/adapters/sigma/client-port.ts

@@ -0,0 +1,51 @@
+import type { FetchContext } from '../../types.js';
+import type { SigmaPullConfig, WorkbookFilterInput } from './types.js';
+
+export interface SigmaTestConnectionResult {
+  success: boolean;
+  message?: string;
+  error?: string;
+}
+
+/** Data model summary shape from GET /v2/dataModels list response. */
+export interface SigmaDataModelSummary {
+  dataModelId: string;
+  dataModelUrlId: string;
+  name: string;
+  path: string;
+  latestVersion: number;
+  ownerId: string;
+  createdAt: string;
+  updatedAt: string;
+  isArchived?: boolean;
+}
+
+/** Workbook summary shape from GET /v2/workbooks list response. */
+export interface SigmaWorkbookSummary {
+  workbookId: string;
+  workbookUrlId: string;
+  name: string;
+  path: string;
+  latestVersion: number;
+  ownerId: string;
+  createdAt: string;
+  updatedAt: string;
+  isArchived?: boolean;
+  description?: string;
+}
+
+/** Re-exported so callers can reference the type without importing from types.ts directly. */
+export type { WorkbookFilterInput as ListWorkbooksOptions } from './types.js';
+
+export interface SigmaRuntimeClient {
+  testConnection(): Promise<SigmaTestConnectionResult>;
+  listDataModels(): Promise<SigmaDataModelSummary[]>;
+  listWorkbooks(opts?: WorkbookFilterInput): Promise<SigmaWorkbookSummary[]>;
+  /** Returns the raw spec object from GET /v2/dataModels/{id}/spec. */
+  getDataModelSpec(dataModelId: string): Promise<unknown>;
+  cleanup(): Promise<void>;
+}
+
+export interface SigmaClientFactory {
+  createClient(config: SigmaPullConfig, ctx: FetchContext): Promise<SigmaRuntimeClient> | SigmaRuntimeClient;
+}

packages/cli/src/context/ingest/adapters/sigma/client.ts

@@ -0,0 +1,231 @@
+import type {
+  ListWorkbooksOptions,
+  SigmaDataModelSummary,
+  SigmaRuntimeClient,
+  SigmaTestConnectionResult,
+  SigmaWorkbookSummary,
+} from './client-port.js';
+
+export interface SigmaClientRuntimeConfig {
+  apiUrl: string;
+  clientId: string;
+  clientSecret: string;
+}
+
+export interface SigmaClientConfig {
+  maxRetries: number;
+  baseDelayMs: number;
+  maxDelayMs: number;
+  timeoutMs: number;
+}
+
+export const DEFAULT_SIGMA_CLIENT_CONFIG: SigmaClientConfig = {
+  maxRetries: 3,
+  baseDelayMs: 500,
+  maxDelayMs: 10_000,
+  timeoutMs: 30_000,
+};
+
+interface TokenResponse {
+  access_token: string;
+  refresh_token?: string;
+  token_type: string;
+  expires_in: number;
+}
+
+interface PaginatedResponse<T> {
+  entries: T[];
+  nextPage: string | null;
+  total?: number;
+}
+
+function isNonRetryable500(text: string): boolean {
+  try {
+    const body = JSON.parse(text) as Record<string, unknown>;
+    // service_error indicates a deterministic Sigma rejection (e.g. unsupported data
+    // source subtype). Retrying will not help, so throw immediately.
+    return body['code'] === 'service_error';
+  } catch {
+    return false;
+  }
+}
+
+export class DefaultSigmaClient implements SigmaRuntimeClient {
+  private accessToken: string | null = null;
+  private refreshToken: string | null = null;
+  private tokenExpiresAt = 0;
+  private tokenInflight: Promise<void> | null = null;
+
+  constructor(
+    private readonly runtimeConfig: SigmaClientRuntimeConfig,
+    private readonly clientConfig: SigmaClientConfig = DEFAULT_SIGMA_CLIENT_CONFIG,
+  ) {}
+
+  private get apiUrl(): string {
+    return this.runtimeConfig.apiUrl.replace(/\/$/, '');
+  }
+
+  private basicAuthHeader(): string {
+    const credentials = Buffer.from(
+      `${this.runtimeConfig.clientId}:${this.runtimeConfig.clientSecret}`,
+    ).toString('base64');
+    return `Basic ${credentials}`;
+  }
+
+  private async fetchToken(body: URLSearchParams): Promise<TokenResponse> {
+    const res = await fetch(`${this.apiUrl}/v2/auth/token`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        Authorization: this.basicAuthHeader(),
+      },
+      body: body.toString(),
+      signal: AbortSignal.timeout(this.clientConfig.timeoutMs),
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      throw new Error(`Sigma auth failed (${res.status}): ${text}`);
+    }
+    return res.json() as Promise<TokenResponse>;
+  }
+
+  private async ensureToken(): Promise<void> {
+    const now = Date.now();
+    // Refresh 60 s before expiry so in-flight requests don't get 401.
+    if (this.accessToken && now < this.tokenExpiresAt - 60_000) {
+      return;
+    }
+    if (this.tokenInflight) return this.tokenInflight;
+    const body = new URLSearchParams();
+    if (this.refreshToken) {
+      body.set('grant_type', 'refresh_token');
+      body.set('refresh_token', this.refreshToken);
+    } else {
+      body.set('grant_type', 'client_credentials');
+    }
+    this.tokenInflight = this.fetchToken(body)
+      .then((data) => {
+        this.accessToken = data.access_token;
+        this.refreshToken = data.refresh_token ?? null;
+        this.tokenExpiresAt = Date.now() + data.expires_in * 1000;
+      })
+      .finally(() => {
+        this.tokenInflight = null;
+      });
+    return this.tokenInflight;
+  }
+
+  private async request<T>(path: string, query?: Record<string, string>): Promise<T> {
+    await this.ensureToken();
+
+    const url = new URL(`${this.apiUrl}${path}`);
+    if (query) {
+      for (const [k, v] of Object.entries(query)) {
+        url.searchParams.set(k, v);
+      }
+    }
+
+    let lastError: Error | null = null;
+    for (let attempt = 0; attempt <= this.clientConfig.maxRetries; attempt++) {
+      if (attempt > 0) {
+        const delay = Math.min(
+          this.clientConfig.baseDelayMs * 2 ** (attempt - 1),
+          this.clientConfig.maxDelayMs,
+        );
+        await new Promise<void>((resolve) => setTimeout(resolve, delay));
+      }
+
+      const res = await fetch(url.toString(), {
+        headers: { Authorization: `Bearer ${this.accessToken}` },
+        signal: AbortSignal.timeout(this.clientConfig.timeoutMs),
+      });
+
+      if (res.status === 401) {
+        // Token rejected — force full re-auth and retry once.
+        this.accessToken = null;
+        this.refreshToken = null;
+        this.tokenExpiresAt = 0;
+        await this.ensureToken();
+        const retried = await fetch(url.toString(), {
+          headers: { Authorization: `Bearer ${this.accessToken}` },
+          signal: AbortSignal.timeout(this.clientConfig.timeoutMs),
+        });
+        if (!retried.ok) {
+          const text = await retried.text().catch(() => '');
+          throw new Error(`Sigma API error after token refresh (${retried.status}): ${text}`);
+        }
+        return retried.json() as Promise<T>;
+      }
+
+      if (res.status === 429 || res.status >= 500) {
+        const text = await res.text().catch(() => '');
+        lastError = new Error(`Sigma API error (${res.status}): ${text}`);
+        if (isNonRetryable500(text)) throw lastError;
+        continue;
+      }
+
+      if (!res.ok) {
+        const text = await res.text().catch(() => '');
+        throw new Error(`Sigma API error (${res.status}): ${text}`);
+      }
+
+      return res.json() as Promise<T>;
+    }
+
+    throw lastError ?? new Error('Sigma API request failed after retries');
+  }
+
+  private async paginateAll<T>(path: string, query: Record<string, string> = {}): Promise<T[]> {
+    const all: T[] = [];
+    let page: string | null = null;
+    do {
+      const q: Record<string, string> = { ...query, limit: '1000' };
+      if (page) {
+        q['page'] = page;
+      }
+      const res = await this.request<PaginatedResponse<T>>(path, q);
+      all.push(...res.entries);
+      page = res.nextPage ?? null;
+    } while (page !== null);
+    return all;
+  }
+
+  async testConnection(): Promise<SigmaTestConnectionResult> {
+    try {
+      await this.ensureToken();
+      return { success: true };
+    } catch (err) {
+      return { success: false, error: err instanceof Error ? err.message : String(err) };
+    }
+  }
+
+  async listDataModels(): Promise<SigmaDataModelSummary[]> {
+    return this.paginateAll<SigmaDataModelSummary>('/v2/dataModels');
+  }
+
+  async listWorkbooks(opts: ListWorkbooksOptions = {}): Promise<SigmaWorkbookSummary[]> {
+    const query: Record<string, string> = {};
+    if (!opts.includeExplorations) query['excludeExplorations'] = 'true';
+
+    let results = await this.paginateAll<SigmaWorkbookSummary>('/v2/workbooks', query);
+
+    if (!opts.includeArchived) {
+      results = results.filter((wb) => !wb.isArchived);
+    }
+    if (opts.updatedSince) {
+      const since = new Date(opts.updatedSince).getTime();
+      results = results.filter((wb) => new Date(wb.updatedAt).getTime() >= since);
+    }
+    return results;
+  }
+
+  async getDataModelSpec(dataModelId: string): Promise<unknown> {
+    return this.request<unknown>(`/v2/dataModels/${encodeURIComponent(dataModelId)}/spec`);
+  }
+
+  async cleanup(): Promise<void> {
+    this.accessToken = null;
+    this.refreshToken = null;
+    this.tokenExpiresAt = 0;
+  }
+}

packages/cli/src/context/ingest/adapters/sigma/detect.ts

@@ -0,0 +1,21 @@
+import { readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { STAGED_FILES } from './types.js';
+
+export async function detectSigmaStagedDir(stagedDir: string): Promise<boolean> {
+  try {
+    await stat(join(stagedDir, STAGED_FILES.manifest));
+  } catch {
+    return false;
+  }
+  for (const subdir of [STAGED_FILES.dataModelsDir, STAGED_FILES.workbooksDir]) {
+    let entries: string[];
+    try {
+      entries = await readdir(join(stagedDir, subdir));
+    } catch {
+      continue;
+    }
+    if (entries.some((name) => name.endsWith('.json'))) return true;
+  }
+  return false;
+}

packages/cli/src/context/ingest/adapters/sigma/fetch.ts

@@ -0,0 +1,241 @@
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { FetchContext } from '../../types.js';
+import type { SigmaClientFactory } from './client-port.js';
+import {
+  type SigmaManifest,
+  type SigmaProjectionConfig,
+  type StagedDataModelFile,
+  type StagedWorkbookFile,
+  parseSigmaPullConfig,
+  stagedDataModelFileSchema,
+  stagedWorkbookFileSchema,
+  STAGED_FILES,
+} from './types.js';
+
+export interface SigmaFetchLogger {
+  log(message: string): void;
+  warn(message: string): void;
+}
+
+const noopLogger: SigmaFetchLogger = { log: () => undefined, warn: () => undefined };
+
+export interface FetchSigmaBundleParams {
+  pullConfig: unknown;
+  stagedDir: string;
+  ctx: FetchContext;
+  clientFactory: SigmaClientFactory;
+  logger?: SigmaFetchLogger;
+}
+
+async function loadExistingStagedFiles(stagedDir: string): Promise<Map<string, StagedDataModelFile>> {
+  const existing = new Map<string, StagedDataModelFile>();
+  const dmDir = join(stagedDir, STAGED_FILES.dataModelsDir);
+  let entries: string[];
+  try {
+    entries = await readdir(dmDir);
+  } catch {
+    return existing;
+  }
+  for (const entry of entries) {
+    if (!entry.endsWith('.json')) continue;
+    try {
+      const body = await readFile(join(dmDir, entry), 'utf-8');
+      const parsed = stagedDataModelFileSchema.parse(JSON.parse(body));
+      existing.set(parsed.sigmaId, parsed);
+    } catch {
+      // Skip malformed files.
+    }
+  }
+  return existing;
+}
+
+async function loadExistingWorkbookFiles(stagedDir: string): Promise<Map<string, StagedWorkbookFile>> {
+  const existing = new Map<string, StagedWorkbookFile>();
+  const wbDir = join(stagedDir, STAGED_FILES.workbooksDir);
+  let entries: string[];
+  try {
+    entries = await readdir(wbDir);
+  } catch {
+    return existing;
+  }
+  for (const entry of entries) {
+    if (!entry.endsWith('.json')) continue;
+    try {
+      const body = await readFile(join(wbDir, entry), 'utf-8');
+      const parsed = stagedWorkbookFileSchema.parse(JSON.parse(body));
+      existing.set(parsed.sigmaId, parsed);
+    } catch {
+      // Skip malformed files.
+    }
+  }
+  return existing;
+}
+
+export async function fetchSigmaBundle({
+  pullConfig,
+  stagedDir,
+  ctx,
+  clientFactory,
+  logger = noopLogger,
+}: FetchSigmaBundleParams): Promise<void> {
+  const config = parseSigmaPullConfig(pullConfig);
+  const client = await clientFactory.createClient(config, ctx);
+
+  try {
+    await mkdir(join(stagedDir, STAGED_FILES.dataModelsDir), { recursive: true });
+    await mkdir(join(stagedDir, STAGED_FILES.workbooksDir), { recursive: true });
+
+    // Load existing staged files to enable incremental sync.
+    const existingByModelId = await loadExistingStagedFiles(stagedDir);
+    const existingByWorkbookId = await loadExistingWorkbookFiles(stagedDir);
+
+    logger.log('Listing Sigma data models...');
+    const summaries = await client.listDataModels();
+    const nonArchived = summaries.filter((dm) => !dm.isArchived);
+    const nonArchivedIds = new Set(nonArchived.map((dm) => dm.dataModelId));
+    let active = nonArchived;
+    if (config.dataModelFilter?.updatedSince) {
+      const since = new Date(config.dataModelFilter.updatedSince).getTime();
+      active = active.filter((dm) => new Date(dm.updatedAt).getTime() >= since);
+    }
+    logger.log(`Found ${active.length} active data model(s) (${summaries.length} total).`);
+
+    let fetched = 0;
+    let skipped = 0;
+
+    const SPEC_CONCURRENCY = 10;
+    const queue = [...active];
+    await Promise.all(
+      Array.from({ length: Math.min(SPEC_CONCURRENCY, queue.length) }, async () => {
+        let summary;
+        while ((summary = queue.shift()) !== undefined) {
+          const existing = existingByModelId.get(summary.dataModelId);
+
+          // Only skip when the cached spec was successfully fetched. spec: null means
+          // the previous attempt failed transiently — retry regardless of updatedAt.
+          if (existing && existing.updatedAt === summary.updatedAt && existing.spec !== null) {
+            logger.log(`Unchanged: ${summary.name}`);
+            skipped++;
+            continue;
+          }
+
+          let spec: unknown = null;
+          try {
+            spec = await client.getDataModelSpec(summary.dataModelId);
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (msg.includes('dataSource subtype not supported')) {
+              logger.warn(
+                `Skipping spec for "${summary.name}" (${summary.dataModelId}): data source type not supported by Sigma spec export API.`,
+              );
+            } else {
+              logger.warn(`Failed to fetch spec for "${summary.name}" (${summary.dataModelId}): ${msg}`);
+            }
+          }
+
+          const staged: StagedDataModelFile = {
+            sigmaId: summary.dataModelId,
+            name: summary.name,
+            path: summary.path,
+            latestVersion: summary.latestVersion,
+            updatedAt: summary.updatedAt,
+            isArchived: summary.isArchived ?? false,
+            dataModelUrlId: summary.dataModelUrlId,
+            spec,
+          };
+
+          const filePath = join(stagedDir, STAGED_FILES.dataModelsDir, `${summary.dataModelId}.json`);
+          await writeFile(filePath, JSON.stringify(staged, null, 2), 'utf-8');
+          logger.log(`Staged data model: ${summary.name}`);
+          fetched++;
+        }
+      }),
+    );
+
+    // Remove staged files for models that are archived or deleted — but not those merely outside the filter window.
+    for (const [modelId] of existingByModelId) {
+      if (nonArchivedIds.has(modelId)) continue;
+      try {
+        await rm(join(stagedDir, STAGED_FILES.dataModelsDir, `${modelId}.json`));
+        logger.log(`Removed stale staged file for model ${modelId}.`);
+      } catch {
+        // Best-effort removal.
+      }
+    }
+
+    // Fetch workbooks (summary metadata only — no separate spec endpoint).
+    // Fetch the full non-archived/non-exploration universe first so eviction is based on
+    // all known workbooks, not just the updatedSince slice. Mirrors the data-model path.
+    logger.log('Listing Sigma workbooks...');
+    const { updatedSince, ...filterWithoutSince } = config.workbookFilter ?? {};
+    const allWorkbooks = await client.listWorkbooks(filterWithoutSince);
+    const nonArchivedWorkbookIds = new Set(allWorkbooks.map((wb) => wb.workbookId));
+    const activeWorkbooks = updatedSince
+      ? allWorkbooks.filter((wb) => new Date(wb.updatedAt).getTime() >= new Date(updatedSince).getTime())
+      : allWorkbooks;
+    logger.log(`Found ${activeWorkbooks.length} workbook(s) to process (${allWorkbooks.length} total).`);
+
+    let workbooksFetched = 0;
+    let workbooksSkipped = 0;
+
+    for (const wb of activeWorkbooks) {
+      const existing = existingByWorkbookId.get(wb.workbookId);
+
+      if (existing && existing.updatedAt === wb.updatedAt) {
+        workbooksSkipped++;
+        continue;
+      }
+
+      const staged: StagedWorkbookFile = {
+        sigmaId: wb.workbookId,
+        name: wb.name,
+        path: wb.path,
+        latestVersion: wb.latestVersion,
+        updatedAt: wb.updatedAt,
+        isArchived: wb.isArchived ?? false,
+        workbookUrlId: wb.workbookUrlId,
+        description: wb.description,
+      };
+
+      const filePath = join(stagedDir, STAGED_FILES.workbooksDir, `${wb.workbookId}.json`);
+      await writeFile(filePath, JSON.stringify(staged, null, 2), 'utf-8');
+      logger.log(`Staged workbook: ${wb.name}`);
+      workbooksFetched++;
+    }
+
+    // Evict only workbooks that are archived or deleted — not those outside the updatedSince window.
+    for (const [workbookId] of existingByWorkbookId) {
+      if (nonArchivedWorkbookIds.has(workbookId)) continue;
+      try {
+        await rm(join(stagedDir, STAGED_FILES.workbooksDir, `${workbookId}.json`));
+        logger.log(`Removed stale staged file for workbook ${workbookId}.`);
+      } catch {
+        // Best-effort removal.
+      }
+    }
+
+    const projectionConfig: SigmaProjectionConfig = {
+      connectionMappings: config.connectionMappings ?? {},
+      workbookFilter: config.workbookFilter ?? { includeArchived: false, includeExplorations: false },
+    };
+    await writeFile(
+      join(stagedDir, STAGED_FILES.projectionConfig),
+      JSON.stringify(projectionConfig, null, 2),
+      'utf-8',
+    );
+
+    const manifest: SigmaManifest = {
+      sigmaConnectionId: config.sigmaConnectionId,
+      fetchedAt: new Date().toISOString(),
+      dataModelCount: active.length,
+      workbookCount: activeWorkbooks.length,
+    };
+    await writeFile(join(stagedDir, STAGED_FILES.manifest), JSON.stringify(manifest, null, 2), 'utf-8');
+    logger.log(
+      `Sigma fetch complete. Data models: ${fetched} fetched, ${skipped} unchanged. Workbooks: ${workbooksFetched} fetched, ${workbooksSkipped} unchanged.`,
+    );
+  } finally {
+    await client.cleanup();
+  }
+}

packages/cli/src/context/ingest/adapters/sigma/local-sigma.adapter.ts

@@ -0,0 +1,76 @@
+import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js';
+import type { KtxLocalProject } from '../../../../context/project/project.js';
+import { resolveKtxConfigReference } from '../../../core/config-reference.js';
+import { DEFAULT_SIGMA_CLIENT_CONFIG, DefaultSigmaClient, type SigmaClientConfig } from './client.js';
+import type { SigmaClientFactory, SigmaRuntimeClient } from './client-port.js';
+import type { SigmaFetchLogger } from './fetch.js';
+import type { SigmaPullConfig } from './types.js';
+import { SigmaSourceAdapter } from './sigma.adapter.js';
+import type { FetchContext } from '../../types.js';
+
+function stringField(value: unknown): string | null {
+  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
+}
+
+export function sigmaRuntimeConfigFromLocalConnection(
+  connectionId: string,
+  connection: KtxProjectConnectionConfig | undefined,
+  env: NodeJS.ProcessEnv = process.env,
+): { apiUrl: string; clientId: string; clientSecret: string } {
+  if (!connection || String(connection.driver).toLowerCase() !== 'sigma') {
+    throw new Error(`Connection "${connectionId}" is not a Sigma connection`);
+  }
+
+  const apiUrl = stringField(connection.api_url) ?? 'https://api.sigmacomputing.com';
+  const clientId = stringField(connection.client_id);
+  const literalSecret = stringField(connection.client_secret);
+  const secretRef = stringField(connection.client_secret_ref);
+  const clientSecret =
+    literalSecret ?? (secretRef ? (resolveKtxConfigReference(secretRef, env) ?? null) : null);
+
+  if (!clientId) {
+    throw new Error(`Connection "${connectionId}" is missing Sigma client_id`);
+  }
+  if (!clientSecret) {
+    throw new Error(
+      `Connection "${connectionId}" is missing Sigma client_secret or client_secret_ref`,
+    );
+  }
+
+  return { apiUrl, clientId, clientSecret };
+}
+
+interface CreateLocalSigmaSourceAdapterOptions {
+  env?: NodeJS.ProcessEnv;
+  defaultClientConfig?: SigmaClientConfig;
+  logger?: SigmaFetchLogger;
+}
+
+class LocalSigmaClientFactory implements SigmaClientFactory {
+  constructor(
+    private readonly project: KtxLocalProject,
+    private readonly options: CreateLocalSigmaSourceAdapterOptions,
+  ) {}
+
+  createClient(config: SigmaPullConfig, _ctx: FetchContext): SigmaRuntimeClient {
+    const runtimeConfig = sigmaRuntimeConfigFromLocalConnection(
+      config.sigmaConnectionId,
+      this.project.config.connections[config.sigmaConnectionId],
+      this.options.env,
+    );
+    return new DefaultSigmaClient(
+      runtimeConfig,
+      this.options.defaultClientConfig ?? DEFAULT_SIGMA_CLIENT_CONFIG,
+    );
+  }
+}
+
+export function createLocalSigmaSourceAdapter(
+  project: KtxLocalProject,
+  options: CreateLocalSigmaSourceAdapterOptions = {},
+): SigmaSourceAdapter {
+  return new SigmaSourceAdapter({
+    clientFactory: new LocalSigmaClientFactory(project, options),
+    ...(options.logger ? { logger: options.logger } : {}),
+  });
+}

packages/cli/src/context/ingest/adapters/sigma/project.ts

@@ -0,0 +1,231 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { z } from 'zod';
+import type { SemanticLayerService } from '../../../../context/sl/semantic-layer.service.js';
+import type { SemanticLayerSource } from '../../../../context/sl/types.js';
+import type { DeterministicProjectionContext, ProjectionResult } from '../../types.js';
+import { sigmaProjectionConfigSchema, stagedDataModelFileSchema, STAGED_FILES } from './types.js';
+
+async function readProjectionConfig(stagedDir: string): Promise<Record<string, string>> {
+  try {
+    const body = await readFile(join(stagedDir, STAGED_FILES.projectionConfig), 'utf-8');
+    return sigmaProjectionConfigSchema.parse(JSON.parse(body)).connectionMappings;
+  } catch {
+    return {};
+  }
+}
+
+const SIGMA_AUTHOR = { name: 'Sigma', email: 'system@kaelio.dev' } as const;
+
+// Best-effort schema for the raw spec blob stored in staged data model files.
+const warehouseTableSourceSchema = z.object({
+  kind: z.literal('warehouse-table'),
+  connectionId: z.string(),
+  path: z.array(z.string()),
+});
+
+const specColumnSchema = z
+  .object({
+    id: z.string(),
+    formula: z.string().optional(),
+    name: z.string().optional(),
+    hidden: z.boolean().optional(),
+    description: z.string().optional(),
+    format: z.object({ kind: z.string() }).passthrough().optional(),
+  })
+  .passthrough();
+
+const specElementSchema = z
+  .object({
+    id: z.string(),
+    kind: z.string().optional(),
+    name: z.string().optional(),
+    hidden: z.boolean().optional(),
+    source: z.object({ kind: z.string() }).passthrough().optional(),
+    columns: z.array(specColumnSchema).optional(),
+  })
+  .passthrough();
+
+const specPageSchema = z
+  .object({
+    id: z.string(),
+    name: z.string().optional(),
+    elements: z.array(specElementSchema).optional(),
+  })
+  .passthrough();
+
+const sigmaSpecSchema = z
+  .object({
+    name: z.string().optional(),
+    pages: z.array(specPageSchema).optional(),
+  })
+  .passthrough();
+
+type SpecColumn = z.infer<typeof specColumnSchema>;
+
+/** Extract the column name from a bracket formula like `[TABLE/Column Name]` or `[Column]`. */
+function extractColumnName(formula: string): string | null {
+  const match = /\[(?:[^\]/]+\/)?([^\]]+)\]/.exec(formula.trim());
+  return match?.[1] ?? null;
+}
+
+function slugify(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '_')
+    .replace(/^_+|_+$/g, '');
+}
+
+function inferColumnType(col: SpecColumn): string {
+  const kind = col.format?.kind;
+  if (kind === 'datetime' || kind === 'date') return 'time';
+  if (kind === 'number' || kind === 'currency' || kind === 'percent') return 'number';
+  return 'string';
+}
+
+function buildSourceFromElement(
+  dataModelName: string,
+  elementName: string | undefined,
+  elementId: string,
+  warehousePath: string[],
+  columns: SpecColumn[],
+): SemanticLayerSource | null {
+  const table = warehousePath.join('.');
+  if (!table) return null;
+
+  const modelSlug = slugify(dataModelName || elementId);
+  const elemSlug = elementName ? slugify(elementName) : '';
+  const sourceName = elemSlug && elemSlug !== modelSlug ? `${modelSlug}_${elemSlug}` : modelSlug;
+  if (!sourceName) return null;
+
+  const slColumns: SemanticLayerSource['columns'] = [];
+
+  for (const col of columns) {
+    if (col.hidden) continue;
+    if (!col.formula) continue;
+    // Aggregation formulas (Sum, Count, etc.) are Sigma-specific expressions that don't map to
+    // warehouse columns — skip them silently. The sigma_ingest skill surfaces them as wiki candidates.
+    if (/^[A-Za-z]+\(/.test(col.formula.trim())) continue;
+
+    const displayName = col.name ?? extractColumnName(col.formula);
+    if (!displayName) continue;
+    const colSlug = slugify(displayName);
+    if (!colSlug) continue;
+
+    slColumns.push({
+      name: colSlug,
+      type: inferColumnType(col),
+      ...(col.description ? { descriptions: { user: col.description } } : {}),
+    });
+  }
+
+  const source: SemanticLayerSource = {
+    name: sourceName,
+    table,
+    grain: [],
+    columns: slColumns,
+    joins: [],
+    measures: [],
+  };
+
+  if (dataModelName) {
+    source.descriptions = { user: dataModelName };
+  }
+
+  return source;
+}
+
+type SlService = Pick<SemanticLayerService, 'writeSource'> & {
+  forWorktree(workdir: string): Pick<SemanticLayerService, 'writeSource'>;
+};
+
+/** @internal */
+export async function projectSigmaDataModels(
+  ctx: DeterministicProjectionContext,
+  slService: SlService,
+): Promise<ProjectionResult> {
+  const svc = ctx.workdir ? slService.forWorktree(ctx.workdir) : slService;
+  const warnings: string[] = [];
+  const errors: string[] = [];
+  const touchedSources: Array<{ connectionId: string; sourceName: string }> = [];
+
+  const connectionMappings = await readProjectionConfig(ctx.stagedDir);
+
+  const dmDir = join(ctx.stagedDir, STAGED_FILES.dataModelsDir);
+  let entries: string[];
+  try {
+    entries = await readdir(dmDir);
+  } catch {
+    return { warnings, errors, touchedSources, changedWikiPageKeys: [] };
+  }
+
+  for (const entry of entries) {
+    if (!entry.endsWith('.json')) continue;
+
+    let stagedFile: z.infer<typeof stagedDataModelFileSchema>;
+    try {
+      const body = await readFile(join(dmDir, entry), 'utf-8');
+      stagedFile = stagedDataModelFileSchema.parse(JSON.parse(body));
+    } catch {
+      warnings.push(`Skipping malformed staged file: ${entry}`);
+      continue;
+    }
+
+    if (!stagedFile.spec) continue;
+
+    let spec: z.infer<typeof sigmaSpecSchema>;
+    try {
+      spec = sigmaSpecSchema.parse(stagedFile.spec);
+    } catch {
+      warnings.push(`Skipping unparseable spec for data model "${stagedFile.name}"`);
+      continue;
+    }
+
+    for (const page of spec.pages ?? []) {
+      for (const element of page.elements ?? []) {
+        if (element.hidden) continue;
+
+        const warehouseSource = warehouseTableSourceSchema.safeParse(element.source);
+        if (!warehouseSource.success) continue;
+
+        const source = buildSourceFromElement(
+          stagedFile.name,
+          element.name,
+          element.id,
+          warehouseSource.data.path,
+          element.columns ?? [],
+        );
+        if (!source) continue;
+
+        // Only write SL sources for elements whose Sigma connection is mapped to a warehouse connection.
+        // Writing under an unmapped connection produces gate failures because the Sigma connection
+        // is not a warehouse connection and cannot be validated.
+        const targetConnectionId = connectionMappings[warehouseSource.data.connectionId];
+        if (!targetConnectionId) {
+          warnings.push(
+            `Skipping SL source for "${stagedFile.name}" / "${element.name ?? element.id}": ` +
+              `no connectionMappings entry for Sigma connection ${warehouseSource.data.connectionId}. ` +
+              `Add a connectionMappings entry in ktx.yaml to enable SL projection for this element.`,
+          );
+          continue;
+        }
+
+        try {
+          const result = await svc.writeSource(
+            targetConnectionId,
+            source,
+            SIGMA_AUTHOR.name,
+            SIGMA_AUTHOR.email,
+            `Sigma: import data model "${stagedFile.name}"`,
+          );
+          touchedSources.push({ connectionId: targetConnectionId, sourceName: source.name });
+          warnings.push(...result.warnings);
+        } catch (err) {
+          errors.push(`Failed to write source "${source.name}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+      }
+    }
+  }
+
+  return { warnings, errors, touchedSources, changedWikiPageKeys: [] };
+}