From 3c4fcc27c790624d45ebaff23e84722d22b3573f Mon Sep 17 00:00:00 2001 From: Kevin Messiaen <114553769+kevinmessiaen@users.noreply.github.com> Date: Wed, 1 Jul 2026 19:06:02 +0700 Subject: [PATCH 01/11] feat: Add duckdb connector (#308) * refactor(duckdb): extract shared json-safe bigint helper Co-Authored-By: Claude Opus 4.8 * feat(duckdb): add and register the duckdb primary connector Add KtxDuckDbDialect, KtxDuckDbScanConnector (local file-backed, read-only, never-create, main-schema introspection via information_schema and duckdb_constraints() for foreign keys), and register the duckdb driver across the dialect factory, driver registry, connection-type enum, warehouse descriptor, config schema, scan normalization, connection test drivers, and status display. Co-Authored-By: Claude Opus 4.8 * feat(duckdb): route live-database ingest through the DuckDB connector Add the DuckDB live-database introspection bridge and dispatch duckdb connections to it in local-adapters, matching the SQLite path. Repoint the config-rejection test off duckdb (now a valid driver) onto the no-driver case. Co-Authored-By: Claude Opus 4.8 * feat(duckdb): add duckdb to the setup database flow Offer DuckDB in the interactive checklist and via ktx setup --database duckdb, with a file-path prompt and duckdb-local default connection id, parallel to SQLite. Co-Authored-By: Claude Opus 4.8 * feat(duckdb): attach native duckdb files in federation Native .duckdb members ATTACH with (READ_ONLY) and no TYPE/INSTALL/LOAD, since the duckdb format needs no extension. attachTypeForDriver returns null for the native case; buildAttachStatements builds load statements from non-null types only and emits a conditional ATTACH clause. Co-Authored-By: Claude Opus 4.8 * docs(duckdb): document the duckdb primary-source connector Add a DuckDB section to the primary-sources integration page (config, read-only never-create behavior, main-schema scope, federation) and update the supported-driver assertion in dialects.test.ts to include duckdb. Co-Authored-By: Claude Opus 4.8 * fix(duckdb): use single-namespace display shape for main-only refs DuckDB v1 introspects the main schema and sets db=null on every table, so its display refs are single-namespace like SQLite. The ansi shape emitted a 1-part table display it then refused to parse, breaking column-level display resolution. Switch the dialect to the sqlite display shape and add a round-trip test plus a composite-foreign-key test that were missing. Co-Authored-By: Claude Opus 4.8 * refactor(duckdb): resolve connector dialect via getDialectForDriver Route the connector's dialect through the shared factory like every other connector, now that duckdb is registered. Single construction path. Co-Authored-By: Claude Opus 4.8 * fix(duckdb): skip schema picker for single-file duckdb setup DuckDB is a single-file, single-namespace ('main') database like SQLite, but the setup scope step only skipped the schema picker for sqlite. DuckDB fell into the multi-schema path with an empty schema list, rendering a broken picker ("No matches found" for main). Extend the file-based-driver early-return to cover duckdb so it ingests every table directly. Co-Authored-By: Claude Opus 4.8 * refactor(duckdb): reuse shared config helper and derive scope skip Route duckdb path resolution through the shared resolveStringReference helper instead of a local third copy of env:/file: handling. Derive the setup scope-picker skip from SCOPE_DISCOVERY_SPECS membership rather than a hardcoded sqlite/duckdb driver list. Co-Authored-By: Claude Opus 4.8 * test(duckdb): use a genuinely unknown driver in the rejection test The merged "rejects unknown drivers" test used `driver: duckdb` as its unknown-driver stand-in, which stopped being unknown once this branch added the duckdb connector. Switch to `nonsense` so it again exercises the unsupported-driver config error. Co-Authored-By: Claude Opus 4.8 * test(duckdb): cover dialect, connector, and live-introspection branches Codecov flagged uncovered branches as dead code; all are real connector, dialect, and live-ingest behavior. Add unit tests instead of removing them. - dialect: precedence ladder, sample/clause builders, profiling expressions - connector: url/env config forms, error throws, never-create guard, cardinality cap branches, table-scope empty/non-empty paths - live-introspection: full-schema and table-scope extraction Functions 100%, lines ~99% across the duckdb connector dir. Co-Authored-By: Claude Opus 4.8 * docs: add DuckDB to supported-driver references The DuckDB connector PR documented the connector itself but left the scattered supported-driver enumerations stale. Add duckdb to the federation concept page (participation table, activation, table naming, limitations), the ktx setup CLI reference, the ktx.yaml warehouse-driver table, the primary-sources field reference, and the quickstart driver list (which also restores the missing ClickHouse entry). --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: Andrey Avtomonov --- .../content/docs/cli-reference/ktx-setup.mdx | 4 +- .../concepts/cross-database-federation.mdx | 24 +- .../content/docs/configuration/ktx-yaml.mdx | 1 + .../docs/getting-started/quickstart.mdx | 3 +- .../docs/integrations/primary-sources.mdx | 52 ++- packages/cli/src/commands/setup-commands.ts | 1 + packages/cli/src/connection-drivers.ts | 1 + packages/cli/src/connection.ts | 1 + .../cli/src/connectors/duckdb/connector.ts | 395 ++++++++++++++++++ packages/cli/src/connectors/duckdb/dialect.ts | 192 +++++++++ .../src/connectors/duckdb/federated-attach.ts | 7 + .../connectors/duckdb/federated-executor.ts | 28 +- .../duckdb/live-database-introspection.ts | 40 ++ .../src/connectors/shared/duckdb-json-safe.ts | 14 + .../context/connections/connection-type.ts | 1 + .../cli/src/context/connections/dialects.ts | 2 + .../cli/src/context/connections/drivers.ts | 21 + .../cli/src/context/connections/federation.ts | 13 +- .../connections/local-warehouse-descriptor.ts | 1 + .../cli/src/context/project/driver-schemas.ts | 2 + packages/cli/src/context/scan/local-scan.ts | 3 +- packages/cli/src/context/scan/types.ts | 1 + .../src/context/sql-analysis/dialect-notes.ts | 4 +- .../context/sql-analysis/dialects/duckdb.md | 10 + packages/cli/src/local-adapters.ts | 9 + packages/cli/src/setup-databases.ts | 21 +- packages/cli/src/status-project.ts | 3 +- packages/cli/test/connection.test.ts | 2 +- .../test/connectors/duckdb/connector.test.ts | 280 +++++++++++++ .../test/connectors/duckdb/dialect.test.ts | 108 +++++ .../duckdb/federated-attach.test.ts | 8 + .../duckdb/federated-executor.test.ts | 23 + .../live-database-introspection.test.ts | 45 ++ .../shared/duckdb-json-safe.test.ts | 17 + .../test/context/connections/dialects.test.ts | 2 +- .../test/context/connections/drivers.test.ts | 4 +- .../test/context/mcp/dialect-notes.test.ts | 3 +- .../cli/test/local-scan-connectors.test.ts | 10 +- packages/cli/test/setup-databases.test.ts | 69 +++ 39 files changed, 1366 insertions(+), 59 deletions(-) create mode 100644 packages/cli/src/connectors/duckdb/connector.ts create mode 100644 packages/cli/src/connectors/duckdb/dialect.ts create mode 100644 packages/cli/src/connectors/duckdb/live-database-introspection.ts create mode 100644 packages/cli/src/connectors/shared/duckdb-json-safe.ts create mode 100644 packages/cli/src/context/sql-analysis/dialects/duckdb.md create mode 100644 packages/cli/test/connectors/duckdb/connector.test.ts create mode 100644 packages/cli/test/connectors/duckdb/dialect.test.ts create mode 100644 packages/cli/test/connectors/duckdb/live-database-introspection.test.ts create mode 100644 packages/cli/test/connectors/shared/duckdb-json-safe.test.ts diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index a424626f..d127525a 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -120,9 +120,9 @@ runtime features are missing. | Flag | Description | |------|-------------| -| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` | +| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `duckdb`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` | | `--database-connection-id ` | Existing selected connection id; repeatable. With `--database` or `--database-url`, connection id for the new connection. | -| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite path | +| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite or DuckDB path | | `--database-schema ` | Database schema or dataset to include; repeatable | | `--skip-databases` | Leave database setup incomplete | diff --git a/docs-site/content/docs/concepts/cross-database-federation.mdx b/docs-site/content/docs/concepts/cross-database-federation.mdx index ee3884f2..b4a055d1 100644 --- a/docs-site/content/docs/concepts/cross-database-federation.mdx +++ b/docs-site/content/docs/concepts/cross-database-federation.mdx @@ -1,6 +1,6 @@ --- title: Cross-database federation -description: How ktx federates postgres, mysql, and sqlite connections so a single read-only SQL query can join across them without copying data. +description: How ktx federates postgres, mysql, sqlite, and duckdb connections so a single read-only SQL query can join across them without copying data. --- Cross-database federation lets a single read-only SQL query join tables that @@ -20,13 +20,14 @@ block to add. With zero or one compatible connection the behavior is unchanged. ## Which connections participate -The v1 federation engine supports three drivers: +The v1 federation engine supports four drivers: | Driver | Participates in federation | |--------|---------------------------| | `postgres` | Yes | | `mysql` | Yes | | `sqlite` | Yes | +| `duckdb` | Yes | | `snowflake` | No — standalone connection | | `bigquery` | No — standalone connection | | `clickhouse` | No — standalone connection | @@ -38,7 +39,7 @@ queried independently; they do not appear as federation members. ## How it activates **ktx** inspects the connections in `ktx.yaml` at startup. When it finds two or -more connections whose driver is `postgres`, `mysql`, or `sqlite`, it +more connections whose driver is `postgres`, `mysql`, `sqlite`, or `duckdb`, it instantiates the DuckDB federation engine and attaches each one read-only. There is no `federation:` key, no opt-in flag, and no connection-level setting to enable. The engine is derived entirely from what is already declared. @@ -60,9 +61,10 @@ Two attach-compatible connections are present, so federation is active. ## Table naming in federated queries Inside a federated query, postgres and mysql tables use a three-part name: -`connectionId.schema.table`. SQLite tables, which have no schema layer in -DuckDB, use the two-part form `connectionId.table`. In both cases the -connection's `id` field in `ktx.yaml` becomes the catalog name inside DuckDB. +`connectionId.schema.table`. SQLite and DuckDB tables use the two-part form +`connectionId.table`, since ktx addresses both as single-namespace members. In +both cases the connection's `id` field in `ktx.yaml` becomes the catalog name +inside DuckDB. If a connection `id` is not a bare SQL identifier — for example it contains a hyphen, like `books-db` — double-quote it in the query the same way DuckDB @@ -131,8 +133,8 @@ ktx sql -c _ktx_federated \ Table names follow the rules from [Table naming in federated queries](#table-naming-in-federated-queries): three-part `connectionId.schema.table` for postgres and mysql, two-part -`connectionId.table` for sqlite. The `_ktx_federated` id is virtual — it is -never written to `ktx.yaml` and only exists when two or more attach-compatible +`connectionId.table` for sqlite and duckdb. The `_ktx_federated` id is virtual — +it is never written to `ktx.yaml` and only exists when two or more attach-compatible connections are declared. It surfaces in `ktx connection` and in the agent's connection list so the id is discoverable. Querying a single member database directly with its own connection id (`ktx sql -c pg_books ...`) is unchanged. @@ -149,6 +151,6 @@ database through the federation engine. them in a source's `joins:` block and automatic discovery of cross-database relationships are not available yet. Intra-database relationship discovery for each member connection is unchanged. -- **postgres, mysql, and sqlite only.** Other drivers (snowflake, bigquery, - clickhouse, sqlserver) do not participate in federation in this version. They - remain usable as standalone connections. +- **postgres, mysql, sqlite, and duckdb only.** Other drivers (snowflake, + bigquery, clickhouse, sqlserver) do not participate in federation in this + version. They remain usable as standalone connections. diff --git a/docs-site/content/docs/configuration/ktx-yaml.mdx b/docs-site/content/docs/configuration/ktx-yaml.mdx index 2ff54cbd..0f67877d 100644 --- a/docs-site/content/docs/configuration/ktx-yaml.mdx +++ b/docs-site/content/docs/configuration/ktx-yaml.mdx @@ -109,6 +109,7 @@ context-source drivers share the map. | `postgres` | Warehouse | `driver` | `url`, `enabled_tables`, `historicSql`, `context.queryHistory` | | `mysql` | Warehouse | `driver` | `url`, `enabled_tables` | | `sqlite` | Warehouse | `driver` | `url` or `path`, `enabled_tables` | +| `duckdb` | Warehouse | `driver` | `url` or `path`, `enabled_tables` | | `sqlserver` | Warehouse | `driver` | `url`, `enabled_tables` | | `bigquery` | Warehouse | `driver` | `credentials_json`, `dataset_ids`, `enabled_tables`, `historicSql` | | `snowflake` | Warehouse | `driver` | `schema_names`, `enabled_tables`, `historicSql` | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 2b8cbdfb..16669794 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -218,7 +218,8 @@ The wizard walks you through everything **ktx** needs in one pass: 3. **Embeddings** - picks an embeddings backend. Choose OpenAI for hosted embeddings or `sentence-transformers` to run locally without an API key. 4. **Database** - adds at least one primary connection. Supported drivers: - SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, and Snowflake. + PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, and + DuckDB. 5. **Context sources** - optionally adds dbt, MetricFlow, LookML, Looker, Metabase, or Notion. You can skip and add them later. 6. **Build** - offers to run the first ingest so semantic sources and wiki diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 40af30e0..270275f1 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -1,6 +1,6 @@ --- title: Primary Sources -description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, or MongoDB. +description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, or MongoDB. --- **ktx** connects to your data warehouse or database to build schema context, @@ -26,14 +26,14 @@ Agents should prefer environment or file references over literal secrets. | Field | Required | Applies to | Description | |-------|----------|------------|-------------| -| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, or `mongodb` | +| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, or `mongodb` | | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | | `databases` | No | ClickHouse, MongoDB | List of databases to scan | | `sample_size`, `order_by` | No | MongoDB | Schema-inference sampling controls (recent documents, sort field) | | `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it | -| `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference | +| `path` | Yes for path-style SQLite/DuckDB | SQLite, DuckDB | Local SQLite or DuckDB database path or `env:NAME` reference | | `max_bytes_billed` | No | BigQuery | Maximum bytes billed per query job | | `query_timeout_ms` | No | all warehouses | Maximum execution time for a single read-only query, in milliseconds (default 30000). A query exceeding it is cancelled server-side (or, for SQLite, by terminating the off-process executor) and returns a `query exceeded Ns` error so the agent can revise. | | `project_id` | No | BigQuery | Optional local descriptor and mapping metadata; not used for BigQuery authentication | @@ -545,6 +545,52 @@ No authentication required - SQLite is file-based. The file must be readable by --- +## DuckDB + +File-based connector using the DuckDB Node.js API. Ideal for local analytics, embedded warehouses, and cross-database federation. + +### Connection config + +```yaml title="ktx.yaml" +connections: + warehouse: + driver: duckdb + path: data/warehouse.duckdb +``` + +`path` is resolved relative to the project directory. The `.duckdb` file must already exist — **ktx** never creates a missing database file. + +### Authentication + +No authentication required — DuckDB is file-based. The `.duckdb` file must be readable by the process running **ktx**. + +### Features + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Tables & views | Yes | Via `information_schema` on the `main` schema | +| Primary keys | Yes | Via `information_schema.table_constraints` | +| Foreign keys | Yes | Via DuckDB's `duckdb_constraints()` catalog function | +| Row count estimates | Yes | Exact count via `SELECT COUNT(*)` | +| Column statistics | No | - | +| Query history | No | - | +| Table sampling | Yes | - | +| Nested analysis | No | - | + +### Dialect notes + +- Introspection scans the `main` schema only +- Execution is read-only; **ktx** opens the file without write access +- Parameter binding uses positional `?` placeholders +- Uses `LIMIT X OFFSET Y` for pagination +- Database file must exist before `ktx connection test` or ingest runs + +### Cross-database federation + +When a project declares two or more attach-compatible connections — any combination of `postgres`, `mysql`, `sqlite`, and `duckdb` — **ktx** derives a cross-database federation connection. That connection can ATTACH a native `.duckdb` file, allowing semantic queries to join across sources without manually copying data. + +--- + ## MongoDB Connects to MongoDB as a primary context source. **ktx** treats each collection diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index ff8f015d..d838e472 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -38,6 +38,7 @@ function llmBackend(value: string): KtxSetupLlmBackend { function databaseDriver(value: string): KtxSetupDatabaseDriver { if ( value === 'sqlite' || + value === 'duckdb' || value === 'postgres' || value === 'mysql' || value === 'clickhouse' || diff --git a/packages/cli/src/connection-drivers.ts b/packages/cli/src/connection-drivers.ts index 511637f5..836c8458 100644 --- a/packages/cli/src/connection-drivers.ts +++ b/packages/cli/src/connection-drivers.ts @@ -3,6 +3,7 @@ import type { KtxProjectConnectionConfig } from './context/project/config.js'; /** @internal Canonical SQL-warehouse driver ids; the dialect-notes coverage test derives its required coverage from this set. */ export const KTX_DATABASE_DRIVER_IDS = [ 'sqlite', + 'duckdb', 'postgres', 'mysql', 'clickhouse', diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts index f134a4a4..809eea96 100644 --- a/packages/cli/src/connection.ts +++ b/packages/cli/src/connection.ts @@ -51,6 +51,7 @@ export interface KtxConnectionDeps { const SUPPORTED_TEST_DRIVERS = [ 'sqlite', + 'duckdb', 'postgres', 'mysql', 'clickhouse', diff --git a/packages/cli/src/connectors/duckdb/connector.ts b/packages/cli/src/connectors/duckdb/connector.ts new file mode 100644 index 00000000..5b904c14 --- /dev/null +++ b/packages/cli/src/connectors/duckdb/connector.ts @@ -0,0 +1,395 @@ +import { DuckDBInstance, type DuckDBConnection } from '@duckdb/node-api'; +import { existsSync, statSync } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { resolveStringReference } from '../shared/string-reference.js'; +import { getSqlDialectForDriver } from '../../context/connections/dialects.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { normalizeQueryRows } from '../../context/connections/query-executor.js'; +import { toJsonSafeRows } from '../shared/duckdb-json-safe.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxConnectorTestResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxSchemaForeignKey, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; +import { scopedTableNames } from '../../context/scan/table-ref.js'; + +const MAIN_SCHEMA = 'main'; + +export interface KtxDuckDbConnectionConfig { + driver?: string; + path?: string; + url?: string; + [key: string]: unknown; +} + +/** @internal */ +export interface DuckDbDatabasePathInput { + connectionId: string; + projectDir?: string; + connection: KtxDuckDbConnectionConfig | undefined; +} + +export interface KtxDuckDbScanConnectorOptions extends DuckDbDatabasePathInput { + now?: () => Date; +} + +export interface KtxDuckDbColumnDistinctValuesOptions { + maxCardinality: number; + limit: number; + sampleSize?: number; +} + +export interface KtxDuckDbColumnDistinctValuesResult { + values: string[] | null; + cardinality: number; +} + +interface InfoSchemaTableRow { + table_name: string; + table_type: string; +} + +interface InfoSchemaColumnRow { + column_name: string; + data_type: string; + is_nullable: string; +} + +// `path` may be an env:/file: reference; `url` resolves env: only, since file: +// on a url is a native URI form (handled by duckDbPathFromUrl), not a file read. +function stringConfigValue( + connection: KtxDuckDbConnectionConfig | undefined, + key: 'path' | 'url', +): string | undefined { + const value = connection?.[key]; + if (typeof value !== 'string' || value.trim().length === 0) { + return undefined; + } + const trimmed = value.trim(); + if (key === 'url') { + return trimmed.startsWith('env:') ? (process.env[trimmed.slice('env:'.length)] ?? '') : trimmed; + } + return resolveStringReference(trimmed, process.env); +} + +function duckDbPathFromUrl(url: string): string { + if (url.startsWith('file:')) { + return fileURLToPath(url); + } + if (url.startsWith('duckdb:')) { + const parsed = new URL(url); + return decodeURIComponent(parsed.pathname); + } + return url; +} + +export function isKtxDuckDbConnectionConfig( + connection: KtxDuckDbConnectionConfig | undefined, +): connection is KtxDuckDbConnectionConfig { + return String(connection?.driver ?? '').toLowerCase() === 'duckdb'; +} + +/** @internal */ +export function duckDbDatabasePathFromConfig(input: DuckDbDatabasePathInput): string { + const inputDriver = input.connection?.driver ?? 'unknown'; + if (!isKtxDuckDbConnectionConfig(input.connection)) { + throw new Error(`Native DuckDB connector cannot run driver "${inputDriver}"`); + } + const configuredPath = + stringConfigValue(input.connection, 'path') ?? duckDbPathFromUrl(stringConfigValue(input.connection, 'url') ?? ''); + if (!configuredPath) { + throw new Error(`Native DuckDB connector requires connections.${input.connectionId}.path or url`); + } + return isAbsolute(configuredPath) ? configuredPath : resolve(input.projectDir ?? process.cwd(), configuredPath); +} + +export class KtxDuckDbScanConnector implements KtxScanConnector { + readonly id: string; + readonly driver = 'duckdb' as const; + readonly capabilities = createKtxConnectorCapabilities({ + tableSampling: true, + columnSampling: true, + columnStats: false, + readOnlySql: true, + nestedAnalysis: false, + formalForeignKeys: true, + estimatedRowCounts: true, + }); + + private readonly connectionId: string; + private readonly dbPath: string; + private readonly now: () => Date; + private readonly dialect = getSqlDialectForDriver('duckdb'); + private instance: DuckDBInstance | null = null; + private connection: DuckDBConnection | null = null; + + constructor(options: KtxDuckDbScanConnectorOptions) { + this.connectionId = options.connectionId; + this.dbPath = duckDbDatabasePathFromConfig(options); + this.now = options.now ?? (() => new Date()); + this.id = `duckdb:${options.connectionId}`; + } + + async testConnection(): Promise { + try { + if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) { + return { success: false, error: `File not found: ${this.dbPath}` }; + } + await this.query('SELECT 1'); + return { success: true }; + } catch (error) { + return connectorTestFailure(error); + } + } + + async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: null, db: null }) : null; + const tableRows = await this.readTableRows(scopedNames); + const tables: KtxSchemaTable[] = []; + for (const row of tableRows) { + tables.push(await this.readTable(row)); + } + return { + connectionId: this.connectionId, + driver: 'duckdb' as const, + extractedAt: this.now().toISOString(), + scope: {}, + metadata: { + file_path: this.dbPath, + table_count: tables.length, + total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0), + }, + tables, + }; + } + + async listSchemas(): Promise { + return [MAIN_SCHEMA]; + } + + async listTables(_schemas?: string[]): Promise { + const rows = await this.readTableRows(null); + return rows.map((row) => ({ + catalog: null, + schema: MAIN_SCHEMA, + name: row.table_name, + kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const), + })); + } + + async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query( + this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns), + ); + return { headers: result.headers, rows: result.rows, totalRows: result.totalRows }; + } + + async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query( + this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit), + ); + const values = result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]); + return { values, nullCount: null, distinctCount: null }; + } + + async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise { + return null; + } + + async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query(limitSqlForExecution(input.sql, input.maxRows)); + return { ...result, rowCount: result.rows.length }; + } + + async getColumnDistinctValues( + table: KtxTableRef, + columnName: string, + options: KtxDuckDbColumnDistinctValuesOptions, + ): Promise { + const sampleSize = options.sampleSize ?? 10000; + const tableName = this.qTableName(table); + const quotedColumn = this.dialect.quoteIdentifier(columnName); + const cardinalityResult = await this.query( + this.dialect.generateCardinalitySampleQuery(tableName, quotedColumn, sampleSize), + ); + if (cardinalityResult.rows.length === 0) { + return null; + } + const cardinality = Number(cardinalityResult.rows[0][0]); + if (Number.isNaN(cardinality)) { + return null; + } + if (cardinality === 0) { + return { values: [], cardinality: 0 }; + } + if (cardinality > options.maxCardinality) { + return { values: null, cardinality }; + } + const valuesResult = await this.query( + this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit), + ); + return { + values: valuesResult.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => String(row[0])), + cardinality, + }; + } + + async getTableRowCount(tableName: string): Promise { + const result = await this.query(`SELECT COUNT(*) AS count FROM ${this.dialect.quoteIdentifier(tableName)}`); + return Number(result.rows[0]?.[0] ?? 0); + } + + qTableName(table: Pick): string { + return this.dialect.formatTableName(table); + } + + quoteIdentifier(identifier: string): string { + return this.dialect.quoteIdentifier(identifier); + } + + async cleanup(): Promise { + this.connection?.closeSync(); + this.instance?.closeSync(); + this.connection = null; + this.instance = null; + } + + private async db(): Promise { + if (!this.connection) { + // DuckDBInstance.create() creates the file if missing, so this pre-check + // enforces the never-create rule. Do not remove it. + if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) { + throw new Error(`File not found: ${this.dbPath}`); + } + this.instance = await DuckDBInstance.create(this.dbPath, { access_mode: 'read_only' }); + this.connection = await this.instance.connect(); + } + return this.connection; + } + + private async query(sql: string): Promise> { + const connection = await this.db(); + const reader = await connection.runAndReadAll(assertReadOnlySql(sql)); + const rows = toJsonSafeRows(normalizeQueryRows(reader.getRows())); + return { + headers: reader.columnNames(), + rows, + totalRows: rows.length, + }; + } + + private async readTableRows(scopedNames: string[] | null): Promise { + if (scopedNames && scopedNames.length === 0) { + return []; + } + const scopeClause = scopedNames + ? `AND table_name IN (${scopedNames.map((name) => `'${name.replaceAll("'", "''")}'`).join(', ')})` + : ''; + const result = await this.query( + `SELECT table_name, table_type + FROM information_schema.tables + WHERE table_schema = '${MAIN_SCHEMA}' ${scopeClause} + ORDER BY table_name`, + ); + return result.rows.map((row) => ({ table_name: String(row[0]), table_type: String(row[1]) })); + } + + private async readTable(table: InfoSchemaTableRow): Promise { + const columnsResult = await this.query( + `SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = '${MAIN_SCHEMA}' AND table_name = '${table.table_name.replaceAll("'", "''")}' + ORDER BY ordinal_position`, + ); + const columns = columnsResult.rows.map((row) => ({ + column_name: String(row[0]), + data_type: String(row[1]), + is_nullable: String(row[2]), + })); + const primaryKeys = await this.readPrimaryKeyColumns(table.table_name); + const isView = table.table_type === 'VIEW'; + const estimatedRows = isView ? null : await this.getTableRowCount(table.table_name); + return { + catalog: null, + db: null, + name: table.table_name, + kind: isView ? 'view' : 'table', + comment: null, + estimatedRows, + columns: columns.map((column) => ({ + name: column.column_name, + nativeType: column.data_type, + normalizedType: this.dialect.mapDataType(column.data_type), + dimensionType: this.dialect.mapToDimensionType(column.data_type), + nullable: column.is_nullable === 'YES' && !primaryKeys.has(column.column_name), + primaryKey: primaryKeys.has(column.column_name), + comment: null, + })), + foreignKeys: await this.readForeignKeys(table.table_name), + }; + } + + private async readPrimaryKeyColumns(tableName: string): Promise> { + const result = await this.query( + `SELECT kcu.column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema + WHERE tc.table_schema = '${MAIN_SCHEMA}' + AND tc.table_name = '${tableName.replaceAll("'", "''")}' + AND tc.constraint_type = 'PRIMARY KEY'`, + ); + return new Set(result.rows.map((row) => String(row[0]))); + } + + private async readForeignKeys(tableName: string): Promise { + // information_schema.constraint_column_usage in DuckDB returns the constrained + // columns (source), not the referenced columns. Use duckdb_constraints() which + // exposes constraint_column_names and referenced_column_names directly. + const result = await this.query( + `SELECT unnest(constraint_column_names) AS from_column, + referenced_table, + unnest(referenced_column_names) AS to_column, + constraint_name + FROM duckdb_constraints() + WHERE schema_name = '${MAIN_SCHEMA}' + AND table_name = '${tableName.replaceAll("'", "''")}' + AND constraint_type = 'FOREIGN KEY'`, + ); + return result.rows.map((row) => ({ + fromColumn: String(row[0]), + toCatalog: null, + toDb: null, + toTable: String(row[1]), + toColumn: String(row[2]), + constraintName: row[3] === null ? null : String(row[3]), + })); + } + + private assertConnection(connectionId: string): void { + if (connectionId !== this.connectionId) { + throw new Error(`ktx DuckDB connector ${this.id} cannot serve connection ${connectionId}`); + } + } +} diff --git a/packages/cli/src/connectors/duckdb/dialect.ts b/packages/cli/src/connectors/duckdb/dialect.ts new file mode 100644 index 00000000..9fef67dd --- /dev/null +++ b/packages/cli/src/connectors/duckdb/dialect.ts @@ -0,0 +1,192 @@ +import type { KtxSqlDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; + +type DuckDbTableNameRef = Pick & Partial>; + +/** @internal */ +export class KtxDuckDbDialect implements KtxSqlDialect { + readonly type = 'duckdb' as const; + + private readonly typeMappings: Record = { + TIMESTAMP: 'time', + 'TIMESTAMP WITH TIME ZONE': 'time', + TIMESTAMPTZ: 'time', + DATE: 'time', + TIME: 'time', + BIGINT: 'number', + INTEGER: 'number', + SMALLINT: 'number', + TINYINT: 'number', + HUGEINT: 'number', + UBIGINT: 'number', + UINTEGER: 'number', + DECIMAL: 'number', + NUMERIC: 'number', + REAL: 'number', + FLOAT: 'number', + DOUBLE: 'number', + VARCHAR: 'string', + CHAR: 'string', + TEXT: 'string', + BLOB: 'string', + UUID: 'string', + BOOLEAN: 'boolean', + BOOL: 'boolean', + }; + + quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; + } + + // v1 introspects the `main` schema only and sets db=null on every table, so + // refs are single-namespace like SQLite — use the matching display shape. + formatTableName(table: DuckDbTableNameRef): string { + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'sqlite'); + } + + formatDisplayRef(table: DuckDbTableNameRef): string { + return formatDialectDisplayRef(table, 'sqlite'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'sqlite'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('sqlite'); + } + + mapDataType(nativeType: string): string { + return nativeType; + } + + mapToDimensionType(nativeType: string): KtxSchemaDimensionType { + if (!nativeType) { + return 'string'; + } + let normalized = nativeType.toUpperCase().trim(); + if (normalized.includes('(')) { + normalized = normalized.split('(')[0].trim(); + } + if (this.typeMappings[normalized]) { + return this.typeMappings[normalized]; + } + if (normalized.includes('TIME') || normalized.includes('DATE')) { + return 'time'; + } + if ( + normalized.includes('INT') || + normalized.includes('DEC') || + normalized.includes('NUM') || + normalized.includes('REAL') || + normalized.includes('FLOAT') || + normalized.includes('DOUBLE') + ) { + return 'number'; + } + if (normalized.includes('BOOL')) { + return 'boolean'; + } + return 'string'; + } + + generateSampleQuery(tableName: string, limit: number, columns?: string[]): string { + const columnList = + columns && columns.length > 0 ? columns.map((column) => this.quoteIdentifier(column)).join(', ') : '*'; + return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`; + } + + generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string { + const quoted = this.quoteIdentifier(columnName); + return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL AND TRIM(CAST(${quoted} AS VARCHAR)) != '' LIMIT ${limit}`; + } + + getRandomSampleFilter(samplePct: number): string { + if (samplePct <= 0 || samplePct >= 1) { + return ''; + } + return `RANDOM() < ${samplePct}`; + } + + getTableSampleClause(samplePct: number): string { + if (samplePct <= 0 || samplePct >= 1) { + return ''; + } + return `USING SAMPLE ${Math.round(samplePct * 100)} PERCENT (bernoulli)`; + } + + getLimitOffsetClause(limit: number, offset?: number): string { + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; + } + + getNullCountExpression(column: string): string { + return `SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END)`; + } + + getDistinctCountExpression(column: string): string { + return `COUNT(DISTINCT ${column})`; + } + + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS VARCHAR))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS VARCHAR)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT STRING_AGG(CAST(value AS VARCHAR), chr(31)) FROM (${innerSql}) AS relationship_profile_values)`; + } + + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + WITH sampled AS ( + SELECT ${columnName} AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + LIMIT ${sampleSize} + ) + SELECT COUNT(DISTINCT val) AS cardinality + FROM sampled + `; + } + + generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string { + return ` + SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY val + LIMIT ${limit} + `; + } + + generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null { + return null; + } + + generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + WITH sampled AS ( + SELECT ${columnName} AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + USING SAMPLE ${sampleSize} ROWS + ) + SELECT COUNT(DISTINCT val) AS cardinality + FROM sampled + `; + } +} diff --git a/packages/cli/src/connectors/duckdb/federated-attach.ts b/packages/cli/src/connectors/duckdb/federated-attach.ts index edcb94eb..c0b9cda0 100644 --- a/packages/cli/src/connectors/duckdb/federated-attach.ts +++ b/packages/cli/src/connectors/duckdb/federated-attach.ts @@ -4,6 +4,7 @@ import { mysqlConnectionPoolConfigFromConfig, type KtxMysqlConnectionConfig, } from '../mysql/connector.js'; +import { duckDbDatabasePathFromConfig, type KtxDuckDbConnectionConfig } from '../../connectors/duckdb/connector.js'; import type { FederatedMember } from '../../context/connections/federation.js'; function kvKeyword(value: string): string { @@ -74,6 +75,12 @@ function mysqlAttachString(member: FederatedMember, env: NodeJS.ProcessEnv): str */ export function federatedAttachTarget(member: FederatedMember, env: NodeJS.ProcessEnv): string { switch (member.driver.toLowerCase()) { + case 'duckdb': + return duckDbDatabasePathFromConfig({ + connectionId: member.connectionId, + projectDir: member.projectDir, + connection: member.connection as KtxDuckDbConnectionConfig, + }); case 'sqlite': return sqliteDatabasePathFromConfig({ connectionId: member.connectionId, diff --git a/packages/cli/src/connectors/duckdb/federated-executor.ts b/packages/cli/src/connectors/duckdb/federated-executor.ts index 7972166c..58894cc4 100644 --- a/packages/cli/src/connectors/duckdb/federated-executor.ts +++ b/packages/cli/src/connectors/duckdb/federated-executor.ts @@ -7,25 +7,12 @@ import type { import { normalizeQueryRows } from '../../context/connections/query-executor.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; import { attachTypeForDriver, type FederatedMember } from '../../context/connections/federation.js'; +import { toJsonSafeRows } from '../shared/duckdb-json-safe.js'; function quoteDuckdbIdentifier(id: string): string { return `"${id.replaceAll('"', '""')}"`; } -const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER); -const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); - -// DuckDB returns integer columns as JS bigint (unserializable by JSON). Values -// in Number's safe range become Number; larger magnitudes become strings so a -// BIGINT beyond 2^53 keeps its exact value instead of silently rounding. -function jsonSafeBigint(value: bigint): number | string { - return value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT ? Number(value) : value.toString(); -} - -function toJsonSafeRows(rows: unknown[][]): unknown[][] { - return rows.map((row) => row.map((cell) => (typeof cell === 'bigint' ? jsonSafeBigint(cell) : cell))); -} - /** @internal */ export function buildAttachStatements(members: FederatedMember[], env: NodeJS.ProcessEnv): string[] { const attachments = members.map((member) => ({ @@ -34,13 +21,12 @@ export function buildAttachStatements(members: FederatedMember[], env: NodeJS.Pr alias: member.connectionId, })); - const loadStatements = [...new Set(attachments.map((a) => a.type))].map( - (type) => `INSTALL ${type}; LOAD ${type};`, - ); - const attachStatements = attachments.map( - ({ type, url, alias }) => - `ATTACH '${url.replaceAll("'", "''")}' AS ${quoteDuckdbIdentifier(alias)} (TYPE ${type}, READ_ONLY);`, - ); + const loadTypes = [...new Set(attachments.map((a) => a.type).filter((type): type is string => type !== null))]; + const loadStatements = loadTypes.map((type) => `INSTALL ${type}; LOAD ${type};`); + const attachStatements = attachments.map(({ type, url, alias }) => { + const options = type === null ? 'READ_ONLY' : `TYPE ${type}, READ_ONLY`; + return `ATTACH '${url.replaceAll("'", "''")}' AS ${quoteDuckdbIdentifier(alias)} (${options});`; + }); return [...loadStatements, ...attachStatements]; } diff --git a/packages/cli/src/connectors/duckdb/live-database-introspection.ts b/packages/cli/src/connectors/duckdb/live-database-introspection.ts new file mode 100644 index 00000000..4edded29 --- /dev/null +++ b/packages/cli/src/connectors/duckdb/live-database-introspection.ts @@ -0,0 +1,40 @@ +import type { + LiveDatabaseIntrospectionOptions, + LiveDatabaseIntrospectionPort, +} from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; +import { KtxDuckDbScanConnector, type KtxDuckDbConnectionConfig } from './connector.js'; + +export interface CreateDuckDbLiveDatabaseIntrospectionOptions { + projectDir?: string; + connections: Record; + now?: () => Date; +} + +export function createDuckDbLiveDatabaseIntrospection( + options: CreateDuckDbLiveDatabaseIntrospectionOptions, +): LiveDatabaseIntrospectionPort { + return { + async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) { + const connection = options.connections[connectionId] as KtxDuckDbConnectionConfig | undefined; + const connector = new KtxDuckDbScanConnector({ + connectionId, + connection, + projectDir: options.projectDir, + now: options.now, + }); + try { + return await connector.introspect( + { + connectionId, + driver: 'duckdb', + ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}), + }, + { runId: `duckdb-${connectionId}` }, + ); + } finally { + await connector.cleanup(); + } + }, + }; +} diff --git a/packages/cli/src/connectors/shared/duckdb-json-safe.ts b/packages/cli/src/connectors/shared/duckdb-json-safe.ts new file mode 100644 index 00000000..d6ea046f --- /dev/null +++ b/packages/cli/src/connectors/shared/duckdb-json-safe.ts @@ -0,0 +1,14 @@ +const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER); +const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); + +// DuckDB returns integer columns as JS bigint (unserializable by JSON). Values +// in Number's safe range become Number; larger magnitudes become strings so a +// BIGINT beyond 2^53 keeps its exact value instead of silently rounding. +/** @internal */ +export function jsonSafeBigint(value: bigint): number | string { + return value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT ? Number(value) : value.toString(); +} + +export function toJsonSafeRows(rows: unknown[][]): unknown[][] { + return rows.map((row) => row.map((cell) => (typeof cell === 'bigint' ? jsonSafeBigint(cell) : cell))); +} diff --git a/packages/cli/src/context/connections/connection-type.ts b/packages/cli/src/context/connections/connection-type.ts index 6cd48042..d122e950 100644 --- a/packages/cli/src/context/connections/connection-type.ts +++ b/packages/cli/src/context/connections/connection-type.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; export const connectionTypeSchema = z.enum([ 'POSTGRESQL', 'SQLITE', + 'DUCKDB', 'SQLSERVER', 'BIGQUERY', 'SNOWFLAKE', diff --git a/packages/cli/src/context/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts index 30608f3e..ad739d1d 100644 --- a/packages/cli/src/context/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,5 +1,6 @@ import { KtxBigQueryDialect } from '../../connectors/bigquery/dialect.js'; import { KtxClickHouseDialect } from '../../connectors/clickhouse/dialect.js'; +import { KtxDuckDbDialect } from '../../connectors/duckdb/dialect.js'; import { KtxMongoDbDialect } from '../../connectors/mongodb/dialect.js'; import { KtxMysqlDialect } from '../../connectors/mysql/dialect.js'; import { KtxPostgresDialect } from '../../connectors/postgres/dialect.js'; @@ -55,6 +56,7 @@ type KtxSqlDriver = Exclude; const sqlDialectFactories: Record KtxSqlDialect> = { bigquery: () => new KtxBigQueryDialect(), clickhouse: () => new KtxClickHouseDialect(), + duckdb: () => new KtxDuckDbDialect(), mysql: () => new KtxMysqlDialect(), postgres: () => new KtxPostgresDialect(), sqlite: () => new KtxSqliteDialect(), diff --git a/packages/cli/src/context/connections/drivers.ts b/packages/cli/src/context/connections/drivers.ts index 76ac408e..1ba17f54 100644 --- a/packages/cli/src/context/connections/drivers.ts +++ b/packages/cli/src/context/connections/drivers.ts @@ -68,6 +68,27 @@ export const driverRegistrations: Record { + const m = await import('../../connectors/duckdb/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxDuckDbConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection, projectDir }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxDuckDbConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('duckdb'); + } + return new m.KtxDuckDbScanConnector({ connectionId, connection: typedConnection, projectDir }); + }, + }; + }, + }, mongodb: { driver: 'mongodb', scopeConfigKey: 'databases', diff --git a/packages/cli/src/context/connections/federation.ts b/packages/cli/src/context/connections/federation.ts index 74036e2f..21fb6f75 100644 --- a/packages/cli/src/context/connections/federation.ts +++ b/packages/cli/src/context/connections/federation.ts @@ -4,18 +4,19 @@ import type { KtxProjectConnectionConfig } from '../project/config.js'; export const FEDERATED_CONNECTION_ID = '_ktx_federated'; /** - * Drivers DuckDB can ATTACH for federation. The driver name doubles as the - * DuckDB extension/TYPE name, so this set is the single source of truth for - * both membership (a driver participates iff it appears here) and attach type. + * Drivers DuckDB can ATTACH for federation. Membership is governed by this set; + * the attach TYPE is governed by attachTypeForDriver, which returns the driver + * name for extension-backed engines and null for a native DuckDB file (attached + * with no INSTALL/LOAD and no TYPE). */ -const ATTACH_COMPATIBLE_DRIVERS = new Set(['postgres', 'mysql', 'sqlite']); +const ATTACH_COMPATIBLE_DRIVERS = new Set(['postgres', 'mysql', 'sqlite', 'duckdb']); -export function attachTypeForDriver(driver: string): string { +export function attachTypeForDriver(driver: string): string | null { const normalized = driver.toLowerCase(); if (!ATTACH_COMPATIBLE_DRIVERS.has(normalized)) { throw new Error(`Driver "${driver}" cannot be attached by DuckDB federation.`); } - return normalized; + return normalized === 'duckdb' ? null : normalized; } export interface FederatedMember { diff --git a/packages/cli/src/context/connections/local-warehouse-descriptor.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.ts index 0e5d0b9d..b9674e09 100644 --- a/packages/cli/src/context/connections/local-warehouse-descriptor.ts +++ b/packages/cli/src/context/connections/local-warehouse-descriptor.ts @@ -23,6 +23,7 @@ export interface LocalConnectionInfo { const DRIVER_TO_CONNECTION_TYPE: Record = { postgres: 'POSTGRESQL', sqlite: 'SQLITE', + duckdb: 'DUCKDB', sqlserver: 'SQLSERVER', mysql: 'MYSQL', clickhouse: 'CLICKHOUSE', diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index a19fee0f..9f808d00 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -11,6 +11,7 @@ const warehouseDrivers = [ 'snowflake', 'bigquery', 'sqlite', + 'duckdb', 'clickhouse', 'sqlserver', ] as const; @@ -52,6 +53,7 @@ const warehouseConnectionSchemas = [ warehouseConnectionSchema('snowflake'), warehouseConnectionSchema('bigquery'), warehouseConnectionSchema('sqlite'), + warehouseConnectionSchema('duckdb'), warehouseConnectionSchema('clickhouse'), warehouseConnectionSchema('sqlserver'), ] as const; diff --git a/packages/cli/src/context/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts index e9644726..0966d15b 100644 --- a/packages/cli/src/context/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -141,6 +141,7 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver { if ( normalized === 'postgres' || normalized === 'sqlite' || + normalized === 'duckdb' || normalized === 'mysql' || normalized === 'clickhouse' || normalized === 'sqlserver' || @@ -151,7 +152,7 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver { return normalized; } throw new Error( - `Standalone ktx scan supports postgres/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake/mongodb in this phase, received "${driver ?? 'unknown'}"`, + `Standalone ktx scan supports postgres/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake/mongodb in this phase, received "${driver ?? 'unknown'}"`, ); } diff --git a/packages/cli/src/context/scan/types.ts b/packages/cli/src/context/scan/types.ts index 9c6010c5..9bd7d865 100644 --- a/packages/cli/src/context/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -2,6 +2,7 @@ import type { KtxTableRefKey } from './table-ref.js'; export type KtxConnectionDriver = | 'sqlite' + | 'duckdb' | 'postgres' | 'sqlserver' | 'bigquery' diff --git a/packages/cli/src/context/sql-analysis/dialect-notes.ts b/packages/cli/src/context/sql-analysis/dialect-notes.ts index d0a7c634..f0a20a2e 100644 --- a/packages/cli/src/context/sql-analysis/dialect-notes.ts +++ b/packages/cli/src/context/sql-analysis/dialect-notes.ts @@ -6,8 +6,7 @@ import type { SqlAnalysisDialect } from './ports.js'; // dialect), served by the sql_dialect_notes MCP tool. They are package-internal: // copy-runtime-assets.mjs ships them to dist, and they are never installed onto an // agent target. The set covers every dialect reachable from a configured warehouse -// driver; duckdb/databricks are intentionally absent because no connector produces -// them. +// driver; databricks is intentionally absent because no connector produces it. /** @internal Dialects with an authored ./dialects/.md file. */ export const DIALECTS_WITH_NOTES = [ @@ -16,6 +15,7 @@ export const DIALECTS_WITH_NOTES = [ 'snowflake', 'bigquery', 'sqlite', + 'duckdb', 'clickhouse', 'tsql', ] as const; diff --git a/packages/cli/src/context/sql-analysis/dialects/duckdb.md b/packages/cli/src/context/sql-analysis/dialects/duckdb.md new file mode 100644 index 00000000..8569b0a0 --- /dev/null +++ b/packages/cli/src/context/sql-analysis/dialects/duckdb.md @@ -0,0 +1,10 @@ +**duckdb** SQL conventions: +- **FQTN:** `schema.table` within one database (e.g. `main.orders`); a bare `table` resolves against `main`. A second, attached database is `db.schema.table` (e.g. `sales.main.orders`). +- **Identifiers:** case-insensitive; double-quote (`"Name"`) to keep a name with spaces or a reserved word. +- **Date/time:** native `DATE`/`TIMESTAMP`. Bucket with `date_trunc('month', ts)`, pull parts with `EXTRACT(YEAR FROM ts)`, format with `strftime(ts, '%Y-%m')`, and use `CURRENT_DATE`; cast text with `col::DATE` (or `TRY_CAST(col AS DATE)` to null bad values). +- **Series:** `FROM generate_series(DATE '2023-01-01', DATE '2023-12-01', INTERVAL 1 MONTH) AS s(d)` builds a date spine (use `range(...)` for integers), then `LEFT JOIN` the aggregated facts onto it so empty periods still appear. +- **Rolling window over time:** a native calendar-range frame spans real dates and tolerates gaps — `AVG(amount) OVER (ORDER BY day RANGE BETWEEN INTERVAL 29 DAYS PRECEDING AND CURRENT ROW)` is a trailing 30-day average without a spine; guard minimum periods with `COUNT(*) OVER ()`. +- **Integer division:** unlike postgres, `/` is true division (`5 / 2` → `2.5`), so a ratio keeps its fraction; use `//` for floor division (`5 // 2` → `2`) when you want the integer quotient, and round only in the final projection. +- **Safe cast:** duckdb has `TRY_CAST` — `TRY_CAST(x AS DOUBLE)` yields `NULL` for a value that does not parse instead of raising, so counting residual `NULL`s among non-sentinel rows catches an encoding the sample missed without a regex guard. +- **Top-N / windows:** filter a window inline with `QUALIFY` — `SELECT ... QUALIFY ROW_NUMBER() OVER (PARTITION BY key ORDER BY x DESC) = 1` returns one row per key without a wrapping CTE; use `ORDER BY ... LIMIT n` for a global top-N. +- **JSON / semi-structured:** `col->'k'` returns JSON, `col->>'k'` returns text, deep path `json_extract(col, '$.a.b')`; duckdb also has native `STRUCT`, `LIST`, and `MAP` — read a struct field with `col.field` and a list element with `col[1]` (1-indexed). diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index d923a68b..03ff48d2 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -9,6 +9,8 @@ import { isKtxPostgresConnectionConfig, type KtxPostgresConnectionConfig } from import { KtxPostgresHistoricSqlQueryClient } from './connectors/postgres/historic-sql-query-client.js'; import { createSqliteLiveDatabaseIntrospection } from './connectors/sqlite/live-database-introspection.js'; import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js'; +import { createDuckDbLiveDatabaseIntrospection } from './connectors/duckdb/live-database-introspection.js'; +import { isKtxDuckDbConnectionConfig } from './connectors/duckdb/connector.js'; import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js'; import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js'; import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'; @@ -104,6 +106,10 @@ function createKtxCliLiveDatabaseIntrospection( projectDir: project.projectDir, connections: project.config.connections, }); + const duckdb = createDuckDbLiveDatabaseIntrospection({ + projectDir: project.projectDir, + connections: project.config.connections, + }); const mysql = createMysqlLiveDatabaseIntrospection({ connections: project.config.connections, }); @@ -139,6 +145,9 @@ function createKtxCliLiveDatabaseIntrospection( if (isKtxSqliteConnectionConfig(connection)) { return sqlite.extractSchema(connectionId, options); } + if (isKtxDuckDbConnectionConfig(connection)) { + return duckdb.extractSchema(connectionId, options); + } if (isKtxMysqlConnectionConfig(connection)) { return mysql.extractSchema(connectionId, options); } diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 7388f74a..33e3b1e0 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -64,6 +64,7 @@ const execFileAsync = promisify(execFileCallback); export type KtxSetupDatabaseDriver = | 'sqlite' + | 'duckdb' | 'postgres' | 'mysql' | 'clickhouse' @@ -159,6 +160,7 @@ const DRIVER_OPTIONS: Array<{ value: KtxSetupDatabaseDriver; label: string }> = { value: 'sqlserver', label: 'SQL Server' }, { value: 'mongodb', label: 'MongoDB' }, { value: 'sqlite', label: 'SQLite' }, + { value: 'duckdb', label: 'DuckDB' }, ]; const DRIVER_LABELS = Object.fromEntries(DRIVER_OPTIONS.map((option) => [option.value, option.label])) as Record< @@ -174,6 +176,7 @@ const HISTORIC_SQL_DIALECT_BY_DRIVER: Partial = { sqlite: 'sqlite-local', + duckdb: 'duckdb-local', postgres: 'postgres-warehouse', mysql: 'mysql-warehouse', clickhouse: 'clickhouse-warehouse', @@ -813,6 +816,18 @@ async function buildConnectionConfig(input: { if (path === undefined) return 'back'; return path ? { driver: 'sqlite', path } : null; } + if (driver === 'duckdb') { + if (args.inputMode === 'disabled' && !args.databaseUrl) return null; + const path = + args.databaseUrl ?? + (await promptText( + prompts, + 'DuckDB database file\nEnter a relative or absolute path, for example ./warehouse.duckdb.', + stringConfigField(input.existingConnection, 'path'), + )); + if (path === undefined) return 'back'; + return path ? { driver: 'duckdb', path } : null; + } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { return await buildUrlConnectionConfig({ driver, @@ -1417,9 +1432,11 @@ async function maybeConfigureDatabaseScope(input: { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver || driver === 'sqlite') return okValidateResult(); + const spec = driver ? SCOPE_DISCOVERY_SPECS[driver] : undefined; + // Drivers with no scope spec are single-namespace (sqlite, duckdb): there is no + // schema to choose, so skip the scope picker and ingest every table. + if (!driver || !spec) return okValidateResult(); - const spec = SCOPE_DISCOVERY_SPECS[driver]; const existingTables = connection?.enabled_tables; const hasExistingTables = Array.isArray(existingTables) && existingTables.length > 0; const existingScope = spec ? configuredScopeValues(connection, spec) : []; diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index 6d52518b..2acd151a 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -412,6 +412,7 @@ function buildConnectionStatus( const hint = envHint((conn as Record).credentials_json); return warn(hint ? `credentials missing (env: ${hint})` : 'credentials not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); } + case 'duckdb': case 'sqlite': { const path = (conn as Record).path; if (typeof path === 'string' && path.length > 0) return ok(`path: ${path}`); @@ -553,7 +554,7 @@ async function buildQueryHistoryStatus( } const ADAPTER_DRIVER_REQUIREMENT: Record = { - 'live-database': ['postgres', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], + 'live-database': ['postgres', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'duckdb', 'sqlserver'], dbt: ['dbt', 'dbt-core', 'dbt-cloud'], notion: ['notion'], metabase: ['metabase'], diff --git a/packages/cli/test/connection.test.ts b/packages/cli/test/connection.test.ts index beeaad65..0fbc6a41 100644 --- a/packages/cli/test/connection.test.ts +++ b/packages/cli/test/connection.test.ts @@ -688,7 +688,7 @@ describe('runKtxConnection', () => { await initKtxProject({ projectDir }); await writeFile( join(projectDir, 'ktx.yaml'), - 'connections:\n mystery:\n driver: duckdb\n', + 'connections:\n mystery:\n driver: nonsense\n', 'utf-8', ); const io = makeIo(); diff --git a/packages/cli/test/connectors/duckdb/connector.test.ts b/packages/cli/test/connectors/duckdb/connector.test.ts new file mode 100644 index 00000000..9f1c3394 --- /dev/null +++ b/packages/cli/test/connectors/duckdb/connector.test.ts @@ -0,0 +1,280 @@ +import { DuckDBInstance } from '@duckdb/node-api'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { + KtxDuckDbScanConnector, + duckDbDatabasePathFromConfig, + isKtxDuckDbConnectionConfig, +} from '../../../src/connectors/duckdb/connector.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; + +let dir: string; +let dbPath: string; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'ktx-duckdb-')); + dbPath = join(dir, 'warehouse.duckdb'); + const instance = await DuckDBInstance.create(dbPath); + const connection = await instance.connect(); + await connection.run('CREATE TABLE customers (id BIGINT PRIMARY KEY, name VARCHAR, big BIGINT)'); + await connection.run( + `INSERT INTO customers VALUES (1, 'Ada', 9223372036854775807), (2, 'Lin', 10)`, + ); + await connection.run('CREATE TABLE orders (id BIGINT, customer_id BIGINT REFERENCES customers(id))'); + await connection.run('INSERT INTO orders VALUES (1, 1), (2, 2)'); + // Composite primary key + composite foreign key, to exercise the parallel + // unnest() zip of constraint/referenced column names in readForeignKeys. + await connection.run('CREATE TABLE regions (country VARCHAR, code VARCHAR, PRIMARY KEY (country, code))'); + await connection.run( + 'CREATE TABLE stores (id BIGINT, country VARCHAR, code VARCHAR, FOREIGN KEY (country, code) REFERENCES regions(country, code))', + ); + await connection.run('CREATE TABLE empty_table (id BIGINT)'); + connection.closeSync(); + instance.closeSync(); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +function connector(connection: Record = { driver: 'duckdb', path: dbPath }) { + return new KtxDuckDbScanConnector({ connectionId: 'warehouse', connection, projectDir: dir }); +} + +describe('isKtxDuckDbConnectionConfig', () => { + it('accepts duckdb driver, rejects others', () => { + expect(isKtxDuckDbConnectionConfig({ driver: 'duckdb' })).toBe(true); + expect(isKtxDuckDbConnectionConfig({ driver: 'sqlite' })).toBe(false); + }); +}); + +describe('duckDbDatabasePathFromConfig', () => { + it('resolves a relative path against projectDir', () => { + const resolved = duckDbDatabasePathFromConfig({ + connectionId: 'warehouse', + projectDir: dir, + connection: { driver: 'duckdb', path: 'warehouse.duckdb' }, + }); + expect(resolved).toBe(dbPath); + }); + + it('derives the path from a file: url', () => { + const resolved = duckDbDatabasePathFromConfig({ + connectionId: 'warehouse', + connection: { driver: 'duckdb', url: pathToFileURL(dbPath).href }, + }); + expect(resolved).toBe(dbPath); + }); + + it('derives the path from a duckdb: url', () => { + const resolved = duckDbDatabasePathFromConfig({ + connectionId: 'warehouse', + connection: { driver: 'duckdb', url: `duckdb://${dbPath}` }, + }); + expect(resolved).toBe(dbPath); + }); + + it('resolves an env: reference in path', () => { + process.env.KTX_TEST_DUCKDB_PATH = dbPath; + try { + const resolved = duckDbDatabasePathFromConfig({ + connectionId: 'warehouse', + connection: { driver: 'duckdb', path: 'env:KTX_TEST_DUCKDB_PATH' }, + }); + expect(resolved).toBe(dbPath); + } finally { + delete process.env.KTX_TEST_DUCKDB_PATH; + } + }); + + it('rejects a non-duckdb driver', () => { + expect(() => + duckDbDatabasePathFromConfig({ + connectionId: 'warehouse', + connection: { driver: 'sqlite', path: dbPath }, + }), + ).toThrow(/cannot run driver "sqlite"/); + }); + + it('requires a path or url', () => { + expect(() => + duckDbDatabasePathFromConfig({ + connectionId: 'warehouse', + connection: { driver: 'duckdb' }, + }), + ).toThrow(/requires connections\.warehouse\.path or url/); + }); +}); + +describe('KtxDuckDbScanConnector', () => { + it('testConnection succeeds for an existing file', async () => { + const c = connector(); + expect(await c.testConnection()).toEqual({ success: true }); + await c.cleanup(); + }); + + it('testConnection fails (never creating) for a missing file', async () => { + const c = connector({ driver: 'duckdb', path: join(dir, 'absent.duckdb') }); + const result = await c.testConnection(); + expect(result.success).toBe(false); + await c.cleanup(); + }); + + it('introspects main-schema tables, columns, and foreign keys', async () => { + const c = connector(); + const snapshot = await c.introspect({ connectionId: 'warehouse', driver: 'duckdb' }, { runId: 't' }); + const names = snapshot.tables.map((t) => t.name).sort(); + expect(names).toEqual(['customers', 'empty_table', 'orders', 'regions', 'stores']); + const orders = snapshot.tables.find((t) => t.name === 'orders'); + expect(orders?.foreignKeys[0]).toMatchObject({ fromColumn: 'customer_id', toTable: 'customers', toColumn: 'id' }); + await c.cleanup(); + }); + + it('maps a composite foreign key column-for-column to the referenced table', async () => { + const c = connector(); + const snapshot = await c.introspect({ connectionId: 'warehouse', driver: 'duckdb' }, { runId: 't' }); + const stores = snapshot.tables.find((t) => t.name === 'stores'); + const fks = stores?.foreignKeys.map((fk) => ({ fromColumn: fk.fromColumn, toTable: fk.toTable, toColumn: fk.toColumn })); + expect(fks).toEqual([ + { fromColumn: 'country', toTable: 'regions', toColumn: 'country' }, + { fromColumn: 'code', toTable: 'regions', toColumn: 'code' }, + ]); + await c.cleanup(); + }); + + it('lists tables', async () => { + const c = connector(); + const tables = (await c.listTables()).map((t) => t.name).sort(); + expect(tables).toEqual(['customers', 'empty_table', 'orders', 'regions', 'stores']); + await c.cleanup(); + }); + + it('samples a table', async () => { + const c = connector(); + const sample = await c.sampleTable( + { connectionId: 'warehouse', table: { name: 'customers', catalog: null, db: null }, limit: 1 }, + { runId: 't' }, + ); + expect(sample.rows.length).toBe(1); + await c.cleanup(); + }); + + it('stringifies BIGINT beyond 2^53 in read-only results', async () => { + const c = connector(); + const result = await c.executeReadOnly( + { connectionId: 'warehouse', sql: 'SELECT big FROM customers WHERE id = 1', maxRows: 10 }, + { runId: 't' }, + ); + expect(result.rows[0][0]).toBe('9223372036854775807'); + await c.cleanup(); + }); + + it('rejects non-read-only SQL', async () => { + const c = connector(); + await expect( + c.executeReadOnly({ connectionId: 'warehouse', sql: 'DELETE FROM customers', maxRows: 10 }, { runId: 't' }), + ).rejects.toThrow(); + await c.cleanup(); + }); + + it('returns distinct values under the cardinality cap', async () => { + const c = connector(); + const distinct = await c.getColumnDistinctValues({ name: 'customers', catalog: null, db: null }, 'name', { + maxCardinality: 10, + limit: 10, + }); + expect(distinct?.values?.sort()).toEqual(['Ada', 'Lin']); + await c.cleanup(); + }); + + it('withholds values but reports the count when cardinality exceeds the cap', async () => { + const c = connector(); + const distinct = await c.getColumnDistinctValues({ name: 'customers', catalog: null, db: null }, 'name', { + maxCardinality: 1, + limit: 10, + }); + expect(distinct).toEqual({ values: null, cardinality: 2 }); + await c.cleanup(); + }); + + it('samples a single column, dropping null rows', async () => { + const c = connector(); + const sample = await c.sampleColumn( + { connectionId: 'warehouse', table: { name: 'customers', catalog: null, db: null }, column: 'name', limit: 10 }, + { runId: 't' }, + ); + expect(sample.values.sort()).toEqual(['Ada', 'Lin']); + expect(sample.nullCount).toBeNull(); + await c.cleanup(); + }); + + it('counts table rows', async () => { + const c = connector(); + expect(await c.getTableRowCount('customers')).toBe(2); + await c.cleanup(); + }); + + it('lists only the main schema and reports no column stats', async () => { + const c = connector(); + expect(await c.listSchemas()).toEqual(['main']); + expect(await c.columnStats({ connectionId: 'warehouse', table: { name: 'customers', catalog: null, db: null }, column: 'id' }, { runId: 't' })).toBeNull(); + await c.cleanup(); + }); + + it('rejects operations for a mismatched connection id', async () => { + const c = connector(); + await expect( + c.executeReadOnly({ connectionId: 'other', sql: 'SELECT 1', maxRows: 1 }, { runId: 't' }), + ).rejects.toThrow(/cannot serve connection other/); + await c.cleanup(); + }); + + it('exposes the dialect identifier quoting', () => { + expect(connector().quoteIdentifier('a"b')).toBe('"a""b"'); + }); + + // Opening a connection must never create the file: the db() guard throws + // rather than letting DuckDBInstance.create() materialize a missing path. + it('refuses to open (never creating) a missing file when a query runs', async () => { + const c = connector({ driver: 'duckdb', path: join(dir, 'absent.duckdb') }); + await expect(c.listTables()).rejects.toThrow(/File not found/); + await c.cleanup(); + }); + + it('returns no tables for an empty table scope', async () => { + const c = connector(); + const snapshot = await c.introspect( + { connectionId: 'warehouse', driver: 'duckdb', tableScope: new Set() }, + { runId: 't' }, + ); + expect(snapshot.tables).toEqual([]); + await c.cleanup(); + }); + + it('restricts introspection to the named tables in a non-empty scope', async () => { + const c = connector(); + const snapshot = await c.introspect( + { + connectionId: 'warehouse', + driver: 'duckdb', + tableScope: tableRefSet([{ catalog: null, db: null, name: 'customers' }]), + }, + { runId: 't' }, + ); + expect(snapshot.tables.map((t) => t.name)).toEqual(['customers']); + await c.cleanup(); + }); + + it('reports zero cardinality and an empty value list for an empty column', async () => { + const c = connector(); + const distinct = await c.getColumnDistinctValues({ name: 'empty_table', catalog: null, db: null }, 'id', { + maxCardinality: 10, + limit: 10, + }); + expect(distinct).toEqual({ values: [], cardinality: 0 }); + await c.cleanup(); + }); +}); diff --git a/packages/cli/test/connectors/duckdb/dialect.test.ts b/packages/cli/test/connectors/duckdb/dialect.test.ts new file mode 100644 index 00000000..6079dfab --- /dev/null +++ b/packages/cli/test/connectors/duckdb/dialect.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; +import { KtxDuckDbDialect } from '../../../src/connectors/duckdb/dialect.js'; + +describe('KtxDuckDbDialect', () => { + const dialect = new KtxDuckDbDialect(); + + it('quotes identifiers with double quotes and escapes embedded quotes', () => { + expect(dialect.quoteIdentifier('order"s')).toBe('"order""s"'); + }); + + it('maps integer types to number dimension', () => { + expect(dialect.mapToDimensionType('BIGINT')).toBe('number'); + expect(dialect.mapToDimensionType('DOUBLE')).toBe('number'); + }); + + it('maps timestamp types to time dimension', () => { + expect(dialect.mapToDimensionType('TIMESTAMP')).toBe('time'); + expect(dialect.mapToDimensionType('DATE')).toBe('time'); + }); + + it('maps text types to string dimension', () => { + expect(dialect.mapToDimensionType('VARCHAR')).toBe('string'); + }); + + it('maps boolean types to boolean dimension', () => { + expect(dialect.mapToDimensionType('BOOLEAN')).toBe('boolean'); + expect(dialect.mapToDimensionType('BOOL')).toBe('boolean'); + }); + + it('falls back to string for an empty or unknown native type', () => { + expect(dialect.mapToDimensionType('')).toBe('string'); + expect(dialect.mapToDimensionType('JSON')).toBe('string'); + }); + + // The precedence ladder strips parameters before substring rules fire, so a + // parameterized DECIMAL still resolves through the numeric branch rather than + // the string fallback. + it('strips type parameters before resolving the dimension', () => { + expect(dialect.mapToDimensionType('DECIMAL(10,2)')).toBe('number'); + expect(dialect.mapToDimensionType('VARCHAR(255)')).toBe('string'); + }); + + // Types absent from the exact-match table still resolve via substring rules: + // TIMESTAMP_NS (time), UINT128/HUGEINT-like (number), and lowercase input. + it('resolves unlisted types through substring matching, case-insensitively', () => { + expect(dialect.mapToDimensionType('timestamp_ns')).toBe('time'); + expect(dialect.mapToDimensionType('INT128')).toBe('number'); + expect(dialect.mapToDimensionType(' double ')).toBe('number'); + }); + + it('generates a limited sample query', () => { + expect(dialect.generateSampleQuery('"t"', 5)).toBe('SELECT * FROM "t" LIMIT 5'); + }); + + it('quotes selected columns in a sample query', () => { + expect(dialect.generateSampleQuery('"t"', 5, ['a', 'b'])).toBe('SELECT "a", "b" FROM "t" LIMIT 5'); + }); + + it('builds a non-null, non-blank column sample query', () => { + expect(dialect.generateColumnSampleQuery('"t"', 'email', 3)).toBe( + `SELECT "email" FROM "t" WHERE "email" IS NOT NULL AND TRIM(CAST("email" AS VARCHAR)) != '' LIMIT 3`, + ); + }); + + // A degenerate sample percentage (<=0 or >=1) means "no sampling", so both the + // random filter and the TABLESAMPLE clause must collapse to an empty string. + it('returns empty sample clauses outside the (0,1) range and real clauses inside it', () => { + expect(dialect.getRandomSampleFilter(0)).toBe(''); + expect(dialect.getRandomSampleFilter(1)).toBe(''); + expect(dialect.getRandomSampleFilter(0.25)).toBe('RANDOM() < 0.25'); + expect(dialect.getTableSampleClause(0)).toBe(''); + expect(dialect.getTableSampleClause(0.1)).toBe('USING SAMPLE 10 PERCENT (bernoulli)'); + }); + + // A type missing from the exact-match table but containing BOOL still resolves + // through the substring branch rather than the string fallback. + it('resolves a BOOL-substring type to boolean', () => { + expect(dialect.mapToDimensionType('MYBOOL')).toBe('boolean'); + }); + + it('builds limit/offset, sample-value aggregation, and randomized cardinality clauses', () => { + expect(dialect.getLimitOffsetClause(10, 5)).toContain('LIMIT 10'); + expect(dialect.getSampleValueAggregation('SELECT 1')).toContain('STRING_AGG'); + expect(dialect.generateRandomizedCardinalitySampleQuery('"t"', 'c', 100)).toContain('USING SAMPLE 100 ROWS'); + }); + + it('exposes profiling expressions and a null column-statistics query', () => { + expect(dialect.getNullCountExpression('c')).toBe('SUM(CASE WHEN c IS NULL THEN 1 ELSE 0 END)'); + expect(dialect.getDistinctCountExpression('c')).toBe('COUNT(DISTINCT c)'); + expect(dialect.textLengthExpression('c')).toBe('LENGTH(CAST(c AS VARCHAR))'); + expect(dialect.castToText('c')).toBe('CAST(c AS VARCHAR)'); + expect(dialect.mapDataType('BIGINT')).toBe('BIGINT'); + expect(dialect.getTopClause(5)).toBe(''); + expect(dialect.generateColumnStatisticsQuery('main', 't')).toBeNull(); + }); + + // Guards the single-namespace (db=null) display shape: v1 introspects only + // `main`, so a display ref must round-trip as a bare table name. An ANSI shape + // would emit a 1-part name it then refuses to parse, breaking column lookups. + it('round-trips a single-namespace display ref and reports a 1-part column shape', () => { + const table = { catalog: null, db: null, name: 'orders' }; + const display = dialect.formatDisplayRef(table); + expect(display).toBe('orders'); + expect(dialect.parseDisplayRef(display)).toMatchObject({ name: 'orders' }); + expect(dialect.columnDisplayTablePartCount()).toBe(1); + expect(dialect.formatTableName(table)).toBe('"orders"'); + }); +}); diff --git a/packages/cli/test/connectors/duckdb/federated-attach.test.ts b/packages/cli/test/connectors/duckdb/federated-attach.test.ts index 7d16fb47..e5c30090 100644 --- a/packages/cli/test/connectors/duckdb/federated-attach.test.ts +++ b/packages/cli/test/connectors/duckdb/federated-attach.test.ts @@ -135,6 +135,14 @@ describe('federatedAttachTarget', () => { expect(target).toContain('ssl_mode=REQUIRED'); }); + it('resolves a duckdb member to its database file path', () => { + const target = federatedAttachTarget( + { connectionId: 'dux', driver: 'duckdb', projectDir: '/p', connection: { driver: 'duckdb', path: 'a.duckdb' } }, + {}, + ); + expect(target).toBe('/p/a.duckdb'); + }); + it('throws for an unsupported driver', () => { expect(() => federatedAttachTarget(member({ driver: 'snowflake', connection: { driver: 'snowflake' } }), {})).toThrow( /cannot be attached/i, diff --git a/packages/cli/test/connectors/duckdb/federated-executor.test.ts b/packages/cli/test/connectors/duckdb/federated-executor.test.ts index 0cc07dc4..96638c08 100644 --- a/packages/cli/test/connectors/duckdb/federated-executor.test.ts +++ b/packages/cli/test/connectors/duckdb/federated-executor.test.ts @@ -67,4 +67,27 @@ describe('buildAttachStatements', () => { ); expect(stmts.at(-1)).toBe('ATTACH \'postgresql://u:it\'\'s@h/db\' AS "pg" (TYPE postgres, READ_ONLY);'); }); + + it('attaches a native duckdb member with no TYPE and no INSTALL/LOAD', () => { + const statements = buildAttachStatements( + [{ connectionId: 'dux', driver: 'duckdb', projectDir: '/p', connection: { driver: 'duckdb', path: '/p/a.duckdb' } }], + {}, + ); + expect(statements.some((s) => s.startsWith('INSTALL'))).toBe(false); + expect(statements.find((s) => s.startsWith('ATTACH'))).toContain('(READ_ONLY)'); + expect(statements.find((s) => s.startsWith('ATTACH'))).not.toContain('TYPE'); + }); + + it('mixes a duckdb member with a postgres member, loading only postgres', () => { + const statements = buildAttachStatements( + [ + { connectionId: 'dux', driver: 'duckdb', projectDir: '/p', connection: { driver: 'duckdb', path: '/p/a.duckdb' } }, + { connectionId: 'pg', driver: 'postgres', projectDir: '/p', connection: { driver: 'postgres', url: 'postgres://h/db' } }, + ], + {}, + ); + expect(statements).toContain('INSTALL postgres; LOAD postgres;'); + expect(statements.some((s) => s.includes('INSTALL duckdb'))).toBe(false); + expect(statements.filter((s) => s.startsWith('ATTACH')).length).toBe(2); + }); }); diff --git a/packages/cli/test/connectors/duckdb/live-database-introspection.test.ts b/packages/cli/test/connectors/duckdb/live-database-introspection.test.ts new file mode 100644 index 00000000..61d4d0e2 --- /dev/null +++ b/packages/cli/test/connectors/duckdb/live-database-introspection.test.ts @@ -0,0 +1,45 @@ +import { DuckDBInstance } from '@duckdb/node-api'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { createDuckDbLiveDatabaseIntrospection } from '../../../src/connectors/duckdb/live-database-introspection.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; + +let dir: string; +let dbPath: string; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'ktx-duckdb-live-')); + dbPath = join(dir, 'warehouse.duckdb'); + const instance = await DuckDBInstance.create(dbPath); + const connection = await instance.connect(); + await connection.run('CREATE TABLE customers (id BIGINT, name VARCHAR)'); + await connection.run('CREATE TABLE orders (id BIGINT)'); + connection.closeSync(); + instance.closeSync(); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +function port() { + return createDuckDbLiveDatabaseIntrospection({ + projectDir: dir, + connections: { warehouse: { driver: 'duckdb', path: dbPath } }, + }); +} + +describe('createDuckDbLiveDatabaseIntrospection', () => { + it('extracts the full schema for a connection', async () => { + const snapshot = await port().extractSchema('warehouse'); + expect(snapshot.tables.map((t) => t.name).sort()).toEqual(['customers', 'orders']); + }); + + it('restricts extraction to a table scope', async () => { + const tableScope = tableRefSet([{ catalog: null, db: null, name: 'customers' }]); + const snapshot = await port().extractSchema('warehouse', { tableScope }); + expect(snapshot.tables.map((t) => t.name)).toEqual(['customers']); + }); +}); diff --git a/packages/cli/test/connectors/shared/duckdb-json-safe.test.ts b/packages/cli/test/connectors/shared/duckdb-json-safe.test.ts new file mode 100644 index 00000000..cd114a53 --- /dev/null +++ b/packages/cli/test/connectors/shared/duckdb-json-safe.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { jsonSafeBigint, toJsonSafeRows } from '../../../src/connectors/shared/duckdb-json-safe.js'; + +describe('duckdb json-safe bigint', () => { + it('keeps safe-range bigints as numbers', () => { + expect(jsonSafeBigint(42n)).toBe(42); + }); + + it('stringifies bigints beyond Number.MAX_SAFE_INTEGER', () => { + const big = BigInt(Number.MAX_SAFE_INTEGER) + 10n; + expect(jsonSafeBigint(big)).toBe(big.toString()); + }); + + it('converts only bigint cells in a row matrix', () => { + expect(toJsonSafeRows([[1n, 'a', null]])).toEqual([[1, 'a', null]]); + }); +}); diff --git a/packages/cli/test/context/connections/dialects.test.ts b/packages/cli/test/context/connections/dialects.test.ts index cc7eaa59..74f95cc2 100644 --- a/packages/cli/test/context/connections/dialects.test.ts +++ b/packages/cli/test/context/connections/dialects.test.ts @@ -305,7 +305,7 @@ describe('getDialectForDriver', () => { it('throws with a supported-driver list for unknown drivers', () => { expect(() => getDialectForDriver('oracle')).toThrow( - 'Unsupported driver "oracle". Supported drivers: bigquery, clickhouse, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', + 'Unsupported driver "oracle". Supported drivers: bigquery, clickhouse, duckdb, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', ); }); diff --git a/packages/cli/test/context/connections/drivers.test.ts b/packages/cli/test/context/connections/drivers.test.ts index 65bbca4b..38a0f48f 100644 --- a/packages/cli/test/context/connections/drivers.test.ts +++ b/packages/cli/test/context/connections/drivers.test.ts @@ -22,6 +22,7 @@ const connectionFixtures: Record = { schemas: ['public'], }), sqlite: () => ({ driver: 'sqlite', path: 'warehouse.db' }), + duckdb: (projectDir) => ({ driver: 'duckdb', path: join(projectDir, 'warehouse.duckdb') }), mongodb: () => ({ driver: 'mongodb', url: 'mongodb://localhost:27017/app', @@ -101,6 +102,7 @@ describe('driverRegistrations', () => { expect(listSupportedDrivers()).toEqual([ 'bigquery', 'clickhouse', + 'duckdb', 'mongodb', 'mysql', 'postgres', @@ -138,7 +140,7 @@ describe('driverRegistrations', () => { expect(connector.listTables).toEqual(expect.any(Function)); await connector.cleanup?.(); - if (registration.driver === 'sqlite') { + if (registration.driver === 'sqlite' || registration.driver === 'duckdb') { expect(registration.scopeConfigKey).toBeNull(); } else { expect(registration.scopeConfigKey).not.toBeNull(); diff --git a/packages/cli/test/context/mcp/dialect-notes.test.ts b/packages/cli/test/context/mcp/dialect-notes.test.ts index 27e9d922..7cf36d7f 100644 --- a/packages/cli/test/context/mcp/dialect-notes.test.ts +++ b/packages/cli/test/context/mcp/dialect-notes.test.ts @@ -33,8 +33,7 @@ describe('per-dialect SQL notes', () => { }); it('does not author notes for unreachable dialects', () => { - // duckdb/databricks appear in the resolver map but no connector produces them. - expect(DIALECTS_WITH_NOTES).not.toContain('duckdb'); + // databricks appears in the resolver map but no connector produces it. expect(DIALECTS_WITH_NOTES).not.toContain('databricks'); }); diff --git a/packages/cli/test/local-scan-connectors.test.ts b/packages/cli/test/local-scan-connectors.test.ts index 827b4ba1..6cb2d8cd 100644 --- a/packages/cli/test/local-scan-connectors.test.ts +++ b/packages/cli/test/local-scan-connectors.test.ts @@ -92,7 +92,7 @@ describe('createKtxCliScanConnector', () => { expect(bigQueryMock.constructorInputs[0]).not.toHaveProperty('maxBytesBilled'); }); - it('rejects daemon-only fallback driver configs at config parse time', async () => { + it('resolves a duckdb connection to the DuckDB scan connector', async () => { await initKtxProject({ projectDir: tempDir }); await writeFile( join(tempDir, 'ktx.yaml'), @@ -105,10 +105,12 @@ describe('createKtxCliScanConnector', () => { ].join('\n'), 'utf-8', ); + const project = await loadKtxProject({ projectDir: tempDir }); - await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow( - /connections\.warehouse\.driver:.*Invalid discriminator value/, - ); + const connector = await createKtxCliScanConnector(project, 'warehouse'); + + expect(connector.id).toBe('duckdb:warehouse'); + expect(connector.driver).toBe('duckdb'); }); it('rejects connection blocks with no driver field at config parse time', async () => { diff --git a/packages/cli/test/setup-databases.test.ts b/packages/cli/test/setup-databases.test.ts index 3f363805..bcf1db79 100644 --- a/packages/cli/test/setup-databases.test.ts +++ b/packages/cli/test/setup-databases.test.ts @@ -245,6 +245,7 @@ describe('setup databases step', () => { { value: 'sqlserver', label: 'SQL Server' }, { value: 'mongodb', label: 'MongoDB' }, { value: 'sqlite', label: 'SQLite' }, + { value: 'duckdb', label: 'DuckDB' }, ], required: true, }); @@ -3515,4 +3516,72 @@ describe('setup databases step', () => { expect(io.stdout()).toContain('ktx cannot work until you add a database.'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); }); + + it('adds one non-interactive DuckDB connection from --database-url without prompting', async () => { + const io = makeIo(); + const prompts = makePromptAdapter({}); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseDrivers: ['duckdb'], + databaseConnectionId: 'duckdb-local', + databaseUrl: './warehouse.duckdb', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { prompts, testConnection, scanConnection }, + ); + + expect(result.status).toBe('ready'); + expect(prompts.text).not.toHaveBeenCalled(); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'duckdb-local', expect.anything()); + expect(scanConnection).toHaveBeenCalledWith(tempDir, 'duckdb-local', expect.anything()); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections['duckdb-local']).toEqual({ + driver: 'duckdb', + path: './warehouse.duckdb', + }); + expect(config.setup).toEqual({ + database_connection_ids: ['duckdb-local'], + }); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases'); + }); + + it('adds an interactive DuckDB connection without prompting for a schema', async () => { + const prompts = makePromptAdapter({ + selectValues: ['no'], + textValues: ['', './warehouse.duckdb'], + }); + const pickers = makePickerStubs(); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseDrivers: ['duckdb'], + databaseSchemas: [], + skipDatabases: false, + }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + pickDatabaseScope: pickers.pickDatabaseScope, + }, + ); + + expect(result.status).toBe('ready'); + expect(pickers.scopeCalls).toHaveLength(0); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections['duckdb-local']).toEqual({ + driver: 'duckdb', + path: './warehouse.duckdb', + }); + }); }); From 6d0103074595cfb84a3508782810c2dbf0774774 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Thu, 2 Jul 2026 11:24:18 +0200 Subject: [PATCH 02/11] fix(deps): patch 22 Dependabot security alerts (#328) Bump transitive dependencies to their patched versions to clear all open Dependabot advisories. npm fixes go through the pnpm-workspace.yaml overrides block; the Python fix goes through uv.lock. npm: undici 6.27.0/7.28.0, hono 4.12.25, form-data 4.0.6, ws 8.21.0, vite 8.0.16, esbuild 0.28.1, js-yaml 4.2.0. pip: starlette 1.3.1. --- pnpm-lock.yaml | 502 +++++++++++++++++++++++--------------------- pnpm-workspace.yaml | 10 +- uv.lock | 10 +- 3 files changed, 276 insertions(+), 246 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5521fa08..53a510d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,18 @@ settings: overrides: '@types/node': ^24.3.0 brace-expansion@>=5.0.0 <5.0.6: 5.0.6 + esbuild: 0.28.1 fast-uri: 3.1.2 fast-xml-builder: 1.1.7 - hono: 4.12.21 + form-data: 4.0.6 + hono: 4.12.25 ip-address: 10.1.1 + js-yaml: 4.2.0 postcss: 8.5.10 - ws: 8.20.1 + undici@6: 6.27.0 + undici@7: 7.28.0 + vite: 8.0.16 + ws: 8.21.0 importers: @@ -81,7 +87,7 @@ importers: version: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) fumadocs-mdx: specifier: 15.0.7 - version: 15.0.7(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + version: 15.0.7(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(rolldown@1.0.3)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) fumadocs-ui: specifier: 16.8.10 version: 16.8.10(@tailwindcss/oxide@4.3.0)(@types/mdx@2.0.13)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(tailwindcss@4.3.0) @@ -190,7 +196,7 @@ importers: version: 7.0.3(@types/react@19.2.15)(react@19.2.6) lookml-parser: specifier: 7.1.0 - version: 7.1.0(js-yaml@4.1.1) + version: 7.1.0(js-yaml@4.2.0) minimatch: specifier: ^10.2.5 version: 10.2.5 @@ -205,7 +211,7 @@ importers: version: 3.22.3(@types/node@24.12.4) openai: specifier: ^6.38.0 - version: 6.38.0(ws@8.20.1)(zod@4.4.3) + version: 6.38.0(ws@8.21.0)(zod@4.4.3) p-limit: specifier: ^7.3.0 version: 7.3.0 @@ -278,7 +284,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.7 - version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) packages: @@ -741,6 +747,7 @@ packages: '@clickhouse/client-common@1.18.5': resolution: {integrity: sha512-g9LwcS1dvkatKDsIjT1PwUHldsiYzwdKAB0nXfd9APLd+t4PrNJa+my+dXcqJdmcWyhWjKLP/2/ztBwgxp+sbQ==} + deprecated: 'Deprecated: import from @clickhouse/client or @clickhouse/client-web instead.' '@clickhouse/client@1.18.5': resolution: {integrity: sha512-4FfoyMkFWhsdNMuXsoEL6l3c12svA63BBJBtDo9SrxRZ14RdmN6jLr/rF3f84BK8cFoxETZCSeKlsbk6NNYebw==} @@ -830,158 +837,158 @@ packages: '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} - '@esbuild/aix-ppc64@0.28.0': - resolution: {integrity: sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==} + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.28.0': - resolution: {integrity: sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==} + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.28.0': - resolution: {integrity: sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==} + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.28.0': - resolution: {integrity: sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==} + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.28.0': - resolution: {integrity: sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==} + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.28.0': - resolution: {integrity: sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==} + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.28.0': - resolution: {integrity: sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==} + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.28.0': - resolution: {integrity: sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==} + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.28.0': - resolution: {integrity: sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==} + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.28.0': - resolution: {integrity: sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==} + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.28.0': - resolution: {integrity: sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==} + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.28.0': - resolution: {integrity: sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==} + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.28.0': - resolution: {integrity: sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==} + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.28.0': - resolution: {integrity: sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==} + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.28.0': - resolution: {integrity: sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==} + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.28.0': - resolution: {integrity: sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==} + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.28.0': - resolution: {integrity: sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==} + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/netbsd-arm64@0.28.0': - resolution: {integrity: sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==} + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-x64@0.28.0': - resolution: {integrity: sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==} + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/openbsd-arm64@0.28.0': - resolution: {integrity: sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==} + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-x64@0.28.0': - resolution: {integrity: sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==} + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/openharmony-arm64@0.28.0': - resolution: {integrity: sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==} + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] - '@esbuild/sunos-x64@0.28.0': - resolution: {integrity: sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==} + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.28.0': - resolution: {integrity: sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==} + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.28.0': - resolution: {integrity: sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==} + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.28.0': - resolution: {integrity: sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==} + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -1044,7 +1051,7 @@ packages: resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} engines: {node: '>=18.14.1'} peerDependencies: - hono: 4.12.21 + hono: 4.12.25 '@img/colour@1.1.0': resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} @@ -1551,8 +1558,8 @@ packages: '@oxc-project/types@0.130.0': resolution: {integrity: sha512-ibD2usx9JRu7f5pu2tMKMI4cpA4NgXJQoYRP4pQ7Pxmn1l6k/53qWtQWZayhYy3X4QZkt90Ot+mJEaeXouio6Q==} - '@oxc-project/types@0.132.0': - resolution: {integrity: sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==} + '@oxc-project/types@0.133.0': + resolution: {integrity: sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==} '@oxc-resolver/binding-android-arm-eabi@11.19.1': resolution: {integrity: sha512-aUs47y+xyXHUKlbhqHUjBABjvycq6YSD7bpxSW7vplUmdzAlJ93yXY6ZR0c1o1x5A/QKbENCvs3+NlY8IpIVzg==} @@ -2048,97 +2055,97 @@ packages: '@radix-ui/rect@1.1.1': resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} - '@rolldown/binding-android-arm64@1.0.2': - resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} + '@rolldown/binding-android-arm64@1.0.3': + resolution: {integrity: sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] - '@rolldown/binding-darwin-arm64@1.0.2': - resolution: {integrity: sha512-vdFA9+C/rekyGce7WqHs/xoT0ioZEWaOFyZLIV1mEeNFaFDUQrPIo8Vs2GvJ6eetb3rzDUtUBgzto3ExpXJB3w==} + '@rolldown/binding-darwin-arm64@1.0.3': + resolution: {integrity: sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] - '@rolldown/binding-darwin-x64@1.0.2': - resolution: {integrity: sha512-BewSOwTHazv77DTYiAZXSqqKZ4KP/KonFisDMVU7PImxoWfB2aepnPhd2E4SWz3zDzYgDNbs6jBmTdgNnF02GA==} + '@rolldown/binding-darwin-x64@1.0.3': + resolution: {integrity: sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] - '@rolldown/binding-freebsd-x64@1.0.2': - resolution: {integrity: sha512-m41o7M0YWtUdqk61Tb+jnKb2rN++iRdIASlExkUoKfIAH30DOHCB8fVLzSUpbWHHU8esmEioY62PxzexE8MBuA==} + '@rolldown/binding-freebsd-x64@1.0.3': + resolution: {integrity: sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] - '@rolldown/binding-linux-arm-gnueabihf@1.0.2': - resolution: {integrity: sha512-jcojB9H7W/jS29pMKWAK1N+fU99vXodHDTatS3b3y/XSOCiHo0kkA74pL3jJmkoQtYpOCxDvaKs1fo2Ij/1X5w==} + '@rolldown/binding-linux-arm-gnueabihf@1.0.3': + resolution: {integrity: sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] - '@rolldown/binding-linux-arm64-gnu@1.0.2': - resolution: {integrity: sha512-1jn6qDU5iiOgFgygDzKUuKP0maTi0/f1+sBLgvij/76C77Nm3ts6ufz9Bjg5q5dduxiUIxtq86JIoBvo1xQ4Ig==} + '@rolldown/binding-linux-arm64-gnu@1.0.3': + resolution: {integrity: sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-arm64-musl@1.0.2': - resolution: {integrity: sha512-QVLO/czFMdoMFSqlX3bcswcJNm/23r+qoa/jgtmFc/qEp6/jXmIkDjF/XIo8dPfGaiwy1xfQn8o77L79GeXFgw==} + '@rolldown/binding-linux-arm64-musl@1.0.3': + resolution: {integrity: sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [musl] - '@rolldown/binding-linux-ppc64-gnu@1.0.2': - resolution: {integrity: sha512-hgO5Abm0w5UL6FEa2iFnZqo2KlK7TQ5QhV5x09hujBf7t5KzHQ1VmfPuTpqRy/rNlSxua3eWH374xxiVrP+lcA==} + '@rolldown/binding-linux-ppc64-gnu@1.0.3': + resolution: {integrity: sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-s390x-gnu@1.0.2': - resolution: {integrity: sha512-fy8rXxuYEu602abC8MUNaPjYLIFzReOaEIEMKMUa0rFEUxNpVXhs15KSSQ4qlqSaM7B6rcj9rDZgADh/IGDzLQ==} + '@rolldown/binding-linux-s390x-gnu@1.0.3': + resolution: {integrity: sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-gnu@1.0.2': - resolution: {integrity: sha512-0+bOkiQ779+r1WpoHOWHqncvyySci0vKph+myNDYb+im6meJAzHQXay6oEgnkHuUGouM1LKTZwqKpBow6Kj7CQ==} + '@rolldown/binding-linux-x64-gnu@1.0.3': + resolution: {integrity: sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-musl@1.0.2': - resolution: {integrity: sha512-mjSkrzZK5Qsl0a9d1JgILOiuZOSDTVdKENcSXBoqbzSrspLR/4/IRVDo5wd2GgZjNss/viBFJdeq+j7qH2nypw==} + '@rolldown/binding-linux-x64-musl@1.0.3': + resolution: {integrity: sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [musl] - '@rolldown/binding-openharmony-arm64@1.0.2': - resolution: {integrity: sha512-1v5vHasdfQAZoEHakBV72LIFAC9JjnymsiKxp+GEr/ma3+NJCPSaYK+qavInOovJkgwFrs7GccX2d6IgDA3Z5w==} + '@rolldown/binding-openharmony-arm64@1.0.3': + resolution: {integrity: sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] - '@rolldown/binding-wasm32-wasi@1.0.2': - resolution: {integrity: sha512-mb1VobWn6NheziTk5/WEaR6AKVbrwT5sOi6C7zk3gy/pD1qtJfU1j4PgTo2NJnOtbL9Dl3Aeei8w9jJ7qC2jZQ==} + '@rolldown/binding-wasm32-wasi@1.0.3': + resolution: {integrity: sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [wasm32] - '@rolldown/binding-win32-arm64-msvc@1.0.2': - resolution: {integrity: sha512-SqKonF56vA/L2yHwHYcEp2P34URpOZ7d1fS635cTkpDnUtEGdUbhI6NzsPdqeSWvAAeGDrxjWjNmibDIdFf9/A==} + '@rolldown/binding-win32-arm64-msvc@1.0.3': + resolution: {integrity: sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [win32] - '@rolldown/binding-win32-x64-msvc@1.0.2': - resolution: {integrity: sha512-v7qRI7gXLRINcOGXt+7YmAZ6iFuyZVMIoXAxhd8oP+DR9dLfL9GfNIx7PLMxmhZdvq8waUJBQiWN9EKNy+TRBQ==} + '@rolldown/binding-win32-x64-msvc@1.0.3': + resolution: {integrity: sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] @@ -2623,7 +2630,7 @@ packages: resolution: {integrity: sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==} peerDependencies: msw: ^2.4.9 - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + vite: 8.0.16 peerDependenciesMeta: msw: optional: true @@ -3341,8 +3348,8 @@ packages: esast-util-from-js@2.0.1: resolution: {integrity: sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==} - esbuild@0.28.0: - resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} engines: {node: '>=18'} hasBin: true @@ -3553,8 +3560,8 @@ packages: debug: optional: true - form-data@4.0.5: - resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} engines: {node: '>= 6'} formatly@0.3.0: @@ -3674,7 +3681,7 @@ packages: next: ^15.3.0 || ^16.0.0 react: ^19.2.0 rolldown: '*' - vite: 7.x.x || 8.x.x + vite: 8.0.16 peerDependenciesMeta: '@types/mdast': optional: true @@ -3826,6 +3833,10 @@ packages: resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==} engines: {node: '>= 0.4'} + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} @@ -3863,8 +3874,8 @@ packages: resolution: {integrity: sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==} engines: {node: '>=0.10.0'} - hono@4.12.21: - resolution: {integrity: sha512-uV63apnb0kyPtAUwoWgaGh9HyIFcv8lgmzPZSiTBQAFOFGIzka5EZ1dZocmGnn0XdX0+XTqJ6Tqv7selMuGLRQ==} + hono@4.12.25: + resolution: {integrity: sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==} engines: {node: '>=16.9.0'} hook-std@4.0.0: @@ -4135,8 +4146,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@4.1.1: - resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + js-yaml@4.2.0: + resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} hasBin: true json-bigint@1.0.0: @@ -4320,7 +4331,7 @@ packages: resolution: {integrity: sha512-gaHQ8h3ixOar8OPrv1rY4rZ5S3Tm8+SGn4oQjP7Km1/16u0BOaK4vXeguqsYLQHlmbaxv7yr3jUXINxFNZ1r/Q==} hasBin: true peerDependencies: - js-yaml: ^4.1.0 + js-yaml: 4.2.0 peerDependenciesMeta: js-yaml: optional: true @@ -4892,7 +4903,7 @@ packages: resolution: {integrity: sha512-AoMplt2UalrpgUDMh3L09QWjNRlgJPipclQvA6sYAaeF6nHNBMgmikAZGmcYLn8on4d9sQY9Q8bOLfrBS7Lc8g==} hasBin: true peerDependencies: - ws: 8.20.1 + ws: 8.21.0 zod: ^3.25 || ^4.0 peerDependenciesMeta: ws: @@ -5354,8 +5365,8 @@ packages: resolution: {integrity: sha512-JzFPAfklk1kjR1w76f0QOIhoDkNkSqW8wYKT08n9yysTmZfB+RQ2QoXoTAeOi1HD9ZipTyTAZg3c4pM/jeqgSw==} engines: {node: '>=18'} - rolldown@1.0.2: - resolution: {integrity: sha512-oZx5zVDtVB44AW3eaifgDml1gWRDZGvjcfdxonE4swNPG98PrrXjaO/KrnUjzlMnztCCRVlUueA1kCXhARGk6g==} + rolldown@1.0.3: + resolution: {integrity: sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true @@ -5748,6 +5759,10 @@ packages: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + tinyrainbow@3.1.0: resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} @@ -5831,12 +5846,12 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} - undici@6.25.0: - resolution: {integrity: sha512-ZgpWDC5gmNiuY9CnLVXEH8rl50xhRCuLNA97fAUnKi8RRuV4E6KG31pDTsLVUKnohJE0I3XDrTeEydAXRw47xg==} + undici@6.27.0: + resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} engines: {node: '>=18.17'} - undici@7.25.0: - resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} + undici@7.28.0: + resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} engines: {node: '>=20.18.1'} unicode-emoji-modifier-base@1.0.0: @@ -5942,14 +5957,14 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} - vite@8.0.14: - resolution: {integrity: sha512-s4BJJ+5y1pYL6Otw51FHhVJQhPnuRinKig64g/1+EUNaJsd3gCKdD31IPFvswUgW9/60QT9oFHbZHbQK5imcxw==} + vite@8.0.16: + resolution: {integrity: sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true peerDependencies: '@types/node': ^24.3.0 '@vitejs/devtools': ^0.1.18 - esbuild: ^0.27.0 || ^0.28.0 + esbuild: 0.28.1 jiti: '>=1.21.0' less: ^4.0.0 sass: ^1.70.0 @@ -6001,7 +6016,7 @@ packages: '@vitest/ui': 4.1.7 happy-dom: '*' jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + vite: 8.0.16 peerDependenciesMeta: '@edge-runtime/vm': optional: true @@ -6088,8 +6103,8 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.20.1: - resolution: {integrity: sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} peerDependencies: bufferutil: ^4.0.1 @@ -6188,7 +6203,7 @@ snapshots: '@actions/http-client@4.0.1': dependencies: tunnel: 0.0.6 - undici: 6.25.0 + undici: 6.27.0 '@actions/io@3.0.2': {} @@ -6201,8 +6216,8 @@ snapshots: '@ai-sdk/devtools@0.0.18': dependencies: '@ai-sdk/provider': 3.0.10 - '@hono/node-server': 1.19.14(hono@4.12.21) - hono: 4.12.21 + '@hono/node-server': 1.19.14(hono@4.12.25) + hono: 4.12.25 '@ai-sdk/gateway@3.0.118(zod@4.4.3)': dependencies: @@ -7015,82 +7030,82 @@ snapshots: tslib: 2.8.1 optional: true - '@esbuild/aix-ppc64@0.28.0': + '@esbuild/aix-ppc64@0.28.1': optional: true - '@esbuild/android-arm64@0.28.0': + '@esbuild/android-arm64@0.28.1': optional: true - '@esbuild/android-arm@0.28.0': + '@esbuild/android-arm@0.28.1': optional: true - '@esbuild/android-x64@0.28.0': + '@esbuild/android-x64@0.28.1': optional: true - '@esbuild/darwin-arm64@0.28.0': + '@esbuild/darwin-arm64@0.28.1': optional: true - '@esbuild/darwin-x64@0.28.0': + '@esbuild/darwin-x64@0.28.1': optional: true - '@esbuild/freebsd-arm64@0.28.0': + '@esbuild/freebsd-arm64@0.28.1': optional: true - '@esbuild/freebsd-x64@0.28.0': + '@esbuild/freebsd-x64@0.28.1': optional: true - '@esbuild/linux-arm64@0.28.0': + '@esbuild/linux-arm64@0.28.1': optional: true - '@esbuild/linux-arm@0.28.0': + '@esbuild/linux-arm@0.28.1': optional: true - '@esbuild/linux-ia32@0.28.0': + '@esbuild/linux-ia32@0.28.1': optional: true - '@esbuild/linux-loong64@0.28.0': + '@esbuild/linux-loong64@0.28.1': optional: true - '@esbuild/linux-mips64el@0.28.0': + '@esbuild/linux-mips64el@0.28.1': optional: true - '@esbuild/linux-ppc64@0.28.0': + '@esbuild/linux-ppc64@0.28.1': optional: true - '@esbuild/linux-riscv64@0.28.0': + '@esbuild/linux-riscv64@0.28.1': optional: true - '@esbuild/linux-s390x@0.28.0': + '@esbuild/linux-s390x@0.28.1': optional: true - '@esbuild/linux-x64@0.28.0': + '@esbuild/linux-x64@0.28.1': optional: true - '@esbuild/netbsd-arm64@0.28.0': + '@esbuild/netbsd-arm64@0.28.1': optional: true - '@esbuild/netbsd-x64@0.28.0': + '@esbuild/netbsd-x64@0.28.1': optional: true - '@esbuild/openbsd-arm64@0.28.0': + '@esbuild/openbsd-arm64@0.28.1': optional: true - '@esbuild/openbsd-x64@0.28.0': + '@esbuild/openbsd-x64@0.28.1': optional: true - '@esbuild/openharmony-arm64@0.28.0': + '@esbuild/openharmony-arm64@0.28.1': optional: true - '@esbuild/sunos-x64@0.28.0': + '@esbuild/sunos-x64@0.28.1': optional: true - '@esbuild/win32-arm64@0.28.0': + '@esbuild/win32-arm64@0.28.1': optional: true - '@esbuild/win32-ia32@0.28.0': + '@esbuild/win32-ia32@0.28.1': optional: true - '@esbuild/win32-x64@0.28.0': + '@esbuild/win32-x64@0.28.1': optional: true '@floating-ui/core@1.7.5': @@ -7156,9 +7171,9 @@ snapshots: '@google-cloud/promisify@5.0.0': {} - '@hono/node-server@1.19.14(hono@4.12.21)': + '@hono/node-server@1.19.14(hono@4.12.25)': dependencies: - hono: 4.12.21 + hono: 4.12.25 '@img/colour@1.1.0': optional: true @@ -7330,7 +7345,7 @@ snapshots: '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': dependencies: - '@hono/node-server': 1.19.14(hono@4.12.21) + '@hono/node-server': 1.19.14(hono@4.12.25) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -7340,7 +7355,7 @@ snapshots: eventsource-parser: 3.0.8 express: 5.2.1 express-rate-limit: 8.5.2(express@5.2.1) - hono: 4.12.21 + hono: 4.12.25 jose: 6.2.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -7553,7 +7568,7 @@ snapshots: '@oxc-project/types@0.130.0': {} - '@oxc-project/types@0.132.0': {} + '@oxc-project/types@0.133.0': {} '@oxc-resolver/binding-android-arm-eabi@11.19.1': optional: true @@ -7996,53 +8011,53 @@ snapshots: '@radix-ui/rect@1.1.1': {} - '@rolldown/binding-android-arm64@1.0.2': + '@rolldown/binding-android-arm64@1.0.3': optional: true - '@rolldown/binding-darwin-arm64@1.0.2': + '@rolldown/binding-darwin-arm64@1.0.3': optional: true - '@rolldown/binding-darwin-x64@1.0.2': + '@rolldown/binding-darwin-x64@1.0.3': optional: true - '@rolldown/binding-freebsd-x64@1.0.2': + '@rolldown/binding-freebsd-x64@1.0.3': optional: true - '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + '@rolldown/binding-linux-arm-gnueabihf@1.0.3': optional: true - '@rolldown/binding-linux-arm64-gnu@1.0.2': + '@rolldown/binding-linux-arm64-gnu@1.0.3': optional: true - '@rolldown/binding-linux-arm64-musl@1.0.2': + '@rolldown/binding-linux-arm64-musl@1.0.3': optional: true - '@rolldown/binding-linux-ppc64-gnu@1.0.2': + '@rolldown/binding-linux-ppc64-gnu@1.0.3': optional: true - '@rolldown/binding-linux-s390x-gnu@1.0.2': + '@rolldown/binding-linux-s390x-gnu@1.0.3': optional: true - '@rolldown/binding-linux-x64-gnu@1.0.2': + '@rolldown/binding-linux-x64-gnu@1.0.3': optional: true - '@rolldown/binding-linux-x64-musl@1.0.2': + '@rolldown/binding-linux-x64-musl@1.0.3': optional: true - '@rolldown/binding-openharmony-arm64@1.0.2': + '@rolldown/binding-openharmony-arm64@1.0.3': optional: true - '@rolldown/binding-wasm32-wasi@1.0.2': + '@rolldown/binding-wasm32-wasi@1.0.3': dependencies: '@emnapi/core': 1.10.0 '@emnapi/runtime': 1.10.0 '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) optional: true - '@rolldown/binding-win32-arm64-msvc@1.0.2': + '@rolldown/binding-win32-arm64-msvc@1.0.3': optional: true - '@rolldown/binding-win32-x64-msvc@1.0.2': + '@rolldown/binding-win32-x64-msvc@1.0.3': optional: true '@rolldown/pluginutils@1.0.1': {} @@ -8111,7 +8126,7 @@ snapshots: p-filter: 4.1.0 semantic-release: 25.0.3(typescript@6.0.3) tinyglobby: 0.2.16 - undici: 7.25.0 + undici: 7.28.0 url-join: 5.0.0 transitivePeerDependencies: - supports-color @@ -8617,7 +8632,7 @@ snapshots: obug: 2.1.1 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + vitest: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) '@vitest/expect@4.1.7': dependencies: @@ -8628,13 +8643,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.7(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0))': + '@vitest/mocker@4.1.7(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) + vite: 8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) '@vitest/pretty-format@4.1.7': dependencies: @@ -8810,7 +8825,7 @@ snapshots: axios@1.16.1: dependencies: follow-redirects: 1.16.0 - form-data: 4.0.5 + form-data: 4.0.6 https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 transitivePeerDependencies: @@ -9122,7 +9137,7 @@ snapshots: dependencies: env-paths: 2.2.1 import-fresh: 3.3.1 - js-yaml: 4.1.1 + js-yaml: 4.2.0 parse-json: 5.2.0 optionalDependencies: typescript: 6.0.3 @@ -9316,34 +9331,34 @@ snapshots: esast-util-from-estree: 2.0.0 vfile-message: 4.0.3 - esbuild@0.28.0: + esbuild@0.28.1: optionalDependencies: - '@esbuild/aix-ppc64': 0.28.0 - '@esbuild/android-arm': 0.28.0 - '@esbuild/android-arm64': 0.28.0 - '@esbuild/android-x64': 0.28.0 - '@esbuild/darwin-arm64': 0.28.0 - '@esbuild/darwin-x64': 0.28.0 - '@esbuild/freebsd-arm64': 0.28.0 - '@esbuild/freebsd-x64': 0.28.0 - '@esbuild/linux-arm': 0.28.0 - '@esbuild/linux-arm64': 0.28.0 - '@esbuild/linux-ia32': 0.28.0 - '@esbuild/linux-loong64': 0.28.0 - '@esbuild/linux-mips64el': 0.28.0 - '@esbuild/linux-ppc64': 0.28.0 - '@esbuild/linux-riscv64': 0.28.0 - '@esbuild/linux-s390x': 0.28.0 - '@esbuild/linux-x64': 0.28.0 - '@esbuild/netbsd-arm64': 0.28.0 - '@esbuild/netbsd-x64': 0.28.0 - '@esbuild/openbsd-arm64': 0.28.0 - '@esbuild/openbsd-x64': 0.28.0 - '@esbuild/openharmony-arm64': 0.28.0 - '@esbuild/sunos-x64': 0.28.0 - '@esbuild/win32-arm64': 0.28.0 - '@esbuild/win32-ia32': 0.28.0 - '@esbuild/win32-x64': 0.28.0 + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 escalade@3.2.0: {} @@ -9591,12 +9606,12 @@ snapshots: follow-redirects@1.16.0: {} - form-data@4.0.5: + form-data@4.0.6: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 es-set-tostringtag: 2.1.0 - hasown: 2.0.3 + hasown: 2.0.4 mime-types: 2.1.35 formatly@0.3.0: @@ -9640,7 +9655,7 @@ snapshots: github-slugger: 2.0.0 hast-util-to-estree: 3.1.3 hast-util-to-jsx-runtime: 2.3.6 - js-yaml: 4.1.1 + js-yaml: 4.2.0 mdast-util-mdx: 3.0.0 mdast-util-to-markdown: 2.1.2 remark: 15.0.1 @@ -9666,15 +9681,15 @@ snapshots: transitivePeerDependencies: - supports-color - fumadocs-mdx@15.0.7(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)): + fumadocs-mdx@15.0.7(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(rolldown@1.0.3)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): dependencies: '@mdx-js/mdx': 3.1.1 '@standard-schema/spec': 1.1.0 chokidar: 5.0.0 - esbuild: 0.28.0 + esbuild: 0.28.1 estree-util-value-to-estree: 3.5.0 fumadocs-core: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) - js-yaml: 4.1.1 + js-yaml: 4.2.0 mdast-util-mdx: 3.0.0 picocolors: 1.1.1 picomatch: 4.0.4 @@ -9691,8 +9706,8 @@ snapshots: '@types/react': 19.2.15 next: 16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 - rolldown: 1.0.2 - vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) + rolldown: 1.0.3 + vite: 8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) transitivePeerDependencies: - supports-color @@ -9859,6 +9874,10 @@ snapshots: dependencies: function-bind: 1.1.2 + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + hast-util-from-parse5@8.0.3: dependencies: '@types/hast': 3.0.4 @@ -9975,7 +9994,7 @@ snapshots: dependencies: parse-passwd: 1.0.0 - hono@4.12.21: {} + hono@4.12.25: {} hook-std@4.0.0: {} @@ -10111,7 +10130,7 @@ snapshots: type-fest: 5.6.0 widest-line: 6.0.0 wrap-ansi: 10.0.0 - ws: 8.20.1 + ws: 8.21.0 yoga-layout: 3.2.1 optionalDependencies: '@types/react': 19.2.15 @@ -10219,7 +10238,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@4.1.1: + js-yaml@4.2.0: dependencies: argparse: 2.0.1 @@ -10396,14 +10415,14 @@ snapshots: longest-streak@3.1.0: {} - lookml-parser@7.1.0(js-yaml@4.1.1): + lookml-parser@7.1.0(js-yaml@4.2.0): dependencies: bluebird: 3.7.2 glob: 7.2.3 minimist: 1.2.8 pegjs: 0.10.0 optionalDependencies: - js-yaml: 4.1.1 + js-yaml: 4.2.0 lru-cache@10.4.3: {} @@ -11136,9 +11155,9 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 - openai@6.38.0(ws@8.20.1)(zod@4.4.3): + openai@6.38.0(ws@8.21.0)(zod@4.4.3): optionalDependencies: - ws: 8.20.1 + ws: 8.21.0 zod: 4.4.3 oxc-parser@0.130.0: @@ -11691,26 +11710,26 @@ snapshots: transitivePeerDependencies: - supports-color - rolldown@1.0.2: + rolldown@1.0.3: dependencies: - '@oxc-project/types': 0.132.0 + '@oxc-project/types': 0.133.0 '@rolldown/pluginutils': 1.0.1 optionalDependencies: - '@rolldown/binding-android-arm64': 1.0.2 - '@rolldown/binding-darwin-arm64': 1.0.2 - '@rolldown/binding-darwin-x64': 1.0.2 - '@rolldown/binding-freebsd-x64': 1.0.2 - '@rolldown/binding-linux-arm-gnueabihf': 1.0.2 - '@rolldown/binding-linux-arm64-gnu': 1.0.2 - '@rolldown/binding-linux-arm64-musl': 1.0.2 - '@rolldown/binding-linux-ppc64-gnu': 1.0.2 - '@rolldown/binding-linux-s390x-gnu': 1.0.2 - '@rolldown/binding-linux-x64-gnu': 1.0.2 - '@rolldown/binding-linux-x64-musl': 1.0.2 - '@rolldown/binding-openharmony-arm64': 1.0.2 - '@rolldown/binding-wasm32-wasi': 1.0.2 - '@rolldown/binding-win32-arm64-msvc': 1.0.2 - '@rolldown/binding-win32-x64-msvc': 1.0.2 + '@rolldown/binding-android-arm64': 1.0.3 + '@rolldown/binding-darwin-arm64': 1.0.3 + '@rolldown/binding-darwin-x64': 1.0.3 + '@rolldown/binding-freebsd-x64': 1.0.3 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.3 + '@rolldown/binding-linux-arm64-gnu': 1.0.3 + '@rolldown/binding-linux-arm64-musl': 1.0.3 + '@rolldown/binding-linux-ppc64-gnu': 1.0.3 + '@rolldown/binding-linux-s390x-gnu': 1.0.3 + '@rolldown/binding-linux-x64-gnu': 1.0.3 + '@rolldown/binding-linux-x64-musl': 1.0.3 + '@rolldown/binding-openharmony-arm64': 1.0.3 + '@rolldown/binding-wasm32-wasi': 1.0.3 + '@rolldown/binding-win32-arm64-msvc': 1.0.3 + '@rolldown/binding-win32-x64-msvc': 1.0.3 router@2.2.0: dependencies: @@ -12216,6 +12235,11 @@ snapshots: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.4) + picomatch: 4.0.4 + tinyrainbow@3.1.0: {} to-regex-range@5.0.1: @@ -12273,9 +12297,9 @@ snapshots: undici-types@7.16.0: {} - undici@6.25.0: {} + undici@6.27.0: {} - undici@7.25.0: {} + undici@7.28.0: {} unicode-emoji-modifier-base@1.0.0: {} @@ -12382,24 +12406,24 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0): + vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 postcss: 8.5.10 - rolldown: 1.0.2 - tinyglobby: 0.2.16 + rolldown: 1.0.3 + tinyglobby: 0.2.17 optionalDependencies: '@types/node': 24.12.4 - esbuild: 0.28.0 + esbuild: 0.28.1 fsevents: 2.3.3 jiti: 2.7.0 yaml: 2.9.0 - vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)): + vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.7 - '@vitest/mocker': 4.1.7(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + '@vitest/mocker': 4.1.7(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.7 '@vitest/runner': 4.1.7 '@vitest/snapshot': 4.1.7 @@ -12416,7 +12440,7 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) + vite: 8.0.16(@types/node@24.12.4)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.1 @@ -12495,7 +12519,7 @@ snapshots: wrappy@1.0.2: {} - ws@8.20.1: {} + ws@8.21.0: {} wsl-utils@0.1.0: dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c66de723..88d5b013 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,12 +5,18 @@ packages: overrides: "@types/node": ^24.3.0 "brace-expansion@>=5.0.0 <5.0.6": 5.0.6 + esbuild: 0.28.1 fast-uri: 3.1.2 fast-xml-builder: 1.1.7 - hono: 4.12.21 + form-data: 4.0.6 + hono: 4.12.25 ip-address: 10.1.1 + js-yaml: 4.2.0 postcss: 8.5.10 - ws: 8.20.1 + "undici@6": 6.27.0 + "undici@7": 7.28.0 + vite: 8.0.16 + ws: 8.21.0 dedupePeerDependents: false preferWorkspacePackages: true diff --git a/uv.lock b/uv.lock index 81405d31..d37cd781 100644 --- a/uv.lock +++ b/uv.lock @@ -466,7 +466,7 @@ wheels = [ [[package]] name = "ktx-daemon" -version = "0.13.1" +version = "0.15.0" source = { editable = "python/ktx-daemon" } dependencies = [ { name = "fastapi" }, @@ -523,7 +523,7 @@ dev = [ [[package]] name = "ktx-sl" -version = "0.13.1" +version = "0.15.0" source = { editable = "python/ktx-sl" } dependencies = [ { name = "pydantic" }, @@ -1409,14 +1409,14 @@ wheels = [ [[package]] name = "starlette" -version = "1.2.0" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/bf/616a066c2760f6c2b1ae3437cc28149734d069fbb46511712beae118a68c/starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553", size = 2668923, upload-time = "2026-05-28T11:42:50.568Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/85/492183764d5d01d4514be3730fdb8e228a80605783099551c51627578b5d/starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7", size = 73213, upload-time = "2026-05-28T11:42:48.801Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] From fe7e6bd1faced28b977cd4c8d61a13a08dcf7514 Mon Sep 17 00:00:00 2001 From: Patel Dhrit <57010146+d180@users.noreply.github.com> Date: Thu, 2 Jul 2026 06:00:26 -0700 Subject: [PATCH 03/11] feat(connector): add Amazon Athena connector via Glue Data Catalog (#309) * feat(connector): add Amazon Athena connector via Glue Data Catalog * fix(athena): address reviewer feedback * fix(athena): wire scope discovery, fix normalizeDriver, tighten types and tests * fix(athena): honor databases scope, wire sql-analysis dialect, harden config resolution - introspect() limits to the configured `databases` scope instead of scanning every Glue database in the account (docs promised this; connector ignored it) - add athena -> athena to sql-analysis SQLGLOT_DIALECTS so `ktx sql` and MCP read-only validation parse Athena SQL under the Trino grammar, not postgres - stringConfigValue coerces a resolved-empty `env:` reference to undefined so optional fields fall back to their defaults (workgroup 'primary', catalog 'AwsDataCatalog') instead of '' - drop trailing whitespace in dialect.test.ts * fix(athena): integrate with main's SQL/non-SQL dialect split and add dialect notes Rebase onto main, which introduced the KtxDialect (core) vs KtxSqlDialect (SQL-only) split for MongoDB: - KtxAthenaDialect implements KtxSqlDialect; the connector resolves it via getSqlDialectForDriver so SQL-generation methods stay in scope - add authored athena.md SQL notes for the sql_dialect_notes MCP tool, required now that athena resolves to the athena sqlglot dialect (dialect-notes coverage is derived from the warehouse-driver registry) --------- Co-authored-by: Andrey Avtomonov --- .../docs/integrations/primary-sources.mdx | 86 ++- packages/cli/package.json | 2 + packages/cli/src/connection-drivers.ts | 1 + .../cli/src/connectors/athena/connector.ts | 555 +++++++++++++++ packages/cli/src/connectors/athena/dialect.ts | 175 +++++ .../athena/live-database-introspection.ts | 44 ++ .../cli/src/context/connections/dialects.ts | 2 + .../cli/src/context/connections/drivers.ts | 17 + .../cli/src/context/project/driver-schemas.ts | 2 + packages/cli/src/context/scan/local-scan.ts | 3 +- packages/cli/src/context/scan/types.ts | 1 + .../src/context/sql-analysis/dialect-notes.ts | 1 + .../cli/src/context/sql-analysis/dialect.ts | 1 + .../context/sql-analysis/dialects/athena.md | 12 + packages/cli/src/local-adapters.ts | 8 + packages/cli/src/setup-databases.ts | 51 ++ .../test/connectors/athena/connector.test.ts | 630 ++++++++++++++++++ .../test/connectors/athena/dialect.test.ts | 72 ++ .../test/context/connections/dialects.test.ts | 2 +- .../test/context/connections/drivers.test.ts | 6 + .../cli/test/context/scan/local-scan.test.ts | 34 + .../test/context/sql-analysis/dialect.test.ts | 1 + packages/cli/test/setup-databases.test.ts | 58 ++ pnpm-lock.yaml | 289 ++++++++ 24 files changed, 2047 insertions(+), 6 deletions(-) create mode 100644 packages/cli/src/connectors/athena/connector.ts create mode 100644 packages/cli/src/connectors/athena/dialect.ts create mode 100644 packages/cli/src/connectors/athena/live-database-introspection.ts create mode 100644 packages/cli/src/context/sql-analysis/dialects/athena.md create mode 100644 packages/cli/test/connectors/athena/connector.test.ts create mode 100644 packages/cli/test/connectors/athena/dialect.test.ts diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 270275f1..b553a3a7 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -1,6 +1,6 @@ --- title: Primary Sources -description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, or MongoDB. +description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, MongoDB, or Amazon Athena. --- **ktx** connects to your data warehouse or database to build schema context, @@ -26,17 +26,23 @@ Agents should prefer environment or file references over literal secrets. | Field | Required | Applies to | Description | |-------|----------|------------|-------------| -| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, or `mongodb` | +| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, `mongodb`, or `athena` | | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | -| `databases` | No | ClickHouse, MongoDB | List of databases to scan | +| `databases` | No | ClickHouse, MongoDB, Athena | List of databases to scan | | `sample_size`, `order_by` | No | MongoDB | Schema-inference sampling controls (recent documents, sort field) | | `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it | | `path` | Yes for path-style SQLite/DuckDB | SQLite, DuckDB | Local SQLite or DuckDB database path or `env:NAME` reference | | `max_bytes_billed` | No | BigQuery | Maximum bytes billed per query job | | `query_timeout_ms` | No | all warehouses | Maximum execution time for a single read-only query, in milliseconds (default 30000). A query exceeding it is cancelled server-side (or, for SQLite, by terminating the off-process executor) and returns a `query exceeded Ns` error so the agent can revise. | | `project_id` | No | BigQuery | Optional local descriptor and mapping metadata; not used for BigQuery authentication | +| `region` | Yes | Athena | AWS region where the Athena workgroup and Glue catalog reside (e.g. `us-east-1`) | +| `s3_staging_dir` | Yes | Athena | S3 URI for Athena query result storage (e.g. `s3://my-bucket/athena-results/`) | +| `workgroup` | No | Athena | Athena workgroup name (default `primary`) | +| `catalog` | No | Athena | Glue Data Catalog name (default `AwsDataCatalog`) | +| `database` | No | Athena | Default Glue database name passed as the query execution context | +| `databases` | No | Athena | Glue databases to include in schema scans; written by `ktx setup` and read by `ktx ingest` | ## PostgreSQL @@ -304,6 +310,76 @@ staged artifact shape as Postgres and Snowflake. --- +## Amazon Athena + +Connects to Amazon Athena using the AWS Glue Data Catalog for schema introspection and the Athena query API for read-only SQL execution. Authentication uses the standard AWS credential chain — no credentials are embedded in `ktx.yaml`. + +### Connection config + +```yaml title="ktx.yaml" +connections: + my-athena: + driver: athena + region: us-east-1 + s3_staging_dir: s3://my-bucket/athena-results/ +``` + +With optional fields: + +```yaml title="ktx.yaml" +connections: + my-athena: + driver: athena + region: us-east-1 + s3_staging_dir: env:ATHENA_S3_STAGING_DIR + workgroup: analytics + catalog: AwsDataCatalog + database: my_default_database + databases: + - analytics + - raw +``` + +`ktx setup` writes the `databases` array when you select Glue databases during setup. `ktx scan` reads it to limit introspection to those databases. + +### Authentication + +**ktx** uses the AWS SDK default credential chain — no credentials appear in `ktx.yaml`. The chain resolves credentials in this order: + +| Method | How to configure | +|--------|-----------------| +| Environment variables | Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN` | +| Shared credentials file | Configure `~/.aws/credentials` with a `[default]` or named profile; set `AWS_PROFILE` to select a non-default profile | +| IAM instance profile | Attach an IAM role to the EC2 instance or ECS task — no local configuration needed | +| IAM roles for service accounts (EKS) | Annotate the pod's service account with the IAM role ARN | + +The IAM principal must have `athena:StartQueryExecution`, `athena:GetQueryExecution`, `athena:GetQueryResults`, `glue:GetDatabases`, and `glue:GetTables` permissions, plus read access to the S3 results bucket. + +### Features + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Tables & views | Yes | Via AWS Glue Data Catalog | +| Primary keys | No | Glue does not expose constraint metadata | +| Foreign keys | No | Not available in Glue/Athena | +| Row count estimates | No | Glue table statistics are often stale | +| Column statistics | No | - | +| Query history | No | - | +| Table sampling | Yes | `SELECT ... LIMIT n` | + +### Dialect notes + +- SQL dialect is Presto/Trino; identifiers are quoted with double-quotes +- Table names use three-part format: `"catalog"."database"."table"` (e.g. `"AwsDataCatalog"."analytics"."orders"`) +- Partition columns (`PartitionKeys` in Glue) are included after regular columns in the schema and are fully queryable +- Athena does not support `TABLESAMPLE`; random sampling uses `ORDER BY rand()` +- Query execution is asynchronous: **ktx** starts the query, polls until completion, then fetches results from S3 +- Results are stored in `s3_staging_dir`; the IAM principal needs write access to that bucket +- Use `workgroup` to apply per-workgroup cost controls and result configuration +- The connector always uses your account's default Glue Data Catalog; cross-account catalog access (`CatalogId` pointing to another account) is not supported + +--- + ## MySQL Standard MySQL/MariaDB connector with full foreign key support and schema introspection. @@ -675,7 +751,9 @@ nullability from how often the field is present: | Error or symptom | Likely cause | Recovery | |------------------|--------------|----------| | Connection URL appears in git diff | A literal credential URL was written to `ktx.yaml` | Replace it with `env:NAME` or `file:/path/to/secret` and rotate exposed credentials | -| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | +| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions. For Athena, confirm the IAM principal has `glue:GetDatabases` and `glue:GetTables` permissions | | Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest --query-history` or `ktx setup` | | Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on schema-level context without column statistics | | Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags | +| Athena query fails with `ACCESS_DENIED` | IAM principal lacks `athena:StartQueryExecution` or S3 write access to `s3_staging_dir` | Attach a policy granting Athena query permissions and `s3:PutObject` on the staging bucket | +| Athena ingest finds databases but no tables | IAM principal has `glue:GetDatabases` but not `glue:GetTables` | Grant `glue:GetTables` on the relevant Glue catalog resources | diff --git a/packages/cli/package.json b/packages/cli/package.json index 8f41231c..f609b571 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -51,6 +51,8 @@ "@ai-sdk/devtools": "0.0.18", "@ai-sdk/google-vertex": "^4.0.134", "@anthropic-ai/claude-agent-sdk": "0.3.146", + "@aws-sdk/client-athena": "^3.1068.0", + "@aws-sdk/client-glue": "^3.1068.0", "@clack/core": "1.3.1", "@clack/prompts": "1.4.0", "@clickhouse/client": "^1.18.5", diff --git a/packages/cli/src/connection-drivers.ts b/packages/cli/src/connection-drivers.ts index 836c8458..b7b1857d 100644 --- a/packages/cli/src/connection-drivers.ts +++ b/packages/cli/src/connection-drivers.ts @@ -10,6 +10,7 @@ export const KTX_DATABASE_DRIVER_IDS = [ 'sqlserver', 'bigquery', 'snowflake', + 'athena', ] as const; // mongodb is a database driver but has no SQL dialect, so it sits outside the diff --git a/packages/cli/src/connectors/athena/connector.ts b/packages/cli/src/connectors/athena/connector.ts new file mode 100644 index 00000000..00b53cc0 --- /dev/null +++ b/packages/cli/src/connectors/athena/connector.ts @@ -0,0 +1,555 @@ +import { AthenaClient, StartQueryExecutionCommand, GetQueryExecutionCommand, GetQueryResultsCommand } from '@aws-sdk/client-athena'; +import { GlueClient, GetDatabasesCommand, GetTablesCommand } from '@aws-sdk/client-glue'; +import { getSqlDialectForDriver } from '../../context/connections/dialects.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxSchemaColumn, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; +import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { resolveStringReference } from '../shared/string-reference.js'; + +export interface KtxAthenaConnectionConfig { + driver?: string; + region?: string; + s3_staging_dir?: string; + workgroup?: string; + catalog?: string; + database?: string; + databases?: string[]; + [key: string]: unknown; +} + +export interface KtxAthenaResolvedConnectionConfig { + region: string; + s3StagingDir: string; + workgroup: string; + catalog: string; + database: string | undefined; + databases: string[]; +} + +interface KtxAthenaQueryExecutionStatus { + State?: string; + StateChangeReason?: string; +} + +interface KtxAthenaQueryExecution { + Status?: KtxAthenaQueryExecutionStatus; +} + +interface KtxAthenaColumnInfo { + Name?: string; + Type?: string; +} + +interface KtxAthenaDatum { + VarCharValue?: string; +} + +interface KtxAthenaRow { + Data?: KtxAthenaDatum[]; +} + +interface KtxAthenaResultSet { + Rows?: KtxAthenaRow[]; + ResultSetMetadata?: { ColumnInfo?: KtxAthenaColumnInfo[] }; +} + +/** @internal */ +export interface KtxAthenaClient { + startQueryExecution(input: { + QueryString: string; + ResultConfiguration: { OutputLocation: string }; + WorkGroup: string; + QueryExecutionContext?: { Database?: string; Catalog?: string }; + }): Promise<{ QueryExecutionId?: string }>; + getQueryExecution(input: { QueryExecutionId: string }): Promise<{ QueryExecution?: KtxAthenaQueryExecution }>; + getQueryResults(input: { QueryExecutionId: string; NextToken?: string }): Promise<{ + ResultSet?: KtxAthenaResultSet; + NextToken?: string; + }>; +} + +interface KtxGlueColumnDef { + Name?: string; + Type?: string; + Comment?: string; +} + +interface KtxGlueStorageDescriptor { + Columns?: KtxGlueColumnDef[]; +} + +/** @internal */ +export interface KtxGlueTable { + Name?: string; + TableType?: string; + StorageDescriptor?: KtxGlueStorageDescriptor; + PartitionKeys?: KtxGlueColumnDef[]; + Description?: string; + Parameters?: Record; +} + +/** @internal */ +export interface KtxGlueClient { + getDatabases(input: { CatalogId?: string; NextToken?: string }): Promise<{ + DatabaseList?: Array<{ Name?: string }>; + NextToken?: string; + }>; + getTables(input: { DatabaseName: string; CatalogId?: string; NextToken?: string }): Promise<{ + TableList?: KtxGlueTable[]; + NextToken?: string; + }>; +} + +export interface KtxAthenaClientFactory { + createAthenaClient(region: string): KtxAthenaClient; + createGlueClient(region: string): KtxGlueClient; +} + +class DefaultAthenaClientFactory implements KtxAthenaClientFactory { + createAthenaClient(region: string): KtxAthenaClient { + const client = new AthenaClient({ region }); + return { + startQueryExecution: async (input) => { + const result = await client.send( + new StartQueryExecutionCommand({ + QueryString: input.QueryString, + ResultConfiguration: { OutputLocation: input.ResultConfiguration.OutputLocation }, + WorkGroup: input.WorkGroup, + QueryExecutionContext: input.QueryExecutionContext, + }), + ); + return { QueryExecutionId: result.QueryExecutionId }; + }, + getQueryExecution: async (input) => { + const result = await client.send(new GetQueryExecutionCommand({ QueryExecutionId: input.QueryExecutionId })); + return { + QueryExecution: result.QueryExecution + ? { + Status: { + State: result.QueryExecution.Status?.State, + StateChangeReason: result.QueryExecution.Status?.StateChangeReason, + }, + } + : undefined, + }; + }, + getQueryResults: async (input) => { + const result = await client.send( + new GetQueryResultsCommand({ QueryExecutionId: input.QueryExecutionId, NextToken: input.NextToken }), + ); + return { + ResultSet: result.ResultSet as KtxAthenaResultSet | undefined, + NextToken: result.NextToken, + }; + }, + }; + } + + createGlueClient(region: string): KtxGlueClient { + const client = new GlueClient({ region }); + return { + getDatabases: async (input) => { + const result = await client.send(new GetDatabasesCommand({ CatalogId: input.CatalogId, NextToken: input.NextToken })); + return { + DatabaseList: result.DatabaseList?.map((db) => ({ Name: db.Name })), + NextToken: result.NextToken, + }; + }, + getTables: async (input) => { + const result = await client.send( + new GetTablesCommand({ DatabaseName: input.DatabaseName, CatalogId: input.CatalogId, NextToken: input.NextToken }), + ); + return { + TableList: result.TableList as KtxGlueTable[] | undefined, + NextToken: result.NextToken, + }; + }, + }; + } +} + +function stringConfigValue( + connection: KtxAthenaConnectionConfig | undefined, + key: keyof KtxAthenaConnectionConfig, + env: NodeJS.ProcessEnv, +): string | undefined { + const value = connection?.[key]; + if (typeof value !== 'string' || value.trim().length === 0) return undefined; + // Resolve before checking emptiness: an unset `env:` reference resolves to '', + // which must become undefined so `?? default` applies instead of keeping ''. + const resolved = resolveStringReference(value.trim(), env).trim(); + return resolved.length > 0 ? resolved : undefined; +} + +function configuredAthenaDatabases(connection: KtxAthenaConnectionConfig): string[] { + if (!Array.isArray(connection.databases)) return []; + const selected = connection.databases + .filter((database): database is string => typeof database === 'string' && database.trim().length > 0) + .map((database) => database.trim()); + return [...new Set(selected)]; +} + +export function isKtxAthenaConnectionConfig( + connection: unknown, +): connection is KtxAthenaConnectionConfig { + return ( + typeof connection === 'object' && + connection !== null && + String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'athena' + ); +} + +/** @internal */ +export function athenaConnectionConfigFromConfig(input: { + connectionId: string; + connection: KtxAthenaConnectionConfig | undefined; + env?: NodeJS.ProcessEnv; +}): KtxAthenaResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; + if (!isKtxAthenaConnectionConfig(input.connection)) { + throw new Error(`Native Athena connector cannot run driver "${String(inputDriver)}"`); + } + const env = input.env ?? process.env; + const region = stringConfigValue(input.connection, 'region', env); + if (!region) { + throw new Error(`Native Athena connector requires connections.${input.connectionId}.region`); + } + const s3StagingDir = stringConfigValue(input.connection, 's3_staging_dir', env); + if (!s3StagingDir) { + throw new Error(`Native Athena connector requires connections.${input.connectionId}.s3_staging_dir`); + } + return { + region, + s3StagingDir, + workgroup: stringConfigValue(input.connection, 'workgroup', env) ?? 'primary', + catalog: stringConfigValue(input.connection, 'catalog', env) ?? 'AwsDataCatalog', + database: stringConfigValue(input.connection, 'database', env), + databases: configuredAthenaDatabases(input.connection), + }; +} + +function glueTableKind(tableType: string | undefined): 'table' | 'view' { + const t = String(tableType ?? '').toUpperCase(); + if (t === 'VIRTUAL_VIEW') return 'view'; + return 'table'; +} + +const POLL_INTERVAL_MS = 250; +const QUERY_TIMEOUT_MS = 5 * 60 * 1000; + +export interface KtxAthenaScanConnectorOptions { + connectionId: string; + connection: KtxAthenaConnectionConfig | undefined; + clientFactory?: KtxAthenaClientFactory; + env?: NodeJS.ProcessEnv; + now?: () => Date; +} + +export class KtxAthenaScanConnector implements KtxScanConnector { + readonly id: string; + readonly driver = 'athena' as const; + readonly capabilities = createKtxConnectorCapabilities({ + tableSampling: true, + columnSampling: true, + columnStats: false, + readOnlySql: true, + nestedAnalysis: false, + formalForeignKeys: false, + estimatedRowCounts: false, + }); + + private readonly connectionId: string; + private readonly resolved: KtxAthenaResolvedConnectionConfig; + private readonly clientFactory: KtxAthenaClientFactory; + private readonly now: () => Date; + private readonly dialect = getSqlDialectForDriver('athena'); + private athenaClient: KtxAthenaClient | null = null; + private glueClient: KtxGlueClient | null = null; + + constructor(options: KtxAthenaScanConnectorOptions) { + this.connectionId = options.connectionId; + this.resolved = athenaConnectionConfigFromConfig({ + connectionId: options.connectionId, + connection: options.connection, + env: options.env, + }); + this.clientFactory = options.clientFactory ?? new DefaultAthenaClientFactory(); + this.now = options.now ?? (() => new Date()); + this.id = `athena:${options.connectionId}`; + } + + async testConnection(): Promise { + try { + await this.listDatabasesPaginated({ maxResults: 1 }); + return { success: true }; + } catch (error) { + return connectorTestFailure(error); + } + } + + async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + // Honor the configured `databases` scope (written by `ktx setup`); fall back + // to every Glue database only when the scope is unset. + const databases = + this.resolved.databases.length > 0 ? this.resolved.databases : await this.listDatabasesPaginated({}); + const tables: KtxSchemaTable[] = []; + for (const database of databases) { + const scopedNames = input.tableScope + ? scopedTableNames(input.tableScope, { catalog: this.resolved.catalog, db: database }) + : null; + tables.push(...(await this.introspectDatabase(database, scopedNames))); + } + return { + connectionId: this.connectionId, + driver: 'athena', + extractedAt: this.now().toISOString(), + scope: { catalogs: [this.resolved.catalog], datasets: databases }, + metadata: { + catalog: this.resolved.catalog, + databases, + table_count: tables.length, + total_columns: tables.reduce((sum, t) => sum + t.columns.length, 0), + }, + tables, + warnings: [], + }; + } + + async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns)); + return { headers: result.headers, headerTypes: result.headerTypes, rows: result.rows, totalRows: result.totalRows }; + } + + async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query( + this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit), + ); + return { + values: result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]), + nullCount: null, + distinctCount: null, + }; + } + + async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise { + return null; + } + + async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); + const result = await this.query(limitedSql); + return { ...result, rowCount: result.rows.length }; + } + + async listSchemas(): Promise { + return this.listDatabasesPaginated({}); + } + + async listTables(databases?: string[]): Promise { + const targetDatabases = databases && databases.length > 0 ? databases : await this.listDatabasesPaginated({}); + const entries: KtxTableListEntry[] = []; + for (const database of targetDatabases) { + const glueTables = await this.listGlueTablesPaginated(database); + for (const t of glueTables) { + if (!t.Name) continue; + entries.push({ + catalog: this.resolved.catalog, + schema: database, + name: t.Name, + kind: glueTableKind(t.TableType), + }); + } + } + return entries; + } + + async cleanup(): Promise { + this.athenaClient = null; + this.glueClient = null; + } + + qTableName(table: Pick & Partial>): string { + return this.dialect.formatTableName(table); + } + + private getAthenaClient(): KtxAthenaClient { + if (!this.athenaClient) { + this.athenaClient = this.clientFactory.createAthenaClient(this.resolved.region); + } + return this.athenaClient; + } + + private getGlueClient(): KtxGlueClient { + if (!this.glueClient) { + this.glueClient = this.clientFactory.createGlueClient(this.resolved.region); + } + return this.glueClient; + } + + private async listDatabasesPaginated(opts: { maxResults?: number }): Promise { + const names: string[] = []; + let nextToken: string | undefined; + do { + const result = await this.getGlueClient().getDatabases({ NextToken: nextToken }); + for (const db of result.DatabaseList ?? []) { + if (db.Name) names.push(db.Name); + if (opts.maxResults && names.length >= opts.maxResults) return names; + } + nextToken = result.NextToken; + } while (nextToken); + return names; + } + + private async listGlueTablesPaginated(database: string): Promise { + const tables: KtxGlueTable[] = []; + let nextToken: string | undefined; + do { + const result = await this.getGlueClient().getTables({ DatabaseName: database, NextToken: nextToken }); + tables.push(...(result.TableList ?? [])); + nextToken = result.NextToken; + } while (nextToken); + return tables; + } + + private async introspectDatabase(database: string, scopedNames: readonly string[] | null): Promise { + if (scopedNames && scopedNames.length === 0) return []; + const glueTables = await this.listGlueTablesPaginated(database); + const scopeSet = scopedNames ? new Set(scopedNames) : null; + return glueTables + .filter((t): t is KtxGlueTable & { Name: string } => Boolean(t.Name) && (!scopeSet || scopeSet.has(t.Name!))) + .map((t) => ({ + catalog: this.resolved.catalog, + db: database, + name: t.Name, + kind: glueTableKind(t.TableType), + comment: t.Description ?? null, + estimatedRows: null, + columns: this.toSchemaColumns(t), + foreignKeys: [], + })); + } + + private toSchemaColumns(table: KtxGlueTable): KtxSchemaColumn[] { + const columns = [...(table.StorageDescriptor?.Columns ?? []), ...(table.PartitionKeys ?? [])]; + return columns + .filter((col): col is KtxGlueColumnDef & { Name: string } => Boolean(col.Name)) + .map((col) => { + const nativeType = String(col.Type ?? 'string').toLowerCase(); + return { + name: col.Name, + nativeType, + normalizedType: this.dialect.mapDataType(nativeType), + dimensionType: this.dialect.mapToDimensionType(nativeType), + nullable: true, + primaryKey: false, + comment: col.Comment ?? null, + }; + }); + } + + private async query(sql: string): Promise { + const athena = this.getAthenaClient(); + const { QueryExecutionId } = await athena.startQueryExecution({ + QueryString: sql, + ResultConfiguration: { OutputLocation: this.resolved.s3StagingDir }, + WorkGroup: this.resolved.workgroup, + ...(this.resolved.database || this.resolved.catalog + ? { + QueryExecutionContext: { + ...(this.resolved.database ? { Database: this.resolved.database } : {}), + ...(this.resolved.catalog ? { Catalog: this.resolved.catalog } : {}), + }, + } + : {}), + }); + + if (!QueryExecutionId) { + throw new Error('Athena did not return a QueryExecutionId'); + } + + await this.waitForQueryCompletion(athena, QueryExecutionId); + + const rows: unknown[][] = []; + let headers: string[] = []; + let headerTypes: string[] = []; + let nextToken: string | undefined; + let firstPage = true; + + do { + const result = await athena.getQueryResults({ QueryExecutionId, NextToken: nextToken }); + const resultSet = result.ResultSet; + + if (firstPage) { + const columnInfo = resultSet?.ResultSetMetadata?.ColumnInfo ?? []; + headers = columnInfo.map((col) => col.Name ?? ''); + headerTypes = columnInfo.map((col) => String(col.Type ?? 'varchar').toUpperCase()); + firstPage = false; + } + + const pageRows = resultSet?.Rows ?? []; + // Athena includes the header row as the first row of the first page — skip it. + const dataRows = nextToken === undefined ? pageRows.slice(1) : pageRows; + for (const row of dataRows) { + rows.push((row.Data ?? []).map((d) => d.VarCharValue ?? null)); + } + + nextToken = result.NextToken; + } while (nextToken); + + return { + headers, + headerTypes: headerTypes.length > 0 ? headerTypes : undefined, + rows, + totalRows: rows.length, + rowCount: rows.length, + }; + } + + private async waitForQueryCompletion(athena: KtxAthenaClient, queryExecutionId: string): Promise { + const terminalStates = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED']); + const deadline = this.now().getTime() + QUERY_TIMEOUT_MS; + for (;;) { + const { QueryExecution } = await athena.getQueryExecution({ QueryExecutionId: queryExecutionId }); + const state = QueryExecution?.Status?.State ?? ''; + if (state === 'SUCCEEDED') return; + if (terminalStates.has(state)) { + const reason = QueryExecution?.Status?.StateChangeReason ?? state; + throw new Error(`Athena query ${state}: ${reason}`); + } + if (this.now().getTime() >= deadline) { + throw new Error(`Athena query ${queryExecutionId} timed out after ${QUERY_TIMEOUT_MS / 1000}s`); + } + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + } + } + + private assertConnection(connectionId: string): void { + if (connectionId !== this.connectionId) { + throw new Error(`Athena connector ${this.connectionId} cannot scan connection ${connectionId}`); + } + } +} diff --git a/packages/cli/src/connectors/athena/dialect.ts b/packages/cli/src/connectors/athena/dialect.ts new file mode 100644 index 00000000..81349804 --- /dev/null +++ b/packages/cli/src/connectors/athena/dialect.ts @@ -0,0 +1,175 @@ +import type { KtxSqlDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; + +type AthenaTableNameRef = Pick & Partial>; + +/** @internal */ +export class KtxAthenaDialect implements KtxSqlDialect { + readonly type = 'athena' as const; + + private readonly dimensionTypeMappings: Record = { + timestamp: 'time', + date: 'time', + bigint: 'number', + int: 'number', + integer: 'number', + tinyint: 'number', + smallint: 'number', + double: 'number', + float: 'number', + real: 'number', + boolean: 'boolean', + }; + + quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; + } + + formatTableName(table: AthenaTableNameRef): string { + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi'); + } + + formatDisplayRef(table: AthenaTableNameRef): string { + return formatDialectDisplayRef(table, 'ansi'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'ansi'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('ansi'); + } + + mapDataType(nativeType: string): string { + const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim(); + const typeMap: Record = { + string: 'VARCHAR', + varchar: 'VARCHAR', + char: 'CHAR', + binary: 'VARBINARY', + bigint: 'BIGINT', + int: 'INTEGER', + integer: 'INTEGER', + tinyint: 'TINYINT', + smallint: 'SMALLINT', + double: 'DOUBLE', + float: 'FLOAT', + real: 'REAL', + decimal: 'DECIMAL', + boolean: 'BOOLEAN', + timestamp: 'TIMESTAMP', + date: 'DATE', + array: 'ARRAY', + map: 'MAP', + struct: 'STRUCT', + uniontype: 'UNION', + }; + return typeMap[base] ?? nativeType.toUpperCase(); + } + + mapToDimensionType(nativeType: string): KtxSchemaDimensionType { + const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim(); + const mapped = this.dimensionTypeMappings[base]; + if (mapped) return mapped; + if (base.includes('timestamp') || base.includes('date')) return 'time'; + if (base.includes('int') || base.includes('float') || base.includes('double') || base.includes('decimal') || base.includes('real')) return 'number'; + if (base.includes('bool')) return 'boolean'; + return 'string'; + } + + generateSampleQuery(tableName: string, limit: number, columns?: string[]): string { + const columnList = + columns && columns.length > 0 ? columns.map((c) => this.quoteIdentifier(c)).join(', ') : '*'; + return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`; + } + + generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string { + const quoted = this.quoteIdentifier(columnName); + return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL LIMIT ${limit}`; + } + + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + SELECT approx_distinct(${columnName}) AS cardinality + FROM ( + SELECT ${columnName} + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + LIMIT ${sampleSize} + ) + `; + } + + generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + SELECT approx_distinct(${columnName}) AS cardinality + FROM ( + SELECT ${columnName} + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY rand() + LIMIT ${sampleSize} + ) + `; + } + + generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string { + return ` + SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY val + LIMIT ${limit} + `; + } + + generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null { + return null; + } + + getNullCountExpression(column: string): string { + return `COUNT_IF(${column} IS NULL)`; + } + + getDistinctCountExpression(column: string): string { + return `approx_distinct(${column})`; + } + + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS VARCHAR))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS VARCHAR)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT array_join(array_agg(CAST(value AS VARCHAR)), '\u001f') FROM (${innerSql}) AS relationship_profile_values)`; + } + + getLimitOffsetClause(limit: number, offset?: number): string { + const safeLimit = Math.max(1, Math.floor(limit)); + const safeOffset = offset !== undefined ? Math.floor(offset) : 0; + return safeOffset > 0 ? `OFFSET ${safeOffset} LIMIT ${safeLimit}` : `LIMIT ${safeLimit}`; + } + + getTopClause(_limit: number): string { + return ''; + } + + getTableSampleClause(_samplePct: number): string { + return ''; + } + + getRandomSampleFilter(samplePct: number): string { + if (samplePct <= 0 || samplePct >= 1) return ''; + return `rand() < ${samplePct}`; + } +} diff --git a/packages/cli/src/connectors/athena/live-database-introspection.ts b/packages/cli/src/connectors/athena/live-database-introspection.ts new file mode 100644 index 00000000..815285ba --- /dev/null +++ b/packages/cli/src/connectors/athena/live-database-introspection.ts @@ -0,0 +1,44 @@ +import type { + LiveDatabaseIntrospectionOptions, + LiveDatabaseIntrospectionPort, +} from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; +import { + KtxAthenaScanConnector, + type KtxAthenaClientFactory, + type KtxAthenaConnectionConfig, +} from './connector.js'; + +interface CreateAthenaLiveDatabaseIntrospectionOptions { + connections: Record; + clientFactory?: KtxAthenaClientFactory; + now?: () => Date; +} + +export function createAthenaLiveDatabaseIntrospection( + options: CreateAthenaLiveDatabaseIntrospectionOptions, +): LiveDatabaseIntrospectionPort { + return { + async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) { + const connection = options.connections[connectionId] as KtxAthenaConnectionConfig | undefined; + const connector = new KtxAthenaScanConnector({ + connectionId, + connection, + clientFactory: options.clientFactory, + now: options.now, + }); + try { + return await connector.introspect( + { + connectionId, + driver: 'athena', + ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}), + }, + { runId: `athena-${connectionId}` }, + ); + } finally { + await connector.cleanup(); + } + }, + }; +} diff --git a/packages/cli/src/context/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts index ad739d1d..8512ed57 100644 --- a/packages/cli/src/context/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,3 +1,4 @@ +import { KtxAthenaDialect } from '../../connectors/athena/dialect.js'; import { KtxBigQueryDialect } from '../../connectors/bigquery/dialect.js'; import { KtxClickHouseDialect } from '../../connectors/clickhouse/dialect.js'; import { KtxDuckDbDialect } from '../../connectors/duckdb/dialect.js'; @@ -54,6 +55,7 @@ export interface KtxSqlDialect extends KtxDialect { type KtxSqlDriver = Exclude; const sqlDialectFactories: Record KtxSqlDialect> = { + athena: () => new KtxAthenaDialect(), bigquery: () => new KtxBigQueryDialect(), clickhouse: () => new KtxClickHouseDialect(), duckdb: () => new KtxDuckDbDialect(), diff --git a/packages/cli/src/context/connections/drivers.ts b/packages/cli/src/context/connections/drivers.ts index 1ba17f54..6a2d4589 100644 --- a/packages/cli/src/context/connections/drivers.ts +++ b/packages/cli/src/context/connections/drivers.ts @@ -26,6 +26,23 @@ function invalidConnectionConfig(driver: KtxConnectionDriver): Error { /** @internal */ export const driverRegistrations: Record = { + athena: { + driver: 'athena', + scopeConfigKey: 'databases', + hasHistoricSqlReader: false, + load: async () => { + const m = await import('../../connectors/athena/connector.js'); + return { + isConnectionConfig: (connection) => m.isKtxAthenaConnectionConfig(connection), + createScanConnector: ({ connectionId, connection }) => { + if (!m.isKtxAthenaConnectionConfig(connection)) { + throw invalidConnectionConfig('athena'); + } + return new m.KtxAthenaScanConnector({ connectionId, connection }); + }, + }; + }, + }, bigquery: { driver: 'bigquery', scopeConfigKey: 'dataset_ids', diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index 9f808d00..df828065 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -14,6 +14,7 @@ const warehouseDrivers = [ 'duckdb', 'clickhouse', 'sqlserver', + 'athena', ] as const; type WarehouseDriver = (typeof warehouseDrivers)[number]; @@ -56,6 +57,7 @@ const warehouseConnectionSchemas = [ warehouseConnectionSchema('duckdb'), warehouseConnectionSchema('clickhouse'), warehouseConnectionSchema('sqlserver'), + warehouseConnectionSchema('athena'), ] as const; const mongodbConnectionSchema = z diff --git a/packages/cli/src/context/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts index 0966d15b..ac0a90e5 100644 --- a/packages/cli/src/context/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -147,12 +147,13 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver { normalized === 'sqlserver' || normalized === 'bigquery' || normalized === 'snowflake' || + normalized === 'athena' || normalized === 'mongodb' ) { return normalized; } throw new Error( - `Standalone ktx scan supports postgres/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake/mongodb in this phase, received "${driver ?? 'unknown'}"`, + `Standalone ktx scan supports postgres/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake/athena/mongodb in this phase, received "${driver ?? 'unknown'}"`, ); } diff --git a/packages/cli/src/context/scan/types.ts b/packages/cli/src/context/scan/types.ts index 9bd7d865..bf72558c 100644 --- a/packages/cli/src/context/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -9,6 +9,7 @@ export type KtxConnectionDriver = | 'snowflake' | 'mysql' | 'clickhouse' + | 'athena' | 'mongodb'; /** Canonical scan-mode registry. Runtime validation derives its allowlist here. */ diff --git a/packages/cli/src/context/sql-analysis/dialect-notes.ts b/packages/cli/src/context/sql-analysis/dialect-notes.ts index f0a20a2e..7fcffab3 100644 --- a/packages/cli/src/context/sql-analysis/dialect-notes.ts +++ b/packages/cli/src/context/sql-analysis/dialect-notes.ts @@ -18,6 +18,7 @@ export const DIALECTS_WITH_NOTES = [ 'duckdb', 'clickhouse', 'tsql', + 'athena', ] as const; type DialectWithNotes = (typeof DIALECTS_WITH_NOTES)[number]; diff --git a/packages/cli/src/context/sql-analysis/dialect.ts b/packages/cli/src/context/sql-analysis/dialect.ts index 54bc9525..83092233 100644 --- a/packages/cli/src/context/sql-analysis/dialect.ts +++ b/packages/cli/src/context/sql-analysis/dialect.ts @@ -16,6 +16,7 @@ const SQLGLOT_DIALECTS: Record = { duckdb: 'duckdb', clickhouse: 'clickhouse', databricks: 'databricks', + athena: 'athena', }; export function sqlAnalysisDialectForDriver(driver: string | undefined): SqlAnalysisDialect { diff --git a/packages/cli/src/context/sql-analysis/dialects/athena.md b/packages/cli/src/context/sql-analysis/dialects/athena.md new file mode 100644 index 00000000..d7a377cf --- /dev/null +++ b/packages/cli/src/context/sql-analysis/dialects/athena.md @@ -0,0 +1,12 @@ +**athena** SQL conventions (Trino engine over the Glue Data Catalog): +- **FQTN:** `database.table` (e.g. `analytics.orders`); a bare `table` resolves against the query's default database. Cross-catalog is `catalog.database.table` (e.g. `awsdatacatalog.analytics.orders`). +- **Identifiers:** case-insensitive and folded to lowercase; double-quote (`"Name"`) to keep case, spaces, or a reserved word. String literals use single quotes only. +- **Date/time:** native `DATE`/`TIMESTAMP`. Bucket with `date_trunc('month', ts)`, pull parts with `EXTRACT(YEAR FROM ts)`, shift with `date_add('day', -30, current_date)`, difference with `date_diff('day', a, b)`, and format with `date_format(ts, '%Y-%m')`. Parse text with `date_parse(str, '%Y-%m-%d')` or `from_iso8601_timestamp(str)`; `current_date` / `now()` are available. +- **Top-N / windows:** Athena has no `QUALIFY` — wrap the window in a subquery and filter it: `SELECT * FROM (SELECT ..., ROW_NUMBER() OVER (PARTITION BY key ORDER BY x DESC) AS rn FROM t) WHERE rn = 1` returns one row per key. Use `ORDER BY ... LIMIT n` for a global top-N, and paginate with `OFFSET m LIMIT n` (offset first — `LIMIT n OFFSET m` is a syntax error). +- **Series:** `CROSS JOIN UNNEST(sequence(DATE '2023-01-01', DATE '2023-12-01', INTERVAL '1' MONTH)) AS s(d)` expands a generated array into a date spine (use `sequence(1, 12)` for integers), then `LEFT JOIN` the aggregated facts onto it so empty periods still appear. +- **Rolling window over time:** a native `RANGE` frame spans real dates and tolerates gaps — `AVG(amount) OVER (ORDER BY day RANGE BETWEEN INTERVAL '29' DAY PRECEDING AND CURRENT ROW)` is a trailing 30-day average without a spine; guard minimum periods with `COUNT(*) OVER ()`. +- **Approximate aggregates:** `approx_distinct(x)` for cardinality and `approx_percentile(x, 0.5)` for quantiles are far cheaper than exact `COUNT(DISTINCT ...)` on large scans. +- **Arrays & maps:** explode with `CROSS JOIN UNNEST(arr) AS t(x)` (add `WITH ORDINALITY` for an index); build with `array_agg(x)`, join with `array_join(arr, ',')`, index 1-based (`arr[1]`), and read a map with `element_at(m, key)`. +- **Safe cast:** `TRY_CAST(x AS DOUBLE)` yields `NULL` for a value that does not parse instead of raising, so counting residual `NULL`s catches an encoding the sample missed; `TRY(expr)` swallows other runtime errors. +- **Integer division:** `/` between integers truncates (`5 / 2` → `2`); cast an operand (`x / CAST(y AS DOUBLE)`) to keep the fraction, and round only in the final projection. +- **JSON:** `json_extract_scalar(col, '$.a.b')` returns varchar, `json_extract(col, '$.a')` returns json; cast a JSON string with `CAST(json_parse(col) AS ...)`. diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 03ff48d2..9f9ac60d 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -1,3 +1,5 @@ +import { createAthenaLiveDatabaseIntrospection } from './connectors/athena/live-database-introspection.js'; +import { isKtxAthenaConnectionConfig } from './connectors/athena/connector.js'; import { createBigQueryLiveDatabaseIntrospection } from './connectors/bigquery/live-database-introspection.js'; import { isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, type KtxBigQueryConnectionConfig } from './connectors/bigquery/connector.js'; import { createClickHouseLiveDatabaseIntrospection } from './connectors/clickhouse/live-database-introspection.js'; @@ -125,6 +127,9 @@ function createKtxCliLiveDatabaseIntrospection( const bigquery = createBigQueryLiveDatabaseIntrospection({ connections: project.config.connections, }); + const athena = createAthenaLiveDatabaseIntrospection({ + connections: project.config.connections, + }); return { async extractSchema(connectionId: string, options?: LiveDatabaseIntrospectionOptions) { const connection = project.config.connections[connectionId]; @@ -160,6 +165,9 @@ function createKtxCliLiveDatabaseIntrospection( if (isKtxBigQueryConnectionConfig(connection)) { return bigquery.extractSchema(connectionId, options); } + if (isKtxAthenaConnectionConfig(connection)) { + return athena.extractSchema(connectionId, options); + } if (hasSnowflakeDriver(connection)) { const { createSnowflakeLiveDatabaseIntrospection } = await import('./connectors/snowflake/live-database-introspection.js'); const { isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 33e3b1e0..efeba209 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -71,6 +71,7 @@ export type KtxSetupDatabaseDriver = | 'sqlserver' | 'bigquery' | 'snowflake' + | 'athena' | 'mongodb'; export interface KtxSetupDatabasesArgs { @@ -158,6 +159,7 @@ const DRIVER_OPTIONS: Array<{ value: KtxSetupDatabaseDriver; label: string }> = { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, + { value: 'athena', label: 'Amazon Athena' }, { value: 'mongodb', label: 'MongoDB' }, { value: 'sqlite', label: 'SQLite' }, { value: 'duckdb', label: 'DuckDB' }, @@ -183,6 +185,7 @@ const DEFAULT_CONNECTION_IDS: Record = { sqlserver: 'sqlserver-warehouse', bigquery: 'bigquery-warehouse', snowflake: 'snowflake-warehouse', + athena: 'athena-warehouse', mongodb: 'mongodb-source', }; @@ -268,6 +271,13 @@ const SCOPE_DISCOVERY_SPECS: Partial; @@ -968,6 +978,47 @@ async function buildConnectionConfig(input: { ...(role ? { role } : {}), }; } + if (driver === 'athena') { + if (args.inputMode === 'disabled' && !args.databaseUrl) return null; + const region = await promptText( + prompts, + 'AWS region\nFor example us-east-1.', + stringConfigField(input.existingConnection, 'region'), + ); + if (region === undefined) return 'back'; + if (!region) return null; + + const s3StagingDir = await promptText( + prompts, + 'S3 staging directory\nAthena writes query results here. For example s3://my-bucket/athena-results/.', + stringConfigField(input.existingConnection, 's3_staging_dir'), + ); + if (s3StagingDir === undefined) return 'back'; + if (!s3StagingDir) return null; + + const workgroup = await promptText( + prompts, + 'Athena workgroup (optional)\nPress Enter to use the default workgroup "primary".', + stringConfigField(input.existingConnection, 'workgroup'), + ); + if (workgroup === undefined) return 'back'; + + const catalog = await promptText( + prompts, + 'Glue Data Catalog name (optional)\nPress Enter to use the default "AwsDataCatalog".', + stringConfigField(input.existingConnection, 'catalog'), + ); + if (catalog === undefined) return 'back'; + + return { + driver: 'athena', + region, + s3_staging_dir: s3StagingDir, + ...(workgroup ? { workgroup } : {}), + ...(catalog ? { catalog } : {}), + ...scriptedScopeConfigForDriver('athena', args.databaseSchemas), + }; + } throw new Error(`Unsupported database driver: ${driver}`); } diff --git a/packages/cli/test/connectors/athena/connector.test.ts b/packages/cli/test/connectors/athena/connector.test.ts new file mode 100644 index 00000000..95afbd4c --- /dev/null +++ b/packages/cli/test/connectors/athena/connector.test.ts @@ -0,0 +1,630 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + athenaConnectionConfigFromConfig, + isKtxAthenaConnectionConfig, + KtxAthenaScanConnector, + type KtxAthenaClientFactory, + type KtxAthenaClient, + type KtxGlueClient, +} from '../../../src/connectors/athena/connector.js'; +import { createAthenaLiveDatabaseIntrospection } from '../../../src/connectors/athena/live-database-introspection.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; + +function fakeClientFactory(options: { queryState?: string; queryError?: string } = {}): KtxAthenaClientFactory { + const state = options.queryState ?? 'SUCCEEDED'; + const queries = new Map(); + let execCounter = 0; + + const fakeAthenaClient: KtxAthenaClient = { + startQueryExecution: vi.fn(async (input) => { + const id = `exec-${++execCounter}`; + queries.set(id, input.QueryString); + return { QueryExecutionId: id }; + }), + getQueryExecution: vi.fn(async () => ({ + QueryExecution: { + Status: { + State: state, + StateChangeReason: options.queryError, + }, + }, + })), + getQueryResults: vi.fn(async (input) => { + const sql = queries.get(input.QueryExecutionId) ?? ''; + // Column sample query: single-column result for the queried column only. + if (sql.includes('IS NOT NULL')) { + return { + ResultSet: { + ResultSetMetadata: { ColumnInfo: [{ Name: 'status', Type: 'string' }] }, + Rows: [ + { Data: [{ VarCharValue: 'status' }] }, // header row + { Data: [{ VarCharValue: 'paid' }] }, + ], + }, + NextToken: undefined, + }; + } + return { + ResultSet: { + ResultSetMetadata: { + ColumnInfo: [ + { Name: 'id', Type: 'bigint' }, + { Name: 'status', Type: 'string' }, + ], + }, + Rows: [ + // Header row (Athena always includes it on first page) + { Data: [{ VarCharValue: 'id' }, { VarCharValue: 'status' }] }, + // Data row + { Data: [{ VarCharValue: '1' }, { VarCharValue: 'paid' }] }, + ], + }, + NextToken: undefined, + }; + }), + }; + + const fakeGlueClient: KtxGlueClient = { + getDatabases: vi.fn(async () => ({ + DatabaseList: [{ Name: 'analytics' }], + NextToken: undefined, + })), + getTables: vi.fn(async () => ({ + TableList: [ + { + Name: 'orders', + TableType: 'EXTERNAL_TABLE', + Description: 'Orders table', + StorageDescriptor: { + Columns: [ + { Name: 'id', Type: 'bigint', Comment: 'Order id' }, + { Name: 'status', Type: 'string' }, + ], + }, + PartitionKeys: [{ Name: 'dt', Type: 'date', Comment: 'Partition date' }], + }, + ], + NextToken: undefined, + })), + }; + + return { + createAthenaClient: vi.fn(() => fakeAthenaClient), + createGlueClient: vi.fn(() => fakeGlueClient), + }; +} + +const connection = { + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://my-bucket/athena-results/', + workgroup: 'analytics', + catalog: 'AwsDataCatalog', + database: 'analytics', +} as const; + +describe('KtxAthenaScanConnector', () => { + it('identifies athena connection configs correctly', () => { + expect(isKtxAthenaConnectionConfig(connection)).toBe(true); + expect(isKtxAthenaConnectionConfig({ driver: 'bigquery' })).toBe(false); + expect(isKtxAthenaConnectionConfig(null)).toBe(false); + expect(isKtxAthenaConnectionConfig(undefined)).toBe(false); + }); + + it('resolves configuration and throws on missing required fields', () => { + expect(athenaConnectionConfigFromConfig({ connectionId: 'dw', connection })).toMatchObject({ + region: 'us-east-1', + s3StagingDir: 's3://my-bucket/athena-results/', + workgroup: 'analytics', + catalog: 'AwsDataCatalog', + database: 'analytics', + }); + + expect(() => + athenaConnectionConfigFromConfig({ connectionId: 'dw', connection: { driver: 'athena' } }), + ).toThrow('connections.dw.region'); + + expect(() => + athenaConnectionConfigFromConfig({ + connectionId: 'dw', + connection: { driver: 'athena', region: 'us-east-1' }, + }), + ).toThrow('connections.dw.s3_staging_dir'); + }); + + it('applies defaults for optional config fields', () => { + const resolved = athenaConnectionConfigFromConfig({ + connectionId: 'dw', + connection: { driver: 'athena', region: 'us-east-1', s3_staging_dir: 's3://bucket/' }, + }); + expect(resolved.workgroup).toBe('primary'); + expect(resolved.catalog).toBe('AwsDataCatalog'); + expect(resolved.database).toBeUndefined(); + }); + + it('introspects databases, tables, and columns from Glue', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'dw', driver: 'athena' }, + { runId: 'scan-1' }, + ); + + expect(snapshot).toMatchObject({ + connectionId: 'dw', + driver: 'athena', + extractedAt: '2026-06-21T10:00:00.000Z', + scope: { catalogs: ['AwsDataCatalog'], datasets: ['analytics'] }, + metadata: { + catalog: 'AwsDataCatalog', + databases: ['analytics'], + table_count: 1, + total_columns: 3, + }, + }); + + expect(snapshot.tables[0]).toMatchObject({ + catalog: 'AwsDataCatalog', + db: 'analytics', + name: 'orders', + kind: 'table', + comment: 'Orders table', + estimatedRows: null, + foreignKeys: [], + }); + + expect(snapshot.tables[0]?.columns).toEqual([ + { + name: 'id', + nativeType: 'bigint', + normalizedType: 'BIGINT', + dimensionType: 'number', + nullable: true, + primaryKey: false, + comment: 'Order id', + }, + { + name: 'status', + nativeType: 'string', + normalizedType: 'VARCHAR', + dimensionType: 'string', + nullable: true, + primaryKey: false, + comment: null, + }, + { + name: 'dt', + nativeType: 'date', + normalizedType: 'DATE', + dimensionType: 'time', + nullable: true, + primaryKey: false, + comment: 'Partition date', + }, + ]); + }); + + it('respects tableScope and excludes tables not in scope', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const scopedSnapshot = await connector.introspect( + { + connectionId: 'dw', + driver: 'athena', + tableScope: tableRefSet([{ catalog: 'AwsDataCatalog', db: 'analytics', name: 'nonexistent' }]), + }, + { runId: 'scan-1' }, + ); + expect(scopedSnapshot.tables).toHaveLength(0); + + const matchingSnapshot = await connector.introspect( + { + connectionId: 'dw', + driver: 'athena', + tableScope: tableRefSet([{ catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }]), + }, + { runId: 'scan-1' }, + ); + expect(matchingSnapshot.tables).toHaveLength(1); + expect(matchingSnapshot.tables[0]?.name).toBe('orders'); + }); + + it('limits introspection to the configured databases scope', async () => { + const requestedDatabases: string[] = []; + const getDatabases = vi.fn(async () => ({ + DatabaseList: [{ Name: 'analytics' }, { Name: 'raw' }, { Name: 'staging' }], + NextToken: undefined, + })); + const glueClient: KtxGlueClient = { + getDatabases, + getTables: vi.fn(async (input) => { + requestedDatabases.push(input.DatabaseName); + return { + TableList: [ + { + Name: `${input.DatabaseName}_orders`, + TableType: 'EXTERNAL_TABLE', + StorageDescriptor: { Columns: [{ Name: 'id', Type: 'bigint' }] }, + }, + ], + NextToken: undefined, + }; + }), + }; + const clientFactory: KtxAthenaClientFactory = { + createAthenaClient: vi.fn(() => fakeClientFactory().createAthenaClient('us-east-1')), + createGlueClient: vi.fn(() => glueClient), + }; + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection: { ...connection, databases: ['analytics', 'raw'] }, + clientFactory, + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect({ connectionId: 'dw', driver: 'athena' }, { runId: 'scan-1' }); + + // Scope is taken from config, so the account-wide database list is never enumerated. + expect(getDatabases).not.toHaveBeenCalled(); + expect(requestedDatabases).toEqual(['analytics', 'raw']); + expect(snapshot.scope).toMatchObject({ datasets: ['analytics', 'raw'] }); + expect(snapshot.tables.map((t) => t.db)).toEqual(['analytics', 'raw']); + }); + + it('resolves optional env-referenced config to defaults when the variable is unset', () => { + const resolved = athenaConnectionConfigFromConfig({ + connectionId: 'dw', + connection: { + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://bucket/', + workgroup: 'env:ATHENA_WORKGROUP_UNSET', + catalog: 'env:GLUE_CATALOG_UNSET', + }, + env: {}, + }); + expect(resolved.workgroup).toBe('primary'); + expect(resolved.catalog).toBe('AwsDataCatalog'); + }); + + it('samples a table via Athena query execution', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + const result = await connector.sampleTable( + { + connectionId: 'dw', + table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, + columns: ['id', 'status'], + limit: 10, + }, + { runId: 'scan-1' }, + ); + + expect(result).toMatchObject({ + headers: ['id', 'status'], + rows: [['1', 'paid']], + totalRows: 1, + }); + }); + + it('samples a column via Athena query execution', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + const result = await connector.sampleColumn( + { + connectionId: 'dw', + table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, + column: 'status', + limit: 10, + }, + { runId: 'scan-1' }, + ); + + expect(result).toMatchObject({ + values: ['paid'], + nullCount: null, + distinctCount: null, + }); + }); + + it('executes read-only SQL and rejects write statements', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect( + connector.executeReadOnly( + { connectionId: 'dw', sql: 'SELECT id, status FROM "analytics"."orders"', maxRows: 100 }, + { runId: 'scan-1' }, + ), + ).resolves.toMatchObject({ + headers: ['id', 'status'], + rows: [['1', 'paid']], + rowCount: 1, + }); + + await expect( + connector.executeReadOnly({ connectionId: 'dw', sql: 'DELETE FROM orders' }, { runId: 'scan-1' }), + ).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally'); + }); + + it('lists schemas (databases) from Glue', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect(connector.listSchemas()).resolves.toEqual(['analytics']); + }); + + it('lists tables from Glue', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect(connector.listTables(['analytics'])).resolves.toEqual([ + { + catalog: 'AwsDataCatalog', + schema: 'analytics', + name: 'orders', + kind: 'table', + }, + ]); + }); + + it('returns null for columnStats', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect( + connector.columnStats( + { connectionId: 'dw', table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, column: 'status' }, + { runId: 'scan-1' }, + ), + ).resolves.toBeNull(); + }); + + it('tests connection successfully', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect(connector.testConnection()).resolves.toMatchObject({ success: true }); + }); + + it('returns failure result when testConnection throws', async () => { + const factory = fakeClientFactory(); + const glueClient = factory.createGlueClient('us-east-1'); + vi.mocked(glueClient.getDatabases).mockRejectedValue(new Error('Access denied')); + const brokenFactory: KtxAthenaClientFactory = { + createAthenaClient: factory.createAthenaClient, + createGlueClient: vi.fn(() => glueClient), + }; + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: brokenFactory, + }); + + await expect(connector.testConnection()).resolves.toMatchObject({ + success: false, + error: 'Access denied', + }); + }); + + it('cleans up without throwing', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + await connector.listSchemas(); + await expect(connector.cleanup()).resolves.toBeUndefined(); + }); + + it('throws when query execution fails', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory({ queryState: 'FAILED', queryError: 'Syntax error in SQL' }), + }); + + await expect( + connector.executeReadOnly({ connectionId: 'dw', sql: 'SELECT 1' }, { runId: 'scan-1' }), + ).rejects.toThrow('Athena query FAILED: Syntax error in SQL'); + }); + + it('throws when query execution times out', async () => { + let callCount = 0; + // First now() call sets the deadline; second call simulates time past it. + const now = () => (++callCount === 1 ? new Date(0) : new Date(5 * 60 * 1000 + 1)); + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory({ queryState: 'RUNNING' }), + now, + }); + + await expect( + connector.executeReadOnly({ connectionId: 'dw', sql: 'SELECT 1' }, { runId: 'scan-1' }), + ).rejects.toThrow('timed out after 300s'); + }); + + it('passes the exact column list to Athena when sampling specific columns', async () => { + const factory = fakeClientFactory(); + const athenaClient = factory.createAthenaClient('us-east-1'); + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: { createAthenaClient: vi.fn(() => athenaClient), createGlueClient: factory.createGlueClient }, + }); + + await connector.sampleTable( + { + connectionId: 'dw', + table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, + columns: ['id', 'status'], + limit: 5, + }, + { runId: 'scan-1' }, + ); + + expect(vi.mocked(athenaClient.startQueryExecution).mock.calls[0]?.[0].QueryString).toBe( + 'SELECT "id", "status" FROM "AwsDataCatalog"."analytics"."orders" LIMIT 5', + ); + }); + + it('paginates Glue databases and tables across multiple pages', async () => { + const glueClient: KtxGlueClient = { + getDatabases: vi.fn() + .mockResolvedValueOnce({ DatabaseList: [{ Name: 'db1' }], NextToken: 'page2' }) + .mockResolvedValueOnce({ DatabaseList: [{ Name: 'db2' }], NextToken: undefined }), + getTables: vi.fn().mockImplementation(async ({ DatabaseName }: { DatabaseName: string }) => { + if (DatabaseName === 'db1') { + return { + TableList: [ + { + Name: 'table_a', + TableType: 'EXTERNAL_TABLE', + StorageDescriptor: { Columns: [{ Name: 'id', Type: 'bigint' }] }, + }, + ], + NextToken: undefined, + }; + } + return { + TableList: [ + { + Name: 'table_b', + TableType: 'EXTERNAL_TABLE', + StorageDescriptor: { Columns: [{ Name: 'id', Type: 'bigint' }] }, + }, + ], + NextToken: undefined, + }; + }), + }; + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: { + createAthenaClient: vi.fn(() => fakeClientFactory().createAthenaClient('us-east-1')), + createGlueClient: vi.fn(() => glueClient), + }, + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect({ connectionId: 'dw', driver: 'athena' }, { runId: 'scan-1' }); + + expect(vi.mocked(glueClient.getDatabases)).toHaveBeenCalledTimes(2); + expect(snapshot.metadata).toMatchObject({ databases: ['db1', 'db2'], table_count: 2 }); + expect(snapshot.tables.map((t) => t.name)).toEqual(['table_a', 'table_b']); + }); + + it('paginates Athena query results across multiple pages', async () => { + const factory = fakeClientFactory(); + const athenaClient = factory.createAthenaClient('us-east-1'); + vi.mocked(athenaClient.getQueryResults) + .mockResolvedValueOnce({ + ResultSet: { + ResultSetMetadata: { + ColumnInfo: [ + { Name: 'id', Type: 'bigint' }, + { Name: 'status', Type: 'string' }, + ], + }, + Rows: [ + // Header row — only present on the first page + { Data: [{ VarCharValue: 'id' }, { VarCharValue: 'status' }] }, + { Data: [{ VarCharValue: '1' }, { VarCharValue: 'paid' }] }, + { Data: [{ VarCharValue: '2' }, { VarCharValue: 'shipped' }] }, + ], + }, + NextToken: 'page-2', + }) + .mockResolvedValueOnce({ + ResultSet: { + ResultSetMetadata: { ColumnInfo: [] }, + // No header row on subsequent pages + Rows: [{ Data: [{ VarCharValue: '3' }, { VarCharValue: 'pending' }] }], + }, + NextToken: undefined, + }); + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: { createAthenaClient: vi.fn(() => athenaClient), createGlueClient: factory.createGlueClient }, + }); + + const result = await connector.executeReadOnly( + { connectionId: 'dw', sql: 'SELECT id, status FROM "analytics"."orders"', maxRows: 100 }, + { runId: 'scan-1' }, + ); + + expect(result.headers).toEqual(['id', 'status']); + expect(result.rows).toEqual([ + ['1', 'paid'], + ['2', 'shipped'], + ['3', 'pending'], + ]); + expect(result.rowCount).toBe(3); + expect(vi.mocked(athenaClient.getQueryResults)).toHaveBeenCalledTimes(2); + expect(vi.mocked(athenaClient.getQueryResults).mock.calls[1]?.[0].NextToken).toBe('page-2'); + }); + + it('adapts to the live-database introspection port via factory', async () => { + const introspection = createAthenaLiveDatabaseIntrospection({ + connections: { dw: connection }, + clientFactory: fakeClientFactory(), + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + await expect(introspection.extractSchema('dw')).resolves.toMatchObject({ + connectionId: 'dw', + driver: 'athena', + metadata: { catalog: 'AwsDataCatalog' }, + tables: expect.arrayContaining([ + expect.objectContaining({ + db: 'analytics', + name: 'orders', + columns: expect.arrayContaining([ + expect.objectContaining({ name: 'id', dimensionType: 'number' }), + ]), + }), + ]), + }); + }); +}); diff --git a/packages/cli/test/connectors/athena/dialect.test.ts b/packages/cli/test/connectors/athena/dialect.test.ts new file mode 100644 index 00000000..642f61c1 --- /dev/null +++ b/packages/cli/test/connectors/athena/dialect.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; +import { KtxAthenaDialect } from '../../../src/connectors/athena/dialect.js'; + +describe('KtxAthenaDialect', () => { + const dialect = new KtxAthenaDialect(); + + it('quotes identifiers and formats catalog.database.table names', () => { + expect(dialect.quoteIdentifier('my"col')).toBe('"my""col"'); + expect(dialect.formatTableName({ catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' })).toBe( + '"AwsDataCatalog"."analytics"."orders"', + ); + expect(dialect.formatTableName({ db: 'analytics', name: 'orders' })).toBe('"analytics"."orders"'); + expect(dialect.formatTableName({ name: 'orders' })).toBe('"orders"'); + }); + + it('maps native Athena/Glue types to normalized types and dimension types', () => { + expect(dialect.mapDataType('bigint')).toBe('BIGINT'); + expect(dialect.mapDataType('string')).toBe('VARCHAR'); + expect(dialect.mapDataType('array')).toBe('ARRAY'); + expect(dialect.mapDataType('map')).toBe('MAP'); + expect(dialect.mapDataType('struct')).toBe('STRUCT'); + expect(dialect.mapDataType('decimal(18,2)')).toBe('DECIMAL'); + expect(dialect.mapDataType('UNKNOWN_TYPE')).toBe('UNKNOWN_TYPE'); + + expect(dialect.mapToDimensionType('timestamp')).toBe('time'); + expect(dialect.mapToDimensionType('date')).toBe('time'); + expect(dialect.mapToDimensionType('bigint')).toBe('number'); + expect(dialect.mapToDimensionType('double')).toBe('number'); + expect(dialect.mapToDimensionType('decimal(10,2)')).toBe('number'); + expect(dialect.mapToDimensionType('boolean')).toBe('boolean'); + expect(dialect.mapToDimensionType('string')).toBe('string'); + expect(dialect.mapToDimensionType('varchar')).toBe('string'); + }); + + it('generates correct sample and column-sample SQL', () => { + expect(dialect.generateSampleQuery('"analytics"."orders"', 10, ['id', 'status'])).toBe( + 'SELECT "id", "status" FROM "analytics"."orders" LIMIT 10', + ); + expect(dialect.generateSampleQuery('"analytics"."orders"', 5)).toBe( + 'SELECT * FROM "analytics"."orders" LIMIT 5', + ); + expect(dialect.generateColumnSampleQuery('"analytics"."orders"', 'status', 20)).toBe( + 'SELECT "status" FROM "analytics"."orders" WHERE "status" IS NOT NULL LIMIT 20', + ); + }); + + it('generates Presto-style cardinality and distinct-values SQL', () => { + expect(dialect.generateCardinalitySampleQuery('"t"', '"col"', 1000)).toContain('approx_distinct'); + expect(dialect.generateRandomizedCardinalitySampleQuery('"t"', '"col"', 500)).toContain('rand()'); + expect(dialect.generateDistinctValuesQuery('"t"', '"col"', 50)).toContain( + 'SELECT DISTINCT CAST("col" AS VARCHAR) AS val', + ); + }); + + it('returns null for column statistics (unsupported)', () => { + expect(dialect.generateColumnStatisticsQuery('analytics', 'orders')).toBeNull(); + }); + + it('produces Trino-correct OFFSET-before-LIMIT ordering', () => { + expect(dialect.getLimitOffsetClause(10)).toBe('LIMIT 10'); + expect(dialect.getLimitOffsetClause(10, 0)).toBe('LIMIT 10'); + expect(dialect.getLimitOffsetClause(10, 20)).toBe('OFFSET 20 LIMIT 10'); + }); + + it('uses unit-separator (U+001F) as the array_join delimiter', () => { + const sql = dialect.getSampleValueAggregation('SELECT value FROM t'); + const separatorIndex = + sql.indexOf("array_join(array_agg(CAST(value AS VARCHAR)), '") + + "array_join(array_agg(CAST(value AS VARCHAR)), '".length; + expect(sql.charCodeAt(separatorIndex)).toBe(0x1f); + }); +}); diff --git a/packages/cli/test/context/connections/dialects.test.ts b/packages/cli/test/context/connections/dialects.test.ts index 74f95cc2..fad193ed 100644 --- a/packages/cli/test/context/connections/dialects.test.ts +++ b/packages/cli/test/context/connections/dialects.test.ts @@ -305,7 +305,7 @@ describe('getDialectForDriver', () => { it('throws with a supported-driver list for unknown drivers', () => { expect(() => getDialectForDriver('oracle')).toThrow( - 'Unsupported driver "oracle". Supported drivers: bigquery, clickhouse, duckdb, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', + 'Unsupported driver "oracle". Supported drivers: athena, bigquery, clickhouse, duckdb, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', ); }); diff --git a/packages/cli/test/context/connections/drivers.test.ts b/packages/cli/test/context/connections/drivers.test.ts index 38a0f48f..69ac8464 100644 --- a/packages/cli/test/context/connections/drivers.test.ts +++ b/packages/cli/test/context/connections/drivers.test.ts @@ -70,6 +70,11 @@ const connectionFixtures: Record = { database: 'ANALYTICS', schema: 'PUBLIC', }), + athena: () => ({ + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://my-bucket/athena-results/', + }), }; const allowedScopeKeys = new Set(['dataset_ids', 'databases', 'schemas', 'schema_names']); @@ -100,6 +105,7 @@ describe('driverRegistrations', () => { const registryDrivers = Object.keys(driverRegistrations).sort(); expect(listSupportedDrivers()).toEqual(registryDrivers); expect(listSupportedDrivers()).toEqual([ + 'athena', 'bigquery', 'clickhouse', 'duckdb', diff --git a/packages/cli/test/context/scan/local-scan.test.ts b/packages/cli/test/context/scan/local-scan.test.ts index 5bc60922..75f929d7 100644 --- a/packages/cli/test/context/scan/local-scan.test.ts +++ b/packages/cli/test/context/scan/local-scan.test.ts @@ -2175,6 +2175,40 @@ describe('local scan', () => { }; expect(manifest.tables.orders?.joins?.some((join) => join.to === 'accounts')).toBe(true); }); + + it('accepts athena as a native standalone scan driver when the host supplies a live-database adapter', async () => { + await writeFile( + join(project.projectDir, 'ktx.yaml'), + [ + 'connections:', + ' warehouse:', + ' driver: athena', + ' region: us-east-1', + ' s3_staging_dir: s3://my-bucket/athena-results/', + ' databases:', + ' - analytics', + 'ingest:', + ' adapters:', + ' - live-database', + '', + ].join('\n'), + 'utf-8', + ); + project = await loadKtxProject({ projectDir: project.projectDir }); + + const result = await runLocalScan({ + project, + adapters: [fetchOnlyAdapter()], + connectionId: 'warehouse', + jobId: 'scan-run-athena', + now: () => new Date('2026-04-29T17:00:00.000Z'), + }); + + expect(result.report.driver).toBe('athena'); + expect(result.report.artifactPaths.reportPath).toBe( + 'raw-sources/warehouse/live-database/2026-04-29-170000-scan-run-athena/scan-report.json', + ); + }); }); describe('resolveEnabledTables', () => { diff --git a/packages/cli/test/context/sql-analysis/dialect.test.ts b/packages/cli/test/context/sql-analysis/dialect.test.ts index 68a92af5..0166a133 100644 --- a/packages/cli/test/context/sql-analysis/dialect.test.ts +++ b/packages/cli/test/context/sql-analysis/dialect.test.ts @@ -12,6 +12,7 @@ describe('sqlAnalysisDialectForDriver', () => { expect(sqlAnalysisDialectForDriver('duckdb')).toBe('duckdb'); expect(sqlAnalysisDialectForDriver('clickhouse')).toBe('clickhouse'); expect(sqlAnalysisDialectForDriver('databricks')).toBe('databricks'); + expect(sqlAnalysisDialectForDriver('athena')).toBe('athena'); }); it('maps local connection-type spellings to sqlglot dialects', () => { diff --git a/packages/cli/test/setup-databases.test.ts b/packages/cli/test/setup-databases.test.ts index bcf1db79..feff6613 100644 --- a/packages/cli/test/setup-databases.test.ts +++ b/packages/cli/test/setup-databases.test.ts @@ -243,6 +243,7 @@ describe('setup databases step', () => { { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, + { value: 'athena', label: 'Amazon Athena' }, { value: 'mongodb', label: 'MongoDB' }, { value: 'sqlite', label: 'SQLite' }, { value: 'duckdb', label: 'DuckDB' }, @@ -618,6 +619,29 @@ describe('setup databases step', () => { }, ], }, + { + driver: 'athena', + textValues: ['', 'us-east-1', 's3://my-bucket/athena-results/', '', ''], + expectedTextPrompts: [ + { + message: connectionNamePrompt('Amazon Athena'), + placeholder: 'athena-warehouse', + initialValue: 'athena-warehouse', + }, + { + message: 'AWS region\nFor example us-east-1.', + }, + { + message: 'S3 staging directory\nAthena writes query results here. For example s3://my-bucket/athena-results/.', + }, + { + message: 'Athena workgroup (optional)\nPress Enter to use the default workgroup "primary".', + }, + { + message: 'Glue Data Catalog name (optional)\nPress Enter to use the default "AwsDataCatalog".', + }, + ], + }, ]; for (const testCase of cases) { @@ -1967,6 +1991,40 @@ describe('setup databases step', () => { expect(project.config.connections['clickhouse-warehouse']).not.toHaveProperty('schemas'); }); + it('maps Athena scripted database schema input to databases field', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' athena-warehouse:', + ' driver: athena', + ' region: us-east-1', + ' s3_staging_dir: s3://my-bucket/athena-results/', + '', + ].join('\n'), + 'utf-8', + ); + + await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + skipDatabases: false, + databaseConnectionIds: ['athena-warehouse'], + databaseSchemas: ['analytics', 'raw'], + }, + makeIo().io, + { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0) }, + ); + + const project = await loadKtxProject({ projectDir: tempDir }); + expect(project.config.connections['athena-warehouse']).toMatchObject({ + driver: 'athena', + databases: ['analytics', 'raw'], + }); + expect(project.config.connections['athena-warehouse']).not.toHaveProperty('schemas'); + }); + it('does not prompt for a bootstrap BigQuery dataset before scope discovery', async () => { const prompts = makePromptAdapter({ multiselectValues: [['bigquery']], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53a510d4..6e5a1710 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -137,6 +137,12 @@ importers: '@anthropic-ai/claude-agent-sdk': specifier: 0.3.146 version: 0.3.146(@anthropic-ai/sdk@0.97.1(zod@4.4.3))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(zod@4.4.3) + '@aws-sdk/client-athena': + specifier: ^3.1068.0 + version: 3.1068.0 + '@aws-sdk/client-glue': + specifier: ^3.1068.0 + version: 3.1068.0 '@clack/core': specifier: 1.3.1 version: 1.3.1 @@ -437,6 +443,14 @@ packages: '@aws-crypto/util@5.2.0': resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + '@aws-sdk/client-athena@3.1068.0': + resolution: {integrity: sha512-WvoGXgJ5luTY7wRGqRYQFJ1heRzKiLOiuo7gkQJ9tRRau10XZIyJ0c09+wOvcRXG8CYbUdWpdGkcwswDBE2eSQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-glue@3.1068.0': + resolution: {integrity: sha512-T/2aZGVaDDuSSQ3OWhWowBZ3P2hQGIV+16idxiA0Z8WevLWgbzztSGScDyYa7ohe/J/BY0XiYsBavLYtv+yGlg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-s3@3.1045.0': resolution: {integrity: sha512-fsuO3Y6t+3Ro9Bsg41DKj4Sfy53CGSrhnMldNplWmG8Tx0UbYk+YDa4RD1hVlJpERw4JBmPkl0+J9qlxMh1pcA==} engines: {node: '>=20.0.0'} @@ -449,6 +463,10 @@ packages: resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==} engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.974.20': + resolution: {integrity: sha512-7sDi2B2N3mc3nf1nz6FyEx/FCrJ1N1QnBmraHHQNabFaeAh2IaOOLml48/rHOD1bICHgTRkbBgNTvUzEr5Z35g==} + engines: {node: '>=20.0.0'} + '@aws-sdk/crc64-nvme@3.972.8': resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==} engines: {node: '>=20.0.0'} @@ -457,34 +475,66 @@ packages: resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.46': + resolution: {integrity: sha512-+GPXVS2srMOlH74S+SmC1gVuP2TvUZ0siuC0onKO93q+udP+M72dmY8wJfVQ5CX9z/9X5A1HHwz5yRIGBtskvQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.40': resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.48': + resolution: {integrity: sha512-fA5loSdlocacRxyUXtpoHSMuk5rsIKRDzQYVMnMxjcmFeZshaJlJ8lymy/hYKji6sne/UmNGj5pxuEs6kq/Qcg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.42': resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.53': + resolution: {integrity: sha512-ZfdhIOR41q8TcWEnUac+gCOb+O2LBWdHLmjedXpXz4IEFW2ppNuFcm6p0sMTavpM+zD5TYfpH5Gp7guRyqSgsQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.42': resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.52': + resolution: {integrity: sha512-9hu2oR0qH7Fst5Tzdx+UWxm+w5zCXtErTLtOOW5hwwQc170CLwOeniRxyFY6s9mHfGEfC5zFukNBdKBwJR8mhQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.43': resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.55': + resolution: {integrity: sha512-zMGLa/dhESVqmCD7mmIFFKSwSFrJGScvCXcjvBZEVOOMauFS5JRQvLTMukFpMEFWiV6dTAlsen2ATDBulLPtbg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.38': resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.46': + resolution: {integrity: sha512-VUoNFBIjWrUN8NbFiQiuxQEgFjvziAlBRPK+ddh27aj65gk0BYu6bLZnrdrNZwpW6vAihtSUtEMQ1PUJ32QRPA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.42': resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.52': + resolution: {integrity: sha512-nb2/n4o/HQf+FVpVbZe9vCTFngmuDoIsltMgLAtjixaKzvzhB4J8WSDFyWgnErgLHk55ctWH+I4PU+LIHhyffg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.42': resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.52': + resolution: {integrity: sha512-lKj6aRSGbqLmpYmM24bY7a1Xmfcq2vkE3hv8CSPYfc1yCu0BPu/XEJ1L4Fm61MsU6ULLNSG8UGsffNoFUBjESA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/ec2-metadata-service@3.1045.0': resolution: {integrity: sha512-cYjEbjbGScw9l8TmI9AFYde1hIu5c9Wt0Qp7/cbWBHBiOzMfLwmjGhd5+4AUm1RsnmC5HZ/WOA9iGJHfHL4cuA==} engines: {node: '>=20.0.0'} @@ -533,6 +583,10 @@ packages: resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==} engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.20': + resolution: {integrity: sha512-IYJuLpXp2DEILVQpQOy0PMpkftv0AHEOCn52o0atyOaumA0CdWQ3klPyXdViGYLbNpESsVFMVybvHUeZAuiGxA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/region-config-resolver@3.972.16': resolution: {integrity: sha512-/YaivCvKUkEeMN9VTKBSvBn5w/4osAM1YboM58DKaLF/vqFGf/FdJCLmppqiPPJWZaXcASqByVjc3evE7KHKdA==} engines: {node: '>=20.0.0'} @@ -541,10 +595,22 @@ packages: resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==} engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.34': + resolution: {integrity: sha512-mx1L5qlumSOt/nKM3BFaHE2HVkWwz0i4Bw0pyYO42FfX/FeLlo8YI6csC0gSPprEk6fTIqI+CZN9RwUwKd5krQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1049.0': resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==} engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1066.0': + resolution: {integrity: sha512-UqEUJq7dqa44hneLDUcX7UJy95cg8YqEWyakRpvIPnrNS3Mq+UlQHgCDGu5pvwAPtlIW4qcYbvW6reG6++FyvA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.12': + resolution: {integrity: sha512-43ajd1NF0RMgX5k0hxCNUyEdrtFUsb2aHT2QvpktSC/2Eyb2Jr/JPVqdp0XIoaHWikZJq5tNWSLO6kB5q2eMCA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.973.8': resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} engines: {node: '>=20.0.0'} @@ -568,6 +634,10 @@ packages: resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==} engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.29': + resolution: {integrity: sha512-fk0niuGFxfi8yIJuMVM4mhwObkiQSuwZFj3tAPrLVx64Pk3BkrEIpqjzHKY4hKoEBUD6Jg/S74Zj9jy+5F3DnQ==} + engines: {node: '>=20.0.0'} + '@aws/lambda-invoke-store@0.2.4': resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} @@ -2257,10 +2327,18 @@ packages: resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==} engines: {node: '>=18.0.0'} + '@smithy/core@3.24.7': + resolution: {integrity: sha512-KoUi4M1f3BG6kzN1FnCwL7oyFptTbyBJKjR6yhSib+JHRdUmM1o+VwsFtJ66NZCkCzVfJMWRHJNo0R0jznp0Pg==} + engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.3.3': resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==} engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.3.9': + resolution: {integrity: sha512-ZlfJ/4Fa3jYb+3eaohPfG9utX9HmdhFNcFtpoGAhUhdynAOmGXtmigbi7eEiONKM+ykHw8RwKuDEb85Lx7t7fA==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-serde-browser@4.3.3': resolution: {integrity: sha512-LXg5yYJPYnVSrpa6LOZ+/wqpI2OlIccy7j5F16EFNYDbXWmnhry/PFRRPyM30H+hJeqfVgckFuvNGnAGCt56cA==} engines: {node: '>=18.0.0'} @@ -2277,6 +2355,10 @@ packages: resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==} engines: {node: '>=18.0.0'} + '@smithy/fetch-http-handler@5.4.7': + resolution: {integrity: sha512-NslaM2ir0N2hisDmzXLstPaVINZheh8SokyOC++kzFPloZucL2R7Y7bS57mSzx/1Fc/fqmn7twjkeezTTrV0EA==} + engines: {node: '>=18.0.0'} + '@smithy/hash-blob-browser@4.3.3': resolution: {integrity: sha512-TkGfDlYeWOGwYvAunHHHmKgvFtD7DFAl6gWxATI4pv4B6w0Wnx6RK5zCMoXTTqMVd+zPcWm7w8RPTgHytoCDJA==} engines: {node: '>=18.0.0'} @@ -2329,6 +2411,10 @@ packages: resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==} engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.7.8': + resolution: {integrity: sha512-f+DbsWUwSbtMu1a/j8Y93KiU1SRg9nyzfjereqn1BJ33QOTUXxdlYvVXMhAYl1vuR1Kmna5aIJe09KSIfyFNYw==} + engines: {node: '>=18.0.0'} + '@smithy/protocol-http@5.4.3': resolution: {integrity: sha512-P16TBD/d8ZcD9MHQ0ubQ9BbOYSd5HZKbHOLsyFWxKk2oBEoghbRFPfGOoqToZX1yrfLITXRylL16EyPP4IzLPg==} engines: {node: '>=18.0.0'} @@ -2337,6 +2423,10 @@ packages: resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==} engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.4.7': + resolution: {integrity: sha512-LwQZazFayImv+IOm0S0enoLeUJwmAlhGC5O6YCcLWezyu08dF46GOxPOq35OpBIHkgd7OvNvBStIFwVNyrvoBw==} + engines: {node: '>=18.0.0'} + '@smithy/smithy-client@4.13.3': resolution: {integrity: sha512-Z8mQ+YryjP5krDadV6unnp5035L4S1brafXpTiRmjPweKSaQ6X9CYDYWvmEggXjDIa1oufX/2a/bdwu8EIz/lw==} engines: {node: '>=18.0.0'} @@ -2345,6 +2435,10 @@ packages: resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==} engines: {node: '>=18.0.0'} + '@smithy/types@4.14.4': + resolution: {integrity: sha512-B2S9+UGm1+/pHkcx3ZoLVX1a+pmSk8rqxRR+ZsNqZaJ5q9FWX9AFGQVM4qG5+OBeQUZVy99HY8HqW8gK/wgXzQ==} + engines: {node: '>=18.0.0'} + '@smithy/url-parser@4.3.3': resolution: {integrity: sha512-TsMTAOnjuMOv1zJBw8cfYGWhopyc3og8tZX/KuyCPjg7V3ji3f4YjFOVu843UjBmrfS/+X6kwFv5ZKg7sSm1bQ==} engines: {node: '>=18.0.0'} @@ -6361,6 +6455,32 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 + '@aws-sdk/client-athena@3.1068.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.20 + '@aws-sdk/credential-provider-node': 3.972.55 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + + '@aws-sdk/client-glue@3.1068.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.20 + '@aws-sdk/credential-provider-node': 3.972.55 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/client-s3@3.1045.0': dependencies: '@aws-crypto/sha1-browser': 5.2.0 @@ -6473,6 +6593,17 @@ snapshots: bowser: 2.14.1 tslib: 2.8.1 + '@aws-sdk/core@3.974.20': + dependencies: + '@aws-sdk/types': 3.973.12 + '@aws-sdk/xml-builder': 3.972.29 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.24.7 + '@smithy/signature-v4': 5.4.7 + '@smithy/types': 4.14.4 + bowser: 2.14.1 + tslib: 2.8.1 + '@aws-sdk/crc64-nvme@3.972.8': dependencies: '@smithy/types': 4.14.2 @@ -6486,6 +6617,14 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.46': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.40': dependencies: '@aws-sdk/core': 3.974.12 @@ -6496,6 +6635,16 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.48': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6512,6 +6661,22 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.972.53': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/credential-provider-env': 3.972.46 + '@aws-sdk/credential-provider-http': 3.972.48 + '@aws-sdk/credential-provider-login': 3.972.52 + '@aws-sdk/credential-provider-process': 3.972.46 + '@aws-sdk/credential-provider-sso': 3.972.52 + '@aws-sdk/credential-provider-web-identity': 3.972.52 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/credential-provider-imds': 4.3.9 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6521,6 +6686,15 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.52': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.43': dependencies: '@aws-sdk/credential-provider-env': 3.972.38 @@ -6535,6 +6709,20 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.55': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.46 + '@aws-sdk/credential-provider-http': 3.972.48 + '@aws-sdk/credential-provider-ini': 3.972.53 + '@aws-sdk/credential-provider-process': 3.972.46 + '@aws-sdk/credential-provider-sso': 3.972.52 + '@aws-sdk/credential-provider-web-identity': 3.972.52 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/credential-provider-imds': 4.3.9 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.38': dependencies: '@aws-sdk/core': 3.974.12 @@ -6543,6 +6731,14 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.46': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6553,6 +6749,16 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.972.52': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/token-providers': 3.1066.0 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6562,6 +6768,15 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.52': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/ec2-metadata-service@3.1045.0': dependencies: '@aws-sdk/types': 3.973.8 @@ -6654,6 +6869,19 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/nested-clients@3.997.20': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.20 + '@aws-sdk/signature-v4-multi-region': 3.996.34 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/region-config-resolver@3.972.16': dependencies: '@aws-sdk/core': 3.974.12 @@ -6667,6 +6895,13 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.34': + dependencies: + '@aws-sdk/types': 3.973.12 + '@smithy/signature-v4': 5.4.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/token-providers@3.1049.0': dependencies: '@aws-sdk/core': 3.974.12 @@ -6676,6 +6911,20 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/token-providers@3.1066.0': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + + '@aws-sdk/types@3.973.12': + dependencies: + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/types@3.973.8': dependencies: '@smithy/types': 4.14.2 @@ -6708,6 +6957,12 @@ snapshots: fast-xml-parser: 5.7.3 tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.29': + dependencies: + '@smithy/types': 4.14.4 + fast-xml-parser: 5.7.3 + tslib: 2.8.1 + '@aws/lambda-invoke-store@0.2.4': {} '@azure-rest/core-client@2.6.0': @@ -8227,12 +8482,24 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/core@3.24.7': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/credential-provider-imds@4.3.3': dependencies: '@smithy/core': 3.24.3 '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/credential-provider-imds@4.3.9': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/eventstream-serde-browser@4.3.3': dependencies: '@smithy/core': 3.24.3 @@ -8254,6 +8521,12 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/fetch-http-handler@5.4.7': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/hash-blob-browser@4.3.3': dependencies: '@smithy/core': 3.24.3 @@ -8319,6 +8592,12 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/node-http-handler@4.7.8': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/protocol-http@5.4.3': dependencies: '@smithy/core': 3.24.3 @@ -8330,6 +8609,12 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/signature-v4@5.4.7': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/smithy-client@4.13.3': dependencies: '@smithy/core': 3.24.3 @@ -8340,6 +8625,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/types@4.14.4': + dependencies: + tslib: 2.8.1 + '@smithy/url-parser@4.3.3': dependencies: '@smithy/core': 3.24.3 From f310391da53642a98defe19cef9584389ce03ff7 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Thu, 2 Jul 2026 15:10:29 +0200 Subject: [PATCH 04/11] feat(athena): first-class AWS Athena warehouse identity (SL + BI mapping) (#332) * feat: support Athena warehouse identity * fix: support Athena and DuckDB BI table parsing --- .../connections/connection-type-dialect.ts | 24 ++++++++ .../context/connections/connection-type.ts | 10 ---- .../connections/local-warehouse-descriptor.ts | 1 + .../context/ingest/adapters/looker/mapping.ts | 14 +---- .../ingest/adapters/metabase/mapping.ts | 2 +- .../src/context/sl/semantic-layer.service.ts | 24 +------- .../sl/tools/sl-warehouse-validation.ts | 5 +- .../connection-type-dialect.test.ts | 59 ++++++++++++++++++ .../local-warehouse-descriptor.test.ts | 16 +++++ .../ingest/adapters/looker/mapping.test.ts | 41 +++++++++++++ .../ingest/adapters/metabase/mapping.test.ts | 21 +++++++ .../context/sl/semantic-layer.service.test.ts | 60 +++++++++++++++++++ .../semantic_layer/table_identifier_parser.py | 2 + .../tests/test_table_identifier_parser.py | 18 ++++++ 14 files changed, 252 insertions(+), 45 deletions(-) create mode 100644 packages/cli/src/context/connections/connection-type-dialect.ts create mode 100644 packages/cli/test/context/connections/connection-type-dialect.test.ts diff --git a/packages/cli/src/context/connections/connection-type-dialect.ts b/packages/cli/src/context/connections/connection-type-dialect.ts new file mode 100644 index 00000000..f20231de --- /dev/null +++ b/packages/cli/src/context/connections/connection-type-dialect.ts @@ -0,0 +1,24 @@ +import type { ConnectionType } from './connection-type.js'; + +const CONNECTION_TYPE_TO_SQLGLOT = { + POSTGRESQL: 'postgres', + SQLITE: 'sqlite', + DUCKDB: 'duckdb', + SQLSERVER: 'tsql', + BIGQUERY: 'bigquery', + SNOWFLAKE: 'snowflake', + MYSQL: 'mysql', + CLICKHOUSE: 'clickhouse', + ATHENA: 'athena', + METABASE: null, + LOOKER: null, + NOTION: null, +} satisfies Record; + +export function dialectForConnectionType(connectionType: string): string { + return CONNECTION_TYPE_TO_SQLGLOT[connectionType.toUpperCase() as ConnectionType] ?? 'postgres'; +} + +export function warehouseTargetDialect(connectionType: string): string | null { + return CONNECTION_TYPE_TO_SQLGLOT[connectionType.toUpperCase() as ConnectionType] ?? null; +} diff --git a/packages/cli/src/context/connections/connection-type.ts b/packages/cli/src/context/connections/connection-type.ts index d122e950..8b12839c 100644 --- a/packages/cli/src/context/connections/connection-type.ts +++ b/packages/cli/src/context/connections/connection-type.ts @@ -7,22 +7,12 @@ export const connectionTypeSchema = z.enum([ 'SQLSERVER', 'BIGQUERY', 'SNOWFLAKE', - 'CENTRALREACH', - 'EPIC', - 'CERNER', 'ATHENA', - 'QUICKBOOKS', - 'WORKDAY', - 'REST', - 'S3', - 'SLACK', 'METABASE', 'LOOKER', 'NOTION', 'MYSQL', 'CLICKHOUSE', - 'PLAIN', - 'BETTERSTACK', ]); export type ConnectionType = z.infer; diff --git a/packages/cli/src/context/connections/local-warehouse-descriptor.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.ts index b9674e09..bb4235c2 100644 --- a/packages/cli/src/context/connections/local-warehouse-descriptor.ts +++ b/packages/cli/src/context/connections/local-warehouse-descriptor.ts @@ -29,6 +29,7 @@ const DRIVER_TO_CONNECTION_TYPE: Record = { clickhouse: 'CLICKHOUSE', snowflake: 'SNOWFLAKE', bigquery: 'BIGQUERY', + athena: 'ATHENA', }; export function localConnectionToWarehouseDescriptor( diff --git a/packages/cli/src/context/ingest/adapters/looker/mapping.ts b/packages/cli/src/context/ingest/adapters/looker/mapping.ts index 3e451aca..29214a36 100644 --- a/packages/cli/src/context/ingest/adapters/looker/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/looker/mapping.ts @@ -1,4 +1,5 @@ import type { ParsedTargetTable } from '../../parsed-target-table.js'; +import { warehouseTargetDialect } from '../../../connections/connection-type-dialect.js'; import type { LookerWarehouseConnectionInfo } from './client.js'; import type { LookerPullConfig, LookerRuntimeCursors, StagedExploreFile, StagedLookmlModelsFile } from './types.js'; @@ -11,6 +12,7 @@ const LOOKER_DIALECT_TO_CONNECTION_TYPE = { sqlite: 'SQLITE', sqlserver: 'SQLSERVER', clickhouse: 'CLICKHOUSE', + awsathena: 'ATHENA', } as const; /** @internal */ @@ -72,16 +74,6 @@ export interface LookerMappingClient { getExplore(modelName: string, exploreName: string): Promise; } -const SQLGLOT_DIALECT_BY_CONNECTION_TYPE: Partial> = { - BIGQUERY: 'bigquery', - SNOWFLAKE: 'snowflake', - POSTGRESQL: 'postgres', - MYSQL: 'mysql', - SQLITE: 'sqlite', - SQLSERVER: 'tsql', - CLICKHOUSE: 'clickhouse', -}; - export async function discoverLookerConnections( client: Pick, ): Promise { @@ -100,7 +92,7 @@ export function lookerDialectToConnectionType(dialect: string | null): LookerWar /** @internal */ export function sqlglotDialectForConnectionType(connectionType: string): string | null { - return SQLGLOT_DIALECT_BY_CONNECTION_TYPE[connectionType as LookerWarehouseTargetConnectionType] ?? null; + return warehouseTargetDialect(connectionType); } /** @internal */ diff --git a/packages/cli/src/context/ingest/adapters/metabase/mapping.ts b/packages/cli/src/context/ingest/adapters/metabase/mapping.ts index b5ba67a7..3123f0e2 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/mapping.ts @@ -8,9 +8,9 @@ export const METABASE_ENGINE_TO_CONNECTION_TYPE = { snowflake: 'SNOWFLAKE', sqlserver: 'SQLSERVER', mysql: 'MYSQL', + athena: 'ATHENA', } as const; - export interface DiscoveredMetabaseDatabase { id: number; name: string; diff --git a/packages/cli/src/context/sl/semantic-layer.service.ts b/packages/cli/src/context/sl/semantic-layer.service.ts index 99837247..913f62d1 100644 --- a/packages/cli/src/context/sl/semantic-layer.service.ts +++ b/packages/cli/src/context/sl/semantic-layer.service.ts @@ -2,6 +2,7 @@ import YAML from 'yaml'; import type { KtxFileStorePort } from '../../context/core/file-store.js'; import type { KtxLogger } from '../../context/core/config.js'; import { noopLogger } from '../../context/core/config.js'; +import { dialectForConnectionType } from '../connections/connection-type-dialect.js'; import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js'; import type { SlConnectionCatalogPort, SlPythonPort } from './ports.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; @@ -674,7 +675,7 @@ export class SemanticLayerService { if (!connection) { throw new Error(`Data source not found: ${connectionId}`); } - return SemanticLayerService.mapDialect(connection.connectionType); + return dialectForConnectionType(connection.connectionType); } async listFilesForConnection(connectionId: string): Promise { @@ -1107,25 +1108,6 @@ export class SemanticLayerService { return { columns, tables }; } - /** - * All callers should use this instead of maintaining their own dialect maps. - */ - static mapDialect(connectionType: string): string { - const normalized = connectionType.toUpperCase(); - const map: Record = { - POSTGRES: 'postgres', - BIGQUERY: 'bigquery', - SNOWFLAKE: 'snowflake', - MYSQL: 'mysql', - SQLSERVER: 'tsql', - SQLITE: 'sqlite', - DUCKDB: 'duckdb', - CLICKHOUSE: 'clickhouse', - DATABRICKS: 'databricks', - }; - return map[normalized] ?? 'postgres'; - } - /** * Execute a semantic layer query: load composed sources, generate SQL via * the python SL engine, and execute the generated SQL against the data source. @@ -1153,7 +1135,7 @@ export class SemanticLayerService { if (!connection) { throw new Error(`Data source not found: ${connectionId}`); } - const dialect = SemanticLayerService.mapDialect(connection.connectionType); + const dialect = dialectForConnectionType(connection.connectionType); // 3. Generate SQL via python SL engine const { data: slResult, error: slError } = await this.python.query({ diff --git a/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts b/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts index 56f80833..69acc783 100644 --- a/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts +++ b/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts @@ -2,9 +2,10 @@ import YAML from 'yaml'; import type { GitService } from '../../../context/core/git.service.js'; import type { KtxFileListResult, KtxFileReadResult, KtxFileStorePort } from '../../../context/core/file-store.js'; import { SYSTEM_GIT_AUTHOR } from '../../../context/tools/authors.js'; +import { dialectForConnectionType } from '../../connections/connection-type-dialect.js'; import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../ports.js'; import { sourceOverlaySchema } from '../schemas.js'; -import { SemanticLayerService } from '../semantic-layer.service.js'; +import type { SemanticLayerService } from '../semantic-layer.service.js'; import { resolveSlSourceFile, slSourceFilePath } from '../source-files.js'; import type { SemanticLayerSource } from '../types.js'; import { sourceDefinitionSchema } from './base-semantic-layer.tool.js'; @@ -28,7 +29,7 @@ function resolveDialect(warehouse: string | null): string | null { if (!warehouse) { return null; } - return SemanticLayerService.mapDialect(warehouse); + return dialectForConnectionType(warehouse); } function wrapWithZeroRowQuery(sql: string, dialect: string): string { diff --git a/packages/cli/test/context/connections/connection-type-dialect.test.ts b/packages/cli/test/context/connections/connection-type-dialect.test.ts new file mode 100644 index 00000000..93bf57fa --- /dev/null +++ b/packages/cli/test/context/connections/connection-type-dialect.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest'; +import { connectionTypeSchema } from '../../../src/context/connections/connection-type.js'; +import { + dialectForConnectionType, + warehouseTargetDialect, +} from '../../../src/context/connections/connection-type-dialect.js'; + +describe('connection type dialect resolution', () => { + it('maps warehouse connection types to sqlglot dialects', () => { + const cases: Array<[string, string]> = [ + ['POSTGRESQL', 'postgres'], + ['SQLITE', 'sqlite'], + ['DUCKDB', 'duckdb'], + ['SQLSERVER', 'tsql'], + ['BIGQUERY', 'bigquery'], + ['SNOWFLAKE', 'snowflake'], + ['MYSQL', 'mysql'], + ['CLICKHOUSE', 'clickhouse'], + ['ATHENA', 'athena'], + ]; + + for (const [connectionType, dialect] of cases) { + expect(dialectForConnectionType(connectionType)).toBe(dialect); + expect(warehouseTargetDialect(connectionType)).toBe(dialect); + } + }); + + it('normalizes case and preserves the semantic-layer postgres fallback for unknown inputs', () => { + expect(dialectForConnectionType('athena')).toBe('athena'); + expect(dialectForConnectionType('postgresql')).toBe('postgres'); + expect(dialectForConnectionType('not-a-real-connection-type')).toBe('postgres'); + }); + + it('rejects non-SQL targets for BI table parsing', () => { + expect(warehouseTargetDialect('METABASE')).toBeNull(); + expect(warehouseTargetDialect('LOOKER')).toBeNull(); + expect(warehouseTargetDialect('NOTION')).toBeNull(); + expect(warehouseTargetDialect('not-a-real-connection-type')).toBeNull(); + }); + + it('removes inherited non-ktx connection type values while keeping AWS Athena', () => { + expect(connectionTypeSchema.safeParse('ATHENA').success).toBe(true); + + for (const removed of [ + 'CENTRALREACH', + 'EPIC', + 'CERNER', + 'QUICKBOOKS', + 'WORKDAY', + 'REST', + 'S3', + 'SLACK', + 'PLAIN', + 'BETTERSTACK', + ]) { + expect(connectionTypeSchema.safeParse(removed).success).toBe(false); + } + }); +}); diff --git a/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts b/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts index e0a285a9..a368ae5d 100644 --- a/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts +++ b/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts @@ -35,6 +35,21 @@ describe('localConnectionToWarehouseDescriptor', () => { }); }); + it('maps Athena connections to canonical warehouse descriptors', () => { + expect( + localConnectionToWarehouseDescriptor('athena-warehouse', { + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://my-bucket/athena-results/', + database: 'analytics', + }), + ).toMatchObject({ + id: 'athena-warehouse', + connection_type: 'ATHENA', + database: 'analytics', + }); + }); + it('returns null for non-warehouse adapters', () => { expect( localConnectionToWarehouseDescriptor('looker', { @@ -51,6 +66,7 @@ describe('local connection info helpers', () => { expect(localConnectionTypeForConfig('warehouse', { driver: 'postgres' })).toBe('POSTGRESQL'); expect(localConnectionTypeForConfig('bq', { driver: 'bigquery', project_id: 'acme' })).toBe('BIGQUERY'); expect(localConnectionTypeForConfig('snowflake', { driver: 'snowflake' })).toBe('SNOWFLAKE'); + expect(localConnectionTypeForConfig('athena-warehouse', { driver: 'athena' })).toBe('ATHENA'); }); it('keeps removed driver aliases as display-only labels', () => { diff --git a/packages/cli/test/context/ingest/adapters/looker/mapping.test.ts b/packages/cli/test/context/ingest/adapters/looker/mapping.test.ts index a5186a75..dbec700f 100644 --- a/packages/cli/test/context/ingest/adapters/looker/mapping.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/mapping.test.ts @@ -72,6 +72,8 @@ describe('looker dialect and target validation helpers', () => { it('maps Looker dialect names to ktx connection types', () => { expect(lookerDialectToConnectionType('bigquery_standard_sql')).toBe('BIGQUERY'); expect(lookerDialectToConnectionType('postgres')).toBe('POSTGRESQL'); + expect(lookerDialectToConnectionType('awsathena')).toBe('ATHENA'); + expect(lookerDialectToConnectionType('AWSATHENA')).toBe('ATHENA'); expect(lookerDialectToConnectionType('mssql')).toBeNull(); expect(lookerDialectToConnectionType('tsql')).toBeNull(); expect(lookerDialectToConnectionType('unknown')).toBeNull(); @@ -80,6 +82,7 @@ describe('looker dialect and target validation helpers', () => { it('maps supported warehouse connection types to sqlglot dialects', () => { expect(sqlglotDialectForConnectionType('BIGQUERY')).toBe('bigquery'); expect(sqlglotDialectForConnectionType('POSTGRESQL')).toBe('postgres'); + expect(sqlglotDialectForConnectionType('ATHENA')).toBe('athena'); expect(sqlglotDialectForConnectionType('LOOKER')).toBeNull(); }); @@ -259,6 +262,44 @@ describe('collectExploreParseItems and projectParsedIdentifier', () => { }); }); + it('collects Athena parser inputs with the athena sqlglot dialect', () => { + expect( + collectExploreParseItems({ + explore: { + ...mappedExplore, + rawSqlTableName: 'analytics.orders', + joins: [ + { + name: 'line_items', + type: 'left_outer', + relationship: 'one_to_many', + rawSqlTableName: 'analytics.line_items', + sqlOn: null, + from: null, + targetTable: null, + }, + ], + }, + connectionMappings: { b2b_sandbox_bq: 'athena-warehouse' }, + targetConnections: new Map([['athena-warehouse', { id: 'athena-warehouse', connection_type: 'ATHENA' }]]), + }), + ).toEqual({ + parsedTargetTables: {}, + parseItems: [ + { + key: 'b2b.sales_pipeline', + sql_table_name: 'analytics.orders', + dialect: 'athena', + }, + { + key: 'b2b.sales_pipeline.line_items', + sql_table_name: 'analytics.line_items', + dialect: 'athena', + }, + ], + }); + }); + it('projects successful and failed parser rows into ktx parsed target tables', () => { expect( projectParsedIdentifier({ diff --git a/packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts b/packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts index 3fb56442..dda0083e 100644 --- a/packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts @@ -121,6 +121,15 @@ describe('validateMappingPhysicalMatch', () => { expect(reason).toContain('engine'); }); + it('accepts Athena mappings when the target connection type is ATHENA', () => { + expect( + validateMappingPhysicalMatch( + { metabaseEngine: 'athena', metabaseDbName: 'analytics', metabaseHost: null }, + { connection_type: 'ATHENA', database: 'analytics' }, + ), + ).toBeNull(); + }); + it('returns null when Postgres host and database both match after normalization', () => { expect( validateMappingPhysicalMatch( @@ -159,6 +168,18 @@ describe('validateMappingPhysicalMatch', () => { }); }); +describe('METABASE_ENGINE_TO_CONNECTION_TYPE', () => { + it('maps Metabase Athena databases to AWS Athena warehouse connections', () => { + expect(METABASE_ENGINE_TO_CONNECTION_TYPE.athena).toBe('ATHENA'); + expect( + validateMappingPhysicalMatch( + { metabaseEngine: 'athena', metabaseDbName: 'analytics', metabaseHost: null }, + { connection_type: 'POSTGRESQL', database: 'analytics' }, + ), + ).toBe("Metabase database engine 'athena' does not match ktx connection type 'POSTGRESQL'"); + }); +}); + describe('computeMetabaseMappingPhysicalMismatches', () => { it('returns only mismatched physical mappings', () => { expect( diff --git a/packages/cli/test/context/sl/semantic-layer.service.test.ts b/packages/cli/test/context/sl/semantic-layer.service.test.ts index edc31ce1..5532986d 100644 --- a/packages/cli/test/context/sl/semantic-layer.service.test.ts +++ b/packages/cli/test/context/sl/semantic-layer.service.test.ts @@ -960,6 +960,28 @@ describe('validateWithProposedSource', () => { ); }); + it('uses the Athena sqlglot dialect when validating Athena sources', async () => { + pythonPort.validateSources.mockResolvedValue({ + data: { errors: [], warnings: [] }, + }); + service = new SemanticLayerService(configService as never, connectionCatalog('ATHENA'), pythonPort); + + await service.validateWithProposedSource('conn-1', { + name: 'orders', + table: 'analytics.orders', + grain: ['id'], + columns: [{ name: 'id', type: 'number' }], + joins: [], + measures: [], + }); + + expect(pythonPort.validateSources).toHaveBeenCalledWith( + expect.objectContaining({ + dialect: 'athena', + }), + ); + }); + it('composes a bare overlay with its manifest base before validating', async () => { const schemaPath = 'semantic-layer/conn-1/_schema/core.yaml'; const listFilesImpl = (dir: string): Promise<{ files: string[] }> => { @@ -1326,6 +1348,44 @@ describe('validateWithProposedSource', () => { }); }); +describe('executeQuery dialect resolution', () => { + it('passes the Athena sqlglot dialect to the Python query engine for Athena connections', async () => { + pythonPort.query.mockResolvedValue({ data: { sql: 'SELECT * FROM orders' } }); + const catalog = connectionCatalog('ATHENA'); + catalog.executeQuery.mockResolvedValue({ headers: ['id'], rows: [[1]], totalRows: 1 }); + const configService = { + listFiles: vi.fn().mockResolvedValue({ + files: ['semantic-layer/conn-1/orders.yaml'], + }), + readFile: vi.fn().mockResolvedValue({ + content: [ + 'name: orders', + 'table: analytics.orders', + 'grain:', + ' - id', + 'columns:', + ' - name: id', + ' type: number', + 'joins: []', + 'measures:', + ' - name: count', + ' expr: count(*)', + ].join('\n'), + }), + }; + const service = new SemanticLayerService(configService as never, catalog, pythonPort); + + await service.executeQuery('conn-1', { measures: ['count'] } as never); + + expect(pythonPort.query).toHaveBeenCalledWith( + expect.objectContaining({ + dialect: 'athena', + }), + ); + expect(catalog.executeQuery).toHaveBeenCalledWith('conn-1', 'SELECT * FROM orders'); + }); +}); + describe('findDanglingSegmentRefs', () => { it('returns empty when every measure segment resolves', () => { const source = { diff --git a/python/ktx-sl/semantic_layer/table_identifier_parser.py b/python/ktx-sl/semantic_layer/table_identifier_parser.py index f8df6631..a32863e1 100644 --- a/python/ktx-sl/semantic_layer/table_identifier_parser.py +++ b/python/ktx-sl/semantic_layer/table_identifier_parser.py @@ -18,6 +18,8 @@ SUPPORTED_TABLE_IDENTIFIER_DIALECTS = { "sqlite", "tsql", "clickhouse", + "athena", + "duckdb", } ParseTableIdentifierReason = Literal[ diff --git a/python/ktx-sl/tests/test_table_identifier_parser.py b/python/ktx-sl/tests/test_table_identifier_parser.py index ef18e990..6a6cdd03 100644 --- a/python/ktx-sl/tests/test_table_identifier_parser.py +++ b/python/ktx-sl/tests/test_table_identifier_parser.py @@ -23,6 +23,16 @@ def test_parse_table_identifier_supported_dialects_and_aliases() -> None: sql_table_name="RAW.PUBLIC.ORDERS", dialect="snowflake", ), + ParseTableIdentifierItem( + key="athena", + sql_table_name="analytics.orders", + dialect="athena", + ), + ParseTableIdentifierItem( + key="duckdb", + sql_table_name="analytics.orders", + dialect="duckdb", + ), ] ) @@ -37,6 +47,14 @@ def test_parse_table_identifier_supported_dialects_and_aliases() -> None: assert response["sf"].catalog == "RAW" assert response["sf"].schema_ == "PUBLIC" assert response["sf"].name == "ORDERS" + assert response["athena"].ok is True + assert response["athena"].schema_ == "analytics" + assert response["athena"].name == "orders" + assert response["athena"].canonical_table == "analytics.orders" + assert response["duckdb"].ok is True + assert response["duckdb"].schema_ == "analytics" + assert response["duckdb"].name == "orders" + assert response["duckdb"].canonical_table == "analytics.orders" def test_parse_table_identifier_rejects_non_physical_inputs() -> None: From 66768fe009e166d573b5a4c337f30a254ca69f1a Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:07:22 -0700 Subject: [PATCH 05/11] docs: reflect recently added connectors in ingestion diagram and README (#333) * docs(diagram): show recently added connectors in ingestion flow Add Amazon Athena and MongoDB (plus a "& more" chip), Sigma, and Google Drive to the ingestion diagram source cards so it reflects the current connector set. Trim the Databases card body to one line so the added chips fit the fixed-height card without clipping. Co-Authored-By: Claude Opus 4.8 * docs(readme): list newly added connectors in the "Works with" summary Include DuckDB, Amazon Athena, MongoDB, Sigma, and Google Drive so the README matches the integrations reference pages. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- README.md | 5 +++-- docs-site/components/product-mechanics.tsx | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3f704f04..618e9c81 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,9 @@ upkeep and don't absorb the rest of your company's knowledge. - You don't have a SQL warehouse - **ktx** sits on top of one - You only need one ad-hoc query - `psql` or a notebook will do -Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and -SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, and Notion. +Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, +SQLite, DuckDB, Amazon Athena, and MongoDB. Integrates with dbt, MetricFlow, +LookML, Looker, Metabase, Sigma, Notion, and Google Drive. ## Quick Start diff --git a/docs-site/components/product-mechanics.tsx b/docs-site/components/product-mechanics.tsx index 642e1107..55c10272 100644 --- a/docs-site/components/product-mechanics.tsx +++ b/docs-site/components/product-mechanics.tsx @@ -68,14 +68,14 @@ const EDGE_STROKE = "#94a3b8"; const sourceData: SourceNodeData[] = [ { title: "Databases", - body: "Schemas, columns, keys, row counts, and query history.", - items: ["PostgreSQL", "Snowflake", "BigQuery", "SQLite"], + body: "Schemas, keys, row counts, query history.", + items: ["PostgreSQL", "Snowflake", "BigQuery", "Athena", "MongoDB", "& more"], accent: "#3b82f6", }, { title: "BI tools", body: "Dashboards, questions, explores, usage, and trusted examples.", - items: ["Metabase", "Looker"], + items: ["Metabase", "Looker", "Sigma"], accent: "#f97316", }, { @@ -87,7 +87,7 @@ const sourceData: SourceNodeData[] = [ { title: "Docs and notes", body: "Policies, caveats, team definitions, and analyst context.", - items: ["Notion", "Any text"], + items: ["Notion", "Google Drive", "Any text"], accent: "#10b981", }, ]; From a651b82e2f2e160c321ed9a3487d7f2e2f5ebebc Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:54:17 -0700 Subject: [PATCH 06/11] feat: query_policy semantic-layer-only restricts agents to predefined semantic-layer measures (#334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sl): add predefined_measures_only guard to semantic query planning SemanticQuery gains a predefined_measures_only flag; the planner rejects any measure resolved with Provenance.COMPOSED (runtime aggregate expressions and query-time derivations) while predefined measures, predefined derived chains, dimensions, filters, and segments pass. Co-Authored-By: Claude Fable 5 * feat(config): add per-connection query_policy to warehouse connections query_policy: semantic-layer-only | read-only-sql (default) on the warehouse connection schema, plus a policy module with the raw-SQL guard, federated member restriction lookup, and the project-level predicate used to gate sql_execution registration. Co-Authored-By: Claude Fable 5 * feat(cli): enforce query_policy on raw SQL through one shared executor ktx sql and the MCP sql_execution tool now share executeProjectRawSql (resolve, policy check, read-only validation, execute), collapsing their duplicated validate-then-execute paths. Restricted connections are rejected before validation; federated raw SQL is rejected when any member is restricted. sql_execution is not registered when every SQL connection is restricted, and connection_list marks restricted connections so agents route to sl_query. executeProjectReadOnlySql stays generic for ktx-internal SQL (scan, ingest, SL-generated). Co-Authored-By: Claude Fable 5 * feat(sl): compile queries with predefined_measures_only from query_policy compileLocalSlQuery injects the flag from the connection's query_policy, never from caller input, covering both ktx sl query and the MCP sl_query tool through the daemon compile path. Co-Authored-By: Claude Fable 5 * docs: document query_policy semantic-layer-only Co-Authored-By: Claude Fable 5 * fix(sl): close semantic-layer-only bypasses via filters and federated hint The predefined_measures_only guard only inspected query.measures, so a composed aggregate written into `filters` slipped through _classify_filters into a HAVING clause untouched — letting a restricted agent evaluate arbitrary aggregates (e.g. threshold-probing `sum(x) BETWEEN a AND b`). Reject filter clauses that compose an aggregate function; a HAVING that compares a predefined measure by name (`orders.revenue > 100`) still works. Also make the federated sl_query error policy-aware: when a member is restricted, raw federated SQL is disabled too, so stop directing the agent to `ktx sql -c _ktx_federated` / sql_execution (a guaranteed failure) and point to per-connection semantic-layer queries instead. --------- Co-authored-by: Claude Fable 5 Co-authored-by: Andrey Avtomonov --- .../concepts/cross-database-federation.mdx | 5 + .../content/docs/configuration/ktx-yaml.mdx | 35 +++++ .../connections/local-warehouse-descriptor.ts | 9 ++ .../connections/project-sql-executor.ts | 75 +++++++++ .../src/context/connections/query-policy.ts | 61 ++++++++ packages/cli/src/context/mcp/context-tools.ts | 1 + .../src/context/mcp/local-project-ports.ts | 66 +++----- .../cli/src/context/project/driver-schemas.ts | 6 + packages/cli/src/context/sl/local-query.ts | 37 ++++- packages/cli/src/context/sl/types.ts | 3 + packages/cli/src/sql.ts | 23 ++- .../connections/project-sql-executor.test.ts | 102 +++++++++++- .../context/connections/query-policy.test.ts | 144 +++++++++++++++++ .../mcp/__snapshots__/mcp-tools-list.json | 4 + .../context/mcp/local-project-ports.test.ts | 74 +++++++++ .../cli/test/context/sl/local-query.test.ts | 56 +++++++ packages/cli/test/sql.test.ts | 35 +++++ .../ktx-daemon/tests/test_semantic_layer.py | 28 ++++ python/ktx-sl/semantic_layer/models.py | 3 + python/ktx-sl/semantic_layer/planner.py | 40 +++++ .../tests/test_predefined_measures_only.py | 148 ++++++++++++++++++ 21 files changed, 887 insertions(+), 68 deletions(-) create mode 100644 packages/cli/src/context/connections/query-policy.ts create mode 100644 packages/cli/test/context/connections/query-policy.test.ts create mode 100644 python/ktx-sl/tests/test_predefined_measures_only.py diff --git a/docs-site/content/docs/concepts/cross-database-federation.mdx b/docs-site/content/docs/concepts/cross-database-federation.mdx index b4a055d1..78717ff4 100644 --- a/docs-site/content/docs/concepts/cross-database-federation.mdx +++ b/docs-site/content/docs/concepts/cross-database-federation.mdx @@ -139,6 +139,11 @@ connections are declared. It surfaces in `ktx connection` and in the agent's connection list so the id is discoverable. Querying a single member database directly with its own connection id (`ktx sql -c pg_books ...`) is unchanged. +If any member connection sets +[`query_policy: semantic-layer-only`](/docs/configuration/ktx-yaml#query-policy), +raw SQL against `_ktx_federated` is rejected as a whole: a federated query can +touch any member's tables, so one restricted member restricts the federation. + ## Federated queries are read-only DuckDB attaches every member database with read-only access. Federated queries diff --git a/docs-site/content/docs/configuration/ktx-yaml.mdx b/docs-site/content/docs/configuration/ktx-yaml.mdx index 0f67877d..05ebea3e 100644 --- a/docs-site/content/docs/configuration/ktx-yaml.mdx +++ b/docs-site/content/docs/configuration/ktx-yaml.mdx @@ -217,6 +217,41 @@ connections: observed in-scope query history. The block uses `mode: exclude` and remains hand-editable. +### Query policy + +Set `query_policy: semantic-layer-only` on a warehouse connection to stop +agents from authoring SQL against it. The default, `read-only-sql`, allows +parser-validated read-only SQL through `ktx sql` and the `sql_execution` MCP +tool alongside semantic-layer queries. + +```yaml +connections: + warehouse: + driver: snowflake + query_policy: semantic-layer-only +``` + +With `semantic-layer-only`: + +- `ktx sql` and the `sql_execution` MCP tool reject the connection with a + clear error. When every SQL connection in the project is restricted, the + `sql_execution` tool is not registered at all. +- Raw SQL against the federated connection (`_ktx_federated`) is rejected + when any member connection is restricted. +- Semantic-layer queries (`ktx sl query`, the `sl_query` tool) accept only + measures predefined in the semantic-layer sources. Composed aggregate + expressions such as `sum(orders.amount)` are rejected wherever they appear, + including inside `filters` (a `HAVING`-style clause may only compare a + predefined measure by name, e.g. `orders.revenue > 100`). Grouping by + declared dimensions, filtering on columns, and segments remain available. +- `connection_list` marks the connection as restricted so agents route to + `sl_query` instead of burning a failed call. + +The policy governs agent-facing query authorship, not data access: **ktx**'s +own scan, ingest, and semantic-layer-generated SQL still run, and context +tools such as `entity_details` and `dictionary_search` still expose schema +metadata and sampled values. + ### Metabase ```yaml diff --git a/packages/cli/src/context/connections/local-warehouse-descriptor.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.ts index bb4235c2..39cdddbf 100644 --- a/packages/cli/src/context/connections/local-warehouse-descriptor.ts +++ b/packages/cli/src/context/connections/local-warehouse-descriptor.ts @@ -1,5 +1,6 @@ import type { KtxProjectConnectionConfig } from '../project/config.js'; import type { ConnectionType } from './connection-type.js'; +import { connectionQueryPolicy } from './query-policy.js'; export interface LocalWarehouseDescriptor { id: string; @@ -18,6 +19,7 @@ export interface LocalConnectionInfo { connectionType: string; members?: string[]; hint?: string; + queryPolicy?: 'semantic-layer-only'; } const DRIVER_TO_CONNECTION_TYPE: Record = { @@ -92,10 +94,17 @@ export function localConnectionInfoFromConfig( if (!connection) { return null; } + const restricted = connectionQueryPolicy(connection) === 'semantic-layer-only'; return { id, name: id, connectionType: localConnectionTypeForConfig(id, connection), + ...(restricted + ? { + queryPolicy: 'semantic-layer-only' as const, + hint: 'Raw SQL is disabled on this connection; query it with sl_query using predefined measures.', + } + : {}), }; } diff --git a/packages/cli/src/context/connections/project-sql-executor.ts b/packages/cli/src/context/connections/project-sql-executor.ts index 0c2da04e..acdf79e0 100644 --- a/packages/cli/src/context/connections/project-sql-executor.ts +++ b/packages/cli/src/context/connections/project-sql-executor.ts @@ -1,8 +1,15 @@ import { executeFederatedQuery } from '../../connectors/duckdb/federated-executor.js'; +import { KtxExpectedError, KtxQueryError, isNativeProgrammingFault } from '../../errors.js'; +import { sqlAnalysisDialectForDriver } from '../sql-analysis/dialect.js'; +import type { SqlAnalysisPort } from '../sql-analysis/ports.js'; +import { assertSafeConnectionId } from '../sl/source-files.js'; import type { KtxLocalProject } from '../project/project.js'; import type { KtxScanConnector, KtxScanContext } from '../scan/types.js'; +import { assertSqlQueryableConnection } from './dialects.js'; import { deriveFederatedConnection, FEDERATED_CONNECTION_ID } from './federation.js'; +import { assertRawSqlAllowed } from './query-policy.js'; import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutionResult } from './query-executor.js'; +import { resolveConfiguredConnection } from './resolve-connection.js'; export interface ExecuteProjectReadOnlySqlDeps { project: KtxLocalProject; @@ -56,3 +63,71 @@ export async function executeProjectReadOnlySql( await connector?.cleanup?.(); } } + +type RawSqlProgressCallback = (event: { progress: number; message: string }) => void | Promise; + +export interface ExecuteProjectRawSqlDeps { + project: KtxLocalProject; + connectionId: string; + sql: string; + maxRows: number; + sqlAnalysis: SqlAnalysisPort; + createConnector: (connectionId: string) => Promise | KtxScanConnector; + executeFederated?: typeof executeFederatedQuery; + runId: string; + onProgress?: RawSqlProgressCallback; +} + +/** + * Single guarded path for user-authored (raw) SQL — `ktx sql` and the MCP + * sql_execution tool. Enforces the connection's query_policy and the parser + * read-only guard before executing; ktx-internal SQL (semantic-layer, ingest) + * calls executeProjectReadOnlySql directly and is not subject to query_policy. + */ +export async function executeProjectRawSql(deps: ExecuteProjectRawSqlDeps): Promise { + const { project } = deps; + await deps.onProgress?.({ progress: 0, message: 'Validating SQL' }); + + const isFederated = deps.connectionId === FEDERATED_CONNECTION_ID; + const connectionId = isFederated ? deps.connectionId : assertSafeConnectionId(deps.connectionId); + const connection = isFederated ? undefined : resolveConfiguredConnection(project.config, connectionId); + if (!isFederated) { + assertSqlQueryableConnection(connectionId, connection!.driver); + } + assertRawSqlAllowed(project.config, project.projectDir, connectionId); + + const dialect = sqlAnalysisDialectForDriver(isFederated ? 'duckdb' : connection!.driver); + const validation = await deps.sqlAnalysis.validateReadOnly(deps.sql, dialect); + if (!validation.ok) { + // A read-only guard rejecting the caller's SQL is an expected outcome, not a + // ktx fault: classify it so reportException keeps it out of Error Tracking. + throw new KtxQueryError(validation.error ?? 'SQL is not read-only.'); + } + + await deps.onProgress?.({ progress: 0.3, message: 'Executing' }); + const result = await executeProjectReadOnlySql({ + project, + input: { + connectionId, + projectDir: project.projectDir, + connection, + sql: deps.sql, + maxRows: deps.maxRows, + }, + createConnector: deps.createConnector, + executeFederated: deps.executeFederated, + runId: deps.runId, + }).catch((error: unknown) => { + // A warehouse/driver rejection (e.g. the caller's SQL failed to compile) is a + // surfaced operational outcome, not a ktx fault: mark it expected while + // preserving the warehouse's own diagnostics. A native JS error (TypeError, + // etc.) signals a bug in connector code — let it propagate unchanged so Error + // Tracking still sees it. + if (isNativeProgrammingFault(error) || error instanceof KtxExpectedError) { + throw error; + } + throw new KtxQueryError(error instanceof Error ? error.message : String(error), { cause: error }); + }); + await deps.onProgress?.({ progress: 1, message: `Fetched ${result.rowCount ?? result.rows.length} rows` }); + return result; +} diff --git a/packages/cli/src/context/connections/query-policy.ts b/packages/cli/src/context/connections/query-policy.ts new file mode 100644 index 00000000..ce369735 --- /dev/null +++ b/packages/cli/src/context/connections/query-policy.ts @@ -0,0 +1,61 @@ +import { KtxQueryError } from '../../errors.js'; +import type { KtxProjectConfig, KtxProjectConnectionConfig } from '../project/config.js'; +import { deriveFederatedConnection, FEDERATED_CONNECTION_ID } from './federation.js'; +import { isSqlQueryableDriver } from './dialects.js'; + +export type KtxConnectionQueryPolicy = 'read-only-sql' | 'semantic-layer-only'; + +export function connectionQueryPolicy( + connection: KtxProjectConnectionConfig | undefined, +): KtxConnectionQueryPolicy { + return connection !== undefined && connection.query_policy === 'semantic-layer-only' + ? 'semantic-layer-only' + : 'read-only-sql'; +} + +/** Member ids whose policy blocks raw SQL through the federated connection. */ +export function restrictedFederatedMemberIds(config: KtxProjectConfig, projectDir: string): string[] { + const descriptor = deriveFederatedConnection(config.connections, projectDir); + if (!descriptor) { + return []; + } + return descriptor.members + .filter((member) => connectionQueryPolicy(member.connection) === 'semantic-layer-only') + .map((member) => member.connectionId); +} + +export function assertRawSqlAllowed(config: KtxProjectConfig, projectDir: string, connectionId: string): void { + if (connectionId === FEDERATED_CONNECTION_ID) { + const restricted = restrictedFederatedMemberIds(config, projectDir); + if (restricted.length > 0) { + throw new KtxQueryError( + `Federated SQL execution is disabled: member connection(s) ${restricted + .map((id) => `"${id}"`) + .join(', ')} are restricted to semantic-layer queries (query_policy: semantic-layer-only in ktx.yaml).`, + ); + } + return; + } + if (connectionQueryPolicy(config.connections[connectionId]) === 'semantic-layer-only') { + throw new KtxQueryError( + `Connection "${connectionId}" is restricted to semantic-layer queries (query_policy: semantic-layer-only in ktx.yaml); ` + + 'raw SQL execution is disabled. Query it through the semantic layer with predefined measures instead ' + + '(the sl_query tool or `ktx sl query`).', + ); + } +} + +/** + * False only when the project has SQL-queryable connections and every one of + * them is restricted — then no raw-SQL surface can succeed and the + * sql_execution tool should not be offered at all. + */ +export function projectAllowsRawSql(config: KtxProjectConfig): boolean { + const sqlConnections = Object.values(config.connections).filter((connection) => + isSqlQueryableDriver(connection.driver), + ); + if (sqlConnections.length === 0) { + return true; + } + return sqlConnections.some((connection) => connectionQueryPolicy(connection) === 'read-only-sql'); +} diff --git a/packages/cli/src/context/mcp/context-tools.ts b/packages/cli/src/context/mcp/context-tools.ts index 525f9540..106167e1 100644 --- a/packages/cli/src/context/mcp/context-tools.ts +++ b/packages/cli/src/context/mcp/context-tools.ts @@ -243,6 +243,7 @@ const connectionListOutputSchema = z.object({ connectionType: z.string(), members: z.array(z.string()).optional(), hint: z.string().optional(), + queryPolicy: z.literal('semantic-layer-only').optional(), }), ), }); diff --git a/packages/cli/src/context/mcp/local-project-ports.ts b/packages/cli/src/context/mcp/local-project-ports.ts index fc44e7da..9f8db652 100644 --- a/packages/cli/src/context/mcp/local-project-ports.ts +++ b/packages/cli/src/context/mcp/local-project-ports.ts @@ -1,12 +1,11 @@ import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; -import { KtxExpectedError, KtxQueryError, isNativeProgrammingFault } from '../../errors.js'; +import { KtxExpectedError } from '../../errors.js'; import { isDatabaseDriver, normalizeConnectionDriver } from '../../connection-drivers.js'; import { sqlDialectNotes } from '../../context/sql-analysis/dialect-notes.js'; import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; -import { executeProjectReadOnlySql } from '../../context/connections/project-sql-executor.js'; -import { FEDERATED_CONNECTION_ID, federatedConnectionListing } from '../../context/connections/federation.js'; -import { assertSqlQueryableConnection } from '../../context/connections/dialects.js'; -import { resolveConfiguredConnection } from '../../context/connections/resolve-connection.js'; +import { executeProjectRawSql } from '../../context/connections/project-sql-executor.js'; +import { federatedConnectionListing } from '../../context/connections/federation.js'; +import { projectAllowsRawSql, restrictedFederatedMemberIds } from '../../context/connections/query-policy.js'; import { type LocalConnectionInfo, localConnectionInfoFromConfig, @@ -41,7 +40,6 @@ async function executeValidatedReadOnlySql( input: { connectionId: string; sql: string; maxRows: number }, onProgress?: KtxMcpProgressCallback, ): Promise { - await onProgress?.({ progress: 0, message: 'Validating SQL' }); if (!options.sqlAnalysis) { throw new Error('sql_execution requires parser-backed SQL validation.'); } @@ -50,52 +48,22 @@ async function executeValidatedReadOnlySql( throw new Error('sql_execution requires a local scan connector factory.'); } - const isFederated = input.connectionId === FEDERATED_CONNECTION_ID; - const connectionId = isFederated ? input.connectionId : assertSafeConnectionId(input.connectionId); - const connection = isFederated ? undefined : resolveConfiguredConnection(project.config, connectionId); - if (!isFederated) { - assertSqlQueryableConnection(connectionId, connection!.driver); - } - - const dialect = sqlAnalysisDialectForDriver(isFederated ? 'duckdb' : connection!.driver); - const validation = await options.sqlAnalysis.validateReadOnly(input.sql, dialect); - if (!validation.ok) { - // A read-only guard rejecting the agent's SQL is an expected outcome, not a - // ktx fault: classify it so reportException keeps it out of Error Tracking. - throw new KtxQueryError(validation.error ?? 'SQL is not read-only.'); - } - - await onProgress?.({ progress: 0.3, message: 'Executing' }); - const result = await executeProjectReadOnlySql({ + const result = await executeProjectRawSql({ project, - input: { - connectionId, - projectDir: project.projectDir, - connection, - sql: input.sql, - maxRows: input.maxRows, - }, + connectionId: input.connectionId, + sql: input.sql, + maxRows: input.maxRows, + sqlAnalysis: options.sqlAnalysis, createConnector, runId: 'mcp-sql-execution', - }).catch((error: unknown) => { - // A warehouse/driver rejection (e.g. the agent's SQL failed to compile) is a - // surfaced operational outcome, not a ktx fault: mark it expected while - // preserving the warehouse's own diagnostics. A native JS error (TypeError, - // etc.) signals a bug in connector code — let it propagate unchanged so Error - // Tracking still sees it. - if (isNativeProgrammingFault(error) || error instanceof KtxExpectedError) { - throw error; - } - throw new KtxQueryError(error instanceof Error ? error.message : String(error), { cause: error }); + onProgress, }); - const response = { + return { headers: result.headers, ...(result.headerTypes ? { headerTypes: result.headerTypes } : {}), rows: result.rows, rowCount: result.rowCount ?? result.rows.length, }; - await onProgress?.({ progress: 1, message: `Fetched ${response.rowCount} rows` }); - return response; } /** @internal Resolves a connection's dialect SQL notes; throws KtxExpectedError for an unknown or non-SQL-warehouse connection. */ @@ -130,12 +98,17 @@ export function createLocalProjectMcpContextPorts( .sort((a, b) => a.id.localeCompare(b.id)); const federated = federatedConnectionListing(project.config.connections, project.projectDir); if (federated) { + const restricted = restrictedFederatedMemberIds(project.config, project.projectDir); configured.push({ id: federated.id, name: federated.id, connectionType: 'DUCKDB', members: federated.members, - hint: federated.hint, + hint: + restricted.length > 0 + ? `Federated SQL is disabled: member connection(s) ${restricted.join(', ')} have query_policy: semantic-layer-only.` + : federated.hint, + ...(restricted.length > 0 ? { queryPolicy: 'semantic-layer-only' as const } : {}), }); } return configured; @@ -231,7 +204,10 @@ export function createLocalProjectMcpContextPorts( }, }; - if (options.sqlAnalysis && options.localScan?.createConnector) { + // Register sql_execution only when some connection can accept raw SQL; in + // mixed projects the tool stays and executeProjectRawSql rejects restricted + // connection ids at request time. + if (options.sqlAnalysis && options.localScan?.createConnector && projectAllowsRawSql(project.config)) { ports.sqlExecution = { async execute(input, executionOptions) { return executeValidatedReadOnlySql(project, options, input, executionOptions?.onProgress); diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index df828065..72d1a133 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -42,6 +42,12 @@ function warehouseConnectionSchema(driver: .describe( 'Maximum execution time for a single read-only query, in milliseconds (default 30000). Enforced as a server-side statement timeout for remote engines and by SIGKILL-ing a forked query subprocess for in-process SQLite. A query exceeding it is cancelled and returns a "query exceeded Ns" error so the agent can revise.', ), + query_policy: z + .enum(['read-only-sql', 'semantic-layer-only']) + .optional() + .describe( + 'Agent-facing query authorship policy (default "read-only-sql"). "read-only-sql" allows parser-validated read-only SQL plus semantic-layer queries. "semantic-layer-only" rejects raw SQL on this connection (`ktx sql`, the sql_execution tool, and federated queries that include it) and restricts semantic-layer queries to measures predefined in the semantic-layer sources. ktx-internal scan and ingest queries are unaffected.', + ), }) .describe( `${driver} warehouse connection. Additional driver-tunable fields (e.g. context.queryHistory) are accepted and passed through.`, diff --git a/packages/cli/src/context/sl/local-query.ts b/packages/cli/src/context/sl/local-query.ts index fb8b9439..a1ab17be 100644 --- a/packages/cli/src/context/sl/local-query.ts +++ b/packages/cli/src/context/sl/local-query.ts @@ -4,6 +4,7 @@ import type { KtxMcpProgressCallback } from '../mcp/types.js'; import type { KtxLocalProject } from '../../context/project/project.js'; import { isSqlQueryableDriver } from '../connections/dialects.js'; import { FEDERATED_CONNECTION_ID } from '../connections/federation.js'; +import { connectionQueryPolicy, restrictedFederatedMemberIds } from '../connections/query-policy.js'; import { resolveRequiredConnectionId } from '../connections/resolve-connection.js'; import { sqlAnalysisDialectForDriver } from '../sql-analysis/dialect.js'; import { loadLocalSlSourceRecords } from './local-sl.js'; @@ -14,11 +15,31 @@ import type { SemanticLayerQueryExecutionResult, SemanticLayerQueryInput, Semant const COMPILE_ONLY_REASON = 'Local semantic-layer query compiled SQL but no data-source execution adapter is configured.'; -const FEDERATED_SL_QUERY_UNSUPPORTED = - `Semantic-layer queries are per-connection and cannot target the federated connection '${FEDERATED_CONNECTION_ID}'. ` + - `Run a cross-database query as read-only SQL instead — ktx sql -c ${FEDERATED_CONNECTION_ID} "SELECT ..." or the sql_execution tool — ` + - 'using catalog-qualified table names (connectionId.schema.table, or connectionId.table for sqlite; ' + - 'double-quote ids that are not bare identifiers, e.g. "books-db".public.books).'; +const FEDERATED_SL_QUERY_PREFIX = + `Semantic-layer queries are per-connection and cannot target the federated connection '${FEDERATED_CONNECTION_ID}'. `; + +// The raw-SQL fallback is only valid when federated raw SQL is allowed; when a +// member is restricted (query_policy: semantic-layer-only), assertRawSqlAllowed +// rejects the same path, so directing the agent there would burn a guaranteed +// failure. Derive the message from the restricted-member set instead. +function federatedSlQueryUnsupportedMessage(project: KtxLocalProject): string { + const restricted = restrictedFederatedMemberIds(project.config, project.projectDir); + if (restricted.length > 0) { + return ( + FEDERATED_SL_QUERY_PREFIX + + `Cross-database SQL through '${FEDERATED_CONNECTION_ID}' is also disabled because member connection(s) ` + + `${restricted.map((id) => `'${id}'`).join(', ')} are restricted to semantic-layer queries ` + + '(query_policy: semantic-layer-only). Query each connection on its own through the semantic layer ' + + '(the sl_query tool or `ktx sl query` with its connection id).' + ); + } + return ( + FEDERATED_SL_QUERY_PREFIX + + `Run a cross-database query as read-only SQL instead — ktx sql -c ${FEDERATED_CONNECTION_ID} "SELECT ..." or the sql_execution tool — ` + + 'using catalog-qualified table names (connectionId.schema.table, or connectionId.table for sqlite; ' + + 'double-quote ids that are not bare identifiers, e.g. "books-db".public.books).' + ); +} export interface CompileLocalSlQueryOptions { connectionId?: string; @@ -79,7 +100,7 @@ export async function compileLocalSlQuery( options: CompileLocalSlQueryOptions, ): Promise { if (options.connectionId === FEDERATED_CONNECTION_ID) { - throw new Error(FEDERATED_SL_QUERY_UNSUPPORTED); + throw new Error(federatedSlQueryUnsupportedMessage(project)); } await options.onProgress?.({ progress: 0, message: 'Compiling query' }); const connectionId = resolveLocalConnectionId(project, options.connectionId); @@ -93,11 +114,13 @@ export async function compileLocalSlQuery( const dialect = sqlAnalysisDialectForDriver(driver); const sources = await loadComputableSources(project, connectionId); + const predefinedMeasuresOnly = + connectionQueryPolicy(project.config.connections[connectionId]) === 'semantic-layer-only'; await options.onProgress?.({ progress: 0.3, message: 'Generating SQL' }); const response = await options.compute.query({ sources, dialect, - query: options.query, + query: { ...options.query, predefined_measures_only: predefinedMeasuresOnly }, }); if (!options.execute) { diff --git a/packages/cli/src/context/sl/types.ts b/packages/cli/src/context/sl/types.ts index cc9575b7..1f132c4b 100644 --- a/packages/cli/src/context/sl/types.ts +++ b/packages/cli/src/context/sl/types.ts @@ -81,6 +81,9 @@ export interface SemanticLayerQueryInput { order_by?: Array; limit?: number; include_empty?: boolean; + // Set by compileLocalSlQuery from the connection's query_policy, never from + // caller input: the planner rejects runtime-composed measures when true. + predefined_measures_only?: boolean; } export interface SemanticLayerQueryExecutionResult { diff --git a/packages/cli/src/sql.ts b/packages/cli/src/sql.ts index 78cfb796..fbf05975 100644 --- a/packages/cli/src/sql.ts +++ b/packages/cli/src/sql.ts @@ -1,6 +1,6 @@ import { executeFederatedQuery } from './connectors/duckdb/federated-executor.js'; import { FEDERATED_CONNECTION_ID } from './context/connections/federation.js'; -import { executeProjectReadOnlySql } from './context/connections/project-sql-executor.js'; +import { executeProjectRawSql } from './context/connections/project-sql-executor.js'; import type { KtxSqlQueryExecutionResult } from './context/connections/query-executor.js'; import { assertSqlQueryableConnection } from './context/connections/dialects.js'; import { resolveConfiguredConnection } from './context/connections/resolve-connection.js'; @@ -137,6 +137,8 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps: const connection = isFederated ? undefined : resolveConfiguredConnection(project.config, args.connectionId); driver = isFederated ? 'duckdb' : String(connection?.driver ?? 'unknown').toLowerCase(); demoConnection = isFederated ? false : isDemoConnection(args.connectionId, connection); + // Fail fast before creating the SQL-analysis daemon port; executeProjectRawSql + // re-asserts this for every caller. if (!isFederated) { assertSqlQueryableConnection(args.connectionId, connection?.driver); } @@ -152,26 +154,19 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps: })); const analysisPort = createSqlAnalysis(); const dialect: SqlAnalysisDialect = isFederated ? 'duckdb' : sqlAnalysisDialectForDriver(connection?.driver); - const validation = await analysisPort.validateReadOnly(args.sql, dialect); - if (!validation.ok) { - throw new Error(validation.error ?? 'SQL is not read-only.'); - } - const referencedTableCount = await safeReferencedTableCount(analysisPort, args.sql, dialect); const createScanConnector = deps.createScanConnector ?? createKtxCliScanConnector; - const result = await executeProjectReadOnlySql({ + const result = await executeProjectRawSql({ project, - input: { - connectionId: args.connectionId, - projectDir: args.projectDir, - connection, - sql: args.sql, - maxRows: args.maxRows, - }, + connectionId: args.connectionId, + sql: args.sql, + maxRows: args.maxRows, + sqlAnalysis: analysisPort, createConnector: (connectionId) => createScanConnector(project!, connectionId), executeFederated: deps.executeFederated, runId: 'cli-sql', }); + const referencedTableCount = await safeReferencedTableCount(analysisPort, args.sql, dialect); const mode = resolveOutputMode({ explicit: args.output, json: args.json, io }); printSqlResult(resultOutput(args.connectionId, result), mode, io); await emitTelemetryEvent({ diff --git a/packages/cli/test/context/connections/project-sql-executor.test.ts b/packages/cli/test/context/connections/project-sql-executor.test.ts index 899875a8..bcf6acd9 100644 --- a/packages/cli/test/context/connections/project-sql-executor.test.ts +++ b/packages/cli/test/context/connections/project-sql-executor.test.ts @@ -1,10 +1,14 @@ import { describe, expect, it, vi } from 'vitest'; import type { executeFederatedQuery } from '../../../src/connectors/duckdb/federated-executor.js'; -import { executeProjectReadOnlySql } from '../../../src/context/connections/project-sql-executor.js'; +import { executeProjectRawSql, executeProjectReadOnlySql } from '../../../src/context/connections/project-sql-executor.js'; import type { KtxLocalProject } from '../../../src/context/project/project.js'; import type { KtxScanConnector } from '../../../src/context/scan/types.js'; +import type { SqlAnalysisPort } from '../../../src/context/sql-analysis/ports.js'; +import { KtxQueryError } from '../../../src/errors.js'; -function fakeProject(connections: Record): KtxLocalProject { +function fakeProject( + connections: Record, +): KtxLocalProject { return { projectDir: '/tmp/proj', configPath: '/tmp/proj/ktx.yaml', @@ -114,3 +118,97 @@ describe('executeProjectReadOnlySql headerTypes', () => { expect(result.headerTypes).toEqual(['INTEGER']); }); }); + +function fakeSqlAnalysis(validation: { ok: boolean; error: string | null }): SqlAnalysisPort { + return { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(), + validateReadOnly: vi.fn(async () => validation), + } as unknown as SqlAnalysisPort; +} + +describe('executeProjectRawSql', () => { + it('validates then executes raw SQL on an unrestricted connection', async () => { + const project = fakeProject({ pg: { driver: 'postgres' } }); + const sqlAnalysis = fakeSqlAnalysis({ ok: true, error: null }); + const connector = connectorReturning({ + headers: ['id'], + rows: [[1]], + totalRows: 1, + rowCount: 1, + }); + + const result = await executeProjectRawSql({ + project, + connectionId: 'pg', + sql: 'SELECT id FROM orders', + maxRows: 25, + sqlAnalysis, + createConnector: () => connector, + runId: 'test-raw-sql', + }); + + expect(result.rows).toEqual([[1]]); + expect(sqlAnalysis.validateReadOnly).toHaveBeenCalledWith('SELECT id FROM orders', 'postgres'); + }); + + it('rejects a restricted connection before validation or execution', async () => { + const project = fakeProject({ pg: { driver: 'postgres', query_policy: 'semantic-layer-only' } }); + const sqlAnalysis = fakeSqlAnalysis({ ok: true, error: null }); + const createConnector = vi.fn(); + + const execution = executeProjectRawSql({ + project, + connectionId: 'pg', + sql: 'SELECT 1', + maxRows: 25, + sqlAnalysis, + createConnector: createConnector as never, + runId: 'test-raw-sql', + }); + await expect(execution).rejects.toBeInstanceOf(KtxQueryError); + await expect(execution).rejects.toThrow(/query_policy: semantic-layer-only/); + expect(sqlAnalysis.validateReadOnly).not.toHaveBeenCalled(); + expect(createConnector).not.toHaveBeenCalled(); + }); + + it('rejects federated raw SQL when a member connection is restricted', async () => { + const project = fakeProject({ + pg: { driver: 'postgres', query_policy: 'semantic-layer-only' }, + lite: { driver: 'sqlite' }, + }); + const executeFederated = vi.fn(); + + await expect( + executeProjectRawSql({ + project, + connectionId: '_ktx_federated', + sql: 'SELECT 1', + maxRows: 25, + sqlAnalysis: fakeSqlAnalysis({ ok: true, error: null }), + createConnector: vi.fn() as never, + executeFederated: executeFederated as never, + runId: 'test-raw-sql', + }), + ).rejects.toThrow(/"pg"/); + expect(executeFederated).not.toHaveBeenCalled(); + }); + + it('classifies a read-only validation failure as an expected query error', async () => { + const project = fakeProject({ pg: { driver: 'postgres' } }); + const createConnector = vi.fn(); + + const execution = executeProjectRawSql({ + project, + connectionId: 'pg', + sql: 'DROP TABLE orders', + maxRows: 25, + sqlAnalysis: fakeSqlAnalysis({ ok: false, error: 'SQL is not read-only: DROP.' }), + createConnector: createConnector as never, + runId: 'test-raw-sql', + }); + await expect(execution).rejects.toBeInstanceOf(KtxQueryError); + await expect(execution).rejects.toThrow('SQL is not read-only: DROP.'); + expect(createConnector).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/test/context/connections/query-policy.test.ts b/packages/cli/test/context/connections/query-policy.test.ts new file mode 100644 index 00000000..e2d2c7e1 --- /dev/null +++ b/packages/cli/test/context/connections/query-policy.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from 'vitest'; +import { + assertRawSqlAllowed, + connectionQueryPolicy, + projectAllowsRawSql, + restrictedFederatedMemberIds, +} from '../../../src/context/connections/query-policy.js'; +import { parseKtxProjectConfig } from '../../../src/context/project/config.js'; +import { KtxQueryError } from '../../../src/errors.js'; + +const PROJECT_DIR = '/tmp/proj'; + +function config(yaml: string) { + return parseKtxProjectConfig(yaml); +} + +describe('connectionQueryPolicy', () => { + it('defaults to read-only-sql when the field is absent or the connection is unknown', () => { + const parsed = config(` +connections: + warehouse: + driver: sqlite + url: file:warehouse.db +`); + expect(connectionQueryPolicy(parsed.connections.warehouse)).toBe('read-only-sql'); + expect(connectionQueryPolicy(undefined)).toBe('read-only-sql'); + }); + + it('reads semantic-layer-only from ktx.yaml', () => { + const parsed = config(` +connections: + warehouse: + driver: snowflake + url: env:SNOWFLAKE_URL + query_policy: semantic-layer-only +`); + expect(connectionQueryPolicy(parsed.connections.warehouse)).toBe('semantic-layer-only'); + }); + + it('rejects unknown query_policy values at config parse time', () => { + expect(() => + config(` +connections: + warehouse: + driver: sqlite + url: file:warehouse.db + query_policy: everything-goes +`), + ).toThrow(); + }); +}); + +describe('assertRawSqlAllowed', () => { + it('allows raw SQL on an unrestricted connection', () => { + const parsed = config(` +connections: + warehouse: + driver: sqlite + url: file:warehouse.db +`); + expect(() => assertRawSqlAllowed(parsed, PROJECT_DIR, 'warehouse')).not.toThrow(); + }); + + it('rejects raw SQL on a restricted connection with an expected error naming the policy', () => { + const parsed = config(` +connections: + warehouse: + driver: sqlite + url: file:warehouse.db + query_policy: semantic-layer-only +`); + expect(() => assertRawSqlAllowed(parsed, PROJECT_DIR, 'warehouse')).toThrow(KtxQueryError); + expect(() => assertRawSqlAllowed(parsed, PROJECT_DIR, 'warehouse')).toThrow( + /query_policy: semantic-layer-only/, + ); + }); + + it('rejects federated raw SQL when any member connection is restricted', () => { + const parsed = config(` +connections: + sales: + driver: sqlite + url: file:sales.db + query_policy: semantic-layer-only + events: + driver: sqlite + url: file:events.db +`); + expect(restrictedFederatedMemberIds(parsed, PROJECT_DIR)).toEqual(['sales']); + expect(() => assertRawSqlAllowed(parsed, PROJECT_DIR, '_ktx_federated')).toThrow(/"sales"/); + }); + + it('allows federated raw SQL when no member is restricted', () => { + const parsed = config(` +connections: + sales: + driver: sqlite + url: file:sales.db + events: + driver: sqlite + url: file:events.db +`); + expect(restrictedFederatedMemberIds(parsed, PROJECT_DIR)).toEqual([]); + expect(() => assertRawSqlAllowed(parsed, PROJECT_DIR, '_ktx_federated')).not.toThrow(); + }); +}); + +describe('projectAllowsRawSql', () => { + it('is true when at least one SQL connection is unrestricted', () => { + const parsed = config(` +connections: + finance: + driver: postgres + url: env:FINANCE_URL + query_policy: semantic-layer-only + warehouse: + driver: sqlite + url: file:warehouse.db +`); + expect(projectAllowsRawSql(parsed)).toBe(true); + }); + + it('is false when every SQL connection is restricted', () => { + const parsed = config(` +connections: + finance: + driver: postgres + url: env:FINANCE_URL + query_policy: semantic-layer-only +`); + expect(projectAllowsRawSql(parsed)).toBe(false); + }); + + it('is true for projects with no SQL-queryable connections', () => { + const parsed = config(` +connections: + docs: + driver: mongodb + url: mongodb://localhost:27017/app +`); + expect(projectAllowsRawSql(parsed)).toBe(true); + expect(projectAllowsRawSql(config('connections: {}'))).toBe(true); + }); +}); diff --git a/packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json b/packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json index 6015b006..914f1f46 100644 --- a/packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json +++ b/packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json @@ -33,6 +33,10 @@ }, "hint": { "type": "string" + }, + "queryPolicy": { + "type": "string", + "const": "semantic-layer-only" } }, "required": [ diff --git a/packages/cli/test/context/mcp/local-project-ports.test.ts b/packages/cli/test/context/mcp/local-project-ports.test.ts index a20445ff..ee28dd38 100644 --- a/packages/cli/test/context/mcp/local-project-ports.test.ts +++ b/packages/cli/test/context/mcp/local-project-ports.test.ts @@ -247,6 +247,80 @@ describe('createLocalProjectMcpContextPorts', () => { expect(connector.cleanup).toHaveBeenCalled(); }); + it('omits sql_execution when every SQL connection is semantic-layer-only', async () => { + const project = await initKtxProject({ projectDir: tempDir }); + project.config.connections.warehouse = { + driver: 'postgres', + url: 'env:DATABASE_URL', + query_policy: 'semantic-layer-only', + }; + const ports = createLocalProjectMcpContextPorts(project, { + sqlAnalysis: { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(), + validateReadOnly: vi.fn(async () => ({ ok: true, error: null })), + }, + localScan: { createConnector: vi.fn(async () => testConnector()) }, + embeddingService: null, + }); + + expect(ports.sqlExecution).toBeUndefined(); + }); + + it('keeps sql_execution in mixed projects but rejects restricted connections and flags them in connection_list', async () => { + const project = await initKtxProject({ projectDir: tempDir }); + project.config.connections.warehouse = { + driver: 'postgres', + url: 'env:DATABASE_URL', + }; + project.config.connections.finance = { + driver: 'postgres', + url: 'env:FINANCE_URL', + query_policy: 'semantic-layer-only', + }; + const createConnector = vi.fn(async () => testConnector()); + const sqlAnalysis = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(), + validateReadOnly: vi.fn(async () => ({ ok: true, error: null })), + }; + const ports = createLocalProjectMcpContextPorts(project, { + sqlAnalysis, + localScan: { createConnector }, + embeddingService: null, + }); + + expect(ports.sqlExecution).toBeDefined(); + const execution = ports.sqlExecution?.execute({ + connectionId: 'finance', + sql: 'select 1', + maxRows: 5, + }); + await expect(execution).rejects.toBeInstanceOf(KtxQueryError); + await expect(execution).rejects.toThrow(/query_policy: semantic-layer-only/); + expect(sqlAnalysis.validateReadOnly).not.toHaveBeenCalled(); + expect(createConnector).not.toHaveBeenCalled(); + + // Both postgres members federate, so the restricted member also blocks federated raw SQL. + await expect( + ports.sqlExecution?.execute({ connectionId: '_ktx_federated', sql: 'select 1', maxRows: 5 }), + ).rejects.toThrow(/"finance"/); + + await expect(ports.connections?.list()).resolves.toEqual([ + expect.objectContaining({ + id: 'finance', + queryPolicy: 'semantic-layer-only', + hint: expect.stringContaining('sl_query'), + }), + expect.objectContaining({ id: 'warehouse' }), + expect.objectContaining({ + id: '_ktx_federated', + queryPolicy: 'semantic-layer-only', + hint: expect.stringContaining('finance'), + }), + ]); + }); + it('rejects sql_execution against an unconfigured connection with an actionable expected error', async () => { const project = await initKtxProject({ projectDir: tempDir }); project.config.connections.warehouse = { diff --git a/packages/cli/test/context/sl/local-query.test.ts b/packages/cli/test/context/sl/local-query.test.ts index 5a5891e4..e532ba8c 100644 --- a/packages/cli/test/context/sl/local-query.test.ts +++ b/packages/cli/test/context/sl/local-query.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { KtxSemanticLayerComputePort } from '../../../src/context/daemon/semantic-layer-compute.js'; +import { FEDERATED_CONNECTION_ID } from '../../../src/context/connections/federation.js'; import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; import { compileLocalSlQuery } from '../../../src/context/sl/local-query.js'; @@ -67,6 +68,59 @@ grain: [] await rm(tempDir, { recursive: true, force: true }); }); + it('injects predefined_measures_only when the connection query_policy is semantic-layer-only', async () => { + project.config.connections.warehouse = { driver: 'postgres', query_policy: 'semantic-layer-only' }; + + await compileLocalSlQuery(project, { + connectionId: 'warehouse', + query: { measures: ['orders.order_count'], dimensions: [], limit: 10 }, + compute, + }); + + expect(compute.query).toHaveBeenCalledWith( + expect.objectContaining({ + query: expect.objectContaining({ predefined_measures_only: true }), + }), + ); + }); + + it('rejects a federated sl_query, pointing to per-connection SL when a member is restricted', async () => { + project.config.connections.warehouse = { driver: 'postgres', query_policy: 'semantic-layer-only' }; + project.config.connections.analytics = { driver: 'postgres' }; + + let message = ''; + try { + await compileLocalSlQuery(project, { + connectionId: FEDERATED_CONNECTION_ID, + query: { measures: ['orders.order_count'], dimensions: [] }, + compute, + }); + throw new Error('expected compileLocalSlQuery to reject'); + } catch (e) { + message = (e as Error).message; + } + + expect(message).toContain("member connection(s) 'warehouse'"); + expect(message).toContain('query_policy: semantic-layer-only'); + // Must not send the agent down the raw-SQL path that assertRawSqlAllowed rejects. + expect(message).not.toContain(`ktx sql -c ${FEDERATED_CONNECTION_ID}`); + expect(message).not.toContain('sql_execution'); + expect(compute.query).not.toHaveBeenCalled(); + }); + + it('rejects a federated sl_query, pointing to raw federated SQL when no member is restricted', async () => { + project.config.connections.analytics = { driver: 'postgres' }; + + await expect( + compileLocalSlQuery(project, { + connectionId: FEDERATED_CONNECTION_ID, + query: { measures: ['orders.order_count'], dimensions: [] }, + compute, + }), + ).rejects.toThrow(`ktx sql -c ${FEDERATED_CONNECTION_ID}`); + expect(compute.query).not.toHaveBeenCalled(); + }); + it('refuses a non-SQL (context-only) connection instead of compiling it as Postgres', async () => { project.config.connections['mongo-prod'] = { driver: 'mongodb', url: 'mongodb://localhost:27017/app' }; await expect( @@ -109,6 +163,7 @@ grain: [] measures: ['orders.order_count'], dimensions: ['orders.status'], limit: 25, + predefined_measures_only: false, }, }); expect(result).toEqual({ @@ -190,6 +245,7 @@ grain: [] query: { measures: ['sum(payments.amount)'], dimensions: [], + predefined_measures_only: false, }, }); }); diff --git a/packages/cli/test/sql.test.ts b/packages/cli/test/sql.test.ts index 4ffc56ce..5e72ff48 100644 --- a/packages/cli/test/sql.test.ts +++ b/packages/cli/test/sql.test.ts @@ -141,6 +141,41 @@ describe('runKtxSql', () => { expect(io.stderr()).toBe(''); }); + it('refuses raw SQL when the connection query_policy is semantic-layer-only', async () => { + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir }); + await writeConnections(projectDir, { + warehouse: { driver: 'sqlite', path: 'warehouse.db', query_policy: 'semantic-layer-only' }, + }); + const sqlAnalysis = makeSqlAnalysis({ ok: true, error: null }); + const createScanConnector = vi.fn(async () => makeConnector()); + const io = makeIo(); + + await expect( + runKtxSql( + { + command: 'execute', + projectDir, + connectionId: 'warehouse', + sql: 'select id from orders', + maxRows: 1000, + output: 'pretty', + json: false, + cliVersion: '0.0.0-test', + }, + io.io, + { + createSqlAnalysis: () => sqlAnalysis, + createScanConnector, + }, + ), + ).resolves.toBe(1); + + expect(io.stderr()).toContain('query_policy: semantic-layer-only'); + expect(sqlAnalysis.validateReadOnly).not.toHaveBeenCalled(); + expect(createScanConnector).not.toHaveBeenCalled(); + }); + it('emits debug telemetry for SQL without raw query text', async () => { vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); vi.stubEnv('CI', ''); diff --git a/python/ktx-daemon/tests/test_semantic_layer.py b/python/ktx-daemon/tests/test_semantic_layer.py index 72040df9..2a1f1da9 100644 --- a/python/ktx-daemon/tests/test_semantic_layer.py +++ b/python/ktx-daemon/tests/test_semantic_layer.py @@ -51,6 +51,34 @@ def test_query_semantic_layer_generates_sql_and_plan() -> None: assert response.plan["sources_used"] == ["orders"] +def test_query_semantic_layer_enforces_predefined_measures_only() -> None: + with pytest.raises(ValueError, match="composed measure"): + query_semantic_layer( + SemanticLayerQueryRequest( + sources=[ORDERS_SOURCE], + dialect="postgres", + query={ + "measures": ["sum(orders.amount)"], + "predefined_measures_only": True, + }, + ) + ) + + +def test_query_semantic_layer_allows_predefined_measures_under_policy() -> None: + response = query_semantic_layer( + SemanticLayerQueryRequest( + sources=[ORDERS_SOURCE], + dialect="postgres", + query={ + "measures": ["orders.revenue"], + "predefined_measures_only": True, + }, + ) + ) + assert "public.orders" in response.sql + + def test_query_semantic_layer_emits_plan_and_sql_debug_events( tmp_path: Path, monkeypatch, diff --git a/python/ktx-sl/semantic_layer/models.py b/python/ktx-sl/semantic_layer/models.py index 08e9fd42..339da263 100644 --- a/python/ktx-sl/semantic_layer/models.py +++ b/python/ktx-sl/semantic_layer/models.py @@ -169,6 +169,9 @@ class SemanticQuery(BaseModel): order_by: list[str | dict[str, Any]] = [] limit: int = 1000 include_empty: bool = True + # Set by ktx from the connection's query_policy, never by agent input: + # reject runtime-composed measures so only predefined measures execute. + predefined_measures_only: bool = False @model_validator(mode="after") def _validate_limit(self) -> SemanticQuery: diff --git a/python/ktx-sl/semantic_layer/planner.py b/python/ktx-sl/semantic_layer/planner.py index 8f0c9cca..2e7f0c74 100644 --- a/python/ktx-sl/semantic_layer/planner.py +++ b/python/ktx-sl/semantic_layer/planner.py @@ -62,6 +62,10 @@ class QueryPlanner: # 2. Resolve measures (parse, look up pre-defined, classify) raw_measures = self._resolve_measures(query.measures) + if query.predefined_measures_only: + self._reject_composed_measures(raw_measures) + self._reject_composed_filter_aggregates(query.filters) + # 3. Topological sort for derived measures measures = self._topological_sort_measures(raw_measures) @@ -239,6 +243,42 @@ class QueryPlanner: measures = self._qualify_duplicate_names(measures) return measures + def _reject_composed_measures(self, measures: list[ResolvedMeasure]) -> None: + composed = [m for m in measures if m.provenance is Provenance.COMPOSED] + if not composed: + return + rejected = ", ".join(f"'{m.expr}'" for m in composed) + raise ValueError( + f"Only predefined semantic-layer measures can be queried on this " + f"connection (query_policy: semantic-layer-only); composed measure " + f"expressions are not allowed: {rejected}. Use measures declared " + f"on the semantic-layer sources, referenced as 'source.measure'." + ) + + def _reject_composed_filter_aggregates(self, filters: list[str]) -> None: + # Predefined measures are compared in HAVING by name (e.g. + # `orders.revenue > 100`), which parses as a plain column ref. Any + # aggregate function written into a filter is therefore a runtime-composed + # aggregate the policy forbids — without this guard it would slip through + # `_classify_filters` into HAVING and evaluate an arbitrary aggregate. + composed: list[str] = [] + for f in filters: + if not f or not f.strip(): + continue + for clause in self._split_top_level_and(f): + if self.parser.parse(clause).is_aggregate: + composed.append(clause) + if not composed: + return + rejected = ", ".join(f"'{c}'" for c in composed) + raise ValueError( + f"Only predefined semantic-layer measures can be queried on this " + f"connection (query_policy: semantic-layer-only); filters may not " + f"compose aggregate expressions: {rejected}. Filter on declared " + f"dimensions or columns, or compare a predefined measure by name " + f"(e.g. 'orders.revenue > 100')." + ) + def _collect_colliding_predefined_names(self, raw: list[str | dict]) -> set[str]: counts: Counter[str] = Counter() for item in raw: diff --git a/python/ktx-sl/tests/test_predefined_measures_only.py b/python/ktx-sl/tests/test_predefined_measures_only.py new file mode 100644 index 00000000..f01215ec --- /dev/null +++ b/python/ktx-sl/tests/test_predefined_measures_only.py @@ -0,0 +1,148 @@ +"""predefined_measures_only rejects runtime-composed measures while leaving +predefined measures, predefined derived chains, dimensions, and filters usable.""" + +from __future__ import annotations + +import pytest + +from semantic_layer.engine import SemanticEngine +from semantic_layer.models import SourceDefinition + + +def _engine() -> SemanticEngine: + orders = SourceDefinition( + name="orders", + table="public.orders", + grain=["id"], + columns=[ + {"name": "id", "type": "number"}, + {"name": "amount", "type": "number"}, + {"name": "status", "type": "string"}, + ], + measures=[ + {"name": "revenue", "expr": "sum(amount)"}, + {"name": "order_count", "expr": "count(*)"}, + {"name": "aov", "expr": "revenue / order_count"}, + ], + ) + return SemanticEngine.from_sources({"orders": orders}) + + +def test_rejects_composed_string_measure() -> None: + with pytest.raises(ValueError, match="composed measure") as excinfo: + _engine().query( + { + "measures": ["sum(orders.amount)"], + "predefined_measures_only": True, + } + ) + assert "sum(orders.amount)" in str(excinfo.value) + assert "query_policy: semantic-layer-only" in str(excinfo.value) + + +def test_rejects_composed_dict_measure() -> None: + with pytest.raises(ValueError, match="composed measure"): + _engine().query( + { + "measures": [{"expr": "avg(orders.amount)", "name": "avg_amount"}], + "predefined_measures_only": True, + } + ) + + +def test_rejects_query_time_derivation_over_predefined_measures() -> None: + with pytest.raises(ValueError, match="composed measure"): + _engine().query( + { + "measures": [ + {"expr": "orders.revenue / orders.order_count", "name": "ratio"} + ], + "predefined_measures_only": True, + } + ) + + +def test_rejects_composed_aggregate_in_filter() -> None: + # A HAVING-classified filter must not smuggle a runtime aggregate the + # measures guard would reject (threshold-probing bypass). + with pytest.raises(ValueError, match="compose aggregate expressions") as excinfo: + _engine().query( + { + "measures": ["orders.revenue"], + "dimensions": ["orders.status"], + "filters": ["avg(orders.amount) > 100"], + "predefined_measures_only": True, + } + ) + assert "avg(orders.amount) > 100" in str(excinfo.value) + assert "query_policy: semantic-layer-only" in str(excinfo.value) + + +def test_rejects_composed_aggregate_in_compound_filter() -> None: + with pytest.raises(ValueError, match="compose aggregate expressions"): + _engine().query( + { + "measures": ["orders.revenue"], + "filters": ["orders.status = 'active' AND sum(orders.amount) > 5000"], + "predefined_measures_only": True, + } + ) + + +def test_allows_predefined_measure_having_filter() -> None: + result = _engine().query( + { + "measures": ["orders.revenue"], + "dimensions": ["orders.status"], + "filters": ["orders.revenue > 100"], + "predefined_measures_only": True, + } + ) + assert "having" in result.sql.lower() + + +def test_composed_aggregate_filter_allowed_when_flag_absent() -> None: + result = _engine().query( + { + "measures": ["orders.revenue"], + "filters": ["avg(orders.amount) > 100"], + } + ) + assert "having" in result.sql.lower() + + +def test_allows_predefined_measure_with_dimensions_and_filters() -> None: + result = _engine().query( + { + "measures": ["orders.revenue"], + "dimensions": ["orders.status"], + "filters": ["orders.status != 'cancelled'"], + "predefined_measures_only": True, + } + ) + assert "sum" in result.sql.lower() + + +def test_allows_unqualified_predefined_measure() -> None: + result = _engine().query( + { + "measures": ["revenue"], + "predefined_measures_only": True, + } + ) + assert "sum" in result.sql.lower() + + +def test_allows_predefined_derived_measure_chain() -> None: + result = _engine().query( + { + "measures": ["orders.aov"], + "predefined_measures_only": True, + } + ) + assert "sum" in result.sql.lower() + + +def test_composed_measures_allowed_when_flag_absent() -> None: + result = _engine().query({"measures": ["sum(orders.amount)"]}) + assert "sum" in result.sql.lower() From 5d17469601f7891fb124612ce4b718c0365e3ab0 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Fri, 3 Jul 2026 22:39:34 +0200 Subject: [PATCH 07/11] fix(telemetry): classify daemon query rejections as expected, not faults (#335) * fix(telemetry): classify daemon query rejections as expected, not faults Semantic-layer query rejections and warehouse-execution rejections from the sl_query MCP tool were wrapped as generic Errors, so reportException filed them as PostHog $exception faults indistinguishable from real ktx bugs. The daemon already separates a caller rejection (planner ValueError -> exit 3 / HTTP 400) from a crash. The Node runner now carries that distinction as a typed KtxDaemonComputeError, and a shared throwClassifiedQueryError promotes daemon input-rejections and warehouse rejections to KtxQueryError while daemon crashes and native JS faults still reach Error Tracking. query_semantic_layer stops report_exception-ing expected ValueErrors, and a missing 'file:' secret now raises KtxExpectedError so absent .ktx/secrets/-password stops filing faults. * chore: sync uv.lock to ktx-daemon/ktx-sl 0.15.0 --- .../cli/src/context/core/config-reference.ts | 11 ++- .../context/daemon/semantic-layer-compute.ts | 53 +++++++++++- .../src/context/mcp/local-project-ports.ts | 46 +++++++--- .../context/core/config-reference.test.ts | 12 +++ .../daemon/semantic-layer-compute.test.ts | 79 +++++++++++++++++- .../context/mcp/local-project-ports.test.ts | 83 +++++++++++++++++++ python/ktx-daemon/src/ktx_daemon/__main__.py | 16 +++- .../src/ktx_daemon/semantic_layer.py | 21 +++-- python/ktx-daemon/tests/test_cli.py | 16 ++++ .../ktx-daemon/tests/test_semantic_layer.py | 38 ++++++++- 10 files changed, 347 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/context/core/config-reference.ts b/packages/cli/src/context/core/config-reference.ts index f1f235ca..c6565988 100644 --- a/packages/cli/src/context/core/config-reference.ts +++ b/packages/cli/src/context/core/config-reference.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; +import { KtxExpectedError } from '../../errors.js'; /** @internal */ export function resolveKtxHomePath(path: string): string { @@ -28,7 +29,15 @@ export function resolveKtxConfigReference(value: string | undefined, env: NodeJS if (value.startsWith('file:')) { const filePath = resolveKtxHomePath(value.slice('file:'.length).trim()); - const fileValue = readFileSync(filePath, 'utf8').trim(); + let fileValue: string; + try { + fileValue = readFileSync(filePath, 'utf8').trim(); + } catch (error) { + // A `file:` secret whose target is missing or unreadable is a config + // problem (secrets/ is gitignored, so absent after a clone), not a ktx + // fault — classify it so it stays out of Error Tracking. + throw new KtxExpectedError(`Could not read the secret file referenced in ktx.yaml: ${filePath}`, { cause: error }); + } return fileValue.length > 0 ? fileValue : undefined; } diff --git a/packages/cli/src/context/daemon/semantic-layer-compute.ts b/packages/cli/src/context/daemon/semantic-layer-compute.ts index c590c3fa..5db2bb27 100644 --- a/packages/cli/src/context/daemon/semantic-layer-compute.ts +++ b/packages/cli/src/context/daemon/semantic-layer-compute.ts @@ -4,6 +4,44 @@ import { URL } from 'node:url'; import { spawn } from 'node:child_process'; import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js'; +// The daemon signals "the caller sent a well-formed request I refused as +// invalid" with a distinct subprocess exit code (INPUT_REJECTED_EXIT_CODE in +// python/ktx-daemon/__main__.py) or HTTP 400. Kept in sync across the boundary. +const DAEMON_INPUT_REJECTED_EXIT_CODE = 3; +const DAEMON_INPUT_REJECTED_HTTP_STATUS = 400; + +/** + * A ktx-daemon compute call failed. `inputRejected` is true when the daemon + * refused the request as invalid (a caller-driven outcome) rather than faulting; + * callers can promote those to an expected KtxQueryError while letting genuine + * daemon crashes reach Error Tracking. `detail` is the daemon's own message, + * unwrapped for surfacing to the caller. + */ +export class KtxDaemonComputeError extends Error { + readonly inputRejected: boolean; + readonly detail: string; + + constructor(message: string, options: { inputRejected: boolean; detail: string; cause?: unknown }) { + super(message, options.cause !== undefined ? { cause: options.cause } : undefined); + this.name = 'KtxDaemonComputeError'; + this.inputRejected = options.inputRejected; + this.detail = options.detail; + } +} + +/** The daemon reports HTTP failures as `{ "detail": "…" }`; fall back to the raw body. */ +function daemonHttpErrorDetail(raw: string): string { + try { + const parsed = JSON.parse(raw) as unknown; + if (parsed && typeof parsed === 'object' && typeof (parsed as { detail?: unknown }).detail === 'string') { + return (parsed as { detail: string }).detail; + } + } catch { + // Non-JSON body (e.g. a proxy error page) — surface it verbatim. + } + return raw; +} + interface KtxSemanticLayerComputeQueryResult { sql: string; dialect: string; @@ -128,7 +166,13 @@ function runProcessJson( const stdoutText = Buffer.concat(stdout).toString('utf8').trim(); const stderrText = Buffer.concat(stderr).toString('utf8').trim(); if (code !== 0) { - reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`)); + const detail = stderrText || `exit code ${code}`; + reject( + new KtxDaemonComputeError(`ktx-daemon ${subcommand} failed: ${detail}`, { + inputRejected: code === DAEMON_INPUT_REJECTED_EXIT_CODE, + detail, + }), + ); return; } try { @@ -168,7 +212,12 @@ function postJson(baseUrl: string): KtxDaemonHttpJsonRunner { const text = Buffer.concat(chunks).toString('utf8'); const statusCode = response.statusCode ?? 0; if (statusCode < 200 || statusCode >= 300) { - reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`)); + reject( + new KtxDaemonComputeError(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`, { + inputRejected: statusCode === DAEMON_INPUT_REJECTED_HTTP_STATUS, + detail: daemonHttpErrorDetail(text), + }), + ); return; } try { diff --git a/packages/cli/src/context/mcp/local-project-ports.ts b/packages/cli/src/context/mcp/local-project-ports.ts index 9f8db652..684c002c 100644 --- a/packages/cli/src/context/mcp/local-project-ports.ts +++ b/packages/cli/src/context/mcp/local-project-ports.ts @@ -1,5 +1,5 @@ import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; -import { KtxExpectedError } from '../../errors.js'; +import { KtxExpectedError, KtxQueryError, isNativeProgrammingFault } from '../../errors.js'; import { isDatabaseDriver, normalizeConnectionDriver } from '../../connection-drivers.js'; import { sqlDialectNotes } from '../../context/sql-analysis/dialect-notes.js'; import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; @@ -11,7 +11,7 @@ import { localConnectionInfoFromConfig, } from '../../context/connections/local-warehouse-descriptor.js'; import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; -import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { KtxDaemonComputeError, type KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; import type { KtxLocalProject } from '../../context/project/project.js'; import { createKtxEntityDetailsService } from '../../context/scan/entity-details.js'; import type { LocalScanMcpOptions } from '../../context/scan/local-scan.js'; @@ -34,6 +34,26 @@ interface CreateLocalProjectMcpContextPortsOptions { embeddingService: KtxEmbeddingPort | null; } +/** + * Reclassify a query-path failure. Warehouse/driver rejections and daemon + * input-rejections are caller-driven outcomes (KtxQueryError, kept out of Error + * Tracking while preserving the underlying diagnostics); native JS faults, daemon + * crashes, and already-expected errors propagate unchanged so genuine ktx bugs + * still reach Error Tracking. + */ +function throwClassifiedQueryError(error: unknown): never { + if (error instanceof KtxDaemonComputeError) { + if (error.inputRejected) { + throw new KtxQueryError(error.detail, { cause: error }); + } + throw error; + } + if (isNativeProgrammingFault(error) || error instanceof KtxExpectedError) { + throw error; + } + throw new KtxQueryError(error instanceof Error ? error.message : String(error), { cause: error }); +} + async function executeValidatedReadOnlySql( project: KtxLocalProject, options: CreateLocalProjectMcpContextPortsOptions, @@ -170,15 +190,19 @@ export function createLocalProjectMcpContextPorts( if (!options.semanticLayerCompute) { throw new Error('sl_query requires a semantic-layer query adapter.'); } - return compileLocalSlQuery(project, { - connectionId: input.connectionId, - query: input.query, - compute: options.semanticLayerCompute, - execute: Boolean(options.queryExecutor), - maxRows: input.query.limit, - queryExecutor: options.queryExecutor, - onProgress: executionOptions?.onProgress, - }); + try { + return await compileLocalSlQuery(project, { + connectionId: input.connectionId, + query: input.query, + compute: options.semanticLayerCompute, + execute: Boolean(options.queryExecutor), + maxRows: input.query.limit, + queryExecutor: options.queryExecutor, + onProgress: executionOptions?.onProgress, + }); + } catch (error) { + throwClassifiedQueryError(error); + } }, }, entityDetails: { diff --git a/packages/cli/test/context/core/config-reference.test.ts b/packages/cli/test/context/core/config-reference.test.ts index d7871cc7..ec538003 100644 --- a/packages/cli/test/context/core/config-reference.test.ts +++ b/packages/cli/test/context/core/config-reference.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { resolveKtxConfigReference, resolveKtxHomePath } from '../../../src/context/core/config-reference.js'; +import { KtxExpectedError } from '../../../src/errors.js'; describe('ktx config references', () => { it('resolves env references without returning empty values', () => { @@ -22,6 +23,17 @@ describe('ktx config references', () => { expect(resolveKtxConfigReference(`file:${keyPath}`, {})).toBe('file-gateway-key'); }); + it('raises an expected error when a file reference is missing', () => { + const missing = join(tmpdir(), `ktx-config-reference-missing-${process.pid}`, 'absent-password'); + try { + resolveKtxConfigReference(`file:${missing}`, {}); + expect.unreachable('expected a thrown error for the missing secret file'); + } catch (error) { + expect(error).toBeInstanceOf(KtxExpectedError); + expect((error as Error).message).toContain(missing); + } + }); + it('returns literal values unchanged after trimming blank-only values', () => { expect(resolveKtxConfigReference('provider/model', {})).toBe('provider/model'); expect(resolveKtxConfigReference(' ', {})).toBeUndefined(); diff --git a/packages/cli/test/context/daemon/semantic-layer-compute.test.ts b/packages/cli/test/context/daemon/semantic-layer-compute.test.ts index e6bbddbc..c79d8ad9 100644 --- a/packages/cli/test/context/daemon/semantic-layer-compute.test.ts +++ b/packages/cli/test/context/daemon/semantic-layer-compute.test.ts @@ -1,7 +1,11 @@ import { once } from 'node:events'; import { createServer } from 'node:http'; import { describe, expect, it, vi } from 'vitest'; -import { createHttpSemanticLayerComputePort, createPythonSemanticLayerComputePort } from '../../../src/context/daemon/semantic-layer-compute.js'; +import { + createHttpSemanticLayerComputePort, + createPythonSemanticLayerComputePort, + KtxDaemonComputeError, +} from '../../../src/context/daemon/semantic-layer-compute.js'; const source = { name: 'orders', @@ -174,6 +178,79 @@ describe('createPythonSemanticLayerComputePort', () => { }); }); +describe('KtxDaemonComputeError classification', () => { + const query = { sources: [source], dialect: 'postgres', query: { measures: ['count(*)'], dimensions: [] } }; + + function exitingPort(code: number, stderr: string) { + return createPythonSemanticLayerComputePort({ + command: process.execPath, + args: [ + '-e', + `process.stdin.on('data',()=>{});process.stdin.on('end',()=>{process.stderr.write(${JSON.stringify(stderr)});process.exit(${code})});`, + ], + }); + } + + async function rejection(promise: Promise): Promise { + const error = await promise.then( + () => null, + (thrown: unknown) => thrown, + ); + expect(error).toBeInstanceOf(KtxDaemonComputeError); + return error as KtxDaemonComputeError; + } + + it('marks a subprocess input-rejection (exit 3) as inputRejected', async () => { + const error = await rejection(exitingPort(3, 'Measure expr does not reference any source').query(query)); + expect(error.inputRejected).toBe(true); + expect(error.detail).toContain('does not reference any source'); + }); + + it('marks a subprocess fault (exit 1) as not inputRejected', async () => { + const error = await rejection(exitingPort(1, 'Traceback: boom').query(query)); + expect(error.inputRejected).toBe(false); + expect(error.detail).toContain('boom'); + }); + + async function statusPort(statusCode: number, body: string): Promise<{ port: ReturnType; close: () => void }> { + const server = createServer((_request, response) => { + response.writeHead(statusCode, { 'content-type': 'application/json' }); + response.end(body); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('expected TCP server address'); + } + return { + port: createHttpSemanticLayerComputePort({ baseUrl: `http://127.0.0.1:${address.port}` }), + close: () => server.close(), + }; + } + + it('marks an HTTP 400 as inputRejected and unwraps the daemon detail', async () => { + const { port, close } = await statusPort(400, JSON.stringify({ detail: 'Measure expr does not reference any source' })); + try { + const error = await rejection(port.query(query)); + expect(error.inputRejected).toBe(true); + expect(error.detail).toBe('Measure expr does not reference any source'); + } finally { + close(); + } + }); + + it('marks an HTTP 500 as not inputRejected', async () => { + const { port, close } = await statusPort(500, JSON.stringify({ detail: 'Daemon request failed: boom' })); + try { + const error = await rejection(port.query(query)); + expect(error.inputRejected).toBe(false); + } finally { + close(); + } + }); +}); + describe('createHttpSemanticLayerComputePort', () => { it('calls semantic query and validate HTTP endpoints through an injected runner', async () => { const requestJson = vi.fn(async (path: string) => { diff --git a/packages/cli/test/context/mcp/local-project-ports.test.ts b/packages/cli/test/context/mcp/local-project-ports.test.ts index ee28dd38..1220adf6 100644 --- a/packages/cli/test/context/mcp/local-project-ports.test.ts +++ b/packages/cli/test/context/mcp/local-project-ports.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { initKtxProject } from '../../../src/context/project/project.js'; import { KtxExpectedError, KtxQueryError } from '../../../src/errors.js'; +import { KtxDaemonComputeError } from '../../../src/context/daemon/semantic-layer-compute.js'; import { createKtxConnectorCapabilities, type KtxQueryResult, type KtxScanConnector, type KtxSchemaSnapshot } from '../../../src/context/scan/types.js'; import { SemanticLayerService } from '../../../src/context/sl/semantic-layer.service.js'; import type { SemanticLayerSource } from '../../../src/context/sl/types.js'; @@ -1177,4 +1178,86 @@ describe('createLocalProjectMcpContextPorts', () => { }), ); }); + + async function seedOrdersWarehouse() { + const project = await initKtxProject({ projectDir: tempDir }); + project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL' }; + await seedSlSourceFile(project, { + connectionId: 'warehouse', + sourceName: 'orders', + yaml: ['name: orders', 'table: public.orders', 'grain:', ' - id', 'columns:', ' - name: id', ' type: number', 'joins: []', 'measures: []', ''].join('\n'), + }); + return project; + } + + it('promotes a daemon input-rejection to an expected KtxQueryError carrying the daemon detail', async () => { + const project = await seedOrdersWarehouse(); + const semanticLayerCompute = { + validateSources: vi.fn(), + generateSources: vi.fn(), + query: vi.fn(async () => { + throw new KtxDaemonComputeError("ktx-daemon semantic-query failed: Measure expr 'count(*)' does not reference any source", { + inputRejected: true, + detail: "Measure expr 'count(*)' does not reference any source", + }); + }), + }; + const ports = createLocalProjectMcpContextPorts(project, { semanticLayerCompute, embeddingService: null }); + + const rejection = ports.semanticLayer?.query({ + connectionId: 'warehouse', + query: { measures: [{ expr: 'count(*)', name: 'n' }], dimensions: [] }, + }); + await expect(rejection).rejects.toBeInstanceOf(KtxQueryError); + await expect(rejection).rejects.toThrow("Measure expr 'count(*)' does not reference any source"); + }); + + it('leaves a daemon crash as an unexpected fault', async () => { + const project = await seedOrdersWarehouse(); + const semanticLayerCompute = { + validateSources: vi.fn(), + generateSources: vi.fn(), + query: vi.fn(async () => { + throw new KtxDaemonComputeError('ktx-daemon semantic-query failed: KeyError: boom', { + inputRejected: false, + detail: 'KeyError: boom', + }); + }), + }; + const ports = createLocalProjectMcpContextPorts(project, { semanticLayerCompute, embeddingService: null }); + + const rejection = ports.semanticLayer?.query({ + connectionId: 'warehouse', + query: { measures: ['orders.order_count'], dimensions: [] }, + }); + await expect(rejection).rejects.toBeInstanceOf(KtxDaemonComputeError); + await expect(rejection).rejects.not.toBeInstanceOf(KtxQueryError); + }); + + it('wraps a warehouse execution rejection from sl_query as KtxQueryError', async () => { + const project = await seedOrdersWarehouse(); + const semanticLayerCompute = { + validateSources: vi.fn(), + generateSources: vi.fn(), + query: vi.fn(async () => ({ + sql: 'select count(*) from public.orders', + dialect: 'postgres', + columns: [{ name: 'orders.order_count' }], + plan: {}, + })), + }; + const queryExecutor = { + execute: vi.fn(async () => { + throw new Error("Unknown column '검사 유형' in 'SELECT'"); + }), + }; + const ports = createLocalProjectMcpContextPorts(project, { semanticLayerCompute, queryExecutor, embeddingService: null }); + + const rejection = ports.semanticLayer?.query({ + connectionId: 'warehouse', + query: { measures: ['orders.order_count'], dimensions: [], limit: 5 }, + }); + await expect(rejection).rejects.toBeInstanceOf(KtxQueryError); + await expect(rejection).rejects.toThrow("Unknown column '검사 유형' in 'SELECT'"); + }); }); diff --git a/python/ktx-daemon/src/ktx_daemon/__main__.py b/python/ktx-daemon/src/ktx_daemon/__main__.py index 83fe0367..2b344c82 100644 --- a/python/ktx-daemon/src/ktx_daemon/__main__.py +++ b/python/ktx-daemon/src/ktx_daemon/__main__.py @@ -35,6 +35,12 @@ from ktx_daemon.source_generation import ( generate_sources_response, ) +# The caller (the Node client) sent a well-formed request the compute layer +# refused as invalid — e.g. the planner rejecting an agent's query. A distinct +# exit code lets the client classify it as an expected outcome rather than a +# ktx fault. Kept in sync with DAEMON_INPUT_REJECTED_EXIT_CODE on the Node side. +INPUT_REJECTED_EXIT_CODE = 3 + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="ktx-daemon") @@ -210,9 +216,17 @@ def main(argv: list[str] | None = None) -> int: return 2 sys.stdout.write(response.model_dump_json() + "\n") return 0 - except (json.JSONDecodeError, ValidationError, ValueError) as error: + except (json.JSONDecodeError, ValidationError) as error: + # Malformed request envelope — ktx sent bad JSON or a mis-shaped payload. + # That is a ktx fault, so keep the generic non-zero code (JSONDecodeError + # subclasses ValueError, so this clause must precede the ValueError one). sys.stderr.write(f"{error}\n") return 1 + except ValueError as error: + # The compute layer refused a well-formed request as invalid (e.g. the + # planner rejecting the agent's measures). Expected, not a fault. + sys.stderr.write(f"{error}\n") + return INPUT_REJECTED_EXIT_CODE except Exception as error: from ktx_daemon.telemetry import report_exception diff --git a/python/ktx-daemon/src/ktx_daemon/semantic_layer.py b/python/ktx-daemon/src/ktx_daemon/semantic_layer.py index b7e7f133..f0e7a2f4 100644 --- a/python/ktx-daemon/src/ktx_daemon/semantic_layer.py +++ b/python/ktx-daemon/src/ktx_daemon/semantic_layer.py @@ -6,7 +6,7 @@ import time from typing import Any from ktx_daemon.telemetry import error_class, report_exception, track_telemetry_event -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, ValidationError from semantic_layer.duplicate_check import validate_measure_duplicates from semantic_layer.engine import SemanticEngine from semantic_layer.models import QueryResult, SourceDefinition @@ -150,13 +150,18 @@ def query_semantic_layer( track_telemetry_event( "sql_gen_completed", sql_fields, project_id=request.project_id ) - report_exception( - error, - source="semantic-query", - handled=True, - fatal=False, - project_id=request.project_id, - ) + # A ValueError is the engine refusing the caller's query — an expected + # rejection surfaced to the agent, not a ktx fault. Keep it out of Error + # Tracking (a pydantic ValidationError subclasses ValueError but means a + # malformed request envelope, so it stays a reported fault). + if not isinstance(error, ValueError) or isinstance(error, ValidationError): + report_exception( + error, + source="semantic-query", + handled=True, + fatal=False, + project_id=request.project_id, + ) raise diff --git a/python/ktx-daemon/tests/test_cli.py b/python/ktx-daemon/tests/test_cli.py index 76376320..32a98f70 100644 --- a/python/ktx-daemon/tests/test_cli.py +++ b/python/ktx-daemon/tests/test_cli.py @@ -91,6 +91,22 @@ def test_command_returns_nonzero_for_invalid_json() -> None: assert "Expecting property name enclosed in double quotes" in result.stderr +def test_semantic_query_rejects_invalid_query_with_input_rejected_code() -> None: + # A planner ValueError (agent's measure references no source) is an expected + # input rejection, distinguished from a fault by INPUT_REJECTED_EXIT_CODE (3). + result = run_daemon_command( + "semantic-query", + { + "sources": [ORDERS_SOURCE], + "dialect": "postgres", + "query": {"measures": ["count(*)"], "dimensions": []}, + }, + ) + + assert result.returncode == 3, result.stderr + assert "does not reference any source" in result.stderr + + def test_serve_http_command_starts_uvicorn_without_reading_stdin( monkeypatch, ) -> None: diff --git a/python/ktx-daemon/tests/test_semantic_layer.py b/python/ktx-daemon/tests/test_semantic_layer.py index 2a1f1da9..a8fa8f39 100644 --- a/python/ktx-daemon/tests/test_semantic_layer.py +++ b/python/ktx-daemon/tests/test_semantic_layer.py @@ -125,7 +125,7 @@ def test_query_semantic_layer_emits_plan_and_sql_debug_events( assert "public.orders" not in captured.err -def test_query_semantic_layer_reports_exception(monkeypatch) -> None: +def test_query_semantic_layer_reports_unexpected_fault(monkeypatch) -> None: from ktx_daemon import semantic_layer as semantic_layer_module reports: list[dict[str, object]] = [] @@ -133,12 +133,16 @@ def test_query_semantic_layer_reports_exception(monkeypatch) -> None: def fake_report(exception: BaseException, **kwargs: object) -> None: reports.append({"exception": exception, **kwargs}) - monkeypatch.setattr(semantic_layer_module, "report_exception", fake_report) + def boom(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("engine construction failed") - with pytest.raises(ValueError): + monkeypatch.setattr(semantic_layer_module, "report_exception", fake_report) + monkeypatch.setattr(semantic_layer_module.SemanticEngine, "from_sources", boom) + + with pytest.raises(RuntimeError): query_semantic_layer( SemanticLayerQueryRequest( - sources=[ORDERS_SOURCE, ORDERS_SOURCE], + sources=[ORDERS_SOURCE], dialect="postgres", projectId="a" * 64, query={"measures": ["orders.order_count"]}, @@ -152,6 +156,32 @@ def test_query_semantic_layer_reports_exception(monkeypatch) -> None: assert reports[0]["project_id"] == "a" * 64 +def test_query_semantic_layer_does_not_report_expected_query_rejection( + monkeypatch, +) -> None: + from ktx_daemon import semantic_layer as semantic_layer_module + + reports: list[dict[str, object]] = [] + monkeypatch.setattr( + semantic_layer_module, + "report_exception", + lambda *_args, **kwargs: reports.append(kwargs), + ) + + # A planner ValueError is the engine refusing the agent's query — surfaced to + # the caller and re-raised, but never filed as a ktx fault. + with pytest.raises(ValueError, match="does not reference any source"): + query_semantic_layer( + SemanticLayerQueryRequest( + sources=[ORDERS_SOURCE], + dialect="postgres", + query={"measures": ["count(*)"]}, + ) + ) + + assert reports == [] + + def test_semantic_layer_request_rejects_project_id_field_name() -> None: with pytest.raises(ValueError): SemanticLayerQueryRequest( From a0d19ba26fd98a6277e6789f9dd8e0a0e1413f0c Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Fri, 3 Jul 2026 23:17:33 +0200 Subject: [PATCH 08/11] fix(sl): classify semantic-query request rejections as expected, not faults (#339) The daemon rejects an invalid semantic-query request (unknown source, ambiguous measure, no join path) with a plain ValueError; the Node compute port now maps the daemon's exit code 3 / HTTP 400 to KtxExpectedError so these routine, caller-driven rejections stay out of Error Tracking. A dedicated SemanticLayerRequestError(ValueError) is raised only for engine rejections and routed through both daemon transports and the HTTP handler. Because pydantic v2 ValidationError subclasses ValueError, malformed sources or responses (contract faults) are kept as faults: they are reported and mapped to exit 1 / HTTP 500 / plain Error on every path. Non-object stdin is likewise a fault (exit 1), not exit 3. --- python/ktx-daemon/src/ktx_daemon/app.py | 9 +++++++++ python/ktx-daemon/tests/test_app.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/python/ktx-daemon/src/ktx_daemon/app.py b/python/ktx-daemon/src/ktx_daemon/app.py index 63487439..6c790931 100644 --- a/python/ktx-daemon/src/ktx_daemon/app.py +++ b/python/ktx-daemon/src/ktx_daemon/app.py @@ -12,6 +12,7 @@ from typing import Any from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse, Response +from pydantic import ValidationError from ktx_daemon import VERSION from ktx_daemon.code_execution import ( @@ -268,6 +269,14 @@ def create_app( ) -> SemanticLayerQueryResponse: try: return query_semantic_layer(request) + except ValidationError as error: + # A malformed source or response is a ktx contract fault, not a + # caller rejection; surface it as a server fault (500) so the Node + # client does not classify it as an expected query rejection, matching + # the subprocess transport (exit 1) and query_semantic_layer's own + # report. ValidationError subclasses ValueError, so catch it first. + logger.exception("Semantic query failed on a malformed source") + raise HTTPException(status_code=500, detail=str(error)) from error except ValueError as error: logger.warning("Semantic query rejected: %s", error) raise HTTPException(status_code=400, detail=str(error)) from error diff --git a/python/ktx-daemon/tests/test_app.py b/python/ktx-daemon/tests/test_app.py index fffc2899..e6ec9ed6 100644 --- a/python/ktx-daemon/tests/test_app.py +++ b/python/ktx-daemon/tests/test_app.py @@ -424,6 +424,24 @@ def test_semantic_query_endpoint_maps_value_error_to_400() -> None: assert "missing.order_count" in response.json()["detail"] +def test_semantic_query_endpoint_maps_malformed_source_to_fault() -> None: + client = TestClient(create_app(), raise_server_exceptions=False) + invalid_source = { + key: value for key, value in ORDERS_SOURCE.items() if key != "table" + } + + response = client.post( + "/semantic-layer/query", + json={ + "sources": [invalid_source], + "dialect": "postgres", + "query": {"measures": ["orders.order_count"], "dimensions": []}, + }, + ) + + assert response.status_code == 500 + + def test_semantic_validate_endpoint_returns_structured_validation() -> None: client = TestClient(create_app()) invalid_source = { From 4ebce754496dc6cda0637824d44876499efecd22 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Fri, 3 Jul 2026 23:19:33 +0200 Subject: [PATCH 09/11] fix(sl): correct reserved-word/week-grain SQL and classify sl_query errors (#340) Reserved-word columns (like, default, ...) referenced as source.col were quoted with postgres double quotes even on BigQuery/MySQL, where a double-quoted token is a string literal, not an identifier -- the "Unexpected string literal" semantic-layer errors. quote_reserved_identifiers now uses the identifier quote char of the dialect it will be parsed in (backtick for BigQuery/MySQL), threaded through the planner and generator parse sites; week_ granularity now emits WEEK() on BigQuery instead of the invalid WEEK_MONDAY. On the telemetry side, warehouse rejections from the sl_query execution path are classified as expected (KtxQueryError) via a new shared markExpected() helper, so routine agent/warehouse query failures stop reaching PostHog Error Tracking as ktx faults; the sql_execution catch is refactored onto the same helper. The daemon-compile boundary is deliberately left unclassified here so genuine daemon crashes stay visible. --- .../ktx-sl/semantic_layer/duplicate_check.py | 2 +- python/ktx-sl/semantic_layer/generator.py | 32 +++++-- python/ktx-sl/semantic_layer/graph.py | 2 +- python/ktx-sl/semantic_layer/parser.py | 42 +++++++-- python/ktx-sl/semantic_layer/planner.py | 6 +- .../tests/test_dialect_identifier_quoting.py | 93 +++++++++++++++++++ 6 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 python/ktx-sl/tests/test_dialect_identifier_quoting.py diff --git a/python/ktx-sl/semantic_layer/duplicate_check.py b/python/ktx-sl/semantic_layer/duplicate_check.py index 05f91cb4..d2e7658b 100644 --- a/python/ktx-sl/semantic_layer/duplicate_check.py +++ b/python/ktx-sl/semantic_layer/duplicate_check.py @@ -42,7 +42,7 @@ def validate_measure_duplicates( parsed: list[tuple[str, exp.Expression | None, str | None, frozenset[str]]] = [] for m in source.measures: try: - quoted = quote_reserved_identifiers(m.expr) + quoted = quote_reserved_identifiers(m.expr, dialect) tree = sqlglot.parse_one(f"SELECT {quoted}", read=dialect) expr_node = tree.expressions[0] if tree.expressions else None except Exception: diff --git a/python/ktx-sl/semantic_layer/generator.py b/python/ktx-sl/semantic_layer/generator.py index 8207bac1..c44f3764 100644 --- a/python/ktx-sl/semantic_layer/generator.py +++ b/python/ktx-sl/semantic_layer/generator.py @@ -671,7 +671,8 @@ class SqlGenerator: """Apply a measure-level filter by injecting CASE WHEN into each aggregate.""" try: tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", read=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + read=self.dialect, ) select_expr = tree.expressions[0] if isinstance(select_expr, exp.Alias): @@ -723,7 +724,8 @@ class SqlGenerator: def _translate_custom_funcs(self, expr: str) -> str: """Translate custom functions: median(), percentile(), count_distinct().""" tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", read=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + read=self.dialect, ) has_custom = False @@ -767,7 +769,8 @@ class SqlGenerator: def _extract_outer_aggregate(self, expr: str) -> tuple[str | None, str | None]: """Use AST to extract the outer aggregate function name and inner expression.""" tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", read=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + read=self.dialect, ) select_expr = tree.expressions[0] if isinstance(select_expr, exp.Alias): @@ -788,7 +791,8 @@ class SqlGenerator: if not replacements: return expr tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", read=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + read=self.dialect, ) def _replace(node): @@ -857,13 +861,26 @@ class SqlGenerator: return self._time_trunc(dim.granularity, field) return field + _WEEKDAYS = frozenset( + {"sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"} + ) + + def _bigquery_date_part(self, granularity: str) -> str: + # BigQuery spells a week starting on a given weekday as WEEK(MONDAY), + # not the bare WEEK_MONDAY that other systems accept. + if granularity.startswith("week_"): + weekday = granularity[len("week_") :] + if weekday in self._WEEKDAYS: + return f"WEEK({weekday.upper()})" + return granularity.upper() + def _time_trunc(self, granularity: str, field: str) -> str: """Generate dialect-appropriate time truncation expression.""" g = granularity.lower() if self.dialect == "sqlite": return self._sqlite_time_trunc(g, field) if self.dialect == "bigquery": - return f"DATE_TRUNC({field}, {g.upper()})" + return f"DATE_TRUNC({field}, {self._bigquery_date_part(g)})" if self.dialect == "mysql": return self._mysql_time_trunc(g, field) return f"DATE_TRUNC('{g}', {field})" @@ -1257,7 +1274,8 @@ class SqlGenerator: """Qualify bare column references in a computed column expression with the source name.""" try: tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", read=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + read=self.dialect, ) def _qualify(node: exp.Expression) -> exp.Expression: @@ -1403,7 +1421,7 @@ class SqlGenerator: try: # Quote reserved-word identifiers so target dialect parsers do not # confuse them with keywords (e.g. Snowflake's SAMPLE, QUALIFY). - quoted_outer = quote_reserved_identifiers(outer_sql) + quoted_outer = quote_reserved_identifiers(outer_sql, self.dialect) results = sqlglot.transpile( quoted_outer, read=self.dialect, write=self.dialect ) diff --git a/python/ktx-sl/semantic_layer/graph.py b/python/ktx-sl/semantic_layer/graph.py index b37b54d7..fafa7953 100644 --- a/python/ktx-sl/semantic_layer/graph.py +++ b/python/ktx-sl/semantic_layer/graph.py @@ -254,7 +254,7 @@ class JoinGraph: from sqlglot import exp as _exp from semantic_layer.parser import quote_reserved_identifiers - quoted = quote_reserved_identifiers(on_clause) + quoted = quote_reserved_identifiers(on_clause, self.dialect) tree = sqlglot.parse_one( f"SELECT 1 FROM _a JOIN _b ON {quoted}", read=self.dialect ) diff --git a/python/ktx-sl/semantic_layer/parser.py b/python/ktx-sl/semantic_layer/parser.py index b6f2e3d7..1d9c56f3 100644 --- a/python/ktx-sl/semantic_layer/parser.py +++ b/python/ktx-sl/semantic_layer/parser.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field import sqlglot from sqlglot import exp +from sqlglot.dialects.dialect import Dialect # DIALECT CONVENTION: # `ExpressionParser` wraps read-only AST walks over user-authored @@ -154,12 +155,29 @@ def _strip_quotes(name: str) -> str: return name -def quote_reserved_identifiers(expr: str) -> str: +@functools.lru_cache(maxsize=32) +def _identifier_quote(dialect: str) -> str: + """The character `dialect` quotes identifiers with (backtick for BigQuery/MySQL, + double-quote for ANSI dialects). A reserved-word column must be quoted with this so a + backtick dialect does not read a double-quoted identifier as a string literal.""" + try: + start = Dialect.get_or_raise(dialect).IDENTIFIER_START + except Exception: + start = '"' + return start or '"' + + +def quote_reserved_identifiers(expr: str, dialect: str = "postgres") -> str: """Quote source.column references where either part is a SQL reserved word. - String literals are masked before processing to prevent matching - dotted identifiers inside quoted strings like 'group.value'. + Quoted with `dialect`'s identifier character, because the caller parses the + result in that dialect: a double-quoted identifier on BigQuery/MySQL is a + string literal, not an identifier. String literals are masked before + processing to prevent matching dotted identifiers inside quoted strings like + 'group.value'. """ + quote = _identifier_quote(dialect) + # Mask string literals to avoid matching inside them literals: list[str] = [] @@ -172,16 +190,16 @@ def quote_reserved_identifiers(expr: str) -> str: def _quote_match(m: re.Match) -> str: source, col = m.group(1), m.group(2) start = m.start() - if start > 0 and masked[start - 1] == '"': + if start > 0 and masked[start - 1] == quote: return m.group(0) needs_quote = False source_q = source col_q = col if source.lower() in _SQL_RESERVED: - source_q = f'"{source}"' + source_q = f"{quote}{source}{quote}" needs_quote = True if col.lower() in _SQL_RESERVED: - col_q = f'"{col}"' + col_q = f"{quote}{col}{quote}" needs_quote = True if needs_quote: return f"{source_q}.{col_q}" @@ -196,13 +214,13 @@ def quote_reserved_identifiers(expr: str) -> str: return result -def _predicate_select(expr: str) -> str: +def _predicate_select(expr: str, dialect: str = "postgres") -> str: """Wrap a user expression as `SELECT * WHERE …`, quoting reserved identifiers. Predicate, not projection: T-SQL reads a top-level `col = 'value'` projection as the `alias = expression` form and would compile the filter to `'value' AS col`. """ - return f"SELECT * WHERE {quote_reserved_identifiers(expr)}" + return f"SELECT * WHERE {quote_reserved_identifiers(expr, dialect)}" @functools.lru_cache(maxsize=256) @@ -220,7 +238,11 @@ def parse_predicate(expr: str, dialect: str) -> exp.Expression: Uncached, so the result is safe to `.transform()`; raises on unparseable input. """ - return sqlglot.parse_one(_predicate_select(expr), read=dialect).find(exp.Where).this + return ( + sqlglot.parse_one(_predicate_select(expr, dialect), read=dialect) + .find(exp.Where) + .this + ) class ExpressionParser: @@ -237,7 +259,7 @@ class ExpressionParser: def _parse_as_select(self, expr: str) -> exp.Expression: """Parse a user fragment for read-only AST walks, via the parse cache.""" - return _cached_parse_select(_predicate_select(expr), self.dialect) + return _cached_parse_select(_predicate_select(expr, self.dialect), self.dialect) def parse( self, diff --git a/python/ktx-sl/semantic_layer/planner.py b/python/ktx-sl/semantic_layer/planner.py index 2e7f0c74..6ba4f7d3 100644 --- a/python/ktx-sl/semantic_layer/planner.py +++ b/python/ktx-sl/semantic_layer/planner.py @@ -694,7 +694,8 @@ class QueryPlanner: all_dep_names.add(dep_name) named.add(dep_name) tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", dialect=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + dialect=self.dialect, ) def _replace(node): @@ -738,7 +739,8 @@ class QueryPlanner: """Reject expressions with nested aggregate functions (e.g., avg(sum(x))).""" try: tree = sqlglot.parse_one( - f"SELECT {quote_reserved_identifiers(expr)}", dialect=self.dialect + f"SELECT {quote_reserved_identifiers(expr, self.dialect)}", + dialect=self.dialect, ) for agg_node in tree.find_all(exp.AggFunc): # Check if this aggregate contains another aggregate inside diff --git a/python/ktx-sl/tests/test_dialect_identifier_quoting.py b/python/ktx-sl/tests/test_dialect_identifier_quoting.py new file mode 100644 index 00000000..452774ac --- /dev/null +++ b/python/ktx-sl/tests/test_dialect_identifier_quoting.py @@ -0,0 +1,93 @@ +"""Reserved-word identifiers and week-of-day granularity must produce valid SQL +in the connection's native dialect. + +Regression: on backtick dialects (BigQuery, MySQL) a reserved-word column such +as `default` or `like` was quoted with postgres-style double quotes, which those +dialects read as a *string literal* — `WHERE loans.'like' = 'x'` — yielding the +production error `Syntax error: Unexpected string literal "like"`. +""" + +from __future__ import annotations + +import sqlglot + +from conftest import make_engine + + +def _loans_engine(dialect: str): + source = { + "name": "loans", + "table": "mydataset.loans", + "grain": ["id"], + "columns": [ + {"name": "id", "type": "number"}, + {"name": "amount", "type": "number"}, + {"name": "default", "type": "boolean"}, + {"name": "like", "type": "string"}, + {"name": "created_at", "type": "time"}, + ], + "measures": [{"name": "total", "expr": "sum(amount)"}], + } + return make_engine({"loans": source}, dialect=dialect) + + +def test_reserved_word_column_filter_valid_on_bigquery(): + sql = ( + _loans_engine("bigquery") + .query({"measures": ["loans.total"], "filters": ["loans.default = true"]}) + .sql + ) + sqlglot.parse_one(sql, read="bigquery") # must not raise + assert "`default`" in sql + assert "'default'" not in sql + + +def test_reserved_word_column_filter_valid_on_mysql(): + sql = ( + _loans_engine("mysql") + .query({"measures": ["loans.total"], "filters": ["loans.default = true"]}) + .sql + ) + sqlglot.parse_one(sql, read="mysql") + assert "`default`" in sql + assert "'default'" not in sql + + +def test_like_column_filter_valid_on_bigquery(): + # Mirrors the production message: Unexpected string literal "like". + sql = ( + _loans_engine("bigquery") + .query({"measures": ["loans.total"], "filters": ["loans.like = 'x'"]}) + .sql + ) + sqlglot.parse_one(sql, read="bigquery") + assert "`like`" in sql + assert "'like'" not in sql + + +def test_reserved_word_column_still_double_quoted_on_snowflake(): + sql = ( + _loans_engine("snowflake") + .query({"measures": ["loans.total"], "filters": ["loans.default = true"]}) + .sql + ) + sqlglot.parse_one(sql, read="snowflake") + assert '"default"' in sql + + +def test_week_weekday_granularity_translated_on_bigquery(): + sql = ( + _loans_engine("bigquery") + .query( + { + "measures": ["loans.total"], + "dimensions": [ + {"field": "loans.created_at", "granularity": "week_monday"} + ], + } + ) + .sql + ) + sqlglot.parse_one(sql, read="bigquery") # must not raise + assert "WEEK_MONDAY" not in sql + assert "WEEK(MONDAY)" in sql From a6dd8cf7303d2cbe2b60954816d022928a0eedef Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 3 Jul 2026 21:22:57 +0000 Subject: [PATCH 10/11] chore(release): 0.16.0 [skip ci] ## [0.16.0](https://github.com/Kaelio/ktx/compare/v0.15.0...v0.16.0) (2026-07-03) ### Features * Add duckdb connector ([#308](https://github.com/Kaelio/ktx/issues/308)) ([3c4fcc2](https://github.com/Kaelio/ktx/commit/3c4fcc27c790624d45ebaff23e84722d22b3573f)) * **athena:** first-class AWS Athena warehouse identity (SL + BI mapping) ([#332](https://github.com/Kaelio/ktx/issues/332)) ([f310391](https://github.com/Kaelio/ktx/commit/f310391da53642a98defe19cef9584389ce03ff7)) * **connector:** add Amazon Athena connector via Glue Data Catalog ([#309](https://github.com/Kaelio/ktx/issues/309)) ([fe7e6bd](https://github.com/Kaelio/ktx/commit/fe7e6bd1faced28b977cd4c8d61a13a08dcf7514)) * query_policy semantic-layer-only restricts agents to predefined semantic-layer measures ([#334](https://github.com/Kaelio/ktx/issues/334)) ([a651b82](https://github.com/Kaelio/ktx/commit/a651b82e2f2e160c321ed9a3487d7f2e2f5ebebc)) ### Bug Fixes * **deps:** patch 22 Dependabot security alerts ([#328](https://github.com/Kaelio/ktx/issues/328)) ([6d01030](https://github.com/Kaelio/ktx/commit/6d0103074595cfb84a3508782810c2dbf0774774)) * **sl:** classify semantic-query request rejections as expected, not faults ([#339](https://github.com/Kaelio/ktx/issues/339)) ([a0d19ba](https://github.com/Kaelio/ktx/commit/a0d19ba26fd98a6277e6789f9dd8e0a0e1413f0c)) * **sl:** correct reserved-word/week-grain SQL and classify sl_query errors ([#340](https://github.com/Kaelio/ktx/issues/340)) ([4ebce75](https://github.com/Kaelio/ktx/commit/4ebce754496dc6cda0637824d44876499efecd22)) * **telemetry:** classify daemon query rejections as expected, not faults ([#335](https://github.com/Kaelio/ktx/issues/335)) ([5d17469](https://github.com/Kaelio/ktx/commit/5d17469601f7891fb124612ce4b718c0365e3ab0)) ### Documentation * reflect recently added connectors in ingestion diagram and README ([#333](https://github.com/Kaelio/ktx/issues/333)) ([66768fe](https://github.com/Kaelio/ktx/commit/66768fe009e166d573b5a4c337f30a254ca69f1a)) --- package.json | 2 +- packages/cli/package.json | 2 +- python/ktx-daemon/pyproject.toml | 2 +- python/ktx-sl/pyproject.toml | 2 +- release-policy.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7c55c07f..e9f94149 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ktx-workspace", - "version": "0.15.0", + "version": "0.16.0", "description": "Workspace root for ktx packages", "private": true, "type": "module", diff --git a/packages/cli/package.json b/packages/cli/package.json index f609b571..af4bf65c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@kaelio/ktx", - "version": "0.15.0", + "version": "0.16.0", "description": "Standalone ktx context layer for data agents", "author": { "name": "Kaelio", diff --git a/python/ktx-daemon/pyproject.toml b/python/ktx-daemon/pyproject.toml index d75099fd..52bb6ba0 100644 --- a/python/ktx-daemon/pyproject.toml +++ b/python/ktx-daemon/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ktx-daemon" -version = "0.15.0" +version = "0.16.0" description = "Portable compute package for ktx semantic-layer operations" readme = "README.md" requires-python = ">=3.13" diff --git a/python/ktx-sl/pyproject.toml b/python/ktx-sl/pyproject.toml index 2aabe545..fee2453d 100644 --- a/python/ktx-sl/pyproject.toml +++ b/python/ktx-sl/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ktx-sl" -version = "0.15.0" +version = "0.16.0" description = "Agent-first semantic layer engine with aggregate locality" readme = "README.md" requires-python = ">=3.13" diff --git a/release-policy.json b/release-policy.json index b7572bc9..e4cffaa2 100644 --- a/release-policy.json +++ b/release-policy.json @@ -19,7 +19,7 @@ }, "publishedPackageSmoke": { "packageName": "@kaelio/ktx", - "version": "0.15.0", + "version": "0.16.0", "registry": null }, "runtimeInstaller": { From 49a4ae6f5b28c904da921afc0ee72da6db7ac202 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Tue, 7 Jul 2026 14:38:33 +0200 Subject: [PATCH 11/11] fix(ci): repair slow sl test and sync uv.lock to 0.16.0 (#345) * test(sl): assert predefined_measures_only in slow sl query test The slow-only sl query test asserted the outgoing compute.query() request with an exact nested query object, so the predefined_measures_only flag added by the semantic-layer-only query policy (#334) broke the match and turned the Slow TypeScript tests CI job red on main. * chore: sync uv.lock ktx-sl/ktx-daemon to 0.16.0 The 0.16.0 release bumped the pyproject versions but left uv.lock pinning the editable ktx-sl/ktx-daemon packages at 0.15.0. --- packages/cli/test/sl.test.ts | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/test/sl.test.ts b/packages/cli/test/sl.test.ts index ca37747e..0bef2071 100644 --- a/packages/cli/test/sl.test.ts +++ b/packages/cli/test/sl.test.ts @@ -717,7 +717,7 @@ joins: [] expect(query).toHaveBeenCalledWith( expect.objectContaining({ - query: { measures: ['orders.order_count'], dimensions: [] }, + query: { measures: ['orders.order_count'], dimensions: [], predefined_measures_only: false }, }), ); expect(JSON.parse(String(stdout.write.mock.calls[0][0]))).toMatchObject({ diff --git a/uv.lock b/uv.lock index d37cd781..1c4c2af5 100644 --- a/uv.lock +++ b/uv.lock @@ -466,7 +466,7 @@ wheels = [ [[package]] name = "ktx-daemon" -version = "0.15.0" +version = "0.16.0" source = { editable = "python/ktx-daemon" } dependencies = [ { name = "fastapi" }, @@ -523,7 +523,7 @@ dev = [ [[package]] name = "ktx-sl" -version = "0.15.0" +version = "0.16.0" source = { editable = "python/ktx-sl" } dependencies = [ { name = "pydantic" },