From fe7e6bd1faced28b977cd4c8d61a13a08dcf7514 Mon Sep 17 00:00:00 2001 From: Patel Dhrit <57010146+d180@users.noreply.github.com> Date: Thu, 2 Jul 2026 06:00:26 -0700 Subject: [PATCH] feat(connector): add Amazon Athena connector via Glue Data Catalog (#309) * feat(connector): add Amazon Athena connector via Glue Data Catalog * fix(athena): address reviewer feedback * fix(athena): wire scope discovery, fix normalizeDriver, tighten types and tests * fix(athena): honor databases scope, wire sql-analysis dialect, harden config resolution - introspect() limits to the configured `databases` scope instead of scanning every Glue database in the account (docs promised this; connector ignored it) - add athena -> athena to sql-analysis SQLGLOT_DIALECTS so `ktx sql` and MCP read-only validation parse Athena SQL under the Trino grammar, not postgres - stringConfigValue coerces a resolved-empty `env:` reference to undefined so optional fields fall back to their defaults (workgroup 'primary', catalog 'AwsDataCatalog') instead of '' - drop trailing whitespace in dialect.test.ts * fix(athena): integrate with main's SQL/non-SQL dialect split and add dialect notes Rebase onto main, which introduced the KtxDialect (core) vs KtxSqlDialect (SQL-only) split for MongoDB: - KtxAthenaDialect implements KtxSqlDialect; the connector resolves it via getSqlDialectForDriver so SQL-generation methods stay in scope - add authored athena.md SQL notes for the sql_dialect_notes MCP tool, required now that athena resolves to the athena sqlglot dialect (dialect-notes coverage is derived from the warehouse-driver registry) --------- Co-authored-by: Andrey Avtomonov --- .../docs/integrations/primary-sources.mdx | 86 ++- packages/cli/package.json | 2 + packages/cli/src/connection-drivers.ts | 1 + .../cli/src/connectors/athena/connector.ts | 555 +++++++++++++++ packages/cli/src/connectors/athena/dialect.ts | 175 +++++ .../athena/live-database-introspection.ts | 44 ++ .../cli/src/context/connections/dialects.ts | 2 + .../cli/src/context/connections/drivers.ts | 17 + .../cli/src/context/project/driver-schemas.ts | 2 + packages/cli/src/context/scan/local-scan.ts | 3 +- packages/cli/src/context/scan/types.ts | 1 + .../src/context/sql-analysis/dialect-notes.ts | 1 + .../cli/src/context/sql-analysis/dialect.ts | 1 + .../context/sql-analysis/dialects/athena.md | 12 + packages/cli/src/local-adapters.ts | 8 + packages/cli/src/setup-databases.ts | 51 ++ .../test/connectors/athena/connector.test.ts | 630 ++++++++++++++++++ .../test/connectors/athena/dialect.test.ts | 72 ++ .../test/context/connections/dialects.test.ts | 2 +- .../test/context/connections/drivers.test.ts | 6 + .../cli/test/context/scan/local-scan.test.ts | 34 + .../test/context/sql-analysis/dialect.test.ts | 1 + packages/cli/test/setup-databases.test.ts | 58 ++ pnpm-lock.yaml | 289 ++++++++ 24 files changed, 2047 insertions(+), 6 deletions(-) create mode 100644 packages/cli/src/connectors/athena/connector.ts create mode 100644 packages/cli/src/connectors/athena/dialect.ts create mode 100644 packages/cli/src/connectors/athena/live-database-introspection.ts create mode 100644 packages/cli/src/context/sql-analysis/dialects/athena.md create mode 100644 packages/cli/test/connectors/athena/connector.test.ts create mode 100644 packages/cli/test/connectors/athena/dialect.test.ts diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 270275f1..b553a3a7 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -1,6 +1,6 @@ --- title: Primary Sources -description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, or MongoDB. +description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, DuckDB, MongoDB, or Amazon Athena. --- **ktx** connects to your data warehouse or database to build schema context, @@ -26,17 +26,23 @@ Agents should prefer environment or file references over literal secrets. | Field | Required | Applies to | Description | |-------|----------|------------|-------------| -| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, or `mongodb` | +| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, `duckdb`, `mongodb`, or `athena` | | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | -| `databases` | No | ClickHouse, MongoDB | List of databases to scan | +| `databases` | No | ClickHouse, MongoDB, Athena | List of databases to scan | | `sample_size`, `order_by` | No | MongoDB | Schema-inference sampling controls (recent documents, sort field) | | `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it | | `path` | Yes for path-style SQLite/DuckDB | SQLite, DuckDB | Local SQLite or DuckDB database path or `env:NAME` reference | | `max_bytes_billed` | No | BigQuery | Maximum bytes billed per query job | | `query_timeout_ms` | No | all warehouses | Maximum execution time for a single read-only query, in milliseconds (default 30000). A query exceeding it is cancelled server-side (or, for SQLite, by terminating the off-process executor) and returns a `query exceeded Ns` error so the agent can revise. | | `project_id` | No | BigQuery | Optional local descriptor and mapping metadata; not used for BigQuery authentication | +| `region` | Yes | Athena | AWS region where the Athena workgroup and Glue catalog reside (e.g. `us-east-1`) | +| `s3_staging_dir` | Yes | Athena | S3 URI for Athena query result storage (e.g. `s3://my-bucket/athena-results/`) | +| `workgroup` | No | Athena | Athena workgroup name (default `primary`) | +| `catalog` | No | Athena | Glue Data Catalog name (default `AwsDataCatalog`) | +| `database` | No | Athena | Default Glue database name passed as the query execution context | +| `databases` | No | Athena | Glue databases to include in schema scans; written by `ktx setup` and read by `ktx ingest` | ## PostgreSQL @@ -304,6 +310,76 @@ staged artifact shape as Postgres and Snowflake. --- +## Amazon Athena + +Connects to Amazon Athena using the AWS Glue Data Catalog for schema introspection and the Athena query API for read-only SQL execution. Authentication uses the standard AWS credential chain — no credentials are embedded in `ktx.yaml`. + +### Connection config + +```yaml title="ktx.yaml" +connections: + my-athena: + driver: athena + region: us-east-1 + s3_staging_dir: s3://my-bucket/athena-results/ +``` + +With optional fields: + +```yaml title="ktx.yaml" +connections: + my-athena: + driver: athena + region: us-east-1 + s3_staging_dir: env:ATHENA_S3_STAGING_DIR + workgroup: analytics + catalog: AwsDataCatalog + database: my_default_database + databases: + - analytics + - raw +``` + +`ktx setup` writes the `databases` array when you select Glue databases during setup. `ktx scan` reads it to limit introspection to those databases. + +### Authentication + +**ktx** uses the AWS SDK default credential chain — no credentials appear in `ktx.yaml`. The chain resolves credentials in this order: + +| Method | How to configure | +|--------|-----------------| +| Environment variables | Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN` | +| Shared credentials file | Configure `~/.aws/credentials` with a `[default]` or named profile; set `AWS_PROFILE` to select a non-default profile | +| IAM instance profile | Attach an IAM role to the EC2 instance or ECS task — no local configuration needed | +| IAM roles for service accounts (EKS) | Annotate the pod's service account with the IAM role ARN | + +The IAM principal must have `athena:StartQueryExecution`, `athena:GetQueryExecution`, `athena:GetQueryResults`, `glue:GetDatabases`, and `glue:GetTables` permissions, plus read access to the S3 results bucket. + +### Features + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Tables & views | Yes | Via AWS Glue Data Catalog | +| Primary keys | No | Glue does not expose constraint metadata | +| Foreign keys | No | Not available in Glue/Athena | +| Row count estimates | No | Glue table statistics are often stale | +| Column statistics | No | - | +| Query history | No | - | +| Table sampling | Yes | `SELECT ... LIMIT n` | + +### Dialect notes + +- SQL dialect is Presto/Trino; identifiers are quoted with double-quotes +- Table names use three-part format: `"catalog"."database"."table"` (e.g. `"AwsDataCatalog"."analytics"."orders"`) +- Partition columns (`PartitionKeys` in Glue) are included after regular columns in the schema and are fully queryable +- Athena does not support `TABLESAMPLE`; random sampling uses `ORDER BY rand()` +- Query execution is asynchronous: **ktx** starts the query, polls until completion, then fetches results from S3 +- Results are stored in `s3_staging_dir`; the IAM principal needs write access to that bucket +- Use `workgroup` to apply per-workgroup cost controls and result configuration +- The connector always uses your account's default Glue Data Catalog; cross-account catalog access (`CatalogId` pointing to another account) is not supported + +--- + ## MySQL Standard MySQL/MariaDB connector with full foreign key support and schema introspection. @@ -675,7 +751,9 @@ nullability from how often the field is present: | Error or symptom | Likely cause | Recovery | |------------------|--------------|----------| | Connection URL appears in git diff | A literal credential URL was written to `ktx.yaml` | Replace it with `env:NAME` or `file:/path/to/secret` and rotate exposed credentials | -| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | +| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions. For Athena, confirm the IAM principal has `glue:GetDatabases` and `glue:GetTables` permissions | | Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest --query-history` or `ktx setup` | | Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on schema-level context without column statistics | | Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags | +| Athena query fails with `ACCESS_DENIED` | IAM principal lacks `athena:StartQueryExecution` or S3 write access to `s3_staging_dir` | Attach a policy granting Athena query permissions and `s3:PutObject` on the staging bucket | +| Athena ingest finds databases but no tables | IAM principal has `glue:GetDatabases` but not `glue:GetTables` | Grant `glue:GetTables` on the relevant Glue catalog resources | diff --git a/packages/cli/package.json b/packages/cli/package.json index 8f41231c..f609b571 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -51,6 +51,8 @@ "@ai-sdk/devtools": "0.0.18", "@ai-sdk/google-vertex": "^4.0.134", "@anthropic-ai/claude-agent-sdk": "0.3.146", + "@aws-sdk/client-athena": "^3.1068.0", + "@aws-sdk/client-glue": "^3.1068.0", "@clack/core": "1.3.1", "@clack/prompts": "1.4.0", "@clickhouse/client": "^1.18.5", diff --git a/packages/cli/src/connection-drivers.ts b/packages/cli/src/connection-drivers.ts index 836c8458..b7b1857d 100644 --- a/packages/cli/src/connection-drivers.ts +++ b/packages/cli/src/connection-drivers.ts @@ -10,6 +10,7 @@ export const KTX_DATABASE_DRIVER_IDS = [ 'sqlserver', 'bigquery', 'snowflake', + 'athena', ] as const; // mongodb is a database driver but has no SQL dialect, so it sits outside the diff --git a/packages/cli/src/connectors/athena/connector.ts b/packages/cli/src/connectors/athena/connector.ts new file mode 100644 index 00000000..00b53cc0 --- /dev/null +++ b/packages/cli/src/connectors/athena/connector.ts @@ -0,0 +1,555 @@ +import { AthenaClient, StartQueryExecutionCommand, GetQueryExecutionCommand, GetQueryResultsCommand } from '@aws-sdk/client-athena'; +import { GlueClient, GetDatabasesCommand, GetTablesCommand } from '@aws-sdk/client-glue'; +import { getSqlDialectForDriver } from '../../context/connections/dialects.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxSchemaColumn, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; +import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { resolveStringReference } from '../shared/string-reference.js'; + +export interface KtxAthenaConnectionConfig { + driver?: string; + region?: string; + s3_staging_dir?: string; + workgroup?: string; + catalog?: string; + database?: string; + databases?: string[]; + [key: string]: unknown; +} + +export interface KtxAthenaResolvedConnectionConfig { + region: string; + s3StagingDir: string; + workgroup: string; + catalog: string; + database: string | undefined; + databases: string[]; +} + +interface KtxAthenaQueryExecutionStatus { + State?: string; + StateChangeReason?: string; +} + +interface KtxAthenaQueryExecution { + Status?: KtxAthenaQueryExecutionStatus; +} + +interface KtxAthenaColumnInfo { + Name?: string; + Type?: string; +} + +interface KtxAthenaDatum { + VarCharValue?: string; +} + +interface KtxAthenaRow { + Data?: KtxAthenaDatum[]; +} + +interface KtxAthenaResultSet { + Rows?: KtxAthenaRow[]; + ResultSetMetadata?: { ColumnInfo?: KtxAthenaColumnInfo[] }; +} + +/** @internal */ +export interface KtxAthenaClient { + startQueryExecution(input: { + QueryString: string; + ResultConfiguration: { OutputLocation: string }; + WorkGroup: string; + QueryExecutionContext?: { Database?: string; Catalog?: string }; + }): Promise<{ QueryExecutionId?: string }>; + getQueryExecution(input: { QueryExecutionId: string }): Promise<{ QueryExecution?: KtxAthenaQueryExecution }>; + getQueryResults(input: { QueryExecutionId: string; NextToken?: string }): Promise<{ + ResultSet?: KtxAthenaResultSet; + NextToken?: string; + }>; +} + +interface KtxGlueColumnDef { + Name?: string; + Type?: string; + Comment?: string; +} + +interface KtxGlueStorageDescriptor { + Columns?: KtxGlueColumnDef[]; +} + +/** @internal */ +export interface KtxGlueTable { + Name?: string; + TableType?: string; + StorageDescriptor?: KtxGlueStorageDescriptor; + PartitionKeys?: KtxGlueColumnDef[]; + Description?: string; + Parameters?: Record; +} + +/** @internal */ +export interface KtxGlueClient { + getDatabases(input: { CatalogId?: string; NextToken?: string }): Promise<{ + DatabaseList?: Array<{ Name?: string }>; + NextToken?: string; + }>; + getTables(input: { DatabaseName: string; CatalogId?: string; NextToken?: string }): Promise<{ + TableList?: KtxGlueTable[]; + NextToken?: string; + }>; +} + +export interface KtxAthenaClientFactory { + createAthenaClient(region: string): KtxAthenaClient; + createGlueClient(region: string): KtxGlueClient; +} + +class DefaultAthenaClientFactory implements KtxAthenaClientFactory { + createAthenaClient(region: string): KtxAthenaClient { + const client = new AthenaClient({ region }); + return { + startQueryExecution: async (input) => { + const result = await client.send( + new StartQueryExecutionCommand({ + QueryString: input.QueryString, + ResultConfiguration: { OutputLocation: input.ResultConfiguration.OutputLocation }, + WorkGroup: input.WorkGroup, + QueryExecutionContext: input.QueryExecutionContext, + }), + ); + return { QueryExecutionId: result.QueryExecutionId }; + }, + getQueryExecution: async (input) => { + const result = await client.send(new GetQueryExecutionCommand({ QueryExecutionId: input.QueryExecutionId })); + return { + QueryExecution: result.QueryExecution + ? { + Status: { + State: result.QueryExecution.Status?.State, + StateChangeReason: result.QueryExecution.Status?.StateChangeReason, + }, + } + : undefined, + }; + }, + getQueryResults: async (input) => { + const result = await client.send( + new GetQueryResultsCommand({ QueryExecutionId: input.QueryExecutionId, NextToken: input.NextToken }), + ); + return { + ResultSet: result.ResultSet as KtxAthenaResultSet | undefined, + NextToken: result.NextToken, + }; + }, + }; + } + + createGlueClient(region: string): KtxGlueClient { + const client = new GlueClient({ region }); + return { + getDatabases: async (input) => { + const result = await client.send(new GetDatabasesCommand({ CatalogId: input.CatalogId, NextToken: input.NextToken })); + return { + DatabaseList: result.DatabaseList?.map((db) => ({ Name: db.Name })), + NextToken: result.NextToken, + }; + }, + getTables: async (input) => { + const result = await client.send( + new GetTablesCommand({ DatabaseName: input.DatabaseName, CatalogId: input.CatalogId, NextToken: input.NextToken }), + ); + return { + TableList: result.TableList as KtxGlueTable[] | undefined, + NextToken: result.NextToken, + }; + }, + }; + } +} + +function stringConfigValue( + connection: KtxAthenaConnectionConfig | undefined, + key: keyof KtxAthenaConnectionConfig, + env: NodeJS.ProcessEnv, +): string | undefined { + const value = connection?.[key]; + if (typeof value !== 'string' || value.trim().length === 0) return undefined; + // Resolve before checking emptiness: an unset `env:` reference resolves to '', + // which must become undefined so `?? default` applies instead of keeping ''. + const resolved = resolveStringReference(value.trim(), env).trim(); + return resolved.length > 0 ? resolved : undefined; +} + +function configuredAthenaDatabases(connection: KtxAthenaConnectionConfig): string[] { + if (!Array.isArray(connection.databases)) return []; + const selected = connection.databases + .filter((database): database is string => typeof database === 'string' && database.trim().length > 0) + .map((database) => database.trim()); + return [...new Set(selected)]; +} + +export function isKtxAthenaConnectionConfig( + connection: unknown, +): connection is KtxAthenaConnectionConfig { + return ( + typeof connection === 'object' && + connection !== null && + String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'athena' + ); +} + +/** @internal */ +export function athenaConnectionConfigFromConfig(input: { + connectionId: string; + connection: KtxAthenaConnectionConfig | undefined; + env?: NodeJS.ProcessEnv; +}): KtxAthenaResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; + if (!isKtxAthenaConnectionConfig(input.connection)) { + throw new Error(`Native Athena connector cannot run driver "${String(inputDriver)}"`); + } + const env = input.env ?? process.env; + const region = stringConfigValue(input.connection, 'region', env); + if (!region) { + throw new Error(`Native Athena connector requires connections.${input.connectionId}.region`); + } + const s3StagingDir = stringConfigValue(input.connection, 's3_staging_dir', env); + if (!s3StagingDir) { + throw new Error(`Native Athena connector requires connections.${input.connectionId}.s3_staging_dir`); + } + return { + region, + s3StagingDir, + workgroup: stringConfigValue(input.connection, 'workgroup', env) ?? 'primary', + catalog: stringConfigValue(input.connection, 'catalog', env) ?? 'AwsDataCatalog', + database: stringConfigValue(input.connection, 'database', env), + databases: configuredAthenaDatabases(input.connection), + }; +} + +function glueTableKind(tableType: string | undefined): 'table' | 'view' { + const t = String(tableType ?? '').toUpperCase(); + if (t === 'VIRTUAL_VIEW') return 'view'; + return 'table'; +} + +const POLL_INTERVAL_MS = 250; +const QUERY_TIMEOUT_MS = 5 * 60 * 1000; + +export interface KtxAthenaScanConnectorOptions { + connectionId: string; + connection: KtxAthenaConnectionConfig | undefined; + clientFactory?: KtxAthenaClientFactory; + env?: NodeJS.ProcessEnv; + now?: () => Date; +} + +export class KtxAthenaScanConnector implements KtxScanConnector { + readonly id: string; + readonly driver = 'athena' as const; + readonly capabilities = createKtxConnectorCapabilities({ + tableSampling: true, + columnSampling: true, + columnStats: false, + readOnlySql: true, + nestedAnalysis: false, + formalForeignKeys: false, + estimatedRowCounts: false, + }); + + private readonly connectionId: string; + private readonly resolved: KtxAthenaResolvedConnectionConfig; + private readonly clientFactory: KtxAthenaClientFactory; + private readonly now: () => Date; + private readonly dialect = getSqlDialectForDriver('athena'); + private athenaClient: KtxAthenaClient | null = null; + private glueClient: KtxGlueClient | null = null; + + constructor(options: KtxAthenaScanConnectorOptions) { + this.connectionId = options.connectionId; + this.resolved = athenaConnectionConfigFromConfig({ + connectionId: options.connectionId, + connection: options.connection, + env: options.env, + }); + this.clientFactory = options.clientFactory ?? new DefaultAthenaClientFactory(); + this.now = options.now ?? (() => new Date()); + this.id = `athena:${options.connectionId}`; + } + + async testConnection(): Promise { + try { + await this.listDatabasesPaginated({ maxResults: 1 }); + return { success: true }; + } catch (error) { + return connectorTestFailure(error); + } + } + + async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + // Honor the configured `databases` scope (written by `ktx setup`); fall back + // to every Glue database only when the scope is unset. + const databases = + this.resolved.databases.length > 0 ? this.resolved.databases : await this.listDatabasesPaginated({}); + const tables: KtxSchemaTable[] = []; + for (const database of databases) { + const scopedNames = input.tableScope + ? scopedTableNames(input.tableScope, { catalog: this.resolved.catalog, db: database }) + : null; + tables.push(...(await this.introspectDatabase(database, scopedNames))); + } + return { + connectionId: this.connectionId, + driver: 'athena', + extractedAt: this.now().toISOString(), + scope: { catalogs: [this.resolved.catalog], datasets: databases }, + metadata: { + catalog: this.resolved.catalog, + databases, + table_count: tables.length, + total_columns: tables.reduce((sum, t) => sum + t.columns.length, 0), + }, + tables, + warnings: [], + }; + } + + async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns)); + return { headers: result.headers, headerTypes: result.headerTypes, rows: result.rows, totalRows: result.totalRows }; + } + + async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.query( + this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit), + ); + return { + values: result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]), + nullCount: null, + distinctCount: null, + }; + } + + async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise { + return null; + } + + async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); + const result = await this.query(limitedSql); + return { ...result, rowCount: result.rows.length }; + } + + async listSchemas(): Promise { + return this.listDatabasesPaginated({}); + } + + async listTables(databases?: string[]): Promise { + const targetDatabases = databases && databases.length > 0 ? databases : await this.listDatabasesPaginated({}); + const entries: KtxTableListEntry[] = []; + for (const database of targetDatabases) { + const glueTables = await this.listGlueTablesPaginated(database); + for (const t of glueTables) { + if (!t.Name) continue; + entries.push({ + catalog: this.resolved.catalog, + schema: database, + name: t.Name, + kind: glueTableKind(t.TableType), + }); + } + } + return entries; + } + + async cleanup(): Promise { + this.athenaClient = null; + this.glueClient = null; + } + + qTableName(table: Pick & Partial>): string { + return this.dialect.formatTableName(table); + } + + private getAthenaClient(): KtxAthenaClient { + if (!this.athenaClient) { + this.athenaClient = this.clientFactory.createAthenaClient(this.resolved.region); + } + return this.athenaClient; + } + + private getGlueClient(): KtxGlueClient { + if (!this.glueClient) { + this.glueClient = this.clientFactory.createGlueClient(this.resolved.region); + } + return this.glueClient; + } + + private async listDatabasesPaginated(opts: { maxResults?: number }): Promise { + const names: string[] = []; + let nextToken: string | undefined; + do { + const result = await this.getGlueClient().getDatabases({ NextToken: nextToken }); + for (const db of result.DatabaseList ?? []) { + if (db.Name) names.push(db.Name); + if (opts.maxResults && names.length >= opts.maxResults) return names; + } + nextToken = result.NextToken; + } while (nextToken); + return names; + } + + private async listGlueTablesPaginated(database: string): Promise { + const tables: KtxGlueTable[] = []; + let nextToken: string | undefined; + do { + const result = await this.getGlueClient().getTables({ DatabaseName: database, NextToken: nextToken }); + tables.push(...(result.TableList ?? [])); + nextToken = result.NextToken; + } while (nextToken); + return tables; + } + + private async introspectDatabase(database: string, scopedNames: readonly string[] | null): Promise { + if (scopedNames && scopedNames.length === 0) return []; + const glueTables = await this.listGlueTablesPaginated(database); + const scopeSet = scopedNames ? new Set(scopedNames) : null; + return glueTables + .filter((t): t is KtxGlueTable & { Name: string } => Boolean(t.Name) && (!scopeSet || scopeSet.has(t.Name!))) + .map((t) => ({ + catalog: this.resolved.catalog, + db: database, + name: t.Name, + kind: glueTableKind(t.TableType), + comment: t.Description ?? null, + estimatedRows: null, + columns: this.toSchemaColumns(t), + foreignKeys: [], + })); + } + + private toSchemaColumns(table: KtxGlueTable): KtxSchemaColumn[] { + const columns = [...(table.StorageDescriptor?.Columns ?? []), ...(table.PartitionKeys ?? [])]; + return columns + .filter((col): col is KtxGlueColumnDef & { Name: string } => Boolean(col.Name)) + .map((col) => { + const nativeType = String(col.Type ?? 'string').toLowerCase(); + return { + name: col.Name, + nativeType, + normalizedType: this.dialect.mapDataType(nativeType), + dimensionType: this.dialect.mapToDimensionType(nativeType), + nullable: true, + primaryKey: false, + comment: col.Comment ?? null, + }; + }); + } + + private async query(sql: string): Promise { + const athena = this.getAthenaClient(); + const { QueryExecutionId } = await athena.startQueryExecution({ + QueryString: sql, + ResultConfiguration: { OutputLocation: this.resolved.s3StagingDir }, + WorkGroup: this.resolved.workgroup, + ...(this.resolved.database || this.resolved.catalog + ? { + QueryExecutionContext: { + ...(this.resolved.database ? { Database: this.resolved.database } : {}), + ...(this.resolved.catalog ? { Catalog: this.resolved.catalog } : {}), + }, + } + : {}), + }); + + if (!QueryExecutionId) { + throw new Error('Athena did not return a QueryExecutionId'); + } + + await this.waitForQueryCompletion(athena, QueryExecutionId); + + const rows: unknown[][] = []; + let headers: string[] = []; + let headerTypes: string[] = []; + let nextToken: string | undefined; + let firstPage = true; + + do { + const result = await athena.getQueryResults({ QueryExecutionId, NextToken: nextToken }); + const resultSet = result.ResultSet; + + if (firstPage) { + const columnInfo = resultSet?.ResultSetMetadata?.ColumnInfo ?? []; + headers = columnInfo.map((col) => col.Name ?? ''); + headerTypes = columnInfo.map((col) => String(col.Type ?? 'varchar').toUpperCase()); + firstPage = false; + } + + const pageRows = resultSet?.Rows ?? []; + // Athena includes the header row as the first row of the first page — skip it. + const dataRows = nextToken === undefined ? pageRows.slice(1) : pageRows; + for (const row of dataRows) { + rows.push((row.Data ?? []).map((d) => d.VarCharValue ?? null)); + } + + nextToken = result.NextToken; + } while (nextToken); + + return { + headers, + headerTypes: headerTypes.length > 0 ? headerTypes : undefined, + rows, + totalRows: rows.length, + rowCount: rows.length, + }; + } + + private async waitForQueryCompletion(athena: KtxAthenaClient, queryExecutionId: string): Promise { + const terminalStates = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED']); + const deadline = this.now().getTime() + QUERY_TIMEOUT_MS; + for (;;) { + const { QueryExecution } = await athena.getQueryExecution({ QueryExecutionId: queryExecutionId }); + const state = QueryExecution?.Status?.State ?? ''; + if (state === 'SUCCEEDED') return; + if (terminalStates.has(state)) { + const reason = QueryExecution?.Status?.StateChangeReason ?? state; + throw new Error(`Athena query ${state}: ${reason}`); + } + if (this.now().getTime() >= deadline) { + throw new Error(`Athena query ${queryExecutionId} timed out after ${QUERY_TIMEOUT_MS / 1000}s`); + } + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + } + } + + private assertConnection(connectionId: string): void { + if (connectionId !== this.connectionId) { + throw new Error(`Athena connector ${this.connectionId} cannot scan connection ${connectionId}`); + } + } +} diff --git a/packages/cli/src/connectors/athena/dialect.ts b/packages/cli/src/connectors/athena/dialect.ts new file mode 100644 index 00000000..81349804 --- /dev/null +++ b/packages/cli/src/connectors/athena/dialect.ts @@ -0,0 +1,175 @@ +import type { KtxSqlDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; + +type AthenaTableNameRef = Pick & Partial>; + +/** @internal */ +export class KtxAthenaDialect implements KtxSqlDialect { + readonly type = 'athena' as const; + + private readonly dimensionTypeMappings: Record = { + timestamp: 'time', + date: 'time', + bigint: 'number', + int: 'number', + integer: 'number', + tinyint: 'number', + smallint: 'number', + double: 'number', + float: 'number', + real: 'number', + boolean: 'boolean', + }; + + quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; + } + + formatTableName(table: AthenaTableNameRef): string { + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi'); + } + + formatDisplayRef(table: AthenaTableNameRef): string { + return formatDialectDisplayRef(table, 'ansi'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'ansi'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('ansi'); + } + + mapDataType(nativeType: string): string { + const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim(); + const typeMap: Record = { + string: 'VARCHAR', + varchar: 'VARCHAR', + char: 'CHAR', + binary: 'VARBINARY', + bigint: 'BIGINT', + int: 'INTEGER', + integer: 'INTEGER', + tinyint: 'TINYINT', + smallint: 'SMALLINT', + double: 'DOUBLE', + float: 'FLOAT', + real: 'REAL', + decimal: 'DECIMAL', + boolean: 'BOOLEAN', + timestamp: 'TIMESTAMP', + date: 'DATE', + array: 'ARRAY', + map: 'MAP', + struct: 'STRUCT', + uniontype: 'UNION', + }; + return typeMap[base] ?? nativeType.toUpperCase(); + } + + mapToDimensionType(nativeType: string): KtxSchemaDimensionType { + const base = nativeType.toLowerCase().trim().split('<')[0]!.split('(')[0]!.trim(); + const mapped = this.dimensionTypeMappings[base]; + if (mapped) return mapped; + if (base.includes('timestamp') || base.includes('date')) return 'time'; + if (base.includes('int') || base.includes('float') || base.includes('double') || base.includes('decimal') || base.includes('real')) return 'number'; + if (base.includes('bool')) return 'boolean'; + return 'string'; + } + + generateSampleQuery(tableName: string, limit: number, columns?: string[]): string { + const columnList = + columns && columns.length > 0 ? columns.map((c) => this.quoteIdentifier(c)).join(', ') : '*'; + return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`; + } + + generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string { + const quoted = this.quoteIdentifier(columnName); + return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL LIMIT ${limit}`; + } + + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + SELECT approx_distinct(${columnName}) AS cardinality + FROM ( + SELECT ${columnName} + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + LIMIT ${sampleSize} + ) + `; + } + + generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + SELECT approx_distinct(${columnName}) AS cardinality + FROM ( + SELECT ${columnName} + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY rand() + LIMIT ${sampleSize} + ) + `; + } + + generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string { + return ` + SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY val + LIMIT ${limit} + `; + } + + generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null { + return null; + } + + getNullCountExpression(column: string): string { + return `COUNT_IF(${column} IS NULL)`; + } + + getDistinctCountExpression(column: string): string { + return `approx_distinct(${column})`; + } + + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS VARCHAR))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS VARCHAR)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT array_join(array_agg(CAST(value AS VARCHAR)), '\u001f') FROM (${innerSql}) AS relationship_profile_values)`; + } + + getLimitOffsetClause(limit: number, offset?: number): string { + const safeLimit = Math.max(1, Math.floor(limit)); + const safeOffset = offset !== undefined ? Math.floor(offset) : 0; + return safeOffset > 0 ? `OFFSET ${safeOffset} LIMIT ${safeLimit}` : `LIMIT ${safeLimit}`; + } + + getTopClause(_limit: number): string { + return ''; + } + + getTableSampleClause(_samplePct: number): string { + return ''; + } + + getRandomSampleFilter(samplePct: number): string { + if (samplePct <= 0 || samplePct >= 1) return ''; + return `rand() < ${samplePct}`; + } +} diff --git a/packages/cli/src/connectors/athena/live-database-introspection.ts b/packages/cli/src/connectors/athena/live-database-introspection.ts new file mode 100644 index 00000000..815285ba --- /dev/null +++ b/packages/cli/src/connectors/athena/live-database-introspection.ts @@ -0,0 +1,44 @@ +import type { + LiveDatabaseIntrospectionOptions, + LiveDatabaseIntrospectionPort, +} from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; +import { + KtxAthenaScanConnector, + type KtxAthenaClientFactory, + type KtxAthenaConnectionConfig, +} from './connector.js'; + +interface CreateAthenaLiveDatabaseIntrospectionOptions { + connections: Record; + clientFactory?: KtxAthenaClientFactory; + now?: () => Date; +} + +export function createAthenaLiveDatabaseIntrospection( + options: CreateAthenaLiveDatabaseIntrospectionOptions, +): LiveDatabaseIntrospectionPort { + return { + async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) { + const connection = options.connections[connectionId] as KtxAthenaConnectionConfig | undefined; + const connector = new KtxAthenaScanConnector({ + connectionId, + connection, + clientFactory: options.clientFactory, + now: options.now, + }); + try { + return await connector.introspect( + { + connectionId, + driver: 'athena', + ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}), + }, + { runId: `athena-${connectionId}` }, + ); + } finally { + await connector.cleanup(); + } + }, + }; +} diff --git a/packages/cli/src/context/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts index ad739d1d..8512ed57 100644 --- a/packages/cli/src/context/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,3 +1,4 @@ +import { KtxAthenaDialect } from '../../connectors/athena/dialect.js'; import { KtxBigQueryDialect } from '../../connectors/bigquery/dialect.js'; import { KtxClickHouseDialect } from '../../connectors/clickhouse/dialect.js'; import { KtxDuckDbDialect } from '../../connectors/duckdb/dialect.js'; @@ -54,6 +55,7 @@ export interface KtxSqlDialect extends KtxDialect { type KtxSqlDriver = Exclude; const sqlDialectFactories: Record KtxSqlDialect> = { + athena: () => new KtxAthenaDialect(), bigquery: () => new KtxBigQueryDialect(), clickhouse: () => new KtxClickHouseDialect(), duckdb: () => new KtxDuckDbDialect(), diff --git a/packages/cli/src/context/connections/drivers.ts b/packages/cli/src/context/connections/drivers.ts index 1ba17f54..6a2d4589 100644 --- a/packages/cli/src/context/connections/drivers.ts +++ b/packages/cli/src/context/connections/drivers.ts @@ -26,6 +26,23 @@ function invalidConnectionConfig(driver: KtxConnectionDriver): Error { /** @internal */ export const driverRegistrations: Record = { + athena: { + driver: 'athena', + scopeConfigKey: 'databases', + hasHistoricSqlReader: false, + load: async () => { + const m = await import('../../connectors/athena/connector.js'); + return { + isConnectionConfig: (connection) => m.isKtxAthenaConnectionConfig(connection), + createScanConnector: ({ connectionId, connection }) => { + if (!m.isKtxAthenaConnectionConfig(connection)) { + throw invalidConnectionConfig('athena'); + } + return new m.KtxAthenaScanConnector({ connectionId, connection }); + }, + }; + }, + }, bigquery: { driver: 'bigquery', scopeConfigKey: 'dataset_ids', diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index 9f808d00..df828065 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -14,6 +14,7 @@ const warehouseDrivers = [ 'duckdb', 'clickhouse', 'sqlserver', + 'athena', ] as const; type WarehouseDriver = (typeof warehouseDrivers)[number]; @@ -56,6 +57,7 @@ const warehouseConnectionSchemas = [ warehouseConnectionSchema('duckdb'), warehouseConnectionSchema('clickhouse'), warehouseConnectionSchema('sqlserver'), + warehouseConnectionSchema('athena'), ] as const; const mongodbConnectionSchema = z diff --git a/packages/cli/src/context/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts index 0966d15b..ac0a90e5 100644 --- a/packages/cli/src/context/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -147,12 +147,13 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver { normalized === 'sqlserver' || normalized === 'bigquery' || normalized === 'snowflake' || + normalized === 'athena' || normalized === 'mongodb' ) { return normalized; } throw new Error( - `Standalone ktx scan supports postgres/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake/mongodb in this phase, received "${driver ?? 'unknown'}"`, + `Standalone ktx scan supports postgres/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake/athena/mongodb in this phase, received "${driver ?? 'unknown'}"`, ); } diff --git a/packages/cli/src/context/scan/types.ts b/packages/cli/src/context/scan/types.ts index 9bd7d865..bf72558c 100644 --- a/packages/cli/src/context/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -9,6 +9,7 @@ export type KtxConnectionDriver = | 'snowflake' | 'mysql' | 'clickhouse' + | 'athena' | 'mongodb'; /** Canonical scan-mode registry. Runtime validation derives its allowlist here. */ diff --git a/packages/cli/src/context/sql-analysis/dialect-notes.ts b/packages/cli/src/context/sql-analysis/dialect-notes.ts index f0a20a2e..7fcffab3 100644 --- a/packages/cli/src/context/sql-analysis/dialect-notes.ts +++ b/packages/cli/src/context/sql-analysis/dialect-notes.ts @@ -18,6 +18,7 @@ export const DIALECTS_WITH_NOTES = [ 'duckdb', 'clickhouse', 'tsql', + 'athena', ] as const; type DialectWithNotes = (typeof DIALECTS_WITH_NOTES)[number]; diff --git a/packages/cli/src/context/sql-analysis/dialect.ts b/packages/cli/src/context/sql-analysis/dialect.ts index 54bc9525..83092233 100644 --- a/packages/cli/src/context/sql-analysis/dialect.ts +++ b/packages/cli/src/context/sql-analysis/dialect.ts @@ -16,6 +16,7 @@ const SQLGLOT_DIALECTS: Record = { duckdb: 'duckdb', clickhouse: 'clickhouse', databricks: 'databricks', + athena: 'athena', }; export function sqlAnalysisDialectForDriver(driver: string | undefined): SqlAnalysisDialect { diff --git a/packages/cli/src/context/sql-analysis/dialects/athena.md b/packages/cli/src/context/sql-analysis/dialects/athena.md new file mode 100644 index 00000000..d7a377cf --- /dev/null +++ b/packages/cli/src/context/sql-analysis/dialects/athena.md @@ -0,0 +1,12 @@ +**athena** SQL conventions (Trino engine over the Glue Data Catalog): +- **FQTN:** `database.table` (e.g. `analytics.orders`); a bare `table` resolves against the query's default database. Cross-catalog is `catalog.database.table` (e.g. `awsdatacatalog.analytics.orders`). +- **Identifiers:** case-insensitive and folded to lowercase; double-quote (`"Name"`) to keep case, spaces, or a reserved word. String literals use single quotes only. +- **Date/time:** native `DATE`/`TIMESTAMP`. Bucket with `date_trunc('month', ts)`, pull parts with `EXTRACT(YEAR FROM ts)`, shift with `date_add('day', -30, current_date)`, difference with `date_diff('day', a, b)`, and format with `date_format(ts, '%Y-%m')`. Parse text with `date_parse(str, '%Y-%m-%d')` or `from_iso8601_timestamp(str)`; `current_date` / `now()` are available. +- **Top-N / windows:** Athena has no `QUALIFY` — wrap the window in a subquery and filter it: `SELECT * FROM (SELECT ..., ROW_NUMBER() OVER (PARTITION BY key ORDER BY x DESC) AS rn FROM t) WHERE rn = 1` returns one row per key. Use `ORDER BY ... LIMIT n` for a global top-N, and paginate with `OFFSET m LIMIT n` (offset first — `LIMIT n OFFSET m` is a syntax error). +- **Series:** `CROSS JOIN UNNEST(sequence(DATE '2023-01-01', DATE '2023-12-01', INTERVAL '1' MONTH)) AS s(d)` expands a generated array into a date spine (use `sequence(1, 12)` for integers), then `LEFT JOIN` the aggregated facts onto it so empty periods still appear. +- **Rolling window over time:** a native `RANGE` frame spans real dates and tolerates gaps — `AVG(amount) OVER (ORDER BY day RANGE BETWEEN INTERVAL '29' DAY PRECEDING AND CURRENT ROW)` is a trailing 30-day average without a spine; guard minimum periods with `COUNT(*) OVER ()`. +- **Approximate aggregates:** `approx_distinct(x)` for cardinality and `approx_percentile(x, 0.5)` for quantiles are far cheaper than exact `COUNT(DISTINCT ...)` on large scans. +- **Arrays & maps:** explode with `CROSS JOIN UNNEST(arr) AS t(x)` (add `WITH ORDINALITY` for an index); build with `array_agg(x)`, join with `array_join(arr, ',')`, index 1-based (`arr[1]`), and read a map with `element_at(m, key)`. +- **Safe cast:** `TRY_CAST(x AS DOUBLE)` yields `NULL` for a value that does not parse instead of raising, so counting residual `NULL`s catches an encoding the sample missed; `TRY(expr)` swallows other runtime errors. +- **Integer division:** `/` between integers truncates (`5 / 2` → `2`); cast an operand (`x / CAST(y AS DOUBLE)`) to keep the fraction, and round only in the final projection. +- **JSON:** `json_extract_scalar(col, '$.a.b')` returns varchar, `json_extract(col, '$.a')` returns json; cast a JSON string with `CAST(json_parse(col) AS ...)`. diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 03ff48d2..9f9ac60d 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -1,3 +1,5 @@ +import { createAthenaLiveDatabaseIntrospection } from './connectors/athena/live-database-introspection.js'; +import { isKtxAthenaConnectionConfig } from './connectors/athena/connector.js'; import { createBigQueryLiveDatabaseIntrospection } from './connectors/bigquery/live-database-introspection.js'; import { isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, type KtxBigQueryConnectionConfig } from './connectors/bigquery/connector.js'; import { createClickHouseLiveDatabaseIntrospection } from './connectors/clickhouse/live-database-introspection.js'; @@ -125,6 +127,9 @@ function createKtxCliLiveDatabaseIntrospection( const bigquery = createBigQueryLiveDatabaseIntrospection({ connections: project.config.connections, }); + const athena = createAthenaLiveDatabaseIntrospection({ + connections: project.config.connections, + }); return { async extractSchema(connectionId: string, options?: LiveDatabaseIntrospectionOptions) { const connection = project.config.connections[connectionId]; @@ -160,6 +165,9 @@ function createKtxCliLiveDatabaseIntrospection( if (isKtxBigQueryConnectionConfig(connection)) { return bigquery.extractSchema(connectionId, options); } + if (isKtxAthenaConnectionConfig(connection)) { + return athena.extractSchema(connectionId, options); + } if (hasSnowflakeDriver(connection)) { const { createSnowflakeLiveDatabaseIntrospection } = await import('./connectors/snowflake/live-database-introspection.js'); const { isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 33e3b1e0..efeba209 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -71,6 +71,7 @@ export type KtxSetupDatabaseDriver = | 'sqlserver' | 'bigquery' | 'snowflake' + | 'athena' | 'mongodb'; export interface KtxSetupDatabasesArgs { @@ -158,6 +159,7 @@ const DRIVER_OPTIONS: Array<{ value: KtxSetupDatabaseDriver; label: string }> = { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, + { value: 'athena', label: 'Amazon Athena' }, { value: 'mongodb', label: 'MongoDB' }, { value: 'sqlite', label: 'SQLite' }, { value: 'duckdb', label: 'DuckDB' }, @@ -183,6 +185,7 @@ const DEFAULT_CONNECTION_IDS: Record = { sqlserver: 'sqlserver-warehouse', bigquery: 'bigquery-warehouse', snowflake: 'snowflake-warehouse', + athena: 'athena-warehouse', mongodb: 'mongodb-source', }; @@ -268,6 +271,13 @@ const SCOPE_DISCOVERY_SPECS: Partial; @@ -968,6 +978,47 @@ async function buildConnectionConfig(input: { ...(role ? { role } : {}), }; } + if (driver === 'athena') { + if (args.inputMode === 'disabled' && !args.databaseUrl) return null; + const region = await promptText( + prompts, + 'AWS region\nFor example us-east-1.', + stringConfigField(input.existingConnection, 'region'), + ); + if (region === undefined) return 'back'; + if (!region) return null; + + const s3StagingDir = await promptText( + prompts, + 'S3 staging directory\nAthena writes query results here. For example s3://my-bucket/athena-results/.', + stringConfigField(input.existingConnection, 's3_staging_dir'), + ); + if (s3StagingDir === undefined) return 'back'; + if (!s3StagingDir) return null; + + const workgroup = await promptText( + prompts, + 'Athena workgroup (optional)\nPress Enter to use the default workgroup "primary".', + stringConfigField(input.existingConnection, 'workgroup'), + ); + if (workgroup === undefined) return 'back'; + + const catalog = await promptText( + prompts, + 'Glue Data Catalog name (optional)\nPress Enter to use the default "AwsDataCatalog".', + stringConfigField(input.existingConnection, 'catalog'), + ); + if (catalog === undefined) return 'back'; + + return { + driver: 'athena', + region, + s3_staging_dir: s3StagingDir, + ...(workgroup ? { workgroup } : {}), + ...(catalog ? { catalog } : {}), + ...scriptedScopeConfigForDriver('athena', args.databaseSchemas), + }; + } throw new Error(`Unsupported database driver: ${driver}`); } diff --git a/packages/cli/test/connectors/athena/connector.test.ts b/packages/cli/test/connectors/athena/connector.test.ts new file mode 100644 index 00000000..95afbd4c --- /dev/null +++ b/packages/cli/test/connectors/athena/connector.test.ts @@ -0,0 +1,630 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + athenaConnectionConfigFromConfig, + isKtxAthenaConnectionConfig, + KtxAthenaScanConnector, + type KtxAthenaClientFactory, + type KtxAthenaClient, + type KtxGlueClient, +} from '../../../src/connectors/athena/connector.js'; +import { createAthenaLiveDatabaseIntrospection } from '../../../src/connectors/athena/live-database-introspection.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; + +function fakeClientFactory(options: { queryState?: string; queryError?: string } = {}): KtxAthenaClientFactory { + const state = options.queryState ?? 'SUCCEEDED'; + const queries = new Map(); + let execCounter = 0; + + const fakeAthenaClient: KtxAthenaClient = { + startQueryExecution: vi.fn(async (input) => { + const id = `exec-${++execCounter}`; + queries.set(id, input.QueryString); + return { QueryExecutionId: id }; + }), + getQueryExecution: vi.fn(async () => ({ + QueryExecution: { + Status: { + State: state, + StateChangeReason: options.queryError, + }, + }, + })), + getQueryResults: vi.fn(async (input) => { + const sql = queries.get(input.QueryExecutionId) ?? ''; + // Column sample query: single-column result for the queried column only. + if (sql.includes('IS NOT NULL')) { + return { + ResultSet: { + ResultSetMetadata: { ColumnInfo: [{ Name: 'status', Type: 'string' }] }, + Rows: [ + { Data: [{ VarCharValue: 'status' }] }, // header row + { Data: [{ VarCharValue: 'paid' }] }, + ], + }, + NextToken: undefined, + }; + } + return { + ResultSet: { + ResultSetMetadata: { + ColumnInfo: [ + { Name: 'id', Type: 'bigint' }, + { Name: 'status', Type: 'string' }, + ], + }, + Rows: [ + // Header row (Athena always includes it on first page) + { Data: [{ VarCharValue: 'id' }, { VarCharValue: 'status' }] }, + // Data row + { Data: [{ VarCharValue: '1' }, { VarCharValue: 'paid' }] }, + ], + }, + NextToken: undefined, + }; + }), + }; + + const fakeGlueClient: KtxGlueClient = { + getDatabases: vi.fn(async () => ({ + DatabaseList: [{ Name: 'analytics' }], + NextToken: undefined, + })), + getTables: vi.fn(async () => ({ + TableList: [ + { + Name: 'orders', + TableType: 'EXTERNAL_TABLE', + Description: 'Orders table', + StorageDescriptor: { + Columns: [ + { Name: 'id', Type: 'bigint', Comment: 'Order id' }, + { Name: 'status', Type: 'string' }, + ], + }, + PartitionKeys: [{ Name: 'dt', Type: 'date', Comment: 'Partition date' }], + }, + ], + NextToken: undefined, + })), + }; + + return { + createAthenaClient: vi.fn(() => fakeAthenaClient), + createGlueClient: vi.fn(() => fakeGlueClient), + }; +} + +const connection = { + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://my-bucket/athena-results/', + workgroup: 'analytics', + catalog: 'AwsDataCatalog', + database: 'analytics', +} as const; + +describe('KtxAthenaScanConnector', () => { + it('identifies athena connection configs correctly', () => { + expect(isKtxAthenaConnectionConfig(connection)).toBe(true); + expect(isKtxAthenaConnectionConfig({ driver: 'bigquery' })).toBe(false); + expect(isKtxAthenaConnectionConfig(null)).toBe(false); + expect(isKtxAthenaConnectionConfig(undefined)).toBe(false); + }); + + it('resolves configuration and throws on missing required fields', () => { + expect(athenaConnectionConfigFromConfig({ connectionId: 'dw', connection })).toMatchObject({ + region: 'us-east-1', + s3StagingDir: 's3://my-bucket/athena-results/', + workgroup: 'analytics', + catalog: 'AwsDataCatalog', + database: 'analytics', + }); + + expect(() => + athenaConnectionConfigFromConfig({ connectionId: 'dw', connection: { driver: 'athena' } }), + ).toThrow('connections.dw.region'); + + expect(() => + athenaConnectionConfigFromConfig({ + connectionId: 'dw', + connection: { driver: 'athena', region: 'us-east-1' }, + }), + ).toThrow('connections.dw.s3_staging_dir'); + }); + + it('applies defaults for optional config fields', () => { + const resolved = athenaConnectionConfigFromConfig({ + connectionId: 'dw', + connection: { driver: 'athena', region: 'us-east-1', s3_staging_dir: 's3://bucket/' }, + }); + expect(resolved.workgroup).toBe('primary'); + expect(resolved.catalog).toBe('AwsDataCatalog'); + expect(resolved.database).toBeUndefined(); + }); + + it('introspects databases, tables, and columns from Glue', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'dw', driver: 'athena' }, + { runId: 'scan-1' }, + ); + + expect(snapshot).toMatchObject({ + connectionId: 'dw', + driver: 'athena', + extractedAt: '2026-06-21T10:00:00.000Z', + scope: { catalogs: ['AwsDataCatalog'], datasets: ['analytics'] }, + metadata: { + catalog: 'AwsDataCatalog', + databases: ['analytics'], + table_count: 1, + total_columns: 3, + }, + }); + + expect(snapshot.tables[0]).toMatchObject({ + catalog: 'AwsDataCatalog', + db: 'analytics', + name: 'orders', + kind: 'table', + comment: 'Orders table', + estimatedRows: null, + foreignKeys: [], + }); + + expect(snapshot.tables[0]?.columns).toEqual([ + { + name: 'id', + nativeType: 'bigint', + normalizedType: 'BIGINT', + dimensionType: 'number', + nullable: true, + primaryKey: false, + comment: 'Order id', + }, + { + name: 'status', + nativeType: 'string', + normalizedType: 'VARCHAR', + dimensionType: 'string', + nullable: true, + primaryKey: false, + comment: null, + }, + { + name: 'dt', + nativeType: 'date', + normalizedType: 'DATE', + dimensionType: 'time', + nullable: true, + primaryKey: false, + comment: 'Partition date', + }, + ]); + }); + + it('respects tableScope and excludes tables not in scope', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const scopedSnapshot = await connector.introspect( + { + connectionId: 'dw', + driver: 'athena', + tableScope: tableRefSet([{ catalog: 'AwsDataCatalog', db: 'analytics', name: 'nonexistent' }]), + }, + { runId: 'scan-1' }, + ); + expect(scopedSnapshot.tables).toHaveLength(0); + + const matchingSnapshot = await connector.introspect( + { + connectionId: 'dw', + driver: 'athena', + tableScope: tableRefSet([{ catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }]), + }, + { runId: 'scan-1' }, + ); + expect(matchingSnapshot.tables).toHaveLength(1); + expect(matchingSnapshot.tables[0]?.name).toBe('orders'); + }); + + it('limits introspection to the configured databases scope', async () => { + const requestedDatabases: string[] = []; + const getDatabases = vi.fn(async () => ({ + DatabaseList: [{ Name: 'analytics' }, { Name: 'raw' }, { Name: 'staging' }], + NextToken: undefined, + })); + const glueClient: KtxGlueClient = { + getDatabases, + getTables: vi.fn(async (input) => { + requestedDatabases.push(input.DatabaseName); + return { + TableList: [ + { + Name: `${input.DatabaseName}_orders`, + TableType: 'EXTERNAL_TABLE', + StorageDescriptor: { Columns: [{ Name: 'id', Type: 'bigint' }] }, + }, + ], + NextToken: undefined, + }; + }), + }; + const clientFactory: KtxAthenaClientFactory = { + createAthenaClient: vi.fn(() => fakeClientFactory().createAthenaClient('us-east-1')), + createGlueClient: vi.fn(() => glueClient), + }; + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection: { ...connection, databases: ['analytics', 'raw'] }, + clientFactory, + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect({ connectionId: 'dw', driver: 'athena' }, { runId: 'scan-1' }); + + // Scope is taken from config, so the account-wide database list is never enumerated. + expect(getDatabases).not.toHaveBeenCalled(); + expect(requestedDatabases).toEqual(['analytics', 'raw']); + expect(snapshot.scope).toMatchObject({ datasets: ['analytics', 'raw'] }); + expect(snapshot.tables.map((t) => t.db)).toEqual(['analytics', 'raw']); + }); + + it('resolves optional env-referenced config to defaults when the variable is unset', () => { + const resolved = athenaConnectionConfigFromConfig({ + connectionId: 'dw', + connection: { + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://bucket/', + workgroup: 'env:ATHENA_WORKGROUP_UNSET', + catalog: 'env:GLUE_CATALOG_UNSET', + }, + env: {}, + }); + expect(resolved.workgroup).toBe('primary'); + expect(resolved.catalog).toBe('AwsDataCatalog'); + }); + + it('samples a table via Athena query execution', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + const result = await connector.sampleTable( + { + connectionId: 'dw', + table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, + columns: ['id', 'status'], + limit: 10, + }, + { runId: 'scan-1' }, + ); + + expect(result).toMatchObject({ + headers: ['id', 'status'], + rows: [['1', 'paid']], + totalRows: 1, + }); + }); + + it('samples a column via Athena query execution', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + const result = await connector.sampleColumn( + { + connectionId: 'dw', + table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, + column: 'status', + limit: 10, + }, + { runId: 'scan-1' }, + ); + + expect(result).toMatchObject({ + values: ['paid'], + nullCount: null, + distinctCount: null, + }); + }); + + it('executes read-only SQL and rejects write statements', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect( + connector.executeReadOnly( + { connectionId: 'dw', sql: 'SELECT id, status FROM "analytics"."orders"', maxRows: 100 }, + { runId: 'scan-1' }, + ), + ).resolves.toMatchObject({ + headers: ['id', 'status'], + rows: [['1', 'paid']], + rowCount: 1, + }); + + await expect( + connector.executeReadOnly({ connectionId: 'dw', sql: 'DELETE FROM orders' }, { runId: 'scan-1' }), + ).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally'); + }); + + it('lists schemas (databases) from Glue', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect(connector.listSchemas()).resolves.toEqual(['analytics']); + }); + + it('lists tables from Glue', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect(connector.listTables(['analytics'])).resolves.toEqual([ + { + catalog: 'AwsDataCatalog', + schema: 'analytics', + name: 'orders', + kind: 'table', + }, + ]); + }); + + it('returns null for columnStats', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect( + connector.columnStats( + { connectionId: 'dw', table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, column: 'status' }, + { runId: 'scan-1' }, + ), + ).resolves.toBeNull(); + }); + + it('tests connection successfully', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + + await expect(connector.testConnection()).resolves.toMatchObject({ success: true }); + }); + + it('returns failure result when testConnection throws', async () => { + const factory = fakeClientFactory(); + const glueClient = factory.createGlueClient('us-east-1'); + vi.mocked(glueClient.getDatabases).mockRejectedValue(new Error('Access denied')); + const brokenFactory: KtxAthenaClientFactory = { + createAthenaClient: factory.createAthenaClient, + createGlueClient: vi.fn(() => glueClient), + }; + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: brokenFactory, + }); + + await expect(connector.testConnection()).resolves.toMatchObject({ + success: false, + error: 'Access denied', + }); + }); + + it('cleans up without throwing', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory(), + }); + await connector.listSchemas(); + await expect(connector.cleanup()).resolves.toBeUndefined(); + }); + + it('throws when query execution fails', async () => { + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory({ queryState: 'FAILED', queryError: 'Syntax error in SQL' }), + }); + + await expect( + connector.executeReadOnly({ connectionId: 'dw', sql: 'SELECT 1' }, { runId: 'scan-1' }), + ).rejects.toThrow('Athena query FAILED: Syntax error in SQL'); + }); + + it('throws when query execution times out', async () => { + let callCount = 0; + // First now() call sets the deadline; second call simulates time past it. + const now = () => (++callCount === 1 ? new Date(0) : new Date(5 * 60 * 1000 + 1)); + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: fakeClientFactory({ queryState: 'RUNNING' }), + now, + }); + + await expect( + connector.executeReadOnly({ connectionId: 'dw', sql: 'SELECT 1' }, { runId: 'scan-1' }), + ).rejects.toThrow('timed out after 300s'); + }); + + it('passes the exact column list to Athena when sampling specific columns', async () => { + const factory = fakeClientFactory(); + const athenaClient = factory.createAthenaClient('us-east-1'); + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: { createAthenaClient: vi.fn(() => athenaClient), createGlueClient: factory.createGlueClient }, + }); + + await connector.sampleTable( + { + connectionId: 'dw', + table: { catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' }, + columns: ['id', 'status'], + limit: 5, + }, + { runId: 'scan-1' }, + ); + + expect(vi.mocked(athenaClient.startQueryExecution).mock.calls[0]?.[0].QueryString).toBe( + 'SELECT "id", "status" FROM "AwsDataCatalog"."analytics"."orders" LIMIT 5', + ); + }); + + it('paginates Glue databases and tables across multiple pages', async () => { + const glueClient: KtxGlueClient = { + getDatabases: vi.fn() + .mockResolvedValueOnce({ DatabaseList: [{ Name: 'db1' }], NextToken: 'page2' }) + .mockResolvedValueOnce({ DatabaseList: [{ Name: 'db2' }], NextToken: undefined }), + getTables: vi.fn().mockImplementation(async ({ DatabaseName }: { DatabaseName: string }) => { + if (DatabaseName === 'db1') { + return { + TableList: [ + { + Name: 'table_a', + TableType: 'EXTERNAL_TABLE', + StorageDescriptor: { Columns: [{ Name: 'id', Type: 'bigint' }] }, + }, + ], + NextToken: undefined, + }; + } + return { + TableList: [ + { + Name: 'table_b', + TableType: 'EXTERNAL_TABLE', + StorageDescriptor: { Columns: [{ Name: 'id', Type: 'bigint' }] }, + }, + ], + NextToken: undefined, + }; + }), + }; + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: { + createAthenaClient: vi.fn(() => fakeClientFactory().createAthenaClient('us-east-1')), + createGlueClient: vi.fn(() => glueClient), + }, + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect({ connectionId: 'dw', driver: 'athena' }, { runId: 'scan-1' }); + + expect(vi.mocked(glueClient.getDatabases)).toHaveBeenCalledTimes(2); + expect(snapshot.metadata).toMatchObject({ databases: ['db1', 'db2'], table_count: 2 }); + expect(snapshot.tables.map((t) => t.name)).toEqual(['table_a', 'table_b']); + }); + + it('paginates Athena query results across multiple pages', async () => { + const factory = fakeClientFactory(); + const athenaClient = factory.createAthenaClient('us-east-1'); + vi.mocked(athenaClient.getQueryResults) + .mockResolvedValueOnce({ + ResultSet: { + ResultSetMetadata: { + ColumnInfo: [ + { Name: 'id', Type: 'bigint' }, + { Name: 'status', Type: 'string' }, + ], + }, + Rows: [ + // Header row — only present on the first page + { Data: [{ VarCharValue: 'id' }, { VarCharValue: 'status' }] }, + { Data: [{ VarCharValue: '1' }, { VarCharValue: 'paid' }] }, + { Data: [{ VarCharValue: '2' }, { VarCharValue: 'shipped' }] }, + ], + }, + NextToken: 'page-2', + }) + .mockResolvedValueOnce({ + ResultSet: { + ResultSetMetadata: { ColumnInfo: [] }, + // No header row on subsequent pages + Rows: [{ Data: [{ VarCharValue: '3' }, { VarCharValue: 'pending' }] }], + }, + NextToken: undefined, + }); + + const connector = new KtxAthenaScanConnector({ + connectionId: 'dw', + connection, + clientFactory: { createAthenaClient: vi.fn(() => athenaClient), createGlueClient: factory.createGlueClient }, + }); + + const result = await connector.executeReadOnly( + { connectionId: 'dw', sql: 'SELECT id, status FROM "analytics"."orders"', maxRows: 100 }, + { runId: 'scan-1' }, + ); + + expect(result.headers).toEqual(['id', 'status']); + expect(result.rows).toEqual([ + ['1', 'paid'], + ['2', 'shipped'], + ['3', 'pending'], + ]); + expect(result.rowCount).toBe(3); + expect(vi.mocked(athenaClient.getQueryResults)).toHaveBeenCalledTimes(2); + expect(vi.mocked(athenaClient.getQueryResults).mock.calls[1]?.[0].NextToken).toBe('page-2'); + }); + + it('adapts to the live-database introspection port via factory', async () => { + const introspection = createAthenaLiveDatabaseIntrospection({ + connections: { dw: connection }, + clientFactory: fakeClientFactory(), + now: () => new Date('2026-06-21T10:00:00.000Z'), + }); + + await expect(introspection.extractSchema('dw')).resolves.toMatchObject({ + connectionId: 'dw', + driver: 'athena', + metadata: { catalog: 'AwsDataCatalog' }, + tables: expect.arrayContaining([ + expect.objectContaining({ + db: 'analytics', + name: 'orders', + columns: expect.arrayContaining([ + expect.objectContaining({ name: 'id', dimensionType: 'number' }), + ]), + }), + ]), + }); + }); +}); diff --git a/packages/cli/test/connectors/athena/dialect.test.ts b/packages/cli/test/connectors/athena/dialect.test.ts new file mode 100644 index 00000000..642f61c1 --- /dev/null +++ b/packages/cli/test/connectors/athena/dialect.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; +import { KtxAthenaDialect } from '../../../src/connectors/athena/dialect.js'; + +describe('KtxAthenaDialect', () => { + const dialect = new KtxAthenaDialect(); + + it('quotes identifiers and formats catalog.database.table names', () => { + expect(dialect.quoteIdentifier('my"col')).toBe('"my""col"'); + expect(dialect.formatTableName({ catalog: 'AwsDataCatalog', db: 'analytics', name: 'orders' })).toBe( + '"AwsDataCatalog"."analytics"."orders"', + ); + expect(dialect.formatTableName({ db: 'analytics', name: 'orders' })).toBe('"analytics"."orders"'); + expect(dialect.formatTableName({ name: 'orders' })).toBe('"orders"'); + }); + + it('maps native Athena/Glue types to normalized types and dimension types', () => { + expect(dialect.mapDataType('bigint')).toBe('BIGINT'); + expect(dialect.mapDataType('string')).toBe('VARCHAR'); + expect(dialect.mapDataType('array')).toBe('ARRAY'); + expect(dialect.mapDataType('map')).toBe('MAP'); + expect(dialect.mapDataType('struct')).toBe('STRUCT'); + expect(dialect.mapDataType('decimal(18,2)')).toBe('DECIMAL'); + expect(dialect.mapDataType('UNKNOWN_TYPE')).toBe('UNKNOWN_TYPE'); + + expect(dialect.mapToDimensionType('timestamp')).toBe('time'); + expect(dialect.mapToDimensionType('date')).toBe('time'); + expect(dialect.mapToDimensionType('bigint')).toBe('number'); + expect(dialect.mapToDimensionType('double')).toBe('number'); + expect(dialect.mapToDimensionType('decimal(10,2)')).toBe('number'); + expect(dialect.mapToDimensionType('boolean')).toBe('boolean'); + expect(dialect.mapToDimensionType('string')).toBe('string'); + expect(dialect.mapToDimensionType('varchar')).toBe('string'); + }); + + it('generates correct sample and column-sample SQL', () => { + expect(dialect.generateSampleQuery('"analytics"."orders"', 10, ['id', 'status'])).toBe( + 'SELECT "id", "status" FROM "analytics"."orders" LIMIT 10', + ); + expect(dialect.generateSampleQuery('"analytics"."orders"', 5)).toBe( + 'SELECT * FROM "analytics"."orders" LIMIT 5', + ); + expect(dialect.generateColumnSampleQuery('"analytics"."orders"', 'status', 20)).toBe( + 'SELECT "status" FROM "analytics"."orders" WHERE "status" IS NOT NULL LIMIT 20', + ); + }); + + it('generates Presto-style cardinality and distinct-values SQL', () => { + expect(dialect.generateCardinalitySampleQuery('"t"', '"col"', 1000)).toContain('approx_distinct'); + expect(dialect.generateRandomizedCardinalitySampleQuery('"t"', '"col"', 500)).toContain('rand()'); + expect(dialect.generateDistinctValuesQuery('"t"', '"col"', 50)).toContain( + 'SELECT DISTINCT CAST("col" AS VARCHAR) AS val', + ); + }); + + it('returns null for column statistics (unsupported)', () => { + expect(dialect.generateColumnStatisticsQuery('analytics', 'orders')).toBeNull(); + }); + + it('produces Trino-correct OFFSET-before-LIMIT ordering', () => { + expect(dialect.getLimitOffsetClause(10)).toBe('LIMIT 10'); + expect(dialect.getLimitOffsetClause(10, 0)).toBe('LIMIT 10'); + expect(dialect.getLimitOffsetClause(10, 20)).toBe('OFFSET 20 LIMIT 10'); + }); + + it('uses unit-separator (U+001F) as the array_join delimiter', () => { + const sql = dialect.getSampleValueAggregation('SELECT value FROM t'); + const separatorIndex = + sql.indexOf("array_join(array_agg(CAST(value AS VARCHAR)), '") + + "array_join(array_agg(CAST(value AS VARCHAR)), '".length; + expect(sql.charCodeAt(separatorIndex)).toBe(0x1f); + }); +}); diff --git a/packages/cli/test/context/connections/dialects.test.ts b/packages/cli/test/context/connections/dialects.test.ts index 74f95cc2..fad193ed 100644 --- a/packages/cli/test/context/connections/dialects.test.ts +++ b/packages/cli/test/context/connections/dialects.test.ts @@ -305,7 +305,7 @@ describe('getDialectForDriver', () => { it('throws with a supported-driver list for unknown drivers', () => { expect(() => getDialectForDriver('oracle')).toThrow( - 'Unsupported driver "oracle". Supported drivers: bigquery, clickhouse, duckdb, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', + 'Unsupported driver "oracle". Supported drivers: athena, bigquery, clickhouse, duckdb, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', ); }); diff --git a/packages/cli/test/context/connections/drivers.test.ts b/packages/cli/test/context/connections/drivers.test.ts index 38a0f48f..69ac8464 100644 --- a/packages/cli/test/context/connections/drivers.test.ts +++ b/packages/cli/test/context/connections/drivers.test.ts @@ -70,6 +70,11 @@ const connectionFixtures: Record = { database: 'ANALYTICS', schema: 'PUBLIC', }), + athena: () => ({ + driver: 'athena', + region: 'us-east-1', + s3_staging_dir: 's3://my-bucket/athena-results/', + }), }; const allowedScopeKeys = new Set(['dataset_ids', 'databases', 'schemas', 'schema_names']); @@ -100,6 +105,7 @@ describe('driverRegistrations', () => { const registryDrivers = Object.keys(driverRegistrations).sort(); expect(listSupportedDrivers()).toEqual(registryDrivers); expect(listSupportedDrivers()).toEqual([ + 'athena', 'bigquery', 'clickhouse', 'duckdb', diff --git a/packages/cli/test/context/scan/local-scan.test.ts b/packages/cli/test/context/scan/local-scan.test.ts index 5bc60922..75f929d7 100644 --- a/packages/cli/test/context/scan/local-scan.test.ts +++ b/packages/cli/test/context/scan/local-scan.test.ts @@ -2175,6 +2175,40 @@ describe('local scan', () => { }; expect(manifest.tables.orders?.joins?.some((join) => join.to === 'accounts')).toBe(true); }); + + it('accepts athena as a native standalone scan driver when the host supplies a live-database adapter', async () => { + await writeFile( + join(project.projectDir, 'ktx.yaml'), + [ + 'connections:', + ' warehouse:', + ' driver: athena', + ' region: us-east-1', + ' s3_staging_dir: s3://my-bucket/athena-results/', + ' databases:', + ' - analytics', + 'ingest:', + ' adapters:', + ' - live-database', + '', + ].join('\n'), + 'utf-8', + ); + project = await loadKtxProject({ projectDir: project.projectDir }); + + const result = await runLocalScan({ + project, + adapters: [fetchOnlyAdapter()], + connectionId: 'warehouse', + jobId: 'scan-run-athena', + now: () => new Date('2026-04-29T17:00:00.000Z'), + }); + + expect(result.report.driver).toBe('athena'); + expect(result.report.artifactPaths.reportPath).toBe( + 'raw-sources/warehouse/live-database/2026-04-29-170000-scan-run-athena/scan-report.json', + ); + }); }); describe('resolveEnabledTables', () => { diff --git a/packages/cli/test/context/sql-analysis/dialect.test.ts b/packages/cli/test/context/sql-analysis/dialect.test.ts index 68a92af5..0166a133 100644 --- a/packages/cli/test/context/sql-analysis/dialect.test.ts +++ b/packages/cli/test/context/sql-analysis/dialect.test.ts @@ -12,6 +12,7 @@ describe('sqlAnalysisDialectForDriver', () => { expect(sqlAnalysisDialectForDriver('duckdb')).toBe('duckdb'); expect(sqlAnalysisDialectForDriver('clickhouse')).toBe('clickhouse'); expect(sqlAnalysisDialectForDriver('databricks')).toBe('databricks'); + expect(sqlAnalysisDialectForDriver('athena')).toBe('athena'); }); it('maps local connection-type spellings to sqlglot dialects', () => { diff --git a/packages/cli/test/setup-databases.test.ts b/packages/cli/test/setup-databases.test.ts index bcf1db79..feff6613 100644 --- a/packages/cli/test/setup-databases.test.ts +++ b/packages/cli/test/setup-databases.test.ts @@ -243,6 +243,7 @@ describe('setup databases step', () => { { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, + { value: 'athena', label: 'Amazon Athena' }, { value: 'mongodb', label: 'MongoDB' }, { value: 'sqlite', label: 'SQLite' }, { value: 'duckdb', label: 'DuckDB' }, @@ -618,6 +619,29 @@ describe('setup databases step', () => { }, ], }, + { + driver: 'athena', + textValues: ['', 'us-east-1', 's3://my-bucket/athena-results/', '', ''], + expectedTextPrompts: [ + { + message: connectionNamePrompt('Amazon Athena'), + placeholder: 'athena-warehouse', + initialValue: 'athena-warehouse', + }, + { + message: 'AWS region\nFor example us-east-1.', + }, + { + message: 'S3 staging directory\nAthena writes query results here. For example s3://my-bucket/athena-results/.', + }, + { + message: 'Athena workgroup (optional)\nPress Enter to use the default workgroup "primary".', + }, + { + message: 'Glue Data Catalog name (optional)\nPress Enter to use the default "AwsDataCatalog".', + }, + ], + }, ]; for (const testCase of cases) { @@ -1967,6 +1991,40 @@ describe('setup databases step', () => { expect(project.config.connections['clickhouse-warehouse']).not.toHaveProperty('schemas'); }); + it('maps Athena scripted database schema input to databases field', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' athena-warehouse:', + ' driver: athena', + ' region: us-east-1', + ' s3_staging_dir: s3://my-bucket/athena-results/', + '', + ].join('\n'), + 'utf-8', + ); + + await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + skipDatabases: false, + databaseConnectionIds: ['athena-warehouse'], + databaseSchemas: ['analytics', 'raw'], + }, + makeIo().io, + { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0) }, + ); + + const project = await loadKtxProject({ projectDir: tempDir }); + expect(project.config.connections['athena-warehouse']).toMatchObject({ + driver: 'athena', + databases: ['analytics', 'raw'], + }); + expect(project.config.connections['athena-warehouse']).not.toHaveProperty('schemas'); + }); + it('does not prompt for a bootstrap BigQuery dataset before scope discovery', async () => { const prompts = makePromptAdapter({ multiselectValues: [['bigquery']], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53a510d4..6e5a1710 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -137,6 +137,12 @@ importers: '@anthropic-ai/claude-agent-sdk': specifier: 0.3.146 version: 0.3.146(@anthropic-ai/sdk@0.97.1(zod@4.4.3))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(zod@4.4.3) + '@aws-sdk/client-athena': + specifier: ^3.1068.0 + version: 3.1068.0 + '@aws-sdk/client-glue': + specifier: ^3.1068.0 + version: 3.1068.0 '@clack/core': specifier: 1.3.1 version: 1.3.1 @@ -437,6 +443,14 @@ packages: '@aws-crypto/util@5.2.0': resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + '@aws-sdk/client-athena@3.1068.0': + resolution: {integrity: sha512-WvoGXgJ5luTY7wRGqRYQFJ1heRzKiLOiuo7gkQJ9tRRau10XZIyJ0c09+wOvcRXG8CYbUdWpdGkcwswDBE2eSQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-glue@3.1068.0': + resolution: {integrity: sha512-T/2aZGVaDDuSSQ3OWhWowBZ3P2hQGIV+16idxiA0Z8WevLWgbzztSGScDyYa7ohe/J/BY0XiYsBavLYtv+yGlg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-s3@3.1045.0': resolution: {integrity: sha512-fsuO3Y6t+3Ro9Bsg41DKj4Sfy53CGSrhnMldNplWmG8Tx0UbYk+YDa4RD1hVlJpERw4JBmPkl0+J9qlxMh1pcA==} engines: {node: '>=20.0.0'} @@ -449,6 +463,10 @@ packages: resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==} engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.974.20': + resolution: {integrity: sha512-7sDi2B2N3mc3nf1nz6FyEx/FCrJ1N1QnBmraHHQNabFaeAh2IaOOLml48/rHOD1bICHgTRkbBgNTvUzEr5Z35g==} + engines: {node: '>=20.0.0'} + '@aws-sdk/crc64-nvme@3.972.8': resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==} engines: {node: '>=20.0.0'} @@ -457,34 +475,66 @@ packages: resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.46': + resolution: {integrity: sha512-+GPXVS2srMOlH74S+SmC1gVuP2TvUZ0siuC0onKO93q+udP+M72dmY8wJfVQ5CX9z/9X5A1HHwz5yRIGBtskvQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.40': resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.48': + resolution: {integrity: sha512-fA5loSdlocacRxyUXtpoHSMuk5rsIKRDzQYVMnMxjcmFeZshaJlJ8lymy/hYKji6sne/UmNGj5pxuEs6kq/Qcg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.42': resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.53': + resolution: {integrity: sha512-ZfdhIOR41q8TcWEnUac+gCOb+O2LBWdHLmjedXpXz4IEFW2ppNuFcm6p0sMTavpM+zD5TYfpH5Gp7guRyqSgsQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.42': resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.52': + resolution: {integrity: sha512-9hu2oR0qH7Fst5Tzdx+UWxm+w5zCXtErTLtOOW5hwwQc170CLwOeniRxyFY6s9mHfGEfC5zFukNBdKBwJR8mhQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.43': resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.55': + resolution: {integrity: sha512-zMGLa/dhESVqmCD7mmIFFKSwSFrJGScvCXcjvBZEVOOMauFS5JRQvLTMukFpMEFWiV6dTAlsen2ATDBulLPtbg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.38': resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.46': + resolution: {integrity: sha512-VUoNFBIjWrUN8NbFiQiuxQEgFjvziAlBRPK+ddh27aj65gk0BYu6bLZnrdrNZwpW6vAihtSUtEMQ1PUJ32QRPA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.42': resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.52': + resolution: {integrity: sha512-nb2/n4o/HQf+FVpVbZe9vCTFngmuDoIsltMgLAtjixaKzvzhB4J8WSDFyWgnErgLHk55ctWH+I4PU+LIHhyffg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.42': resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.52': + resolution: {integrity: sha512-lKj6aRSGbqLmpYmM24bY7a1Xmfcq2vkE3hv8CSPYfc1yCu0BPu/XEJ1L4Fm61MsU6ULLNSG8UGsffNoFUBjESA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/ec2-metadata-service@3.1045.0': resolution: {integrity: sha512-cYjEbjbGScw9l8TmI9AFYde1hIu5c9Wt0Qp7/cbWBHBiOzMfLwmjGhd5+4AUm1RsnmC5HZ/WOA9iGJHfHL4cuA==} engines: {node: '>=20.0.0'} @@ -533,6 +583,10 @@ packages: resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==} engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.20': + resolution: {integrity: sha512-IYJuLpXp2DEILVQpQOy0PMpkftv0AHEOCn52o0atyOaumA0CdWQ3klPyXdViGYLbNpESsVFMVybvHUeZAuiGxA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/region-config-resolver@3.972.16': resolution: {integrity: sha512-/YaivCvKUkEeMN9VTKBSvBn5w/4osAM1YboM58DKaLF/vqFGf/FdJCLmppqiPPJWZaXcASqByVjc3evE7KHKdA==} engines: {node: '>=20.0.0'} @@ -541,10 +595,22 @@ packages: resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==} engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.34': + resolution: {integrity: sha512-mx1L5qlumSOt/nKM3BFaHE2HVkWwz0i4Bw0pyYO42FfX/FeLlo8YI6csC0gSPprEk6fTIqI+CZN9RwUwKd5krQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1049.0': resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==} engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1066.0': + resolution: {integrity: sha512-UqEUJq7dqa44hneLDUcX7UJy95cg8YqEWyakRpvIPnrNS3Mq+UlQHgCDGu5pvwAPtlIW4qcYbvW6reG6++FyvA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.12': + resolution: {integrity: sha512-43ajd1NF0RMgX5k0hxCNUyEdrtFUsb2aHT2QvpktSC/2Eyb2Jr/JPVqdp0XIoaHWikZJq5tNWSLO6kB5q2eMCA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.973.8': resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} engines: {node: '>=20.0.0'} @@ -568,6 +634,10 @@ packages: resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==} engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.29': + resolution: {integrity: sha512-fk0niuGFxfi8yIJuMVM4mhwObkiQSuwZFj3tAPrLVx64Pk3BkrEIpqjzHKY4hKoEBUD6Jg/S74Zj9jy+5F3DnQ==} + engines: {node: '>=20.0.0'} + '@aws/lambda-invoke-store@0.2.4': resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} @@ -2257,10 +2327,18 @@ packages: resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==} engines: {node: '>=18.0.0'} + '@smithy/core@3.24.7': + resolution: {integrity: sha512-KoUi4M1f3BG6kzN1FnCwL7oyFptTbyBJKjR6yhSib+JHRdUmM1o+VwsFtJ66NZCkCzVfJMWRHJNo0R0jznp0Pg==} + engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.3.3': resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==} engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.3.9': + resolution: {integrity: sha512-ZlfJ/4Fa3jYb+3eaohPfG9utX9HmdhFNcFtpoGAhUhdynAOmGXtmigbi7eEiONKM+ykHw8RwKuDEb85Lx7t7fA==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-serde-browser@4.3.3': resolution: {integrity: sha512-LXg5yYJPYnVSrpa6LOZ+/wqpI2OlIccy7j5F16EFNYDbXWmnhry/PFRRPyM30H+hJeqfVgckFuvNGnAGCt56cA==} engines: {node: '>=18.0.0'} @@ -2277,6 +2355,10 @@ packages: resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==} engines: {node: '>=18.0.0'} + '@smithy/fetch-http-handler@5.4.7': + resolution: {integrity: sha512-NslaM2ir0N2hisDmzXLstPaVINZheh8SokyOC++kzFPloZucL2R7Y7bS57mSzx/1Fc/fqmn7twjkeezTTrV0EA==} + engines: {node: '>=18.0.0'} + '@smithy/hash-blob-browser@4.3.3': resolution: {integrity: sha512-TkGfDlYeWOGwYvAunHHHmKgvFtD7DFAl6gWxATI4pv4B6w0Wnx6RK5zCMoXTTqMVd+zPcWm7w8RPTgHytoCDJA==} engines: {node: '>=18.0.0'} @@ -2329,6 +2411,10 @@ packages: resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==} engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.7.8': + resolution: {integrity: sha512-f+DbsWUwSbtMu1a/j8Y93KiU1SRg9nyzfjereqn1BJ33QOTUXxdlYvVXMhAYl1vuR1Kmna5aIJe09KSIfyFNYw==} + engines: {node: '>=18.0.0'} + '@smithy/protocol-http@5.4.3': resolution: {integrity: sha512-P16TBD/d8ZcD9MHQ0ubQ9BbOYSd5HZKbHOLsyFWxKk2oBEoghbRFPfGOoqToZX1yrfLITXRylL16EyPP4IzLPg==} engines: {node: '>=18.0.0'} @@ -2337,6 +2423,10 @@ packages: resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==} engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.4.7': + resolution: {integrity: sha512-LwQZazFayImv+IOm0S0enoLeUJwmAlhGC5O6YCcLWezyu08dF46GOxPOq35OpBIHkgd7OvNvBStIFwVNyrvoBw==} + engines: {node: '>=18.0.0'} + '@smithy/smithy-client@4.13.3': resolution: {integrity: sha512-Z8mQ+YryjP5krDadV6unnp5035L4S1brafXpTiRmjPweKSaQ6X9CYDYWvmEggXjDIa1oufX/2a/bdwu8EIz/lw==} engines: {node: '>=18.0.0'} @@ -2345,6 +2435,10 @@ packages: resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==} engines: {node: '>=18.0.0'} + '@smithy/types@4.14.4': + resolution: {integrity: sha512-B2S9+UGm1+/pHkcx3ZoLVX1a+pmSk8rqxRR+ZsNqZaJ5q9FWX9AFGQVM4qG5+OBeQUZVy99HY8HqW8gK/wgXzQ==} + engines: {node: '>=18.0.0'} + '@smithy/url-parser@4.3.3': resolution: {integrity: sha512-TsMTAOnjuMOv1zJBw8cfYGWhopyc3og8tZX/KuyCPjg7V3ji3f4YjFOVu843UjBmrfS/+X6kwFv5ZKg7sSm1bQ==} engines: {node: '>=18.0.0'} @@ -6361,6 +6455,32 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 + '@aws-sdk/client-athena@3.1068.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.20 + '@aws-sdk/credential-provider-node': 3.972.55 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + + '@aws-sdk/client-glue@3.1068.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.20 + '@aws-sdk/credential-provider-node': 3.972.55 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/client-s3@3.1045.0': dependencies: '@aws-crypto/sha1-browser': 5.2.0 @@ -6473,6 +6593,17 @@ snapshots: bowser: 2.14.1 tslib: 2.8.1 + '@aws-sdk/core@3.974.20': + dependencies: + '@aws-sdk/types': 3.973.12 + '@aws-sdk/xml-builder': 3.972.29 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.24.7 + '@smithy/signature-v4': 5.4.7 + '@smithy/types': 4.14.4 + bowser: 2.14.1 + tslib: 2.8.1 + '@aws-sdk/crc64-nvme@3.972.8': dependencies: '@smithy/types': 4.14.2 @@ -6486,6 +6617,14 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.46': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.40': dependencies: '@aws-sdk/core': 3.974.12 @@ -6496,6 +6635,16 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.48': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6512,6 +6661,22 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.972.53': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/credential-provider-env': 3.972.46 + '@aws-sdk/credential-provider-http': 3.972.48 + '@aws-sdk/credential-provider-login': 3.972.52 + '@aws-sdk/credential-provider-process': 3.972.46 + '@aws-sdk/credential-provider-sso': 3.972.52 + '@aws-sdk/credential-provider-web-identity': 3.972.52 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/credential-provider-imds': 4.3.9 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6521,6 +6686,15 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.52': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.43': dependencies: '@aws-sdk/credential-provider-env': 3.972.38 @@ -6535,6 +6709,20 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.55': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.46 + '@aws-sdk/credential-provider-http': 3.972.48 + '@aws-sdk/credential-provider-ini': 3.972.53 + '@aws-sdk/credential-provider-process': 3.972.46 + '@aws-sdk/credential-provider-sso': 3.972.52 + '@aws-sdk/credential-provider-web-identity': 3.972.52 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/credential-provider-imds': 4.3.9 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.38': dependencies: '@aws-sdk/core': 3.974.12 @@ -6543,6 +6731,14 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.46': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6553,6 +6749,16 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.972.52': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/token-providers': 3.1066.0 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.42': dependencies: '@aws-sdk/core': 3.974.12 @@ -6562,6 +6768,15 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.52': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/ec2-metadata-service@3.1045.0': dependencies: '@aws-sdk/types': 3.973.8 @@ -6654,6 +6869,19 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/nested-clients@3.997.20': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.20 + '@aws-sdk/signature-v4-multi-region': 3.996.34 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/fetch-http-handler': 5.4.7 + '@smithy/node-http-handler': 4.7.8 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/region-config-resolver@3.972.16': dependencies: '@aws-sdk/core': 3.974.12 @@ -6667,6 +6895,13 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.34': + dependencies: + '@aws-sdk/types': 3.973.12 + '@smithy/signature-v4': 5.4.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/token-providers@3.1049.0': dependencies: '@aws-sdk/core': 3.974.12 @@ -6676,6 +6911,20 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@aws-sdk/token-providers@3.1066.0': + dependencies: + '@aws-sdk/core': 3.974.20 + '@aws-sdk/nested-clients': 3.997.20 + '@aws-sdk/types': 3.973.12 + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + + '@aws-sdk/types@3.973.12': + dependencies: + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@aws-sdk/types@3.973.8': dependencies: '@smithy/types': 4.14.2 @@ -6708,6 +6957,12 @@ snapshots: fast-xml-parser: 5.7.3 tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.29': + dependencies: + '@smithy/types': 4.14.4 + fast-xml-parser: 5.7.3 + tslib: 2.8.1 + '@aws/lambda-invoke-store@0.2.4': {} '@azure-rest/core-client@2.6.0': @@ -8227,12 +8482,24 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/core@3.24.7': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/credential-provider-imds@4.3.3': dependencies: '@smithy/core': 3.24.3 '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/credential-provider-imds@4.3.9': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/eventstream-serde-browser@4.3.3': dependencies: '@smithy/core': 3.24.3 @@ -8254,6 +8521,12 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/fetch-http-handler@5.4.7': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/hash-blob-browser@4.3.3': dependencies: '@smithy/core': 3.24.3 @@ -8319,6 +8592,12 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/node-http-handler@4.7.8': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/protocol-http@5.4.3': dependencies: '@smithy/core': 3.24.3 @@ -8330,6 +8609,12 @@ snapshots: '@smithy/types': 4.14.2 tslib: 2.8.1 + '@smithy/signature-v4@5.4.7': + dependencies: + '@smithy/core': 3.24.7 + '@smithy/types': 4.14.4 + tslib: 2.8.1 + '@smithy/smithy-client@4.13.3': dependencies: '@smithy/core': 3.24.3 @@ -8340,6 +8625,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/types@4.14.4': + dependencies: + tslib: 2.8.1 + '@smithy/url-parser@4.3.3': dependencies: '@smithy/core': 3.24.3