mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-13 11:22:11 +02:00
test: split cli tests from source tree (#216)
* feat(cli): define full warehouse dialect contract
* test(cli): keep dialect edge tests focused
* fix(cli): stabilize dialect contract foundation
* refactor(connectors): own read-only query preparation
* refactor(connectors): resolve dialects through registry
* refactor(connectors): keep concrete dialect classes internal
* chore(workspace): enforce dialect import boundary
* refactor(cli): resolve relationship dialect at scan boundary
* refactor(cli): use dialect display parsing for entity details
* refactor(cli): use dialect display parsing for warehouse catalog
* refactor(cli): use dialect SQL in relationship workflows
* test(cli): verify solid dialect scan workflow closure
* test: split cli tests from source tree
* refactor(cli): standardize BigQuery scope listing
* feat(sqlite): implement connector scope listing
* test(connectors): cover required table listing
* feat(cli): add warehouse driver registry
* refactor(setup): route scope discovery through driver registry
* refactor(cli): route local query execution through driver registry
* refactor(historic-sql): route dialect support through driver registry
* refactor(cli): test warehouse connections through driver registry
* fix(cli): close driver registry type export gaps
* Improve setup daemon diagnostics
* refactor(setup): centralize rail-prefixed diagnostics + query-history fallback
Extract errorMessage, writePrefixedLines, and flushPrefixedBufferedCommandOutput
into clack.ts so the setup wizard, managed daemons, and embedding/agent steps
share one rail-formatted writer. setup-databases.ts also adds a
"disable query history and retry" option when the schema-context build fails
and query history is the likely culprit, surfaced via a new
failed-query-history-unavailable status.
* fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match
The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.
Align the picker boundary with the canonical 3-level KtxTableRef:
- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
(resolveEnabledTables already accepts the 3-part shape) and
schemasFromEnabledTables now goes through parseDottedTableEntry so it
recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
reuse.
Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).
* fix(cli): allow debug telemetry under opt-out env
This commit is contained in:
Andrey Avtomonov
GitHub
parent
924868841d
commit
56985b7e09
548 changed files with 5048 additions and 2228 deletions
|
|
@ -1,556 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { FieldPacket, RowDataPacket } from 'mysql2/promise';
|
||||
import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js';
|
||||
import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlConnectionConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js';
|
||||
import { tableRefSet } from '../../context/scan/table-ref.js';
|
||||
|
||||
function mysqlResult(rows: Record<string, unknown>[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] {
|
||||
return [rows as RowDataPacket[], fields as FieldPacket[]];
|
||||
}
|
||||
|
||||
function fakePoolFactory(): KtxMysqlPoolFactory {
|
||||
const query = vi.fn(async (sql: string, params?: unknown): Promise<[RowDataPacket[], FieldPacket[]]> => {
|
||||
if (sql.includes('INFORMATION_SCHEMA.TABLES')) {
|
||||
return mysqlResult(
|
||||
[
|
||||
{ TABLE_NAME: 'customers', TABLE_TYPE: 'BASE TABLE', TABLE_COMMENT: 'Customer table', TABLE_ROWS: 2 },
|
||||
{ TABLE_NAME: 'orders', TABLE_TYPE: 'BASE TABLE', TABLE_COMMENT: 'InnoDB free: 1 kB; Order table', TABLE_ROWS: 2 },
|
||||
{ TABLE_NAME: 'order_summary', TABLE_TYPE: 'VIEW', TABLE_COMMENT: '', TABLE_ROWS: null },
|
||||
],
|
||||
[{ name: 'TABLE_NAME' }, { name: 'TABLE_TYPE' }, { name: 'TABLE_COMMENT' }, { name: 'TABLE_ROWS' }],
|
||||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.COLUMNS')) {
|
||||
return mysqlResult(
|
||||
[
|
||||
{ TABLE_NAME: 'customers', COLUMN_NAME: 'id', DATA_TYPE: 'int', IS_NULLABLE: 'NO', COLUMN_COMMENT: 'PK' },
|
||||
{ TABLE_NAME: 'customers', COLUMN_NAME: 'name', DATA_TYPE: 'varchar', IS_NULLABLE: 'NO', COLUMN_COMMENT: '' },
|
||||
{ TABLE_NAME: 'orders', COLUMN_NAME: 'id', DATA_TYPE: 'int', IS_NULLABLE: 'NO', COLUMN_COMMENT: '' },
|
||||
{ TABLE_NAME: 'orders', COLUMN_NAME: 'customer_id', DATA_TYPE: 'int', IS_NULLABLE: 'NO', COLUMN_COMMENT: '' },
|
||||
{ TABLE_NAME: 'orders', COLUMN_NAME: 'status', DATA_TYPE: 'varchar', IS_NULLABLE: 'YES', COLUMN_COMMENT: '' },
|
||||
{ TABLE_NAME: 'order_summary', COLUMN_NAME: 'status', DATA_TYPE: 'varchar', IS_NULLABLE: 'YES', COLUMN_COMMENT: '' },
|
||||
],
|
||||
[{ name: 'TABLE_NAME' }, { name: 'COLUMN_NAME' }, { name: 'DATA_TYPE' }, { name: 'IS_NULLABLE' }],
|
||||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes("CONSTRAINT_NAME = 'PRIMARY'")) {
|
||||
return mysqlResult([{ TABLE_NAME: 'customers', COLUMN_NAME: 'id' }, { TABLE_NAME: 'orders', COLUMN_NAME: 'id' }], []);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes('REFERENCED_TABLE_NAME IS NOT NULL')) {
|
||||
return mysqlResult(
|
||||
[
|
||||
{
|
||||
TABLE_NAME: 'orders',
|
||||
COLUMN_NAME: 'customer_id',
|
||||
REFERENCED_TABLE_NAME: 'customers',
|
||||
REFERENCED_COLUMN_NAME: 'id',
|
||||
CONSTRAINT_NAME: 'orders_customer_id_fk',
|
||||
},
|
||||
],
|
||||
[],
|
||||
);
|
||||
}
|
||||
if (sql.includes('SELECT `id`, `status` FROM `analytics`.`orders` LIMIT 1')) {
|
||||
return mysqlResult([{ id: 10, status: 'paid' }], [{ name: 'id', type: 3 }, { name: 'status', type: 253 }]);
|
||||
}
|
||||
if (sql.includes('select * from (select id, status from analytics.orders) as ktx_query_result limit 1')) {
|
||||
return mysqlResult([{ id: 10, status: 'paid' }], [{ name: 'id', type: 3 }, { name: 'status', type: 253 }]);
|
||||
}
|
||||
if (sql.includes('SELECT `status` FROM `analytics`.`orders`')) {
|
||||
return mysqlResult([{ status: 'paid' }, { status: 'open' }], [{ name: 'status', type: 253 }]);
|
||||
}
|
||||
if (sql.includes('COUNT(DISTINCT val)')) {
|
||||
return mysqlResult([{ cardinality: 2 }], [{ name: 'cardinality', type: 8 }]);
|
||||
}
|
||||
if (sql.includes('SELECT DISTINCT CAST(`status` AS CHAR) AS val')) {
|
||||
return mysqlResult([{ val: 'open' }, { val: 'paid' }], [{ name: 'val', type: 253 }]);
|
||||
}
|
||||
if (sql.includes('COUNT(*) AS count')) {
|
||||
return mysqlResult([{ count: 2 }], [{ name: 'count', type: 8 }]);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.SCHEMATA')) {
|
||||
return mysqlResult([{ SCHEMA_NAME: 'analytics' }, { SCHEMA_NAME: 'warehouse' }], [{ name: 'SCHEMA_NAME' }]);
|
||||
}
|
||||
if (sql.trim() === 'SELECT 1') {
|
||||
return mysqlResult([{ '1': 1 }], [{ name: '1', type: 8 }]);
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${sql} params=${JSON.stringify(params)}`);
|
||||
});
|
||||
const release = vi.fn();
|
||||
const end = vi.fn(async () => undefined);
|
||||
return {
|
||||
createPool: vi.fn(() => ({
|
||||
getConnection: vi.fn(async () => ({ query, release })),
|
||||
end,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function multiSchemaMysqlPoolFactory(
|
||||
options: { primaryKeyError?: Error; foreignKeyError?: Error } = {},
|
||||
): KtxMysqlPoolFactory {
|
||||
const query = vi.fn(async (sql: string, params?: unknown): Promise<[RowDataPacket[], FieldPacket[]]> => {
|
||||
if (sql.includes('INFORMATION_SCHEMA.TABLES')) {
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
return mysqlResult(
|
||||
[
|
||||
{
|
||||
TABLE_SCHEMA: 'analytics',
|
||||
TABLE_NAME: 'customers',
|
||||
TABLE_TYPE: 'BASE TABLE',
|
||||
TABLE_COMMENT: '',
|
||||
TABLE_ROWS: 2,
|
||||
},
|
||||
{
|
||||
TABLE_SCHEMA: 'mart',
|
||||
TABLE_NAME: 'orders',
|
||||
TABLE_TYPE: 'BASE TABLE',
|
||||
TABLE_COMMENT: '',
|
||||
TABLE_ROWS: 3,
|
||||
},
|
||||
],
|
||||
[
|
||||
{ name: 'TABLE_SCHEMA' },
|
||||
{ name: 'TABLE_NAME' },
|
||||
{ name: 'TABLE_TYPE' },
|
||||
{ name: 'TABLE_COMMENT' },
|
||||
{ name: 'TABLE_ROWS' },
|
||||
],
|
||||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.COLUMNS')) {
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
return mysqlResult(
|
||||
[
|
||||
{
|
||||
TABLE_SCHEMA: 'analytics',
|
||||
TABLE_NAME: 'customers',
|
||||
COLUMN_NAME: 'id',
|
||||
DATA_TYPE: 'int',
|
||||
IS_NULLABLE: 'NO',
|
||||
COLUMN_COMMENT: '',
|
||||
},
|
||||
{
|
||||
TABLE_SCHEMA: 'mart',
|
||||
TABLE_NAME: 'orders',
|
||||
COLUMN_NAME: 'id',
|
||||
DATA_TYPE: 'int',
|
||||
IS_NULLABLE: 'NO',
|
||||
COLUMN_COMMENT: '',
|
||||
},
|
||||
],
|
||||
[],
|
||||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes("CONSTRAINT_NAME = 'PRIMARY'")) {
|
||||
if (options.primaryKeyError) {
|
||||
throw options.primaryKeyError;
|
||||
}
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
return mysqlResult(
|
||||
[
|
||||
{ TABLE_SCHEMA: 'analytics', TABLE_NAME: 'customers', COLUMN_NAME: 'id' },
|
||||
{ TABLE_SCHEMA: 'mart', TABLE_NAME: 'orders', COLUMN_NAME: 'id' },
|
||||
],
|
||||
[],
|
||||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes('REFERENCED_TABLE_NAME IS NOT NULL')) {
|
||||
if (options.foreignKeyError) {
|
||||
throw options.foreignKeyError;
|
||||
}
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
return mysqlResult([], []);
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${sql} params=${JSON.stringify(params)}`);
|
||||
});
|
||||
return {
|
||||
createPool: vi.fn(() => ({
|
||||
getConnection: vi.fn(async () => ({ query, release: vi.fn() })),
|
||||
end: vi.fn(async () => undefined),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
describe('KtxMysqlScanConnector', () => {
|
||||
it('resolves MySQL connection configuration safely', () => {
|
||||
expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true);
|
||||
expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false);
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
port: 3307,
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
ssl: true,
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
host: 'db.example.test',
|
||||
port: 3307,
|
||||
database: 'analytics',
|
||||
user: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
ssl: { rejectUnauthorized: false },
|
||||
});
|
||||
});
|
||||
|
||||
it('defaults and validates MySQL maxConnections', () => {
|
||||
const baseConnection: KtxMysqlConnectionConfig = {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
};
|
||||
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: baseConnection,
|
||||
}),
|
||||
).toMatchObject({ connectionLimit: 10 });
|
||||
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections: 25 },
|
||||
}),
|
||||
).toMatchObject({ connectionLimit: 25 });
|
||||
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections: '12' as never },
|
||||
}),
|
||||
).toMatchObject({ connectionLimit: 12 });
|
||||
|
||||
for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) {
|
||||
expect(() =>
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections },
|
||||
}),
|
||||
).toThrow('connections.warehouse.maxConnections must be a positive integer');
|
||||
}
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory: fakePoolFactory(),
|
||||
now: () => new Date('2026-04-29T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await connector.introspect(
|
||||
{ connectionId: 'warehouse', driver: 'mysql' },
|
||||
{ runId: 'scan-run-1' },
|
||||
);
|
||||
|
||||
expect(snapshot).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
driver: 'mysql',
|
||||
extractedAt: '2026-04-29T12:00:00.000Z',
|
||||
scope: { schemas: ['analytics'] },
|
||||
metadata: {
|
||||
database: 'analytics',
|
||||
host: 'db.example.test',
|
||||
table_count: 3,
|
||||
total_columns: 6,
|
||||
},
|
||||
});
|
||||
expect(snapshot.tables.map((table) => [table.name, table.kind, table.estimatedRows, table.comment])).toEqual([
|
||||
['customers', 'table', 2, 'Customer table'],
|
||||
['orders', 'table', 2, 'Order table'],
|
||||
['order_summary', 'view', null, null],
|
||||
]);
|
||||
expect(snapshot.tables.find((table) => table.name === 'customers')?.columns[0]).toMatchObject({
|
||||
name: 'id',
|
||||
nativeType: 'int',
|
||||
normalizedType: 'int',
|
||||
dimensionType: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
comment: 'PK',
|
||||
});
|
||||
expect(snapshot.tables.find((table) => table.name === 'orders')?.foreignKeys).toEqual([
|
||||
{
|
||||
fromColumn: 'customer_id',
|
||||
toCatalog: null,
|
||||
toDb: 'analytics',
|
||||
toTable: 'customers',
|
||||
toColumn: 'id',
|
||||
constraintName: 'orders_customer_id_fk',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('introspects every configured MySQL schema scope', async () => {
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
schemas: ['analytics', 'mart'],
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory: multiSchemaMysqlPoolFactory(),
|
||||
now: () => new Date('2026-05-21T10:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await connector.introspect(
|
||||
{ connectionId: 'warehouse', driver: 'mysql' },
|
||||
{ runId: 'scan-run-1' },
|
||||
);
|
||||
|
||||
expect(snapshot.scope).toEqual({ schemas: ['analytics', 'mart'] });
|
||||
expect(snapshot.metadata).toMatchObject({ database: 'analytics', schemas: ['analytics', 'mart'] });
|
||||
expect(snapshot.tables.map((table) => `${table.db}.${table.name}`)).toEqual([
|
||||
'analytics.customers',
|
||||
'mart.orders',
|
||||
]);
|
||||
});
|
||||
|
||||
it('soft-fails denied MySQL constraint discovery with one warning per schema and kind', async () => {
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
schemas: ['analytics', 'mart'],
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory: multiSchemaMysqlPoolFactory({
|
||||
primaryKeyError: Object.assign(new Error('select command denied'), {
|
||||
code: 'ER_TABLEACCESS_DENIED_ERROR',
|
||||
errno: 1142,
|
||||
}),
|
||||
foreignKeyError: Object.assign(new Error('database access denied'), {
|
||||
code: 'ER_DBACCESS_DENIED_ERROR',
|
||||
errno: 1044,
|
||||
}),
|
||||
}),
|
||||
now: () => new Date('2026-04-29T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await connector.introspect(
|
||||
{ connectionId: 'warehouse', driver: 'mysql' },
|
||||
{ runId: 'scan-run-mysql-denied-constraints' },
|
||||
);
|
||||
|
||||
expect(snapshot.warnings).toEqual([
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in analytics (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'analytics', kind: 'primary_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in mart (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'mart', kind: 'primary_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'analytics', kind: 'foreign_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in mart (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'mart', kind: 'foreign_key' },
|
||||
},
|
||||
]);
|
||||
expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true);
|
||||
expect(snapshot.tables.every((table) => table.foreignKeys.length === 0)).toBe(true);
|
||||
});
|
||||
|
||||
it('limits introspection to tables in tableScope', async () => {
|
||||
const queries: Array<{ sql: string; params?: unknown }> = [];
|
||||
const poolFactory: KtxMysqlPoolFactory = {
|
||||
createPool: vi.fn(() => ({
|
||||
getConnection: vi.fn(async () => ({
|
||||
query: vi.fn(async (sql: string, params?: unknown): Promise<[RowDataPacket[], FieldPacket[]]> => {
|
||||
queries.push({ sql, params });
|
||||
if (sql.includes('INFORMATION_SCHEMA.TABLES')) {
|
||||
return mysqlResult(
|
||||
[
|
||||
{
|
||||
TABLE_SCHEMA: 'analytics',
|
||||
TABLE_NAME: 'orders',
|
||||
TABLE_TYPE: 'BASE TABLE',
|
||||
TABLE_COMMENT: '',
|
||||
TABLE_ROWS: 2,
|
||||
},
|
||||
],
|
||||
[],
|
||||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.COLUMNS')) {
|
||||
return mysqlResult(
|
||||
[
|
||||
{
|
||||
TABLE_SCHEMA: 'analytics',
|
||||
TABLE_NAME: 'orders',
|
||||
COLUMN_NAME: 'id',
|
||||
DATA_TYPE: 'int',
|
||||
IS_NULLABLE: 'NO',
|
||||
COLUMN_COMMENT: '',
|
||||
},
|
||||
],
|
||||
[],
|
||||
);
|
||||
}
|
||||
return mysqlResult([], []);
|
||||
}),
|
||||
release: vi.fn(),
|
||||
})),
|
||||
end: vi.fn(async () => undefined),
|
||||
})),
|
||||
};
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory,
|
||||
});
|
||||
const scope = tableRefSet([{ catalog: null, db: 'analytics', name: 'orders' }]);
|
||||
const snapshot = await connector.introspect(
|
||||
{ connectionId: 'warehouse', driver: 'mysql', tableScope: scope },
|
||||
{ runId: 'scope-test' },
|
||||
);
|
||||
expect(snapshot.tables.map((table) => table.name)).toEqual(['orders']);
|
||||
const tablesQuery = queries.find((query) => query.sql.includes('INFORMATION_SCHEMA.TABLES'));
|
||||
expect(tablesQuery?.sql).toMatch(/TABLE_NAME IN \(\?\)/);
|
||||
expect(tablesQuery?.params).toEqual(['analytics', 'orders']);
|
||||
});
|
||||
|
||||
it('runs samples, distinct values, read-only SQL, row count, schema list, and cleanup', async () => {
|
||||
const poolFactory = fakePoolFactory();
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory,
|
||||
});
|
||||
|
||||
await expect(
|
||||
connector.sampleTable(
|
||||
{ connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'orders' }, columns: ['id', 'status'], limit: 1 },
|
||||
{ runId: 'scan-run-1' },
|
||||
),
|
||||
).resolves.toEqual({ headers: ['id', 'status'], rows: [[10, 'paid']], totalRows: 1 });
|
||||
|
||||
await expect(
|
||||
connector.sampleColumn(
|
||||
{ connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'orders' }, column: 'status', limit: 5 },
|
||||
{ runId: 'scan-run-1' },
|
||||
),
|
||||
).resolves.toMatchObject({ values: ['paid', 'open'], nullCount: null, distinctCount: null });
|
||||
|
||||
await expect(
|
||||
connector.getColumnDistinctValues(
|
||||
{ catalog: null, db: 'analytics', name: 'orders' },
|
||||
'status',
|
||||
{ maxCardinality: 5, limit: 10, sampleSize: 100 },
|
||||
),
|
||||
).resolves.toEqual({ values: ['open', 'paid'], cardinality: 2 });
|
||||
|
||||
await expect(
|
||||
connector.executeReadOnly(
|
||||
{ connectionId: 'warehouse', sql: 'select id, status from analytics.orders', maxRows: 1 },
|
||||
{ runId: 'scan-run-1' },
|
||||
),
|
||||
).resolves.toMatchObject({ headers: ['id', 'status'], rows: [[10, 'paid']], totalRows: 1, rowCount: 1 });
|
||||
|
||||
await expect(
|
||||
connector.executeReadOnly({ connectionId: 'warehouse', sql: 'delete from orders' }, { runId: 'scan-run-1' }),
|
||||
).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally');
|
||||
|
||||
await expect(connector.getTableRowCount('orders')).resolves.toBe(2);
|
||||
await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']);
|
||||
await expect(connector.columnStats(
|
||||
{ connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'orders' }, column: 'status' },
|
||||
{ runId: 'scan-run-1' },
|
||||
)).resolves.toBeNull();
|
||||
|
||||
await connector.cleanup();
|
||||
});
|
||||
|
||||
it('adapts native MySQL snapshots to live-database introspection for local ingest', async () => {
|
||||
const introspection = createMysqlLiveDatabaseIntrospection({
|
||||
connections: {
|
||||
warehouse: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
},
|
||||
poolFactory: fakePoolFactory(),
|
||||
now: () => new Date('2026-04-29T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await introspection.extractSchema('warehouse');
|
||||
|
||||
expect(snapshot).toMatchObject({
|
||||
connectionId: 'warehouse',
|
||||
extractedAt: '2026-04-29T12:00:00.000Z',
|
||||
});
|
||||
expect(snapshot.tables.find((table) => table.name === 'customers')).toMatchObject({
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'analytics',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
nativeType: 'int',
|
||||
normalizedType: 'int',
|
||||
dimensionType: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
comment: 'PK',
|
||||
},
|
||||
{
|
||||
name: 'name',
|
||||
nativeType: 'varchar',
|
||||
normalizedType: 'varchar',
|
||||
dimensionType: 'string',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
comment: null,
|
||||
},
|
||||
],
|
||||
foreignKeys: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -2,6 +2,7 @@ import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/p
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import {
|
||||
constraintDiscoveryWarning,
|
||||
|
|
@ -30,7 +31,6 @@ import {
|
|||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '../../context/scan/types.js';
|
||||
import { KtxMysqlDialect } from './dialect.js';
|
||||
|
||||
export interface KtxMysqlConnectionConfig {
|
||||
driver?: string;
|
||||
|
|
@ -303,6 +303,25 @@ function queryParams(params: Record<string, unknown> | unknown[] | undefined): u
|
|||
return Array.isArray(params) ? params : Object.values(params);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function prepareMysqlReadOnlyQuery(
|
||||
sql: string,
|
||||
params?: Record<string, unknown>,
|
||||
): { sql: string; params?: unknown[] } {
|
||||
if (!params) {
|
||||
return { sql, params: undefined };
|
||||
}
|
||||
const values: unknown[] = [];
|
||||
const parameterizedQuery = sql.replace(/:([A-Za-z_][A-Za-z0-9_]*)\b/g, (placeholder, key: string) => {
|
||||
if (!(key in params)) {
|
||||
return placeholder;
|
||||
}
|
||||
values.push(params[key]);
|
||||
return '?';
|
||||
});
|
||||
return { sql: parameterizedQuery, params: values };
|
||||
}
|
||||
|
||||
export function isKtxMysqlConnectionConfig(
|
||||
connection: KtxMysqlConnectionConfig | undefined,
|
||||
): connection is KtxMysqlConnectionConfig {
|
||||
|
|
@ -376,7 +395,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
private readonly poolFactory: KtxMysqlPoolFactory;
|
||||
private readonly endpointResolver?: KtxMysqlEndpointResolver;
|
||||
private readonly now: () => Date;
|
||||
private readonly dialect = new KtxMysqlDialect();
|
||||
private readonly dialect = getDialectForDriver('mysql');
|
||||
private pool: KtxMysqlPool | null = null;
|
||||
private resolvedEndpoint: KtxMysqlResolvedEndpoint | null = null;
|
||||
|
||||
|
|
@ -550,7 +569,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows);
|
||||
const prepared = Array.isArray(input.params)
|
||||
? { sql: limitedSql, params: input.params }
|
||||
: this.dialect.prepareQuery(limitedSql, input.params);
|
||||
: prepareMysqlReadOnlyQuery(limitedSql, input.params);
|
||||
const result = await this.query(prepared.sql, prepared.params);
|
||||
return { ...result, rowCount: result.rows.length };
|
||||
}
|
||||
|
|
@ -625,6 +644,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
filterSchemas,
|
||||
);
|
||||
return rows.map((row) => ({
|
||||
catalog: null,
|
||||
schema: row.TABLE_SCHEMA,
|
||||
name: row.TABLE_NAME,
|
||||
kind: row.TABLE_TYPE === 'VIEW' ? ('view' as const) : ('table' as const),
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { KtxMysqlDialect } from './dialect.js';
|
||||
|
||||
describe('KtxMysqlDialect', () => {
|
||||
const dialect = new KtxMysqlDialect();
|
||||
|
||||
it('quotes identifiers and formats database-qualified table names', () => {
|
||||
expect(dialect.quoteIdentifier('orders')).toBe('`orders`');
|
||||
expect(dialect.quoteIdentifier('odd`name')).toBe('`odd``name`');
|
||||
expect(dialect.formatTableName({ catalog: null, db: 'analytics', name: 'orders' })).toBe(
|
||||
'`analytics`.`orders`',
|
||||
);
|
||||
expect(dialect.formatTableName({ catalog: null, db: null, name: 'orders' })).toBe('`orders`');
|
||||
});
|
||||
|
||||
it('maps native MySQL types to KTX dimension types', () => {
|
||||
expect(dialect.mapToDimensionType('tinyint(1)')).toBe('boolean');
|
||||
expect(dialect.mapToDimensionType('int')).toBe('number');
|
||||
expect(dialect.mapToDimensionType('decimal(10,2)')).toBe('number');
|
||||
expect(dialect.mapToDimensionType('timestamp')).toBe('time');
|
||||
expect(dialect.mapToDimensionType('varchar(255)')).toBe('string');
|
||||
expect(dialect.mapToDimensionType('json')).toBe('string');
|
||||
expect(dialect.mapToDimensionType('')).toBe('string');
|
||||
});
|
||||
|
||||
it('builds sampling, distinct-value, pagination, and time SQL', () => {
|
||||
expect(dialect.generateSampleQuery('`analytics`.`orders`', 25, ['id', 'status'])).toBe(
|
||||
'SELECT `id`, `status` FROM `analytics`.`orders` LIMIT 25',
|
||||
);
|
||||
expect(dialect.generateColumnSampleQuery('`analytics`.`orders`', 'status', 10)).toBe(
|
||||
"SELECT `status` FROM `analytics`.`orders` WHERE `status` IS NOT NULL AND TRIM(CAST(`status` AS CHAR)) != '' LIMIT 10",
|
||||
);
|
||||
expect(dialect.generateDistinctValuesQuery('`analytics`.`orders`', '`status`', 5)).toContain(
|
||||
'SELECT DISTINCT CAST(`status` AS CHAR) AS val',
|
||||
);
|
||||
expect(dialect.getLimitOffsetClause(10, 20)).toBe('LIMIT 10 OFFSET 20');
|
||||
expect(dialect.getTimeTruncExpression('created_at', 'month')).toBe("DATE_FORMAT(created_at, '%Y-%m-01')");
|
||||
});
|
||||
|
||||
it('prepares named parameters in deterministic SQL placeholder order', () => {
|
||||
expect(dialect.prepareQuery('select * from orders where id = :id and status = :status', {
|
||||
status: 'paid',
|
||||
id: 10,
|
||||
})).toEqual({
|
||||
sql: 'select * from orders where id = ? and status = ?',
|
||||
params: [10, 'paid'],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,9 +1,18 @@
|
|||
import type { KtxDialect } from '../../context/connections/dialects.js';
|
||||
import {
|
||||
columnDisplayPartCount,
|
||||
formatDialectDisplayRef,
|
||||
formatDialectTableName,
|
||||
limitOffsetClause,
|
||||
parseDialectDisplayRef,
|
||||
} from '../../context/connections/dialect-helpers.js';
|
||||
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
||||
|
||||
type MysqlTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
||||
|
||||
export class KtxMysqlDialect {
|
||||
readonly type = 'mysql';
|
||||
/** @internal */
|
||||
export class KtxMysqlDialect implements KtxDialect {
|
||||
readonly type = 'mysql' as const;
|
||||
|
||||
private readonly typeMappings: Record<string, KtxSchemaDimensionType> = {
|
||||
datetime: 'time',
|
||||
|
|
@ -41,9 +50,19 @@ export class KtxMysqlDialect {
|
|||
}
|
||||
|
||||
formatTableName(table: MysqlTableNameRef): string {
|
||||
return table.db
|
||||
? `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}`
|
||||
: this.quoteIdentifier(table.name);
|
||||
return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi');
|
||||
}
|
||||
|
||||
formatDisplayRef(table: MysqlTableNameRef): string {
|
||||
return formatDialectDisplayRef(table, 'ansi');
|
||||
}
|
||||
|
||||
parseDisplayRef(display: string): KtxTableRef | null {
|
||||
return parseDialectDisplayRef(display, 'ansi');
|
||||
}
|
||||
|
||||
columnDisplayTablePartCount(): 1 | 2 | 3 {
|
||||
return columnDisplayPartCount('ansi');
|
||||
}
|
||||
|
||||
mapDataType(nativeType: string): string {
|
||||
|
|
@ -91,21 +110,6 @@ export class KtxMysqlDialect {
|
|||
return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND TRIM(CAST(${quotedColumn} AS CHAR)) != '' LIMIT ${limit}`;
|
||||
}
|
||||
|
||||
prepareQuery(sql: string, params?: Record<string, unknown>): { sql: string; params?: unknown[] } {
|
||||
if (!params) {
|
||||
return { sql, params: undefined };
|
||||
}
|
||||
const values: unknown[] = [];
|
||||
const parameterizedQuery = sql.replace(/:([A-Za-z_][A-Za-z0-9_]*)\b/g, (placeholder, key: string) => {
|
||||
if (!(key in params)) {
|
||||
return placeholder;
|
||||
}
|
||||
values.push(params[key]);
|
||||
return '?';
|
||||
});
|
||||
return { sql: parameterizedQuery, params: values };
|
||||
}
|
||||
|
||||
getRandomSampleFilter(samplePct: number): string {
|
||||
if (samplePct <= 0 || samplePct >= 1) {
|
||||
return '';
|
||||
|
|
@ -118,7 +122,11 @@ export class KtxMysqlDialect {
|
|||
}
|
||||
|
||||
getLimitOffsetClause(limit: number, offset?: number): string {
|
||||
return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`;
|
||||
return limitOffsetClause(limit, offset);
|
||||
}
|
||||
|
||||
getTopClause(_limit: number): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
getNullCountExpression(column: string): string {
|
||||
|
|
@ -129,6 +137,18 @@ export class KtxMysqlDialect {
|
|||
return `COUNT(DISTINCT ${column})`;
|
||||
}
|
||||
|
||||
textLengthExpression(columnSql: string): string {
|
||||
return `CHAR_LENGTH(CAST(${columnSql} AS CHAR))`;
|
||||
}
|
||||
|
||||
castToText(columnSql: string): string {
|
||||
return `CAST(${columnSql} AS CHAR)`;
|
||||
}
|
||||
|
||||
getSampleValueAggregation(innerSql: string): string {
|
||||
return `(SELECT GROUP_CONCAT(CAST(value AS CHAR) SEPARATOR CHAR(31)) FROM (${innerSql}) AS relationship_profile_values)`;
|
||||
}
|
||||
|
||||
generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string {
|
||||
return `
|
||||
SELECT COUNT(DISTINCT val) AS cardinality
|
||||
|
|
@ -167,36 +187,4 @@ export class KtxMysqlDialect {
|
|||
) AS sampled
|
||||
`;
|
||||
}
|
||||
|
||||
getTimeTruncExpression(
|
||||
column: string,
|
||||
granularity: 'day' | 'week' | 'month' | 'quarter' | 'year',
|
||||
timezone?: string,
|
||||
): string {
|
||||
const col = timezone ? `CONVERT_TZ(${column}, '+00:00', '${timezone}')` : column;
|
||||
switch (granularity) {
|
||||
case 'day':
|
||||
return `DATE(${col})`;
|
||||
case 'week':
|
||||
return `DATE(${col} - INTERVAL WEEKDAY(${col}) DAY)`;
|
||||
case 'month':
|
||||
return `DATE_FORMAT(${col}, '%Y-%m-01')`;
|
||||
case 'quarter':
|
||||
return `MAKEDATE(YEAR(${col}), 1) + INTERVAL (QUARTER(${col}) - 1) QUARTER`;
|
||||
case 'year':
|
||||
return `DATE_FORMAT(${col}, '%Y-01-01')`;
|
||||
}
|
||||
}
|
||||
|
||||
getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string {
|
||||
const col = timezone ? `CONVERT_TZ(${column}, '+00:00', '${timezone}')` : column;
|
||||
const [amount, unit] = interval.split(' ');
|
||||
const originExpr = origin ? `'${origin}'` : `'1970-01-01'`;
|
||||
return `DATE_ADD(${originExpr}, INTERVAL FLOOR(TIMESTAMPDIFF(${unit!.toUpperCase()}, ${originExpr}, ${col}) / ${amount}) * ${amount} ${unit!.toUpperCase()})`;
|
||||
}
|
||||
|
||||
parseIntervalToSql(interval: string): string {
|
||||
const [amount, unit] = interval.split(' ');
|
||||
return `INTERVAL ${amount} ${unit!.toUpperCase()}`;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue