mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-22 11:51:01 +02:00
feat(connectors): generalize readiness and constraint handling (#212)
* feat(connectors): add postgres maxConnections * feat(connectors): add mysql maxConnections * feat(connectors): add sqlserver maxConnections * feat(connectors): rename snowflake pool config * docs: document connector maxConnections * feat(scan): add constraint discovery warning helper * feat(scan): carry structural warnings through reports * feat(postgres): soft-fail denied constraint discovery * feat(mysql): soft-fail denied constraint discovery * feat(sqlserver): soft-fail denied constraint discovery * feat(bigquery): soft-fail denied primary key discovery * feat(snowflake): report denied primary key discovery * test(scan): verify constraint discovery warnings * feat(historic-sql): use shared readiness probes * docs: document query history readiness probes * test(historic-sql): verify readiness probe registry * test(ingest): account for live database warnings artifact * Add skip option for agent setup
This commit is contained in:
Andrey Avtomonov
GitHub
parent
cfd1749ab9
commit
78b8a0c025
42 changed files with 2763 additions and 554 deletions
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { FieldPacket, RowDataPacket } from 'mysql2/promise';
|
||||
import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js';
|
||||
import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js';
|
||||
import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlConnectionConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js';
|
||||
import { tableRefSet } from '../../context/scan/table-ref.js';
|
||||
|
||||
function mysqlResult(rows: Record<string, unknown>[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] {
|
||||
|
|
@ -86,7 +86,9 @@ function fakePoolFactory(): KtxMysqlPoolFactory {
|
|||
};
|
||||
}
|
||||
|
||||
function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory {
|
||||
function multiSchemaMysqlPoolFactory(
|
||||
options: { primaryKeyError?: Error; foreignKeyError?: Error } = {},
|
||||
): KtxMysqlPoolFactory {
|
||||
const query = vi.fn(async (sql: string, params?: unknown): Promise<[RowDataPacket[], FieldPacket[]]> => {
|
||||
if (sql.includes('INFORMATION_SCHEMA.TABLES')) {
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
|
|
@ -141,6 +143,9 @@ function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory {
|
|||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes("CONSTRAINT_NAME = 'PRIMARY'")) {
|
||||
if (options.primaryKeyError) {
|
||||
throw options.primaryKeyError;
|
||||
}
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
return mysqlResult(
|
||||
[
|
||||
|
|
@ -151,6 +156,9 @@ function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory {
|
|||
);
|
||||
}
|
||||
if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes('REFERENCED_TABLE_NAME IS NOT NULL')) {
|
||||
if (options.foreignKeyError) {
|
||||
throw options.foreignKeyError;
|
||||
}
|
||||
expect(params).toEqual(['analytics', 'mart']);
|
||||
return mysqlResult([], []);
|
||||
}
|
||||
|
|
@ -191,6 +199,46 @@ describe('KtxMysqlScanConnector', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('defaults and validates MySQL maxConnections', () => {
|
||||
const baseConnection: KtxMysqlConnectionConfig = {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
};
|
||||
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: baseConnection,
|
||||
}),
|
||||
).toMatchObject({ connectionLimit: 10 });
|
||||
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections: 25 },
|
||||
}),
|
||||
).toMatchObject({ connectionLimit: 25 });
|
||||
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections: '12' as never },
|
||||
}),
|
||||
).toMatchObject({ connectionLimit: 12 });
|
||||
|
||||
for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) {
|
||||
expect(() =>
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections },
|
||||
}),
|
||||
).toThrow('connections.warehouse.maxConnections must be a positive integer');
|
||||
}
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
|
|
@ -276,6 +324,65 @@ describe('KtxMysqlScanConnector', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('soft-fails denied MySQL constraint discovery with one warning per schema and kind', async () => {
|
||||
const connector = new KtxMysqlScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
schemas: ['analytics', 'mart'],
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory: multiSchemaMysqlPoolFactory({
|
||||
primaryKeyError: Object.assign(new Error('select command denied'), {
|
||||
code: 'ER_TABLEACCESS_DENIED_ERROR',
|
||||
errno: 1142,
|
||||
}),
|
||||
foreignKeyError: Object.assign(new Error('database access denied'), {
|
||||
code: 'ER_DBACCESS_DENIED_ERROR',
|
||||
errno: 1044,
|
||||
}),
|
||||
}),
|
||||
now: () => new Date('2026-04-29T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await connector.introspect(
|
||||
{ connectionId: 'warehouse', driver: 'mysql' },
|
||||
{ runId: 'scan-run-mysql-denied-constraints' },
|
||||
);
|
||||
|
||||
expect(snapshot.warnings).toEqual([
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in analytics (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'analytics', kind: 'primary_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in mart (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'mart', kind: 'primary_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'analytics', kind: 'foreign_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in mart (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'mart', kind: 'foreign_key' },
|
||||
},
|
||||
]);
|
||||
expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true);
|
||||
expect(snapshot.tables.every((table) => table.foreignKeys.length === 0)).toBe(true);
|
||||
});
|
||||
|
||||
it('limits introspection to tables in tableScope', async () => {
|
||||
const queries: Array<{ sql: string; params?: unknown }> = [];
|
||||
const poolFactory: KtxMysqlPoolFactory = {
|
||||
|
|
|
|||
|
|
@ -3,8 +3,33 @@ import { readFileSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import {
|
||||
constraintDiscoveryWarning,
|
||||
tryConstraintQuery,
|
||||
type ConstraintDiscoveryKind,
|
||||
} from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxScanWarning,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaForeignKey,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '../../context/scan/types.js';
|
||||
import { KtxMysqlDialect } from './dialect.js';
|
||||
|
||||
export interface KtxMysqlConnectionConfig {
|
||||
|
|
@ -18,6 +43,7 @@ export interface KtxMysqlConnectionConfig {
|
|||
password?: string;
|
||||
url?: string;
|
||||
ssl?: boolean | { rejectUnauthorized?: boolean };
|
||||
maxConnections?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -163,6 +189,23 @@ function maybeNumber(value: unknown): number | undefined {
|
|||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function positiveIntegerConfigValue(input: {
|
||||
connection: KtxMysqlConnectionConfig;
|
||||
key: keyof KtxMysqlConnectionConfig;
|
||||
connectionId: string;
|
||||
defaultValue: number;
|
||||
}): number {
|
||||
const value = input.connection[input.key];
|
||||
if (value === undefined) {
|
||||
return input.defaultValue;
|
||||
}
|
||||
const numberValue = Number(value);
|
||||
if (!Number.isInteger(numberValue) || numberValue < 1) {
|
||||
throw new Error(`connections.${input.connectionId}.${String(input.key)} must be a positive integer`);
|
||||
}
|
||||
return numberValue;
|
||||
}
|
||||
|
||||
function parseMysqlUrl(url: string): Partial<KtxMysqlConnectionConfig> {
|
||||
const parsed = new URL(url);
|
||||
const sslParam = parsed.searchParams.get('ssl') ?? parsed.searchParams.get('sslmode');
|
||||
|
|
@ -231,6 +274,28 @@ function primaryKeyMap(rows: MysqlPrimaryKeyRow[], fallbackDatabase: string): Ma
|
|||
return grouped;
|
||||
}
|
||||
|
||||
function isDeniedError(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const code = (error as { code?: unknown }).code;
|
||||
return (
|
||||
code === 'ER_TABLEACCESS_DENIED_ERROR' ||
|
||||
code === 'ER_SPECIFIC_ACCESS_DENIED_ERROR' ||
|
||||
code === 'ER_DBACCESS_DENIED_ERROR'
|
||||
);
|
||||
}
|
||||
|
||||
function pushConstraintWarnings(
|
||||
warnings: KtxScanWarning[],
|
||||
schemas: readonly string[],
|
||||
kind: ConstraintDiscoveryKind,
|
||||
): void {
|
||||
for (const schema of schemas) {
|
||||
warnings.push(constraintDiscoveryWarning({ schema, kind }));
|
||||
}
|
||||
}
|
||||
|
||||
function queryParams(params: Record<string, unknown> | unknown[] | undefined): unknown[] | undefined {
|
||||
if (!params) {
|
||||
return undefined;
|
||||
|
|
@ -262,6 +327,12 @@ export function mysqlConnectionPoolConfigFromConfig(input: {
|
|||
const host = stringConfigValue(merged, 'host', env);
|
||||
const database = stringConfigValue(merged, 'database', env);
|
||||
const user = stringConfigValue(merged, 'username', env) ?? stringConfigValue(merged, 'user', env);
|
||||
const maxConnections = positiveIntegerConfigValue({
|
||||
connection: merged,
|
||||
key: 'maxConnections',
|
||||
connectionId: input.connectionId,
|
||||
defaultValue: 10,
|
||||
});
|
||||
|
||||
if (!host) {
|
||||
throw new Error(`Native MySQL connector requires connections.${input.connectionId}.host or url`);
|
||||
|
|
@ -280,7 +351,7 @@ export function mysqlConnectionPoolConfigFromConfig(input: {
|
|||
database,
|
||||
user,
|
||||
password: stringConfigValue(merged, 'password', env),
|
||||
connectionLimit: 10,
|
||||
connectionLimit: maxConnections,
|
||||
waitForConnections: true,
|
||||
...(ssl ? { ssl: { rejectUnauthorized: ssl.rejectUnauthorized ?? false } } : {}),
|
||||
};
|
||||
|
|
@ -335,6 +406,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
|
||||
this.assertConnection(input.connectionId);
|
||||
const databases = configuredMysqlSchemas(this.connection, this.poolConfig.database);
|
||||
const snapshotWarnings: KtxScanWarning[] = [];
|
||||
const placeholders = databases.map(() => '?').join(', ');
|
||||
let allScopedTables: string[] | null = null;
|
||||
if (input.tableScope) {
|
||||
|
|
@ -368,8 +440,11 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
`,
|
||||
[...databases, ...tableNameParams],
|
||||
);
|
||||
const primaryKeys = await this.queryRaw<MysqlPrimaryKeyRow>(
|
||||
`
|
||||
const primaryKeysResult = await tryConstraintQuery(
|
||||
{ schema: databases[0] ?? this.poolConfig.database, kind: 'primary_key', isDeniedError },
|
||||
() =>
|
||||
this.queryRaw<MysqlPrimaryKeyRow>(
|
||||
`
|
||||
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
|
||||
WHERE TABLE_SCHEMA IN (${placeholders})
|
||||
|
|
@ -377,10 +452,18 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
${tableNameClause}
|
||||
ORDER BY TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION
|
||||
`,
|
||||
[...databases, ...tableNameParams],
|
||||
[...databases, ...tableNameParams],
|
||||
),
|
||||
);
|
||||
const foreignKeys = await this.queryRaw<MysqlForeignKeyRow>(
|
||||
`
|
||||
const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : [];
|
||||
if (!primaryKeysResult.ok) {
|
||||
pushConstraintWarnings(snapshotWarnings, databases, 'primary_key');
|
||||
}
|
||||
const foreignKeysResult = await tryConstraintQuery(
|
||||
{ schema: databases[0] ?? this.poolConfig.database, kind: 'foreign_key', isDeniedError },
|
||||
() =>
|
||||
this.queryRaw<MysqlForeignKeyRow>(
|
||||
`
|
||||
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, CONSTRAINT_NAME
|
||||
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
|
||||
WHERE TABLE_SCHEMA IN (${placeholders})
|
||||
|
|
@ -388,8 +471,13 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
${tableNameClause}
|
||||
ORDER BY TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME
|
||||
`,
|
||||
[...databases, ...tableNameParams],
|
||||
[...databases, ...tableNameParams],
|
||||
),
|
||||
);
|
||||
const foreignKeys = foreignKeysResult.ok ? foreignKeysResult.value : [];
|
||||
if (!foreignKeysResult.ok) {
|
||||
pushConstraintWarnings(snapshotWarnings, databases, 'foreign_key');
|
||||
}
|
||||
|
||||
const columnsByTable = groupByTable(columns, this.poolConfig.database);
|
||||
const primaryKeysByTable = primaryKeyMap(primaryKeys, this.poolConfig.database);
|
||||
|
|
@ -417,6 +505,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
total_columns: schemaTables.reduce((sum, table) => sum + table.columns.length, 0),
|
||||
},
|
||||
tables: schemaTables,
|
||||
warnings: snapshotWarnings,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue