mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-13 08:15:14 +02:00
feat(connectors): generalize readiness and constraint handling (#212)
* feat(connectors): add postgres maxConnections * feat(connectors): add mysql maxConnections * feat(connectors): add sqlserver maxConnections * feat(connectors): rename snowflake pool config * docs: document connector maxConnections * feat(scan): add constraint discovery warning helper * feat(scan): carry structural warnings through reports * feat(postgres): soft-fail denied constraint discovery * feat(mysql): soft-fail denied constraint discovery * feat(sqlserver): soft-fail denied constraint discovery * feat(bigquery): soft-fail denied primary key discovery * feat(snowflake): report denied primary key discovery * test(scan): verify constraint discovery warnings * feat(historic-sql): use shared readiness probes * docs: document query history readiness probes * test(historic-sql): verify readiness probe registry * test(ingest): account for live database warnings artifact * Add skip option for agent setup
This commit is contained in:
Andrey Avtomonov
GitHub
parent
cfd1749ab9
commit
78b8a0c025
42 changed files with 2763 additions and 554 deletions
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { createSqlServerLiveDatabaseIntrospection } from '../../connectors/sqlserver/live-database-introspection.js';
|
||||
import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js';
|
||||
import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerConnectionConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js';
|
||||
import { tableRefSet } from '../../context/scan/table-ref.js';
|
||||
|
||||
function recordset<T extends Record<string, unknown>>(
|
||||
|
|
@ -16,7 +16,7 @@ function result<T extends Record<string, unknown>>(rows: T[], columnNames: strin
|
|||
return { recordset: recordset(rows, columnNames) };
|
||||
}
|
||||
|
||||
function fakePoolFactory(): KtxSqlServerPoolFactory {
|
||||
function fakePoolFactory(options: { primaryKeyError?: Error; foreignKeyError?: Error } = {}): KtxSqlServerPoolFactory {
|
||||
const query = vi.fn(async (sql: string): Promise<KtxSqlServerQueryResult> => {
|
||||
if (sql.includes('INFORMATION_SCHEMA.TABLES')) {
|
||||
return result(
|
||||
|
|
@ -55,6 +55,9 @@ function fakePoolFactory(): KtxSqlServerPoolFactory {
|
|||
);
|
||||
}
|
||||
if (sql.includes("CONSTRAINT_TYPE = 'PRIMARY KEY'")) {
|
||||
if (options.primaryKeyError) {
|
||||
throw options.primaryKeyError;
|
||||
}
|
||||
return result(
|
||||
[
|
||||
{ table_name: 'customers', column_name: 'id' },
|
||||
|
|
@ -64,6 +67,9 @@ function fakePoolFactory(): KtxSqlServerPoolFactory {
|
|||
);
|
||||
}
|
||||
if (sql.includes('REFERENTIAL_CONSTRAINTS')) {
|
||||
if (options.foreignKeyError) {
|
||||
throw options.foreignKeyError;
|
||||
}
|
||||
return result(
|
||||
[
|
||||
{
|
||||
|
|
@ -164,6 +170,45 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('defaults and validates SQL Server maxConnections', () => {
|
||||
const baseConnection: KtxSqlServerConnectionConfig = {
|
||||
driver: 'sqlserver',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
};
|
||||
|
||||
expect(
|
||||
sqlServerConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: baseConnection,
|
||||
}),
|
||||
).toMatchObject({ pool: { max: 10 } });
|
||||
|
||||
expect(
|
||||
sqlServerConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections: 15 },
|
||||
}),
|
||||
).toMatchObject({ pool: { max: 15 } });
|
||||
|
||||
expect(
|
||||
sqlServerConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections: '12' as never },
|
||||
}),
|
||||
).toMatchObject({ pool: { max: 12 } });
|
||||
|
||||
for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) {
|
||||
expect(() =>
|
||||
sqlServerConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...baseConnection, maxConnections },
|
||||
}),
|
||||
).toThrow('connections.warehouse.maxConnections must be a positive integer');
|
||||
}
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
|
||||
const connector = new KtxSqlServerScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
|
|
@ -222,6 +267,46 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('soft-fails denied SQL Server constraint discovery with scan warnings', async () => {
|
||||
const connector = new KtxSqlServerScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'sqlserver',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
username: 'reader',
|
||||
schema: 'dbo',
|
||||
},
|
||||
poolFactory: fakePoolFactory({
|
||||
primaryKeyError: Object.assign(new Error('SELECT permission denied'), { number: 229 }),
|
||||
foreignKeyError: Object.assign(new Error('EXECUTE permission denied'), { number: 230 }),
|
||||
}),
|
||||
now: () => new Date('2026-04-29T16:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await connector.introspect(
|
||||
{ connectionId: 'warehouse', driver: 'sqlserver' },
|
||||
{ runId: 'scan-run-sqlserver-denied-constraints' },
|
||||
);
|
||||
|
||||
expect(snapshot.warnings).toEqual([
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in dbo (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'dbo', kind: 'primary_key' },
|
||||
},
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in dbo (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'dbo', kind: 'foreign_key' },
|
||||
},
|
||||
]);
|
||||
expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true);
|
||||
expect(snapshot.tables.every((table) => table.foreignKeys.length === 0)).toBe(true);
|
||||
});
|
||||
|
||||
it('runs samples, distinct values, read-only SQL, row count, schema list, and cleanup', async () => {
|
||||
const poolFactory = fakePoolFactory();
|
||||
const connector = new KtxSqlServerScanConnector({
|
||||
|
|
|
|||
|
|
@ -1,6 +1,27 @@
|
|||
import { assertReadOnlySql } from '../../context/connections/read-only-sql.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
type KtxColumnStatsResult,
|
||||
type KtxQueryResult,
|
||||
type KtxReadOnlyQueryInput,
|
||||
type KtxScanConnector,
|
||||
type KtxScanContext,
|
||||
type KtxScanInput,
|
||||
type KtxScanWarning,
|
||||
type KtxSchemaColumn,
|
||||
type KtxSchemaForeignKey,
|
||||
type KtxSchemaSnapshot,
|
||||
type KtxSchemaTable,
|
||||
type KtxTableListEntry,
|
||||
type KtxTableRef,
|
||||
type KtxTableSampleInput,
|
||||
type KtxTableSampleResult,
|
||||
} from '../../context/scan/types.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
|
@ -19,6 +40,7 @@ export interface KtxSqlServerConnectionConfig {
|
|||
schema?: string;
|
||||
schemas?: string[];
|
||||
trustServerCertificate?: boolean;
|
||||
maxConnections?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -197,6 +219,23 @@ function maybeNumber(value: unknown): number | undefined {
|
|||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function positiveIntegerConfigValue(input: {
|
||||
connection: KtxSqlServerConnectionConfig;
|
||||
key: keyof KtxSqlServerConnectionConfig;
|
||||
connectionId: string;
|
||||
defaultValue: number;
|
||||
}): number {
|
||||
const value = input.connection[input.key];
|
||||
if (value === undefined) {
|
||||
return input.defaultValue;
|
||||
}
|
||||
const numberValue = Number(value);
|
||||
if (!Number.isInteger(numberValue) || numberValue < 1) {
|
||||
throw new Error(`connections.${input.connectionId}.${String(input.key)} must be a positive integer`);
|
||||
}
|
||||
return numberValue;
|
||||
}
|
||||
|
||||
function schemaNames(connection: KtxSqlServerConnectionConfig, env: NodeJS.ProcessEnv): string[] {
|
||||
if (Array.isArray(connection.schemas) && connection.schemas.length > 0) {
|
||||
return connection.schemas.filter((schema) => schema.trim().length > 0).map((schema) => resolveStringReference(schema, env));
|
||||
|
|
@ -219,6 +258,14 @@ function firstNumber(value: unknown): number | null {
|
|||
return Number.isFinite(numberValue) ? numberValue : null;
|
||||
}
|
||||
|
||||
function isDeniedError(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const number = (error as { number?: unknown }).number;
|
||||
return number === 229 || number === 230 || number === 297;
|
||||
}
|
||||
|
||||
function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefined): string {
|
||||
const trimmed = assertReadOnlySql(sqlText).replace(/;+\s*$/, '');
|
||||
if (!maxRows) {
|
||||
|
|
@ -254,6 +301,12 @@ export function sqlServerConnectionPoolConfigFromConfig(input: {
|
|||
const server = stringConfigValue(merged, 'host', env);
|
||||
const database = stringConfigValue(merged, 'database', env);
|
||||
const user = stringConfigValue(merged, 'username', env) ?? stringConfigValue(merged, 'user', env);
|
||||
const maxConnections = positiveIntegerConfigValue({
|
||||
connection: merged,
|
||||
key: 'maxConnections',
|
||||
connectionId: input.connectionId,
|
||||
defaultValue: 10,
|
||||
});
|
||||
|
||||
if (!server) {
|
||||
throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.host or url`);
|
||||
|
|
@ -272,7 +325,7 @@ export function sqlServerConnectionPoolConfigFromConfig(input: {
|
|||
user,
|
||||
password: stringConfigValue(merged, 'password', env),
|
||||
options: { encrypt: true, trustServerCertificate: merged.trustServerCertificate ?? true },
|
||||
pool: { max: 10, min: 0, idleTimeoutMillis: 30000 },
|
||||
pool: { max: maxConnections, min: 0, idleTimeoutMillis: 30000 },
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -328,11 +381,12 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
|||
async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot> {
|
||||
this.assertConnection(input.connectionId);
|
||||
const tables: KtxSchemaTable[] = [];
|
||||
const snapshotWarnings: KtxScanWarning[] = [];
|
||||
for (const schemaName of this.schemas) {
|
||||
const scopedNames = input.tableScope
|
||||
? scopedTableNames(input.tableScope, { catalog: this.poolConfig.database, db: schemaName })
|
||||
: null;
|
||||
tables.push(...(await this.introspectSchema(schemaName, scopedNames)));
|
||||
tables.push(...(await this.introspectSchema(schemaName, scopedNames, snapshotWarnings)));
|
||||
}
|
||||
return {
|
||||
connectionId: this.connectionId,
|
||||
|
|
@ -347,6 +401,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
|||
total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
|
||||
},
|
||||
tables,
|
||||
warnings: snapshotWarnings,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -479,7 +534,11 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
|||
}
|
||||
}
|
||||
|
||||
private async introspectSchema(schemaName: string, scopedNames: readonly string[] | null): Promise<KtxSchemaTable[]> {
|
||||
private async introspectSchema(
|
||||
schemaName: string,
|
||||
scopedNames: readonly string[] | null,
|
||||
snapshotWarnings: KtxScanWarning[],
|
||||
): Promise<KtxSchemaTable[]> {
|
||||
if (scopedNames && scopedNames.length === 0) return [];
|
||||
const tableScope = tableScopeSql(scopedNames, 'TABLE_NAME');
|
||||
const tables = await this.queryRaw<{ table_name: string; table_type: string }>(
|
||||
|
|
@ -510,8 +569,22 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
|||
);
|
||||
const tableComments = await this.tableComments(schemaName, scopedNames);
|
||||
const columnComments = await this.columnComments(schemaName, scopedNames);
|
||||
const primaryKeys = await this.primaryKeys(schemaName, scopedNames);
|
||||
const foreignKeys = await this.foreignKeys(schemaName, scopedNames);
|
||||
const primaryKeysResult = await tryConstraintQuery(
|
||||
{ schema: schemaName, kind: 'primary_key', isDeniedError },
|
||||
() => this.primaryKeys(schemaName, scopedNames),
|
||||
);
|
||||
const foreignKeysResult = await tryConstraintQuery(
|
||||
{ schema: schemaName, kind: 'foreign_key', isDeniedError },
|
||||
() => this.foreignKeys(schemaName, scopedNames),
|
||||
);
|
||||
const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : new Map<string, Set<string>>();
|
||||
const foreignKeys = foreignKeysResult.ok ? foreignKeysResult.value : [];
|
||||
if (!primaryKeysResult.ok) {
|
||||
snapshotWarnings.push(primaryKeysResult.warning);
|
||||
}
|
||||
if (!foreignKeysResult.ok) {
|
||||
snapshotWarnings.push(foreignKeysResult.warning);
|
||||
}
|
||||
const rowCounts = await this.rowCounts(schemaName, scopedNames);
|
||||
const columnsByTable = groupByTable(columns);
|
||||
const foreignKeysByTable = groupByTable(foreignKeys);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue