feat(connectors): generalize readiness and constraint handling (#212)

* feat(connectors): add postgres maxConnections

* feat(connectors): add mysql maxConnections

* feat(connectors): add sqlserver maxConnections

* feat(connectors): rename snowflake pool config

* docs: document connector maxConnections

* feat(scan): add constraint discovery warning helper

* feat(scan): carry structural warnings through reports

* feat(postgres): soft-fail denied constraint discovery

* feat(mysql): soft-fail denied constraint discovery

* feat(sqlserver): soft-fail denied constraint discovery

* feat(bigquery): soft-fail denied primary key discovery

* feat(snowflake): report denied primary key discovery

* test(scan): verify constraint discovery warnings

* feat(historic-sql): use shared readiness probes

* docs: document query history readiness probes

* test(historic-sql): verify readiness probe registry

* test(ingest): account for live database warnings artifact

* Add skip option for agent setup
This commit is contained in:
Andrey Avtomonov 2026-05-24 19:30:06 +02:00 committed by GitHub
parent cfd1749ab9
commit 78b8a0c025
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 2763 additions and 554 deletions

View file

@ -6,6 +6,7 @@ import {
detectLiveDatabaseStagedDir,
LIVE_DATABASE_FOREIGN_KEYS_FILE,
LIVE_DATABASE_META_FILE,
LIVE_DATABASE_WARNINGS_FILE,
liveDatabaseTablePath,
readLiveDatabaseTableFiles,
writeLiveDatabaseSnapshot,
@ -145,6 +146,31 @@ describe('live-database staged snapshot files', () => {
expect(connectionJson).not.toContain('pem-value');
});
it('writes redacted scan warnings next to live database metadata', async () => {
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-warning-stage-'));
await writeLiveDatabaseSnapshot(dir, {
...snapshot(),
warnings: [
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: {
schema: 'public',
kind: 'primary_key',
url: 'postgres://reader:secret@example.test/db', // pragma: allowlist secret
},
},
],
});
const warningsJson = await readFile(join(dir, LIVE_DATABASE_WARNINGS_FILE), 'utf8');
expect(warningsJson).toContain('"constraint_discovery_unauthorized"');
expect(warningsJson).toContain('"schema": "public"');
expect(warningsJson).toContain('"url": "<redacted>"');
expect(warningsJson).not.toContain('postgres://reader:secret@example.test/db'); // pragma: allowlist secret
});
it('returns false for a directory that is missing live database metadata', async () => {
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-empty-'));
expect(await detectLiveDatabaseStagedDir(dir)).toBe(false);

View file

@ -7,6 +7,8 @@ import type { KtxSchemaSnapshot, KtxSchemaTable, KtxTableRef } from '../../../sc
export const LIVE_DATABASE_META_FILE = 'connection.json';
export const LIVE_DATABASE_FOREIGN_KEYS_FILE = 'foreign-keys.json';
/** @internal */
export const LIVE_DATABASE_WARNINGS_FILE = 'warnings.json';
const LIVE_DATABASE_TABLES_DIR = 'tables';
interface LiveDatabaseTableFile {
@ -89,6 +91,13 @@ function foreignKeyIndex(snapshot: KtxSchemaSnapshot): ForeignKeyIndexEntry[] {
return entries;
}
function warningArtifact(snapshot: KtxSchemaSnapshot): { warnings: KtxSchemaSnapshot['warnings'] } {
const redacted = redactKtxSensitiveMetadata({ warnings: snapshot.warnings ?? [] });
return {
warnings: Array.isArray(redacted.warnings) ? (redacted.warnings as KtxSchemaSnapshot['warnings']) : [],
};
}
export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KtxSchemaSnapshot): Promise<void> {
await mkdir(join(stagedDir, LIVE_DATABASE_TABLES_DIR), { recursive: true });
const sortedTables = [...snapshot.tables].sort((a, b) => tableSortKey(a).localeCompare(tableSortKey(b)));
@ -105,6 +114,7 @@ export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: Ktx
join(stagedDir, LIVE_DATABASE_FOREIGN_KEYS_FILE),
stableJson({ foreignKeys: foreignKeyIndex(snapshot) }),
);
await writeFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), stableJson(warningArtifact(snapshot)));
for (const table of sortedTables) {
await writeFile(join(stagedDir, liveDatabaseTablePath(table)), stableJson(table));
}

View file

@ -0,0 +1,157 @@
import { describe, expect, it, vi } from 'vitest';
import type { HistoricSqlDialect } from './adapters/historic-sql/types.js';
import {
historicSqlProbeCatalogName,
runHistoricSqlReadinessProbe,
type HistoricSqlProbeRunner,
type HistoricSqlProbeRunnerFactoryEntry,
} from './historic-sql-probes.js';
function fakeRunner(
dialect: HistoricSqlDialect,
catalogName: string,
options: { result?: unknown; error?: unknown } = {},
): HistoricSqlProbeRunner & { runCalls: () => number } {
let calls = 0;
return {
dialect,
catalogName,
async run() {
calls += 1;
if (options.error) {
throw options.error;
}
return options.result ?? { warnings: [], info: [] };
},
formatSuccessDetail() {
return { detail: `${catalogName} ready`, warnings: [] };
},
fixAdvice(error) {
return {
failHeadline: error instanceof Error ? error.message : String(error),
remediation: 'Fix the test probe.',
};
},
runCalls: () => calls,
};
}
function factories(
overrides: Partial<Record<HistoricSqlDialect, HistoricSqlProbeRunner>>,
): Record<HistoricSqlDialect, HistoricSqlProbeRunnerFactoryEntry> {
const postgres = overrides.postgres ?? fakeRunner('postgres', 'pg_stat_statements');
const snowflake =
overrides.snowflake ??
fakeRunner('snowflake', 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY');
const bigquery =
overrides.bigquery ?? fakeRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT');
return {
postgres: {
catalogName: 'pg_stat_statements',
load: vi.fn(async () => postgres),
},
snowflake: {
catalogName: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
load: vi.fn(async () => snowflake),
},
bigquery: {
catalogName: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT',
load: vi.fn(async () => bigquery),
},
};
}
describe('historic-SQL probe registry', () => {
it('returns null when the connection has no query-history dialect', async () => {
const deps = { factories: factories({}), cache: new Map() };
await expect(
runHistoricSqlReadinessProbe(
{
projectDir: '/work/project',
connectionId: 'mysql',
connection: {
driver: 'mysql',
context: { queryHistory: { enabled: true } },
},
env: {},
},
deps,
),
).resolves.toBeNull();
expect(deps.factories.postgres.load).not.toHaveBeenCalled();
expect(deps.factories.snowflake.load).not.toHaveBeenCalled();
expect(deps.factories.bigquery.load).not.toHaveBeenCalled();
});
it('dispatches to the dialect runner and caches the runner instance', async () => {
const runner = fakeRunner('postgres', 'pg_stat_statements', {
result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] },
});
const deps = { factories: factories({ postgres: runner }), cache: new Map() };
const input = {
projectDir: '/work/project',
connectionId: 'warehouse',
connection: {
driver: 'postgres' as const,
url: 'env:DATABASE_URL',
context: { queryHistory: { enabled: true } },
},
env: {},
};
const first = await runHistoricSqlReadinessProbe(input, deps);
const second = await runHistoricSqlReadinessProbe(input, deps);
expect(first).toMatchObject({ ok: true, dialect: 'postgres', runner });
expect(second).toMatchObject({ ok: true, dialect: 'postgres', runner });
expect(deps.factories.postgres.load).toHaveBeenCalledTimes(1);
expect(runner.runCalls()).toBe(2);
});
it('normalizes runner errors into a failed outcome', async () => {
const error = new Error('missing grants');
const runner = fakeRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT', {
error,
});
const deps = { factories: factories({ bigquery: runner }), cache: new Map() };
await expect(
runHistoricSqlReadinessProbe(
{
projectDir: '/work/project',
connectionId: 'bq',
connection: {
driver: 'bigquery',
credentials_json: '{"project_id":"project-1"}',
context: { queryHistory: { enabled: true } },
},
env: {},
},
deps,
),
).resolves.toEqual({
ok: false,
dialect: 'bigquery',
runner,
error,
});
});
it('returns catalog names without loading runner modules', () => {
const deps = { factories: factories({}), cache: new Map() };
expect(historicSqlProbeCatalogName('postgres', deps)).toBe('pg_stat_statements');
expect(historicSqlProbeCatalogName('snowflake', deps)).toBe(
'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
);
expect(historicSqlProbeCatalogName('bigquery', deps)).toBe(
'INFORMATION_SCHEMA.JOBS_BY_PROJECT',
);
expect(deps.factories.postgres.load).not.toHaveBeenCalled();
expect(deps.factories.snowflake.load).not.toHaveBeenCalled();
expect(deps.factories.bigquery.load).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,141 @@
import type { KtxProjectConnectionConfig } from '../project/config.js';
import { queryHistoryDialectForConnection } from './adapters/historic-sql/connection-dialect.js';
import type { HistoricSqlDialect } from './adapters/historic-sql/types.js';
export interface HistoricSqlFixAdvice {
failHeadline: string;
remediation: string;
}
export interface HistoricSqlSuccessDetail {
detail: string;
warnings: string[];
}
export interface HistoricSqlProbeInput {
projectDir: string;
connectionId: string;
connection: KtxProjectConnectionConfig;
env?: NodeJS.ProcessEnv;
}
export interface HistoricSqlProbeRunner {
readonly dialect: HistoricSqlDialect;
readonly catalogName: string;
run(input: HistoricSqlProbeInput): Promise<unknown>;
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail;
fixAdvice(error: unknown): HistoricSqlFixAdvice;
}
/** @internal */
export interface HistoricSqlProbeRunnerFactoryEntry {
readonly catalogName: string;
load(): Promise<HistoricSqlProbeRunner>;
}
export type HistoricSqlProbeOutcome =
| {
ok: true;
dialect: HistoricSqlDialect;
runner: HistoricSqlProbeRunner;
result: unknown;
}
| {
ok: false;
dialect: HistoricSqlDialect;
runner: HistoricSqlProbeRunner;
error: unknown;
};
export type HistoricSqlReadinessProbe = (
input: HistoricSqlProbeInput,
) => Promise<HistoricSqlProbeOutcome | null>;
export interface HistoricSqlProbeRegistryDeps {
factories?: Record<HistoricSqlDialect, HistoricSqlProbeRunnerFactoryEntry>;
cache?: Map<HistoricSqlDialect, HistoricSqlProbeRunner>;
}
const defaultHistoricSqlProbeRunnerFactories: Record<
HistoricSqlDialect,
HistoricSqlProbeRunnerFactoryEntry
> = {
postgres: {
catalogName: 'pg_stat_statements',
load: async () => {
const { PostgresPgssProbeRunner } = await import(
'./historic-sql-probes/postgres-runner.js'
);
return new PostgresPgssProbeRunner();
},
},
snowflake: {
catalogName: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
load: async () => {
const { SnowflakeAccountUsageProbeRunner } = await import(
'./historic-sql-probes/snowflake-runner.js'
);
return new SnowflakeAccountUsageProbeRunner();
},
},
bigquery: {
catalogName: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT',
load: async () => {
const { BigQueryJobsByProjectProbeRunner } = await import(
'./historic-sql-probes/bigquery-runner.js'
);
return new BigQueryJobsByProjectProbeRunner();
},
},
};
const DEFAULT_RUNNER_CACHE = new Map<HistoricSqlDialect, HistoricSqlProbeRunner>();
function registryDeps(input: HistoricSqlProbeRegistryDeps) {
return {
factories: input.factories ?? defaultHistoricSqlProbeRunnerFactories,
cache: input.cache ?? DEFAULT_RUNNER_CACHE,
};
}
export function historicSqlProbeCatalogName(
dialect: HistoricSqlDialect,
deps: HistoricSqlProbeRegistryDeps = {},
): string {
return registryDeps(deps).factories[dialect].catalogName;
}
async function loadHistoricSqlProbeRunner(
dialect: HistoricSqlDialect,
deps: HistoricSqlProbeRegistryDeps = {},
): Promise<HistoricSqlProbeRunner> {
const { factories, cache } = registryDeps(deps);
const cached = cache.get(dialect);
if (cached) {
return cached;
}
const runner = await factories[dialect].load();
cache.set(dialect, runner);
return runner;
}
export async function runHistoricSqlReadinessProbe(
input: HistoricSqlProbeInput,
deps: HistoricSqlProbeRegistryDeps = {},
): Promise<HistoricSqlProbeOutcome | null> {
const dialect = queryHistoryDialectForConnection(input.connection);
if (!dialect) {
return null;
}
const runner = await loadHistoricSqlProbeRunner(dialect, deps);
try {
return {
ok: true,
dialect,
runner,
result: await runner.run(input),
};
} catch (error) {
return { ok: false, dialect, runner, error };
}
}

View file

@ -0,0 +1,110 @@
import { describe, expect, it, vi } from 'vitest';
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
import { BigQueryJobsByProjectProbeRunner } from './bigquery-runner.js';
describe('BigQueryJobsByProjectProbeRunner', () => {
it('creates a region-scoped reader, runs it, and cleans up the connector', async () => {
const cleanup = vi.fn(async () => undefined);
const reader = {
probe: vi.fn(async () => ({ warnings: [], info: ['region: eu'] })),
};
const createReader = vi.fn(() => reader);
const runner = new BigQueryJobsByProjectProbeRunner({
createReader,
createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }),
resolveReference: () => '{"project_id":"project-1"}',
});
await expect(
runner.run({
projectDir: '/work/project',
connectionId: 'bq',
connection: {
driver: 'bigquery',
credentials_json: 'env:BQ_CREDENTIALS_JSON',
location: 'EU',
},
env: {},
}),
).resolves.toEqual({ warnings: [], info: ['region: eu'] });
expect(createReader).toHaveBeenCalledWith({ projectId: 'project-1', region: 'EU' });
expect(reader.probe).toHaveBeenCalledOnce();
expect(cleanup).toHaveBeenCalledOnce();
});
it('uses us as the default BigQuery region', async () => {
const createReader = vi.fn(() => ({
probe: vi.fn(async () => ({ warnings: [], info: [] })),
}));
const runner = new BigQueryJobsByProjectProbeRunner({
createReader,
createClient: () => ({ client: {}, cleanup: vi.fn(async () => undefined) }),
resolveReference: () => '{"project_id":"project-1"}',
});
await runner.run({
projectDir: '/work/project',
connectionId: 'bq',
connection: {
driver: 'bigquery',
credentials_json: '{"project_id":"project-1"}',
},
env: {},
});
expect(createReader).toHaveBeenCalledWith({ projectId: 'project-1', region: 'us' });
});
it('rejects missing BigQuery credentials_json.project_id', async () => {
const runner = new BigQueryJobsByProjectProbeRunner({
createReader: vi.fn(),
createClient: () => ({ client: {}, cleanup: vi.fn() }),
resolveReference: () => '{"client_email":"svc@example.test"}',
});
await expect(
runner.run({
projectDir: '/work/project',
connectionId: 'bq',
connection: {
driver: 'bigquery',
credentials_json: 'env:BQ_CREDENTIALS_JSON',
},
env: {},
}),
).rejects.toThrow('Query history BigQuery connection bq requires credentials_json.project_id');
});
it('formats successful BigQuery details', () => {
const runner = new BigQueryJobsByProjectProbeRunner();
expect(
runner.formatSuccessDetail({
warnings: ['JOBS_BY_PROJECT is delayed'],
info: ['region: us'],
}),
).toEqual({
detail: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT ready; region: us',
warnings: ['JOBS_BY_PROJECT is delayed'],
});
});
it('maps BigQuery grant errors to runner advice', () => {
const runner = new BigQueryJobsByProjectProbeRunner();
expect(
runner.fixAdvice(
new HistoricSqlGrantsMissingError({
dialect: 'bigquery',
message: 'principal cannot query JOBS_BY_PROJECT',
remediation:
'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.',
}),
),
).toEqual({
failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT',
remediation:
'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.',
});
});
});

View file

@ -0,0 +1,160 @@
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
import { BigQueryHistoricSqlQueryHistoryReader } from '../adapters/historic-sql/bigquery-query-history-reader.js';
import {
type HistoricSqlFixAdvice,
type HistoricSqlProbeInput,
type HistoricSqlProbeRunner,
type HistoricSqlSuccessDetail,
} from '../historic-sql-probes.js';
import { resolveKtxConfigReference } from '../../core/config-reference.js';
import {
isKtxBigQueryConnectionConfig,
KtxBigQueryScanConnector,
type KtxBigQueryConnectionConfig,
} from '../../../connectors/bigquery/connector.js';
interface GenericProbeResult {
warnings: string[];
info?: string[];
}
interface ClientHandle {
client: unknown;
cleanup(): Promise<void>;
}
interface BigQueryJobsByProjectProbeRunnerOptions {
createReader?: (options: { projectId: string; region: string }) => {
probe(client: unknown): Promise<GenericProbeResult>;
};
createClient?: (
input: HistoricSqlProbeInput & { connection: KtxBigQueryConnectionConfig },
) => ClientHandle;
resolveReference?: (value: string | undefined, env: NodeJS.ProcessEnv) => string | undefined;
}
function bigQueryProjectId(
connectionId: string,
connection: KtxBigQueryConnectionConfig,
env: NodeJS.ProcessEnv,
resolveReference: (value: string | undefined, env: NodeJS.ProcessEnv) => string | undefined,
): string {
const rawCredentials =
typeof connection.credentials_json === 'string' ? connection.credentials_json : '';
const resolvedCredentials = resolveReference(rawCredentials, env);
if (!resolvedCredentials) {
throw new Error(`Query history BigQuery connection ${connectionId} requires credentials_json`);
}
const parsed = JSON.parse(resolvedCredentials) as { project_id?: unknown };
if (typeof parsed.project_id !== 'string' || parsed.project_id.trim().length === 0) {
throw new Error(
`Query history BigQuery connection ${connectionId} requires credentials_json.project_id`,
);
}
return parsed.project_id;
}
function bigQueryRegion(connection: KtxBigQueryConnectionConfig): string {
return typeof connection.location === 'string' && connection.location.trim().length > 0
? connection.location.trim()
: 'us';
}
function infoSuffix(info: readonly string[] | undefined): string {
return info && info.length > 0 ? `; ${info.join('; ')}` : '';
}
export class BigQueryJobsByProjectProbeRunner implements HistoricSqlProbeRunner {
readonly dialect = 'bigquery' as const;
readonly catalogName = 'INFORMATION_SCHEMA.JOBS_BY_PROJECT';
private readonly createReader: (options: { projectId: string; region: string }) => {
probe(client: unknown): Promise<GenericProbeResult>;
};
private readonly createClient: (
input: HistoricSqlProbeInput & { connection: KtxBigQueryConnectionConfig },
) => ClientHandle;
private readonly resolveReference: (
value: string | undefined,
env: NodeJS.ProcessEnv,
) => string | undefined;
constructor(options: BigQueryJobsByProjectProbeRunnerOptions = {}) {
this.createReader =
options.createReader ??
((readerOptions) => new BigQueryHistoricSqlQueryHistoryReader(readerOptions));
this.createClient =
options.createClient ??
((input) => {
const connector = new KtxBigQueryScanConnector({
connectionId: input.connectionId,
connection: input.connection,
env: input.env,
});
return {
client: {
async executeQuery(sql: string) {
const result = await connector.executeReadOnly(
{ connectionId: input.connectionId, sql },
{} as never,
);
return {
headers: result.headers,
rows: result.rows,
totalRows: result.totalRows,
};
},
},
cleanup: () => connector.cleanup(),
};
});
this.resolveReference = options.resolveReference ?? resolveKtxConfigReference;
}
async run(input: HistoricSqlProbeInput): Promise<GenericProbeResult> {
const inputDriver = input.connection.driver ?? 'unknown';
if (!isKtxBigQueryConnectionConfig(input.connection)) {
throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`);
}
const projectId = bigQueryProjectId(
input.connectionId,
input.connection,
input.env ?? process.env,
this.resolveReference,
);
const reader = this.createReader({
projectId,
region: bigQueryRegion(input.connection),
});
const handle = this.createClient({
...input,
connection: input.connection,
});
try {
return await reader.probe(handle.client);
} finally {
await handle.cleanup();
}
}
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail {
const probeResult = result as GenericProbeResult;
return {
detail: `${this.catalogName} ready${infoSuffix(probeResult.info)}`,
warnings: probeResult.warnings,
};
}
fixAdvice(error: unknown): HistoricSqlFixAdvice {
if (error instanceof HistoricSqlGrantsMissingError) {
return {
failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT',
remediation: error.remediation,
};
}
return {
failHeadline: `${this.catalogName} readiness check failed`,
remediation: error instanceof Error ? error.message : String(error),
};
}
}

View file

@ -0,0 +1,113 @@
import { describe, expect, it, vi } from 'vitest';
import {
HistoricSqlExtensionMissingError,
HistoricSqlGrantsMissingError,
HistoricSqlVersionUnsupportedError,
} from '../adapters/historic-sql/errors.js';
import { PostgresPgssProbeRunner } from './postgres-runner.js';
describe('PostgresPgssProbeRunner', () => {
it('runs the pg_stat_statements reader and cleans up the client', async () => {
const cleanup = vi.fn(async () => undefined);
const reader = {
probe: vi.fn(async () => ({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [],
info: ['tracked statements: 12'],
})),
};
const runner = new PostgresPgssProbeRunner({
reader,
createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }),
});
await expect(
runner.run({
projectDir: '/work/project',
connectionId: 'warehouse',
connection: { driver: 'postgres', url: 'env:DATABASE_URL' },
env: {},
}),
).resolves.toEqual({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [],
info: ['tracked statements: 12'],
});
expect(reader.probe).toHaveBeenCalledOnce();
expect(cleanup).toHaveBeenCalledOnce();
});
it('rejects non-Postgres connections', async () => {
const runner = new PostgresPgssProbeRunner({
reader: { probe: vi.fn() },
createClient: () => ({ client: {}, cleanup: vi.fn() }),
});
await expect(
runner.run({
projectDir: '/work/project',
connectionId: 'warehouse',
connection: { driver: 'snowflake' },
env: {},
}),
).rejects.toThrow('Native PostgreSQL connector cannot run driver "snowflake"');
});
it('formats successful Postgres details', () => {
const runner = new PostgresPgssProbeRunner();
expect(
runner.formatSuccessDetail({
pgServerVersion: 'PostgreSQL 16.4',
warnings: ['pg_stat_statements.track is top'],
info: ['tracked statements: 12'],
}),
).toEqual({
detail: 'pg_stat_statements ready (PostgreSQL 16.4); tracked statements: 12',
warnings: ['pg_stat_statements.track is top'],
});
});
it('maps Postgres probe errors to actionable advice', () => {
const runner = new PostgresPgssProbeRunner();
expect(
runner.fixAdvice(
new HistoricSqlExtensionMissingError({
dialect: 'postgres',
message: 'pg_stat_statements missing',
remediation: 'CREATE EXTENSION pg_stat_statements;',
}),
),
).toEqual({
failHeadline: 'pg_stat_statements extension is missing',
remediation: 'CREATE EXTENSION pg_stat_statements;',
});
expect(
runner.fixAdvice(
new HistoricSqlGrantsMissingError({
dialect: 'postgres',
message: 'missing grants',
remediation: 'GRANT pg_read_all_stats TO <connection role>;',
}),
),
).toEqual({
failHeadline: 'Postgres connection role lacks pg_read_all_stats',
remediation: 'GRANT pg_read_all_stats TO <connection role>;',
});
expect(
runner.fixAdvice(
new HistoricSqlVersionUnsupportedError({
dialect: 'postgres',
detectedVersion: 'PostgreSQL 13.12',
minimumVersion: 'PostgreSQL 14',
}),
),
).toEqual({
failHeadline: 'Postgres version too old',
remediation: 'Use PostgreSQL 14 or newer, or disable query history for this connection',
});
});
});

View file

@ -0,0 +1,111 @@
import {
HistoricSqlExtensionMissingError,
HistoricSqlGrantsMissingError,
HistoricSqlVersionUnsupportedError,
} from '../adapters/historic-sql/errors.js';
import { PostgresPgssReader } from '../adapters/historic-sql/postgres-pgss-reader.js';
import type { PostgresPgssProbeResult } from '../adapters/historic-sql/types.js';
import {
type HistoricSqlFixAdvice,
type HistoricSqlProbeInput,
type HistoricSqlProbeRunner,
type HistoricSqlSuccessDetail,
} from '../historic-sql-probes.js';
import {
isKtxPostgresConnectionConfig,
type KtxPostgresConnectionConfig,
} from '../../../connectors/postgres/connector.js';
import { KtxPostgresHistoricSqlQueryClient } from '../../../connectors/postgres/historic-sql-query-client.js';
interface ClientHandle {
client: unknown;
cleanup(): Promise<void>;
}
interface PostgresPgssProbeRunnerOptions {
reader?: { probe(client: unknown): Promise<PostgresPgssProbeResult> };
createClient?: (
input: HistoricSqlProbeInput & { connection: KtxPostgresConnectionConfig },
) => ClientHandle;
}
function genericAdvice(error: unknown, catalogName: string): HistoricSqlFixAdvice {
return {
failHeadline: `${catalogName} readiness check failed`,
remediation: error instanceof Error ? error.message : String(error),
};
}
function infoSuffix(info: readonly string[] | undefined): string {
return info && info.length > 0 ? `; ${info.join('; ')}` : '';
}
export class PostgresPgssProbeRunner implements HistoricSqlProbeRunner {
readonly dialect = 'postgres' as const;
readonly catalogName = 'pg_stat_statements';
private readonly reader: { probe(client: unknown): Promise<PostgresPgssProbeResult> };
private readonly createClient: (
input: HistoricSqlProbeInput & { connection: KtxPostgresConnectionConfig },
) => ClientHandle;
constructor(options: PostgresPgssProbeRunnerOptions = {}) {
this.reader = options.reader ?? new PostgresPgssReader();
this.createClient =
options.createClient ??
((input) => {
const client = new KtxPostgresHistoricSqlQueryClient({
connectionId: input.connectionId,
connection: input.connection,
env: input.env,
});
return { client, cleanup: () => client.cleanup() };
});
}
async run(input: HistoricSqlProbeInput): Promise<PostgresPgssProbeResult> {
const inputDriver = input.connection.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(input.connection)) {
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
}
const handle = this.createClient({
...input,
connection: input.connection,
});
try {
return await this.reader.probe(handle.client);
} finally {
await handle.cleanup();
}
}
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail {
const pgssResult = result as PostgresPgssProbeResult;
return {
detail: `pg_stat_statements ready (${pgssResult.pgServerVersion})${infoSuffix(pgssResult.info)}`,
warnings: pgssResult.warnings,
};
}
fixAdvice(error: unknown): HistoricSqlFixAdvice {
if (error instanceof HistoricSqlExtensionMissingError) {
return {
failHeadline: 'pg_stat_statements extension is missing',
remediation: error.remediation,
};
}
if (error instanceof HistoricSqlGrantsMissingError) {
return {
failHeadline: 'Postgres connection role lacks pg_read_all_stats',
remediation: error.remediation,
};
}
if (error instanceof HistoricSqlVersionUnsupportedError) {
return {
failHeadline: 'Postgres version too old',
remediation: 'Use PostgreSQL 14 or newer, or disable query history for this connection',
};
}
return genericAdvice(error, this.catalogName);
}
}

View file

@ -0,0 +1,82 @@
import { describe, expect, it, vi } from 'vitest';
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
import { SnowflakeAccountUsageProbeRunner } from './snowflake-runner.js';
describe('SnowflakeAccountUsageProbeRunner', () => {
it('runs the account usage reader and cleans up the client', async () => {
const cleanup = vi.fn(async () => undefined);
const reader = {
probe: vi.fn(async () => ({ warnings: [], info: ['query history available'] })),
};
const runner = new SnowflakeAccountUsageProbeRunner({
reader,
createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }),
});
await expect(
runner.run({
projectDir: '/work/project',
connectionId: 'warehouse',
connection: {
driver: 'snowflake',
account: 'ACCT',
warehouse: 'WH',
database: 'ANALYTICS',
username: 'reader',
},
env: {},
}),
).resolves.toEqual({ warnings: [], info: ['query history available'] });
expect(reader.probe).toHaveBeenCalledOnce();
expect(cleanup).toHaveBeenCalledOnce();
});
it('rejects non-Snowflake connections', async () => {
const runner = new SnowflakeAccountUsageProbeRunner({
reader: { probe: vi.fn() },
createClient: () => ({ client: {}, cleanup: vi.fn() }),
});
await expect(
runner.run({
projectDir: '/work/project',
connectionId: 'warehouse',
connection: { driver: 'postgres' },
env: {},
}),
).rejects.toThrow('Native Snowflake connector cannot run driver "postgres"');
});
it('formats successful Snowflake details', () => {
const runner = new SnowflakeAccountUsageProbeRunner();
expect(
runner.formatSuccessDetail({
warnings: ['query history is delayed'],
info: ['warehouse: WH'],
}),
).toEqual({
detail: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY ready; warehouse: WH',
warnings: ['query history is delayed'],
});
});
it('maps Snowflake grant errors to runner advice', () => {
const runner = new SnowflakeAccountUsageProbeRunner();
expect(
runner.fixAdvice(
new HistoricSqlGrantsMissingError({
dialect: 'snowflake',
message: 'role cannot read account usage',
remediation:
'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE <connection role>;',
}),
),
).toEqual({
failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
remediation:
'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE <connection role>;',
});
});
});

View file

@ -0,0 +1,96 @@
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
import { SnowflakeHistoricSqlQueryHistoryReader } from '../adapters/historic-sql/snowflake-query-history-reader.js';
import {
type HistoricSqlFixAdvice,
type HistoricSqlProbeInput,
type HistoricSqlProbeRunner,
type HistoricSqlSuccessDetail,
} from '../historic-sql-probes.js';
import {
isKtxSnowflakeConnectionConfig,
type KtxSnowflakeConnectionConfig,
} from '../../../connectors/snowflake/connector.js';
import { KtxSnowflakeHistoricSqlQueryClient } from '../../../connectors/snowflake/historic-sql-query-client.js';
interface GenericProbeResult {
warnings: string[];
info?: string[];
}
interface ClientHandle {
client: unknown;
cleanup(): Promise<void>;
}
interface SnowflakeAccountUsageProbeRunnerOptions {
reader?: { probe(client: unknown): Promise<GenericProbeResult> };
createClient?: (
input: HistoricSqlProbeInput & { connection: KtxSnowflakeConnectionConfig },
) => ClientHandle;
}
function infoSuffix(info: readonly string[] | undefined): string {
return info && info.length > 0 ? `; ${info.join('; ')}` : '';
}
export class SnowflakeAccountUsageProbeRunner implements HistoricSqlProbeRunner {
readonly dialect = 'snowflake' as const;
readonly catalogName = 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY';
private readonly reader: { probe(client: unknown): Promise<GenericProbeResult> };
private readonly createClient: (
input: HistoricSqlProbeInput & { connection: KtxSnowflakeConnectionConfig },
) => ClientHandle;
constructor(options: SnowflakeAccountUsageProbeRunnerOptions = {}) {
this.reader = options.reader ?? new SnowflakeHistoricSqlQueryHistoryReader();
this.createClient =
options.createClient ??
((input) => {
const client = new KtxSnowflakeHistoricSqlQueryClient({
connectionId: input.connectionId,
connection: input.connection,
projectDir: input.projectDir,
env: input.env,
});
return { client, cleanup: () => client.cleanup() };
});
}
async run(input: HistoricSqlProbeInput): Promise<GenericProbeResult> {
const inputDriver = input.connection.driver ?? 'unknown';
if (!isKtxSnowflakeConnectionConfig(input.connection)) {
throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`);
}
const handle = this.createClient({
...input,
connection: input.connection,
});
try {
return await this.reader.probe(handle.client);
} finally {
await handle.cleanup();
}
}
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail {
const probeResult = result as GenericProbeResult;
return {
detail: `${this.catalogName} ready${infoSuffix(probeResult.info)}`,
warnings: probeResult.warnings,
};
}
fixAdvice(error: unknown): HistoricSqlFixAdvice {
if (error instanceof HistoricSqlGrantsMissingError) {
return {
failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
remediation: error.remediation,
};
}
return {
failHeadline: `${this.catalogName} readiness check failed`,
remediation: error instanceof Error ? error.message : String(error),
};
}
}

View file

@ -591,7 +591,7 @@ describe('local ingest', () => {
status: 'done',
adapter: 'live-database',
connectionId: 'warehouse',
rawFileCount: 3,
rawFileCount: 4,
workUnitCount: 1,
});
});

View file

@ -0,0 +1,70 @@
import { describe, expect, it } from 'vitest';
import { constraintDiscoveryWarning, tryConstraintQuery } from './constraint-discovery.js';
describe('tryConstraintQuery', () => {
it('returns the query value when the query succeeds', async () => {
await expect(
tryConstraintQuery(
{
schema: 'public',
kind: 'primary_key',
isDeniedError: () => false,
},
async () => ['id'],
),
).resolves.toEqual({ ok: true, value: ['id'] });
});
it('returns a recoverable warning when the classifier recognizes denial', async () => {
const error = Object.assign(new Error('permission denied'), { code: '42501' });
await expect(
tryConstraintQuery(
{
schema: 'analytics',
kind: 'foreign_key',
isDeniedError: (candidate) => candidate === error,
},
async () => {
throw error;
},
),
).resolves.toEqual({
ok: false,
warning: {
code: 'constraint_discovery_unauthorized',
message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'analytics', kind: 'foreign_key' },
},
});
});
it('rethrows non-denial errors unchanged', async () => {
const error = Object.assign(new Error('connection reset'), { code: 'ECONNRESET' });
await expect(
tryConstraintQuery(
{
schema: 'public',
kind: 'primary_key',
isDeniedError: () => false,
},
async () => {
throw error;
},
),
).rejects.toBe(error);
});
});
describe('constraintDiscoveryWarning', () => {
it('formats stable primary-key warning text and metadata', () => {
expect(constraintDiscoveryWarning({ schema: 'public', kind: 'primary_key' })).toEqual({
code: 'constraint_discovery_unauthorized',
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'primary_key' },
});
});
});

View file

@ -0,0 +1,42 @@
import type { KtxScanWarning } from './types.js';
export type ConstraintDiscoveryKind = 'primary_key' | 'foreign_key';
export interface ConstraintQueryContext {
schema: string;
kind: ConstraintDiscoveryKind;
isDeniedError: (error: unknown) => boolean;
}
export type ConstraintQueryOutcome<T> = { ok: true; value: T } | { ok: false; warning: KtxScanWarning };
export function constraintDiscoveryWarning(input: {
schema: string;
kind: ConstraintDiscoveryKind;
}): KtxScanWarning {
return {
code: 'constraint_discovery_unauthorized',
message:
`Skipped ${input.kind === 'primary_key' ? 'primary-key' : 'foreign-key'} ` +
`discovery in ${input.schema} (insufficient grants on system catalogs)`,
recoverable: true,
metadata: { schema: input.schema, kind: input.kind },
};
}
export async function tryConstraintQuery<T>(
ctx: ConstraintQueryContext,
fn: () => Promise<T>,
): Promise<ConstraintQueryOutcome<T>> {
try {
return { ok: true, value: await fn() };
} catch (error) {
if (!ctx.isDeniedError(error)) {
throw error;
}
return {
ok: false,
warning: constraintDiscoveryWarning({ schema: ctx.schema, kind: ctx.kind }),
};
}
}

View file

@ -180,6 +180,13 @@ function fetchOnlyAdapter(options: { extractedAt?: () => string; snapshot?: KtxS
'utf-8',
);
await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8');
if (scanSnapshot.warnings?.length) {
await writeFile(
join(stagedDir, 'warnings.json'),
`${JSON.stringify({ warnings: scanSnapshot.warnings })}\n`,
'utf-8',
);
}
for (const table of scanSnapshot.tables) {
await writeFile(join(stagedDir, 'tables', `${table.name}.json`), `${JSON.stringify(table)}\n`, 'utf-8');
}
@ -336,6 +343,48 @@ describe('local scan', () => {
});
});
it('threads structural snapshot warnings into the final scan report', async () => {
const result = await runLocalScan({
project,
adapters: [
fetchOnlyAdapter({
snapshot: {
...defaultFetchSnapshot(),
warnings: [
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'primary_key' },
},
],
},
}),
],
connectionId: 'warehouse',
jobId: 'scan-run-structural-warnings',
now: () => new Date('2026-04-29T09:01:00.000Z'),
});
expect(result.report.warnings).toEqual([
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'primary_key' },
},
]);
await expect(
readFile(
join(
project.projectDir,
'raw-sources/warehouse/live-database/2026-04-29-090100-scan-run-structural-warnings/scan-report.json',
),
'utf-8',
),
).resolves.toContain('"constraint_discovery_unauthorized"');
});
it('passes enabled_tables as fetch context tableScope and does not post-filter staged snapshots', async () => {
project.config.connections.warehouse = {
...project.config.connections.warehouse,

View file

@ -467,6 +467,9 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
extractedAtFallback: report.createdAt,
});
enrichmentSnapshot = rawSnapshot;
if (rawSnapshot.warnings?.length) {
report.warnings.push(...rawSnapshot.warnings);
}
const manifestArtifacts = await writeLocalScanManifestShards({
project: options.project,
connectionId: options.connectionId,

View file

@ -165,6 +165,61 @@ describe('readLocalScanStructuralSnapshot', () => {
});
});
it('rebuilds scan warnings from persisted live-database warning files', async () => {
const rawRoot = 'raw-sources/warehouse/live-database/sync-warnings';
await project.fileStore.writeFile(
`${rawRoot}/connection.json`,
'{"connectionId":"warehouse","metadata":{}}\n',
'ktx',
'ktx@example.com',
'Seed connection artifact',
);
await project.fileStore.writeFile(
`${rawRoot}/warnings.json`,
`${JSON.stringify(
{
warnings: [
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'foreign_key' },
},
],
},
null,
2,
)}\n`,
'ktx',
'ktx@example.com',
'Seed warning artifact',
);
await project.fileStore.writeFile(
`${rawRoot}/tables/orders.json`,
'{"name":"orders","catalog":null,"db":"public","kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":false,"comment":null}],"foreignKeys":[]}\n',
'ktx',
'ktx@example.com',
'Seed orders artifact',
);
const snapshot = await readLocalScanStructuralSnapshot({
project,
connectionId: 'warehouse',
driver: 'postgres',
rawSourcesDir: rawRoot,
extractedAtFallback: '2026-04-29T13:00:00.000Z',
});
expect(snapshot.warnings).toEqual([
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'foreign_key' },
},
]);
});
it('uses the scan report timestamp when connection.json omits extractedAt', async () => {
const rawRoot = 'raw-sources/warehouse/live-database/sync-2';
await project.fileStore.writeFile(
@ -192,4 +247,32 @@ describe('readLocalScanStructuralSnapshot', () => {
expect(snapshot.extractedAt).toBe('2026-04-29T13:00:00.000Z');
});
it('tolerates older live-database staged directories without warnings.json', async () => {
const rawRoot = 'raw-sources/warehouse/live-database/sync-no-warnings';
await project.fileStore.writeFile(
`${rawRoot}/connection.json`,
'{"connectionId":"warehouse","metadata":{}}\n',
'ktx',
'ktx@example.com',
'Seed connection artifact',
);
await project.fileStore.writeFile(
`${rawRoot}/tables/orders.json`,
'{"name":"orders","catalog":null,"db":null,"kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":true,"comment":null}],"foreignKeys":[]}\n',
'ktx',
'ktx@example.com',
'Seed orders artifact',
);
const snapshot = await readLocalScanStructuralSnapshot({
project,
connectionId: 'warehouse',
driver: 'postgres',
rawSourcesDir: rawRoot,
extractedAtFallback: '2026-04-29T13:00:00.000Z',
});
expect(snapshot.warnings).toEqual([]);
});
});

View file

@ -1,6 +1,7 @@
import type { KtxLocalProject } from '../../context/project/project.js';
import type {
KtxConnectionDriver,
KtxScanWarning,
KtxSchemaColumn,
KtxSchemaForeignKey,
KtxSchemaSnapshot,
@ -30,6 +31,59 @@ function metadataRecord(value: unknown): Record<string, unknown> {
return isRecord(value) ? value : {};
}
const scanWarningCodes = new Set<KtxScanWarning['code']>([
'connector_capability_missing',
'sampling_failed',
'statistics_failed',
'llm_unavailable',
'embedding_unavailable',
'scan_enrichment_backend_not_configured',
'relationship_validation_failed',
'relationship_llm_invalid_reference',
'relationship_llm_proposal_failed',
'credential_redacted',
'enrichment_failed',
'description_fallback_used',
'constraint_discovery_unauthorized',
]);
function parseWarning(rawWarning: unknown, path: string): KtxScanWarning {
if (
!isRecord(rawWarning) ||
typeof rawWarning.code !== 'string' ||
!scanWarningCodes.has(rawWarning.code as KtxScanWarning['code']) ||
typeof rawWarning.message !== 'string' ||
typeof rawWarning.recoverable !== 'boolean'
) {
throw new Error(`Invalid KTX schema warning artifact: ${path}`);
}
return {
code: rawWarning.code as KtxScanWarning['code'],
message: rawWarning.message,
recoverable: rawWarning.recoverable,
...(typeof rawWarning.table === 'string' ? { table: rawWarning.table } : {}),
...(typeof rawWarning.column === 'string' ? { column: rawWarning.column } : {}),
...(isRecord(rawWarning.metadata) ? { metadata: rawWarning.metadata } : {}),
};
}
async function readWarnings(input: ReadLocalScanStructuralSnapshotInput): Promise<KtxScanWarning[]> {
const path = `${input.rawSourcesDir}/warnings.json`;
try {
const warningRaw = await input.project.fileStore.readFile(path);
const parsed = JSON.parse(warningRaw.content) as unknown;
if (!isRecord(parsed) || !Array.isArray(parsed.warnings)) {
throw new Error(`Invalid KTX schema warnings artifact: ${path}`);
}
return parsed.warnings.map((warning) => parseWarning(warning, path));
} catch (error) {
if (error instanceof Error && /not found|ENOENT|no such file/i.test(error.message)) {
return [];
}
throw error;
}
}
function optionalStringOrNull(value: unknown): string | null | undefined {
if (value === undefined) {
return undefined;
@ -113,6 +167,7 @@ export async function readLocalScanStructuralSnapshot(
const tableRaw = await input.project.fileStore.readFile(path);
tables.push(parseTable(tableRaw.content, path));
}
const warnings = await readWarnings(input);
return {
connectionId: typeof connection.connectionId === 'string' ? connection.connectionId : input.connectionId,
@ -121,5 +176,6 @@ export async function readLocalScanStructuralSnapshot(
scope: isRecord(connection.scope) ? connection.scope : {},
metadata: metadataRecord(connection.metadata),
tables,
warnings,
};
}

View file

@ -90,6 +90,7 @@ export interface KtxSchemaSnapshot {
scope: KtxSchemaScope;
tables: KtxSchemaTable[];
metadata: Record<string, unknown>;
warnings?: KtxScanWarning[];
}
interface KtxCredentialEnvReference {
@ -364,7 +365,8 @@ type KtxScanWarningCode =
| 'relationship_llm_proposal_failed'
| 'credential_redacted'
| 'enrichment_failed'
| 'description_fallback_used';
| 'description_fallback_used'
| 'constraint_discovery_unauthorized';
export interface KtxScanWarning {
code: KtxScanWarningCode;