mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-16 08:25:14 +02:00
feat(connectors): generalize readiness and constraint handling (#212)
* feat(connectors): add postgres maxConnections * feat(connectors): add mysql maxConnections * feat(connectors): add sqlserver maxConnections * feat(connectors): rename snowflake pool config * docs: document connector maxConnections * feat(scan): add constraint discovery warning helper * feat(scan): carry structural warnings through reports * feat(postgres): soft-fail denied constraint discovery * feat(mysql): soft-fail denied constraint discovery * feat(sqlserver): soft-fail denied constraint discovery * feat(bigquery): soft-fail denied primary key discovery * feat(snowflake): report denied primary key discovery * test(scan): verify constraint discovery warnings * feat(historic-sql): use shared readiness probes * docs: document query history readiness probes * test(historic-sql): verify readiness probe registry * test(ingest): account for live database warnings artifact * Add skip option for agent setup
This commit is contained in:
parent
cfd1749ab9
commit
78b8a0c025
42 changed files with 2763 additions and 554 deletions
|
|
@ -6,6 +6,7 @@ import {
|
|||
detectLiveDatabaseStagedDir,
|
||||
LIVE_DATABASE_FOREIGN_KEYS_FILE,
|
||||
LIVE_DATABASE_META_FILE,
|
||||
LIVE_DATABASE_WARNINGS_FILE,
|
||||
liveDatabaseTablePath,
|
||||
readLiveDatabaseTableFiles,
|
||||
writeLiveDatabaseSnapshot,
|
||||
|
|
@ -145,6 +146,31 @@ describe('live-database staged snapshot files', () => {
|
|||
expect(connectionJson).not.toContain('pem-value');
|
||||
});
|
||||
|
||||
it('writes redacted scan warnings next to live database metadata', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-warning-stage-'));
|
||||
await writeLiveDatabaseSnapshot(dir, {
|
||||
...snapshot(),
|
||||
warnings: [
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: {
|
||||
schema: 'public',
|
||||
kind: 'primary_key',
|
||||
url: 'postgres://reader:secret@example.test/db', // pragma: allowlist secret
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const warningsJson = await readFile(join(dir, LIVE_DATABASE_WARNINGS_FILE), 'utf8');
|
||||
expect(warningsJson).toContain('"constraint_discovery_unauthorized"');
|
||||
expect(warningsJson).toContain('"schema": "public"');
|
||||
expect(warningsJson).toContain('"url": "<redacted>"');
|
||||
expect(warningsJson).not.toContain('postgres://reader:secret@example.test/db'); // pragma: allowlist secret
|
||||
});
|
||||
|
||||
it('returns false for a directory that is missing live database metadata', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-empty-'));
|
||||
expect(await detectLiveDatabaseStagedDir(dir)).toBe(false);
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import type { KtxSchemaSnapshot, KtxSchemaTable, KtxTableRef } from '../../../sc
|
|||
|
||||
export const LIVE_DATABASE_META_FILE = 'connection.json';
|
||||
export const LIVE_DATABASE_FOREIGN_KEYS_FILE = 'foreign-keys.json';
|
||||
/** @internal */
|
||||
export const LIVE_DATABASE_WARNINGS_FILE = 'warnings.json';
|
||||
const LIVE_DATABASE_TABLES_DIR = 'tables';
|
||||
|
||||
interface LiveDatabaseTableFile {
|
||||
|
|
@ -89,6 +91,13 @@ function foreignKeyIndex(snapshot: KtxSchemaSnapshot): ForeignKeyIndexEntry[] {
|
|||
return entries;
|
||||
}
|
||||
|
||||
function warningArtifact(snapshot: KtxSchemaSnapshot): { warnings: KtxSchemaSnapshot['warnings'] } {
|
||||
const redacted = redactKtxSensitiveMetadata({ warnings: snapshot.warnings ?? [] });
|
||||
return {
|
||||
warnings: Array.isArray(redacted.warnings) ? (redacted.warnings as KtxSchemaSnapshot['warnings']) : [],
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KtxSchemaSnapshot): Promise<void> {
|
||||
await mkdir(join(stagedDir, LIVE_DATABASE_TABLES_DIR), { recursive: true });
|
||||
const sortedTables = [...snapshot.tables].sort((a, b) => tableSortKey(a).localeCompare(tableSortKey(b)));
|
||||
|
|
@ -105,6 +114,7 @@ export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: Ktx
|
|||
join(stagedDir, LIVE_DATABASE_FOREIGN_KEYS_FILE),
|
||||
stableJson({ foreignKeys: foreignKeyIndex(snapshot) }),
|
||||
);
|
||||
await writeFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), stableJson(warningArtifact(snapshot)));
|
||||
for (const table of sortedTables) {
|
||||
await writeFile(join(stagedDir, liveDatabaseTablePath(table)), stableJson(table));
|
||||
}
|
||||
|
|
|
|||
157
packages/cli/src/context/ingest/historic-sql-probes.test.ts
Normal file
157
packages/cli/src/context/ingest/historic-sql-probes.test.ts
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { HistoricSqlDialect } from './adapters/historic-sql/types.js';
|
||||
import {
|
||||
historicSqlProbeCatalogName,
|
||||
runHistoricSqlReadinessProbe,
|
||||
type HistoricSqlProbeRunner,
|
||||
type HistoricSqlProbeRunnerFactoryEntry,
|
||||
} from './historic-sql-probes.js';
|
||||
|
||||
function fakeRunner(
|
||||
dialect: HistoricSqlDialect,
|
||||
catalogName: string,
|
||||
options: { result?: unknown; error?: unknown } = {},
|
||||
): HistoricSqlProbeRunner & { runCalls: () => number } {
|
||||
let calls = 0;
|
||||
return {
|
||||
dialect,
|
||||
catalogName,
|
||||
async run() {
|
||||
calls += 1;
|
||||
if (options.error) {
|
||||
throw options.error;
|
||||
}
|
||||
return options.result ?? { warnings: [], info: [] };
|
||||
},
|
||||
formatSuccessDetail() {
|
||||
return { detail: `${catalogName} ready`, warnings: [] };
|
||||
},
|
||||
fixAdvice(error) {
|
||||
return {
|
||||
failHeadline: error instanceof Error ? error.message : String(error),
|
||||
remediation: 'Fix the test probe.',
|
||||
};
|
||||
},
|
||||
runCalls: () => calls,
|
||||
};
|
||||
}
|
||||
|
||||
function factories(
|
||||
overrides: Partial<Record<HistoricSqlDialect, HistoricSqlProbeRunner>>,
|
||||
): Record<HistoricSqlDialect, HistoricSqlProbeRunnerFactoryEntry> {
|
||||
const postgres = overrides.postgres ?? fakeRunner('postgres', 'pg_stat_statements');
|
||||
const snowflake =
|
||||
overrides.snowflake ??
|
||||
fakeRunner('snowflake', 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY');
|
||||
const bigquery =
|
||||
overrides.bigquery ?? fakeRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT');
|
||||
|
||||
return {
|
||||
postgres: {
|
||||
catalogName: 'pg_stat_statements',
|
||||
load: vi.fn(async () => postgres),
|
||||
},
|
||||
snowflake: {
|
||||
catalogName: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
|
||||
load: vi.fn(async () => snowflake),
|
||||
},
|
||||
bigquery: {
|
||||
catalogName: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT',
|
||||
load: vi.fn(async () => bigquery),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('historic-SQL probe registry', () => {
|
||||
it('returns null when the connection has no query-history dialect', async () => {
|
||||
const deps = { factories: factories({}), cache: new Map() };
|
||||
|
||||
await expect(
|
||||
runHistoricSqlReadinessProbe(
|
||||
{
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'mysql',
|
||||
connection: {
|
||||
driver: 'mysql',
|
||||
context: { queryHistory: { enabled: true } },
|
||||
},
|
||||
env: {},
|
||||
},
|
||||
deps,
|
||||
),
|
||||
).resolves.toBeNull();
|
||||
|
||||
expect(deps.factories.postgres.load).not.toHaveBeenCalled();
|
||||
expect(deps.factories.snowflake.load).not.toHaveBeenCalled();
|
||||
expect(deps.factories.bigquery.load).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('dispatches to the dialect runner and caches the runner instance', async () => {
|
||||
const runner = fakeRunner('postgres', 'pg_stat_statements', {
|
||||
result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] },
|
||||
});
|
||||
const deps = { factories: factories({ postgres: runner }), cache: new Map() };
|
||||
const input = {
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'postgres' as const,
|
||||
url: 'env:DATABASE_URL',
|
||||
context: { queryHistory: { enabled: true } },
|
||||
},
|
||||
env: {},
|
||||
};
|
||||
|
||||
const first = await runHistoricSqlReadinessProbe(input, deps);
|
||||
const second = await runHistoricSqlReadinessProbe(input, deps);
|
||||
|
||||
expect(first).toMatchObject({ ok: true, dialect: 'postgres', runner });
|
||||
expect(second).toMatchObject({ ok: true, dialect: 'postgres', runner });
|
||||
expect(deps.factories.postgres.load).toHaveBeenCalledTimes(1);
|
||||
expect(runner.runCalls()).toBe(2);
|
||||
});
|
||||
|
||||
it('normalizes runner errors into a failed outcome', async () => {
|
||||
const error = new Error('missing grants');
|
||||
const runner = fakeRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT', {
|
||||
error,
|
||||
});
|
||||
const deps = { factories: factories({ bigquery: runner }), cache: new Map() };
|
||||
|
||||
await expect(
|
||||
runHistoricSqlReadinessProbe(
|
||||
{
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'bq',
|
||||
connection: {
|
||||
driver: 'bigquery',
|
||||
credentials_json: '{"project_id":"project-1"}',
|
||||
context: { queryHistory: { enabled: true } },
|
||||
},
|
||||
env: {},
|
||||
},
|
||||
deps,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
ok: false,
|
||||
dialect: 'bigquery',
|
||||
runner,
|
||||
error,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns catalog names without loading runner modules', () => {
|
||||
const deps = { factories: factories({}), cache: new Map() };
|
||||
|
||||
expect(historicSqlProbeCatalogName('postgres', deps)).toBe('pg_stat_statements');
|
||||
expect(historicSqlProbeCatalogName('snowflake', deps)).toBe(
|
||||
'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
|
||||
);
|
||||
expect(historicSqlProbeCatalogName('bigquery', deps)).toBe(
|
||||
'INFORMATION_SCHEMA.JOBS_BY_PROJECT',
|
||||
);
|
||||
expect(deps.factories.postgres.load).not.toHaveBeenCalled();
|
||||
expect(deps.factories.snowflake.load).not.toHaveBeenCalled();
|
||||
expect(deps.factories.bigquery.load).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
141
packages/cli/src/context/ingest/historic-sql-probes.ts
Normal file
141
packages/cli/src/context/ingest/historic-sql-probes.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
import { queryHistoryDialectForConnection } from './adapters/historic-sql/connection-dialect.js';
|
||||
import type { HistoricSqlDialect } from './adapters/historic-sql/types.js';
|
||||
|
||||
export interface HistoricSqlFixAdvice {
|
||||
failHeadline: string;
|
||||
remediation: string;
|
||||
}
|
||||
|
||||
export interface HistoricSqlSuccessDetail {
|
||||
detail: string;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export interface HistoricSqlProbeInput {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
connection: KtxProjectConnectionConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
export interface HistoricSqlProbeRunner {
|
||||
readonly dialect: HistoricSqlDialect;
|
||||
readonly catalogName: string;
|
||||
run(input: HistoricSqlProbeInput): Promise<unknown>;
|
||||
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail;
|
||||
fixAdvice(error: unknown): HistoricSqlFixAdvice;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface HistoricSqlProbeRunnerFactoryEntry {
|
||||
readonly catalogName: string;
|
||||
load(): Promise<HistoricSqlProbeRunner>;
|
||||
}
|
||||
|
||||
export type HistoricSqlProbeOutcome =
|
||||
| {
|
||||
ok: true;
|
||||
dialect: HistoricSqlDialect;
|
||||
runner: HistoricSqlProbeRunner;
|
||||
result: unknown;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
dialect: HistoricSqlDialect;
|
||||
runner: HistoricSqlProbeRunner;
|
||||
error: unknown;
|
||||
};
|
||||
|
||||
export type HistoricSqlReadinessProbe = (
|
||||
input: HistoricSqlProbeInput,
|
||||
) => Promise<HistoricSqlProbeOutcome | null>;
|
||||
|
||||
export interface HistoricSqlProbeRegistryDeps {
|
||||
factories?: Record<HistoricSqlDialect, HistoricSqlProbeRunnerFactoryEntry>;
|
||||
cache?: Map<HistoricSqlDialect, HistoricSqlProbeRunner>;
|
||||
}
|
||||
|
||||
const defaultHistoricSqlProbeRunnerFactories: Record<
|
||||
HistoricSqlDialect,
|
||||
HistoricSqlProbeRunnerFactoryEntry
|
||||
> = {
|
||||
postgres: {
|
||||
catalogName: 'pg_stat_statements',
|
||||
load: async () => {
|
||||
const { PostgresPgssProbeRunner } = await import(
|
||||
'./historic-sql-probes/postgres-runner.js'
|
||||
);
|
||||
return new PostgresPgssProbeRunner();
|
||||
},
|
||||
},
|
||||
snowflake: {
|
||||
catalogName: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
|
||||
load: async () => {
|
||||
const { SnowflakeAccountUsageProbeRunner } = await import(
|
||||
'./historic-sql-probes/snowflake-runner.js'
|
||||
);
|
||||
return new SnowflakeAccountUsageProbeRunner();
|
||||
},
|
||||
},
|
||||
bigquery: {
|
||||
catalogName: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT',
|
||||
load: async () => {
|
||||
const { BigQueryJobsByProjectProbeRunner } = await import(
|
||||
'./historic-sql-probes/bigquery-runner.js'
|
||||
);
|
||||
return new BigQueryJobsByProjectProbeRunner();
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const DEFAULT_RUNNER_CACHE = new Map<HistoricSqlDialect, HistoricSqlProbeRunner>();
|
||||
|
||||
function registryDeps(input: HistoricSqlProbeRegistryDeps) {
|
||||
return {
|
||||
factories: input.factories ?? defaultHistoricSqlProbeRunnerFactories,
|
||||
cache: input.cache ?? DEFAULT_RUNNER_CACHE,
|
||||
};
|
||||
}
|
||||
|
||||
export function historicSqlProbeCatalogName(
|
||||
dialect: HistoricSqlDialect,
|
||||
deps: HistoricSqlProbeRegistryDeps = {},
|
||||
): string {
|
||||
return registryDeps(deps).factories[dialect].catalogName;
|
||||
}
|
||||
|
||||
async function loadHistoricSqlProbeRunner(
|
||||
dialect: HistoricSqlDialect,
|
||||
deps: HistoricSqlProbeRegistryDeps = {},
|
||||
): Promise<HistoricSqlProbeRunner> {
|
||||
const { factories, cache } = registryDeps(deps);
|
||||
const cached = cache.get(dialect);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const runner = await factories[dialect].load();
|
||||
cache.set(dialect, runner);
|
||||
return runner;
|
||||
}
|
||||
|
||||
export async function runHistoricSqlReadinessProbe(
|
||||
input: HistoricSqlProbeInput,
|
||||
deps: HistoricSqlProbeRegistryDeps = {},
|
||||
): Promise<HistoricSqlProbeOutcome | null> {
|
||||
const dialect = queryHistoryDialectForConnection(input.connection);
|
||||
if (!dialect) {
|
||||
return null;
|
||||
}
|
||||
const runner = await loadHistoricSqlProbeRunner(dialect, deps);
|
||||
try {
|
||||
return {
|
||||
ok: true,
|
||||
dialect,
|
||||
runner,
|
||||
result: await runner.run(input),
|
||||
};
|
||||
} catch (error) {
|
||||
return { ok: false, dialect, runner, error };
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
|
||||
import { BigQueryJobsByProjectProbeRunner } from './bigquery-runner.js';
|
||||
|
||||
describe('BigQueryJobsByProjectProbeRunner', () => {
|
||||
it('creates a region-scoped reader, runs it, and cleans up the connector', async () => {
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const reader = {
|
||||
probe: vi.fn(async () => ({ warnings: [], info: ['region: eu'] })),
|
||||
};
|
||||
const createReader = vi.fn(() => reader);
|
||||
const runner = new BigQueryJobsByProjectProbeRunner({
|
||||
createReader,
|
||||
createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }),
|
||||
resolveReference: () => '{"project_id":"project-1"}',
|
||||
});
|
||||
|
||||
await expect(
|
||||
runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'bq',
|
||||
connection: {
|
||||
driver: 'bigquery',
|
||||
credentials_json: 'env:BQ_CREDENTIALS_JSON',
|
||||
location: 'EU',
|
||||
},
|
||||
env: {},
|
||||
}),
|
||||
).resolves.toEqual({ warnings: [], info: ['region: eu'] });
|
||||
expect(createReader).toHaveBeenCalledWith({ projectId: 'project-1', region: 'EU' });
|
||||
expect(reader.probe).toHaveBeenCalledOnce();
|
||||
expect(cleanup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('uses us as the default BigQuery region', async () => {
|
||||
const createReader = vi.fn(() => ({
|
||||
probe: vi.fn(async () => ({ warnings: [], info: [] })),
|
||||
}));
|
||||
const runner = new BigQueryJobsByProjectProbeRunner({
|
||||
createReader,
|
||||
createClient: () => ({ client: {}, cleanup: vi.fn(async () => undefined) }),
|
||||
resolveReference: () => '{"project_id":"project-1"}',
|
||||
});
|
||||
|
||||
await runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'bq',
|
||||
connection: {
|
||||
driver: 'bigquery',
|
||||
credentials_json: '{"project_id":"project-1"}',
|
||||
},
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(createReader).toHaveBeenCalledWith({ projectId: 'project-1', region: 'us' });
|
||||
});
|
||||
|
||||
it('rejects missing BigQuery credentials_json.project_id', async () => {
|
||||
const runner = new BigQueryJobsByProjectProbeRunner({
|
||||
createReader: vi.fn(),
|
||||
createClient: () => ({ client: {}, cleanup: vi.fn() }),
|
||||
resolveReference: () => '{"client_email":"svc@example.test"}',
|
||||
});
|
||||
|
||||
await expect(
|
||||
runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'bq',
|
||||
connection: {
|
||||
driver: 'bigquery',
|
||||
credentials_json: 'env:BQ_CREDENTIALS_JSON',
|
||||
},
|
||||
env: {},
|
||||
}),
|
||||
).rejects.toThrow('Query history BigQuery connection bq requires credentials_json.project_id');
|
||||
});
|
||||
|
||||
it('formats successful BigQuery details', () => {
|
||||
const runner = new BigQueryJobsByProjectProbeRunner();
|
||||
|
||||
expect(
|
||||
runner.formatSuccessDetail({
|
||||
warnings: ['JOBS_BY_PROJECT is delayed'],
|
||||
info: ['region: us'],
|
||||
}),
|
||||
).toEqual({
|
||||
detail: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT ready; region: us',
|
||||
warnings: ['JOBS_BY_PROJECT is delayed'],
|
||||
});
|
||||
});
|
||||
|
||||
it('maps BigQuery grant errors to runner advice', () => {
|
||||
const runner = new BigQueryJobsByProjectProbeRunner();
|
||||
|
||||
expect(
|
||||
runner.fixAdvice(
|
||||
new HistoricSqlGrantsMissingError({
|
||||
dialect: 'bigquery',
|
||||
message: 'principal cannot query JOBS_BY_PROJECT',
|
||||
remediation:
|
||||
'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.',
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT',
|
||||
remediation:
|
||||
'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
|
||||
import { BigQueryHistoricSqlQueryHistoryReader } from '../adapters/historic-sql/bigquery-query-history-reader.js';
|
||||
import {
|
||||
type HistoricSqlFixAdvice,
|
||||
type HistoricSqlProbeInput,
|
||||
type HistoricSqlProbeRunner,
|
||||
type HistoricSqlSuccessDetail,
|
||||
} from '../historic-sql-probes.js';
|
||||
import { resolveKtxConfigReference } from '../../core/config-reference.js';
|
||||
import {
|
||||
isKtxBigQueryConnectionConfig,
|
||||
KtxBigQueryScanConnector,
|
||||
type KtxBigQueryConnectionConfig,
|
||||
} from '../../../connectors/bigquery/connector.js';
|
||||
|
||||
interface GenericProbeResult {
|
||||
warnings: string[];
|
||||
info?: string[];
|
||||
}
|
||||
|
||||
interface ClientHandle {
|
||||
client: unknown;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
interface BigQueryJobsByProjectProbeRunnerOptions {
|
||||
createReader?: (options: { projectId: string; region: string }) => {
|
||||
probe(client: unknown): Promise<GenericProbeResult>;
|
||||
};
|
||||
createClient?: (
|
||||
input: HistoricSqlProbeInput & { connection: KtxBigQueryConnectionConfig },
|
||||
) => ClientHandle;
|
||||
resolveReference?: (value: string | undefined, env: NodeJS.ProcessEnv) => string | undefined;
|
||||
}
|
||||
|
||||
function bigQueryProjectId(
|
||||
connectionId: string,
|
||||
connection: KtxBigQueryConnectionConfig,
|
||||
env: NodeJS.ProcessEnv,
|
||||
resolveReference: (value: string | undefined, env: NodeJS.ProcessEnv) => string | undefined,
|
||||
): string {
|
||||
const rawCredentials =
|
||||
typeof connection.credentials_json === 'string' ? connection.credentials_json : '';
|
||||
const resolvedCredentials = resolveReference(rawCredentials, env);
|
||||
if (!resolvedCredentials) {
|
||||
throw new Error(`Query history BigQuery connection ${connectionId} requires credentials_json`);
|
||||
}
|
||||
const parsed = JSON.parse(resolvedCredentials) as { project_id?: unknown };
|
||||
if (typeof parsed.project_id !== 'string' || parsed.project_id.trim().length === 0) {
|
||||
throw new Error(
|
||||
`Query history BigQuery connection ${connectionId} requires credentials_json.project_id`,
|
||||
);
|
||||
}
|
||||
return parsed.project_id;
|
||||
}
|
||||
|
||||
function bigQueryRegion(connection: KtxBigQueryConnectionConfig): string {
|
||||
return typeof connection.location === 'string' && connection.location.trim().length > 0
|
||||
? connection.location.trim()
|
||||
: 'us';
|
||||
}
|
||||
|
||||
function infoSuffix(info: readonly string[] | undefined): string {
|
||||
return info && info.length > 0 ? `; ${info.join('; ')}` : '';
|
||||
}
|
||||
|
||||
export class BigQueryJobsByProjectProbeRunner implements HistoricSqlProbeRunner {
|
||||
readonly dialect = 'bigquery' as const;
|
||||
readonly catalogName = 'INFORMATION_SCHEMA.JOBS_BY_PROJECT';
|
||||
|
||||
private readonly createReader: (options: { projectId: string; region: string }) => {
|
||||
probe(client: unknown): Promise<GenericProbeResult>;
|
||||
};
|
||||
private readonly createClient: (
|
||||
input: HistoricSqlProbeInput & { connection: KtxBigQueryConnectionConfig },
|
||||
) => ClientHandle;
|
||||
private readonly resolveReference: (
|
||||
value: string | undefined,
|
||||
env: NodeJS.ProcessEnv,
|
||||
) => string | undefined;
|
||||
|
||||
constructor(options: BigQueryJobsByProjectProbeRunnerOptions = {}) {
|
||||
this.createReader =
|
||||
options.createReader ??
|
||||
((readerOptions) => new BigQueryHistoricSqlQueryHistoryReader(readerOptions));
|
||||
this.createClient =
|
||||
options.createClient ??
|
||||
((input) => {
|
||||
const connector = new KtxBigQueryScanConnector({
|
||||
connectionId: input.connectionId,
|
||||
connection: input.connection,
|
||||
env: input.env,
|
||||
});
|
||||
return {
|
||||
client: {
|
||||
async executeQuery(sql: string) {
|
||||
const result = await connector.executeReadOnly(
|
||||
{ connectionId: input.connectionId, sql },
|
||||
{} as never,
|
||||
);
|
||||
return {
|
||||
headers: result.headers,
|
||||
rows: result.rows,
|
||||
totalRows: result.totalRows,
|
||||
};
|
||||
},
|
||||
},
|
||||
cleanup: () => connector.cleanup(),
|
||||
};
|
||||
});
|
||||
this.resolveReference = options.resolveReference ?? resolveKtxConfigReference;
|
||||
}
|
||||
|
||||
async run(input: HistoricSqlProbeInput): Promise<GenericProbeResult> {
|
||||
const inputDriver = input.connection.driver ?? 'unknown';
|
||||
if (!isKtxBigQueryConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
const projectId = bigQueryProjectId(
|
||||
input.connectionId,
|
||||
input.connection,
|
||||
input.env ?? process.env,
|
||||
this.resolveReference,
|
||||
);
|
||||
const reader = this.createReader({
|
||||
projectId,
|
||||
region: bigQueryRegion(input.connection),
|
||||
});
|
||||
const handle = this.createClient({
|
||||
...input,
|
||||
connection: input.connection,
|
||||
});
|
||||
try {
|
||||
return await reader.probe(handle.client);
|
||||
} finally {
|
||||
await handle.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail {
|
||||
const probeResult = result as GenericProbeResult;
|
||||
return {
|
||||
detail: `${this.catalogName} ready${infoSuffix(probeResult.info)}`,
|
||||
warnings: probeResult.warnings,
|
||||
};
|
||||
}
|
||||
|
||||
fixAdvice(error: unknown): HistoricSqlFixAdvice {
|
||||
if (error instanceof HistoricSqlGrantsMissingError) {
|
||||
return {
|
||||
failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT',
|
||||
remediation: error.remediation,
|
||||
};
|
||||
}
|
||||
return {
|
||||
failHeadline: `${this.catalogName} readiness check failed`,
|
||||
remediation: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
HistoricSqlExtensionMissingError,
|
||||
HistoricSqlGrantsMissingError,
|
||||
HistoricSqlVersionUnsupportedError,
|
||||
} from '../adapters/historic-sql/errors.js';
|
||||
import { PostgresPgssProbeRunner } from './postgres-runner.js';
|
||||
|
||||
describe('PostgresPgssProbeRunner', () => {
|
||||
it('runs the pg_stat_statements reader and cleans up the client', async () => {
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const reader = {
|
||||
probe: vi.fn(async () => ({
|
||||
pgServerVersion: 'PostgreSQL 16.4',
|
||||
warnings: [],
|
||||
info: ['tracked statements: 12'],
|
||||
})),
|
||||
};
|
||||
const runner = new PostgresPgssProbeRunner({
|
||||
reader,
|
||||
createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }),
|
||||
});
|
||||
|
||||
await expect(
|
||||
runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres', url: 'env:DATABASE_URL' },
|
||||
env: {},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
pgServerVersion: 'PostgreSQL 16.4',
|
||||
warnings: [],
|
||||
info: ['tracked statements: 12'],
|
||||
});
|
||||
expect(reader.probe).toHaveBeenCalledOnce();
|
||||
expect(cleanup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('rejects non-Postgres connections', async () => {
|
||||
const runner = new PostgresPgssProbeRunner({
|
||||
reader: { probe: vi.fn() },
|
||||
createClient: () => ({ client: {}, cleanup: vi.fn() }),
|
||||
});
|
||||
|
||||
await expect(
|
||||
runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'snowflake' },
|
||||
env: {},
|
||||
}),
|
||||
).rejects.toThrow('Native PostgreSQL connector cannot run driver "snowflake"');
|
||||
});
|
||||
|
||||
it('formats successful Postgres details', () => {
|
||||
const runner = new PostgresPgssProbeRunner();
|
||||
|
||||
expect(
|
||||
runner.formatSuccessDetail({
|
||||
pgServerVersion: 'PostgreSQL 16.4',
|
||||
warnings: ['pg_stat_statements.track is top'],
|
||||
info: ['tracked statements: 12'],
|
||||
}),
|
||||
).toEqual({
|
||||
detail: 'pg_stat_statements ready (PostgreSQL 16.4); tracked statements: 12',
|
||||
warnings: ['pg_stat_statements.track is top'],
|
||||
});
|
||||
});
|
||||
|
||||
it('maps Postgres probe errors to actionable advice', () => {
|
||||
const runner = new PostgresPgssProbeRunner();
|
||||
|
||||
expect(
|
||||
runner.fixAdvice(
|
||||
new HistoricSqlExtensionMissingError({
|
||||
dialect: 'postgres',
|
||||
message: 'pg_stat_statements missing',
|
||||
remediation: 'CREATE EXTENSION pg_stat_statements;',
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
failHeadline: 'pg_stat_statements extension is missing',
|
||||
remediation: 'CREATE EXTENSION pg_stat_statements;',
|
||||
});
|
||||
|
||||
expect(
|
||||
runner.fixAdvice(
|
||||
new HistoricSqlGrantsMissingError({
|
||||
dialect: 'postgres',
|
||||
message: 'missing grants',
|
||||
remediation: 'GRANT pg_read_all_stats TO <connection role>;',
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
failHeadline: 'Postgres connection role lacks pg_read_all_stats',
|
||||
remediation: 'GRANT pg_read_all_stats TO <connection role>;',
|
||||
});
|
||||
|
||||
expect(
|
||||
runner.fixAdvice(
|
||||
new HistoricSqlVersionUnsupportedError({
|
||||
dialect: 'postgres',
|
||||
detectedVersion: 'PostgreSQL 13.12',
|
||||
minimumVersion: 'PostgreSQL 14',
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
failHeadline: 'Postgres version too old',
|
||||
remediation: 'Use PostgreSQL 14 or newer, or disable query history for this connection',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
import {
|
||||
HistoricSqlExtensionMissingError,
|
||||
HistoricSqlGrantsMissingError,
|
||||
HistoricSqlVersionUnsupportedError,
|
||||
} from '../adapters/historic-sql/errors.js';
|
||||
import { PostgresPgssReader } from '../adapters/historic-sql/postgres-pgss-reader.js';
|
||||
import type { PostgresPgssProbeResult } from '../adapters/historic-sql/types.js';
|
||||
import {
|
||||
type HistoricSqlFixAdvice,
|
||||
type HistoricSqlProbeInput,
|
||||
type HistoricSqlProbeRunner,
|
||||
type HistoricSqlSuccessDetail,
|
||||
} from '../historic-sql-probes.js';
|
||||
import {
|
||||
isKtxPostgresConnectionConfig,
|
||||
type KtxPostgresConnectionConfig,
|
||||
} from '../../../connectors/postgres/connector.js';
|
||||
import { KtxPostgresHistoricSqlQueryClient } from '../../../connectors/postgres/historic-sql-query-client.js';
|
||||
|
||||
interface ClientHandle {
|
||||
client: unknown;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
interface PostgresPgssProbeRunnerOptions {
|
||||
reader?: { probe(client: unknown): Promise<PostgresPgssProbeResult> };
|
||||
createClient?: (
|
||||
input: HistoricSqlProbeInput & { connection: KtxPostgresConnectionConfig },
|
||||
) => ClientHandle;
|
||||
}
|
||||
|
||||
function genericAdvice(error: unknown, catalogName: string): HistoricSqlFixAdvice {
|
||||
return {
|
||||
failHeadline: `${catalogName} readiness check failed`,
|
||||
remediation: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
|
||||
function infoSuffix(info: readonly string[] | undefined): string {
|
||||
return info && info.length > 0 ? `; ${info.join('; ')}` : '';
|
||||
}
|
||||
|
||||
export class PostgresPgssProbeRunner implements HistoricSqlProbeRunner {
|
||||
readonly dialect = 'postgres' as const;
|
||||
readonly catalogName = 'pg_stat_statements';
|
||||
|
||||
private readonly reader: { probe(client: unknown): Promise<PostgresPgssProbeResult> };
|
||||
private readonly createClient: (
|
||||
input: HistoricSqlProbeInput & { connection: KtxPostgresConnectionConfig },
|
||||
) => ClientHandle;
|
||||
|
||||
constructor(options: PostgresPgssProbeRunnerOptions = {}) {
|
||||
this.reader = options.reader ?? new PostgresPgssReader();
|
||||
this.createClient =
|
||||
options.createClient ??
|
||||
((input) => {
|
||||
const client = new KtxPostgresHistoricSqlQueryClient({
|
||||
connectionId: input.connectionId,
|
||||
connection: input.connection,
|
||||
env: input.env,
|
||||
});
|
||||
return { client, cleanup: () => client.cleanup() };
|
||||
});
|
||||
}
|
||||
|
||||
async run(input: HistoricSqlProbeInput): Promise<PostgresPgssProbeResult> {
|
||||
const inputDriver = input.connection.driver ?? 'unknown';
|
||||
if (!isKtxPostgresConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
const handle = this.createClient({
|
||||
...input,
|
||||
connection: input.connection,
|
||||
});
|
||||
try {
|
||||
return await this.reader.probe(handle.client);
|
||||
} finally {
|
||||
await handle.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail {
|
||||
const pgssResult = result as PostgresPgssProbeResult;
|
||||
return {
|
||||
detail: `pg_stat_statements ready (${pgssResult.pgServerVersion})${infoSuffix(pgssResult.info)}`,
|
||||
warnings: pgssResult.warnings,
|
||||
};
|
||||
}
|
||||
|
||||
fixAdvice(error: unknown): HistoricSqlFixAdvice {
|
||||
if (error instanceof HistoricSqlExtensionMissingError) {
|
||||
return {
|
||||
failHeadline: 'pg_stat_statements extension is missing',
|
||||
remediation: error.remediation,
|
||||
};
|
||||
}
|
||||
if (error instanceof HistoricSqlGrantsMissingError) {
|
||||
return {
|
||||
failHeadline: 'Postgres connection role lacks pg_read_all_stats',
|
||||
remediation: error.remediation,
|
||||
};
|
||||
}
|
||||
if (error instanceof HistoricSqlVersionUnsupportedError) {
|
||||
return {
|
||||
failHeadline: 'Postgres version too old',
|
||||
remediation: 'Use PostgreSQL 14 or newer, or disable query history for this connection',
|
||||
};
|
||||
}
|
||||
return genericAdvice(error, this.catalogName);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
|
||||
import { SnowflakeAccountUsageProbeRunner } from './snowflake-runner.js';
|
||||
|
||||
describe('SnowflakeAccountUsageProbeRunner', () => {
|
||||
it('runs the account usage reader and cleans up the client', async () => {
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const reader = {
|
||||
probe: vi.fn(async () => ({ warnings: [], info: ['query history available'] })),
|
||||
};
|
||||
const runner = new SnowflakeAccountUsageProbeRunner({
|
||||
reader,
|
||||
createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }),
|
||||
});
|
||||
|
||||
await expect(
|
||||
runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'snowflake',
|
||||
account: 'ACCT',
|
||||
warehouse: 'WH',
|
||||
database: 'ANALYTICS',
|
||||
username: 'reader',
|
||||
},
|
||||
env: {},
|
||||
}),
|
||||
).resolves.toEqual({ warnings: [], info: ['query history available'] });
|
||||
expect(reader.probe).toHaveBeenCalledOnce();
|
||||
expect(cleanup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('rejects non-Snowflake connections', async () => {
|
||||
const runner = new SnowflakeAccountUsageProbeRunner({
|
||||
reader: { probe: vi.fn() },
|
||||
createClient: () => ({ client: {}, cleanup: vi.fn() }),
|
||||
});
|
||||
|
||||
await expect(
|
||||
runner.run({
|
||||
projectDir: '/work/project',
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres' },
|
||||
env: {},
|
||||
}),
|
||||
).rejects.toThrow('Native Snowflake connector cannot run driver "postgres"');
|
||||
});
|
||||
|
||||
it('formats successful Snowflake details', () => {
|
||||
const runner = new SnowflakeAccountUsageProbeRunner();
|
||||
|
||||
expect(
|
||||
runner.formatSuccessDetail({
|
||||
warnings: ['query history is delayed'],
|
||||
info: ['warehouse: WH'],
|
||||
}),
|
||||
).toEqual({
|
||||
detail: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY ready; warehouse: WH',
|
||||
warnings: ['query history is delayed'],
|
||||
});
|
||||
});
|
||||
|
||||
it('maps Snowflake grant errors to runner advice', () => {
|
||||
const runner = new SnowflakeAccountUsageProbeRunner();
|
||||
|
||||
expect(
|
||||
runner.fixAdvice(
|
||||
new HistoricSqlGrantsMissingError({
|
||||
dialect: 'snowflake',
|
||||
message: 'role cannot read account usage',
|
||||
remediation:
|
||||
'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE <connection role>;',
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
|
||||
remediation:
|
||||
'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE <connection role>;',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js';
|
||||
import { SnowflakeHistoricSqlQueryHistoryReader } from '../adapters/historic-sql/snowflake-query-history-reader.js';
|
||||
import {
|
||||
type HistoricSqlFixAdvice,
|
||||
type HistoricSqlProbeInput,
|
||||
type HistoricSqlProbeRunner,
|
||||
type HistoricSqlSuccessDetail,
|
||||
} from '../historic-sql-probes.js';
|
||||
import {
|
||||
isKtxSnowflakeConnectionConfig,
|
||||
type KtxSnowflakeConnectionConfig,
|
||||
} from '../../../connectors/snowflake/connector.js';
|
||||
import { KtxSnowflakeHistoricSqlQueryClient } from '../../../connectors/snowflake/historic-sql-query-client.js';
|
||||
|
||||
interface GenericProbeResult {
|
||||
warnings: string[];
|
||||
info?: string[];
|
||||
}
|
||||
|
||||
interface ClientHandle {
|
||||
client: unknown;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
interface SnowflakeAccountUsageProbeRunnerOptions {
|
||||
reader?: { probe(client: unknown): Promise<GenericProbeResult> };
|
||||
createClient?: (
|
||||
input: HistoricSqlProbeInput & { connection: KtxSnowflakeConnectionConfig },
|
||||
) => ClientHandle;
|
||||
}
|
||||
|
||||
function infoSuffix(info: readonly string[] | undefined): string {
|
||||
return info && info.length > 0 ? `; ${info.join('; ')}` : '';
|
||||
}
|
||||
|
||||
export class SnowflakeAccountUsageProbeRunner implements HistoricSqlProbeRunner {
|
||||
readonly dialect = 'snowflake' as const;
|
||||
readonly catalogName = 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY';
|
||||
|
||||
private readonly reader: { probe(client: unknown): Promise<GenericProbeResult> };
|
||||
private readonly createClient: (
|
||||
input: HistoricSqlProbeInput & { connection: KtxSnowflakeConnectionConfig },
|
||||
) => ClientHandle;
|
||||
|
||||
constructor(options: SnowflakeAccountUsageProbeRunnerOptions = {}) {
|
||||
this.reader = options.reader ?? new SnowflakeHistoricSqlQueryHistoryReader();
|
||||
this.createClient =
|
||||
options.createClient ??
|
||||
((input) => {
|
||||
const client = new KtxSnowflakeHistoricSqlQueryClient({
|
||||
connectionId: input.connectionId,
|
||||
connection: input.connection,
|
||||
projectDir: input.projectDir,
|
||||
env: input.env,
|
||||
});
|
||||
return { client, cleanup: () => client.cleanup() };
|
||||
});
|
||||
}
|
||||
|
||||
async run(input: HistoricSqlProbeInput): Promise<GenericProbeResult> {
|
||||
const inputDriver = input.connection.driver ?? 'unknown';
|
||||
if (!isKtxSnowflakeConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
const handle = this.createClient({
|
||||
...input,
|
||||
connection: input.connection,
|
||||
});
|
||||
try {
|
||||
return await this.reader.probe(handle.client);
|
||||
} finally {
|
||||
await handle.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail {
|
||||
const probeResult = result as GenericProbeResult;
|
||||
return {
|
||||
detail: `${this.catalogName} ready${infoSuffix(probeResult.info)}`,
|
||||
warnings: probeResult.warnings,
|
||||
};
|
||||
}
|
||||
|
||||
fixAdvice(error: unknown): HistoricSqlFixAdvice {
|
||||
if (error instanceof HistoricSqlGrantsMissingError) {
|
||||
return {
|
||||
failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
|
||||
remediation: error.remediation,
|
||||
};
|
||||
}
|
||||
return {
|
||||
failHeadline: `${this.catalogName} readiness check failed`,
|
||||
remediation: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -591,7 +591,7 @@ describe('local ingest', () => {
|
|||
status: 'done',
|
||||
adapter: 'live-database',
|
||||
connectionId: 'warehouse',
|
||||
rawFileCount: 3,
|
||||
rawFileCount: 4,
|
||||
workUnitCount: 1,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
70
packages/cli/src/context/scan/constraint-discovery.test.ts
Normal file
70
packages/cli/src/context/scan/constraint-discovery.test.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { constraintDiscoveryWarning, tryConstraintQuery } from './constraint-discovery.js';
|
||||
|
||||
describe('tryConstraintQuery', () => {
|
||||
it('returns the query value when the query succeeds', async () => {
|
||||
await expect(
|
||||
tryConstraintQuery(
|
||||
{
|
||||
schema: 'public',
|
||||
kind: 'primary_key',
|
||||
isDeniedError: () => false,
|
||||
},
|
||||
async () => ['id'],
|
||||
),
|
||||
).resolves.toEqual({ ok: true, value: ['id'] });
|
||||
});
|
||||
|
||||
it('returns a recoverable warning when the classifier recognizes denial', async () => {
|
||||
const error = Object.assign(new Error('permission denied'), { code: '42501' });
|
||||
|
||||
await expect(
|
||||
tryConstraintQuery(
|
||||
{
|
||||
schema: 'analytics',
|
||||
kind: 'foreign_key',
|
||||
isDeniedError: (candidate) => candidate === error,
|
||||
},
|
||||
async () => {
|
||||
throw error;
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({
|
||||
ok: false,
|
||||
warning: {
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'analytics', kind: 'foreign_key' },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rethrows non-denial errors unchanged', async () => {
|
||||
const error = Object.assign(new Error('connection reset'), { code: 'ECONNRESET' });
|
||||
|
||||
await expect(
|
||||
tryConstraintQuery(
|
||||
{
|
||||
schema: 'public',
|
||||
kind: 'primary_key',
|
||||
isDeniedError: () => false,
|
||||
},
|
||||
async () => {
|
||||
throw error;
|
||||
},
|
||||
),
|
||||
).rejects.toBe(error);
|
||||
});
|
||||
});
|
||||
|
||||
describe('constraintDiscoveryWarning', () => {
|
||||
it('formats stable primary-key warning text and metadata', () => {
|
||||
expect(constraintDiscoveryWarning({ schema: 'public', kind: 'primary_key' })).toEqual({
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'public', kind: 'primary_key' },
|
||||
});
|
||||
});
|
||||
});
|
||||
42
packages/cli/src/context/scan/constraint-discovery.ts
Normal file
42
packages/cli/src/context/scan/constraint-discovery.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import type { KtxScanWarning } from './types.js';
|
||||
|
||||
export type ConstraintDiscoveryKind = 'primary_key' | 'foreign_key';
|
||||
|
||||
export interface ConstraintQueryContext {
|
||||
schema: string;
|
||||
kind: ConstraintDiscoveryKind;
|
||||
isDeniedError: (error: unknown) => boolean;
|
||||
}
|
||||
|
||||
export type ConstraintQueryOutcome<T> = { ok: true; value: T } | { ok: false; warning: KtxScanWarning };
|
||||
|
||||
export function constraintDiscoveryWarning(input: {
|
||||
schema: string;
|
||||
kind: ConstraintDiscoveryKind;
|
||||
}): KtxScanWarning {
|
||||
return {
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message:
|
||||
`Skipped ${input.kind === 'primary_key' ? 'primary-key' : 'foreign-key'} ` +
|
||||
`discovery in ${input.schema} (insufficient grants on system catalogs)`,
|
||||
recoverable: true,
|
||||
metadata: { schema: input.schema, kind: input.kind },
|
||||
};
|
||||
}
|
||||
|
||||
export async function tryConstraintQuery<T>(
|
||||
ctx: ConstraintQueryContext,
|
||||
fn: () => Promise<T>,
|
||||
): Promise<ConstraintQueryOutcome<T>> {
|
||||
try {
|
||||
return { ok: true, value: await fn() };
|
||||
} catch (error) {
|
||||
if (!ctx.isDeniedError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
warning: constraintDiscoveryWarning({ schema: ctx.schema, kind: ctx.kind }),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -180,6 +180,13 @@ function fetchOnlyAdapter(options: { extractedAt?: () => string; snapshot?: KtxS
|
|||
'utf-8',
|
||||
);
|
||||
await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8');
|
||||
if (scanSnapshot.warnings?.length) {
|
||||
await writeFile(
|
||||
join(stagedDir, 'warnings.json'),
|
||||
`${JSON.stringify({ warnings: scanSnapshot.warnings })}\n`,
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
for (const table of scanSnapshot.tables) {
|
||||
await writeFile(join(stagedDir, 'tables', `${table.name}.json`), `${JSON.stringify(table)}\n`, 'utf-8');
|
||||
}
|
||||
|
|
@ -336,6 +343,48 @@ describe('local scan', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('threads structural snapshot warnings into the final scan report', async () => {
|
||||
const result = await runLocalScan({
|
||||
project,
|
||||
adapters: [
|
||||
fetchOnlyAdapter({
|
||||
snapshot: {
|
||||
...defaultFetchSnapshot(),
|
||||
warnings: [
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'public', kind: 'primary_key' },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
connectionId: 'warehouse',
|
||||
jobId: 'scan-run-structural-warnings',
|
||||
now: () => new Date('2026-04-29T09:01:00.000Z'),
|
||||
});
|
||||
|
||||
expect(result.report.warnings).toEqual([
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'public', kind: 'primary_key' },
|
||||
},
|
||||
]);
|
||||
await expect(
|
||||
readFile(
|
||||
join(
|
||||
project.projectDir,
|
||||
'raw-sources/warehouse/live-database/2026-04-29-090100-scan-run-structural-warnings/scan-report.json',
|
||||
),
|
||||
'utf-8',
|
||||
),
|
||||
).resolves.toContain('"constraint_discovery_unauthorized"');
|
||||
});
|
||||
|
||||
it('passes enabled_tables as fetch context tableScope and does not post-filter staged snapshots', async () => {
|
||||
project.config.connections.warehouse = {
|
||||
...project.config.connections.warehouse,
|
||||
|
|
|
|||
|
|
@ -467,6 +467,9 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
|
|||
extractedAtFallback: report.createdAt,
|
||||
});
|
||||
enrichmentSnapshot = rawSnapshot;
|
||||
if (rawSnapshot.warnings?.length) {
|
||||
report.warnings.push(...rawSnapshot.warnings);
|
||||
}
|
||||
const manifestArtifacts = await writeLocalScanManifestShards({
|
||||
project: options.project,
|
||||
connectionId: options.connectionId,
|
||||
|
|
|
|||
|
|
@ -165,6 +165,61 @@ describe('readLocalScanStructuralSnapshot', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('rebuilds scan warnings from persisted live-database warning files', async () => {
|
||||
const rawRoot = 'raw-sources/warehouse/live-database/sync-warnings';
|
||||
await project.fileStore.writeFile(
|
||||
`${rawRoot}/connection.json`,
|
||||
'{"connectionId":"warehouse","metadata":{}}\n',
|
||||
'ktx',
|
||||
'ktx@example.com',
|
||||
'Seed connection artifact',
|
||||
);
|
||||
await project.fileStore.writeFile(
|
||||
`${rawRoot}/warnings.json`,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
warnings: [
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'public', kind: 'foreign_key' },
|
||||
},
|
||||
],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
'ktx',
|
||||
'ktx@example.com',
|
||||
'Seed warning artifact',
|
||||
);
|
||||
await project.fileStore.writeFile(
|
||||
`${rawRoot}/tables/orders.json`,
|
||||
'{"name":"orders","catalog":null,"db":"public","kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":false,"comment":null}],"foreignKeys":[]}\n',
|
||||
'ktx',
|
||||
'ktx@example.com',
|
||||
'Seed orders artifact',
|
||||
);
|
||||
|
||||
const snapshot = await readLocalScanStructuralSnapshot({
|
||||
project,
|
||||
connectionId: 'warehouse',
|
||||
driver: 'postgres',
|
||||
rawSourcesDir: rawRoot,
|
||||
extractedAtFallback: '2026-04-29T13:00:00.000Z',
|
||||
});
|
||||
|
||||
expect(snapshot.warnings).toEqual([
|
||||
{
|
||||
code: 'constraint_discovery_unauthorized',
|
||||
message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
|
||||
recoverable: true,
|
||||
metadata: { schema: 'public', kind: 'foreign_key' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('uses the scan report timestamp when connection.json omits extractedAt', async () => {
|
||||
const rawRoot = 'raw-sources/warehouse/live-database/sync-2';
|
||||
await project.fileStore.writeFile(
|
||||
|
|
@ -192,4 +247,32 @@ describe('readLocalScanStructuralSnapshot', () => {
|
|||
|
||||
expect(snapshot.extractedAt).toBe('2026-04-29T13:00:00.000Z');
|
||||
});
|
||||
|
||||
it('tolerates older live-database staged directories without warnings.json', async () => {
|
||||
const rawRoot = 'raw-sources/warehouse/live-database/sync-no-warnings';
|
||||
await project.fileStore.writeFile(
|
||||
`${rawRoot}/connection.json`,
|
||||
'{"connectionId":"warehouse","metadata":{}}\n',
|
||||
'ktx',
|
||||
'ktx@example.com',
|
||||
'Seed connection artifact',
|
||||
);
|
||||
await project.fileStore.writeFile(
|
||||
`${rawRoot}/tables/orders.json`,
|
||||
'{"name":"orders","catalog":null,"db":null,"kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":true,"comment":null}],"foreignKeys":[]}\n',
|
||||
'ktx',
|
||||
'ktx@example.com',
|
||||
'Seed orders artifact',
|
||||
);
|
||||
|
||||
const snapshot = await readLocalScanStructuralSnapshot({
|
||||
project,
|
||||
connectionId: 'warehouse',
|
||||
driver: 'postgres',
|
||||
rawSourcesDir: rawRoot,
|
||||
extractedAtFallback: '2026-04-29T13:00:00.000Z',
|
||||
});
|
||||
|
||||
expect(snapshot.warnings).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
import type {
|
||||
KtxConnectionDriver,
|
||||
KtxScanWarning,
|
||||
KtxSchemaColumn,
|
||||
KtxSchemaForeignKey,
|
||||
KtxSchemaSnapshot,
|
||||
|
|
@ -30,6 +31,59 @@ function metadataRecord(value: unknown): Record<string, unknown> {
|
|||
return isRecord(value) ? value : {};
|
||||
}
|
||||
|
||||
const scanWarningCodes = new Set<KtxScanWarning['code']>([
|
||||
'connector_capability_missing',
|
||||
'sampling_failed',
|
||||
'statistics_failed',
|
||||
'llm_unavailable',
|
||||
'embedding_unavailable',
|
||||
'scan_enrichment_backend_not_configured',
|
||||
'relationship_validation_failed',
|
||||
'relationship_llm_invalid_reference',
|
||||
'relationship_llm_proposal_failed',
|
||||
'credential_redacted',
|
||||
'enrichment_failed',
|
||||
'description_fallback_used',
|
||||
'constraint_discovery_unauthorized',
|
||||
]);
|
||||
|
||||
function parseWarning(rawWarning: unknown, path: string): KtxScanWarning {
|
||||
if (
|
||||
!isRecord(rawWarning) ||
|
||||
typeof rawWarning.code !== 'string' ||
|
||||
!scanWarningCodes.has(rawWarning.code as KtxScanWarning['code']) ||
|
||||
typeof rawWarning.message !== 'string' ||
|
||||
typeof rawWarning.recoverable !== 'boolean'
|
||||
) {
|
||||
throw new Error(`Invalid KTX schema warning artifact: ${path}`);
|
||||
}
|
||||
return {
|
||||
code: rawWarning.code as KtxScanWarning['code'],
|
||||
message: rawWarning.message,
|
||||
recoverable: rawWarning.recoverable,
|
||||
...(typeof rawWarning.table === 'string' ? { table: rawWarning.table } : {}),
|
||||
...(typeof rawWarning.column === 'string' ? { column: rawWarning.column } : {}),
|
||||
...(isRecord(rawWarning.metadata) ? { metadata: rawWarning.metadata } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function readWarnings(input: ReadLocalScanStructuralSnapshotInput): Promise<KtxScanWarning[]> {
|
||||
const path = `${input.rawSourcesDir}/warnings.json`;
|
||||
try {
|
||||
const warningRaw = await input.project.fileStore.readFile(path);
|
||||
const parsed = JSON.parse(warningRaw.content) as unknown;
|
||||
if (!isRecord(parsed) || !Array.isArray(parsed.warnings)) {
|
||||
throw new Error(`Invalid KTX schema warnings artifact: ${path}`);
|
||||
}
|
||||
return parsed.warnings.map((warning) => parseWarning(warning, path));
|
||||
} catch (error) {
|
||||
if (error instanceof Error && /not found|ENOENT|no such file/i.test(error.message)) {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function optionalStringOrNull(value: unknown): string | null | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
|
|
@ -113,6 +167,7 @@ export async function readLocalScanStructuralSnapshot(
|
|||
const tableRaw = await input.project.fileStore.readFile(path);
|
||||
tables.push(parseTable(tableRaw.content, path));
|
||||
}
|
||||
const warnings = await readWarnings(input);
|
||||
|
||||
return {
|
||||
connectionId: typeof connection.connectionId === 'string' ? connection.connectionId : input.connectionId,
|
||||
|
|
@ -121,5 +176,6 @@ export async function readLocalScanStructuralSnapshot(
|
|||
scope: isRecord(connection.scope) ? connection.scope : {},
|
||||
metadata: metadataRecord(connection.metadata),
|
||||
tables,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ export interface KtxSchemaSnapshot {
|
|||
scope: KtxSchemaScope;
|
||||
tables: KtxSchemaTable[];
|
||||
metadata: Record<string, unknown>;
|
||||
warnings?: KtxScanWarning[];
|
||||
}
|
||||
|
||||
interface KtxCredentialEnvReference {
|
||||
|
|
@ -364,7 +365,8 @@ type KtxScanWarningCode =
|
|||
| 'relationship_llm_proposal_failed'
|
||||
| 'credential_redacted'
|
||||
| 'enrichment_failed'
|
||||
| 'description_fallback_used';
|
||||
| 'description_fallback_used'
|
||||
| 'constraint_discovery_unauthorized';
|
||||
|
||||
export interface KtxScanWarning {
|
||||
code: KtxScanWarningCode;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue