mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
feat(scan): dispatch query-history probes by connection driver
Extract historic-sql dialect resolution into a shared helper so the status-project readiness check and the local ingest factory agree on which connections enable query history and which probe to run. The status command now picks the postgres/snowflake/bigquery probe based on the connection's driver instead of always reporting against postgres, which previously caused snowflake connections with queryHistory.enabled to surface a misleading "driver is snowflake" failure. Also drops a noisy console.warn from Snowflake primary-key discovery — INFORMATION_SCHEMA.KEY_COLUMN_USAGE is commonly ungranted for read-only roles and the FK + profiling paths handle the empty PK map already.
This commit is contained in:
parent
4e654c43c1
commit
d8b2d8936f
6 changed files with 428 additions and 115 deletions
|
|
@ -332,9 +332,7 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
|
||||
expect(snapshot.tables.map((table) => table.name).sort()).toEqual(['ORDERS', 'ORDER_SUMMARY']);
|
||||
expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Snowflake primary-key discovery skipped for ANALYTICS.PUBLIC'),
|
||||
);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -708,11 +708,9 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector {
|
|||
const columnName = String(row[1]);
|
||||
grouped.get(tableName)?.add(columnName);
|
||||
}
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
console.warn(
|
||||
`Snowflake primary-key discovery skipped for ${this.resolved.database}.${schemaName}: ${detail.replace(/\s+/g, ' ').trim()}`,
|
||||
);
|
||||
} catch {
|
||||
// INFORMATION_SCHEMA.KEY_COLUMN_USAGE often isn't granted to read-only roles;
|
||||
// continue with empty PK map and let FK inference + profiling carry the slack.
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import type { HistoricSqlDialect } from './types.js';
|
||||
|
||||
const KNOWN_DIALECTS = ['postgres', 'bigquery', 'snowflake'] as const;
|
||||
|
||||
function isKnownDialect(value: string): value is HistoricSqlDialect {
|
||||
return (KNOWN_DIALECTS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
function recordOrNull(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : null;
|
||||
}
|
||||
|
||||
function historicSqlRecord(connection: unknown): Record<string, unknown> | null {
|
||||
const conn = recordOrNull(connection);
|
||||
return conn ? recordOrNull(conn.historicSql) : null;
|
||||
}
|
||||
|
||||
function queryHistoryRecord(connection: unknown): Record<string, unknown> | null {
|
||||
const conn = recordOrNull(connection);
|
||||
const context = conn ? recordOrNull(conn.context) : null;
|
||||
return context ? recordOrNull(context.queryHistory) : null;
|
||||
}
|
||||
|
||||
export function isQueryHistoryEnabled(connection: unknown): boolean {
|
||||
const queryHistory = queryHistoryRecord(connection);
|
||||
if (queryHistory) {
|
||||
return queryHistory.enabled === true;
|
||||
}
|
||||
return historicSqlRecord(connection)?.enabled === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the query-history dialect for a connection. Returns null when
|
||||
* query history is disabled, or when the connection's driver has no
|
||||
* query-history reader.
|
||||
*/
|
||||
export function queryHistoryDialectForConnection(connection: unknown): HistoricSqlDialect | null {
|
||||
if (!isQueryHistoryEnabled(connection)) {
|
||||
return null;
|
||||
}
|
||||
const conn = recordOrNull(connection);
|
||||
const driver = String(conn?.driver ?? '').toLowerCase();
|
||||
if (driver === 'postgres' || driver === 'postgresql') return 'postgres';
|
||||
if (driver === 'bigquery') return 'bigquery';
|
||||
if (driver === 'snowflake') return 'snowflake';
|
||||
const legacy = String(historicSqlRecord(connection)?.dialect ?? '').toLowerCase();
|
||||
return isKnownDialect(legacy) ? legacy : null;
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js';
|
|||
import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js';
|
||||
import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js';
|
||||
import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js';
|
||||
import { queryHistoryDialectForConnection } from './context/ingest/adapters/historic-sql/connection-dialect.js';
|
||||
import { createDaemonLiveDatabaseIntrospection } from './context/ingest/adapters/live-database/daemon-introspection.js';
|
||||
import { createDefaultLocalIngestAdapters, type DefaultLocalIngestAdaptersOptions } from './context/ingest/local-adapters.js';
|
||||
import type { HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js';
|
||||
|
|
@ -164,47 +165,6 @@ export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdap
|
|||
logger?: KtxOperationalLogger;
|
||||
}
|
||||
|
||||
function historicSqlRecord(connection: unknown): Record<string, unknown> | null {
|
||||
if (
|
||||
connection &&
|
||||
typeof connection === 'object' &&
|
||||
'historicSql' in connection &&
|
||||
typeof (connection as { historicSql?: unknown }).historicSql === 'object' &&
|
||||
(connection as { historicSql?: unknown }).historicSql !== null &&
|
||||
!Array.isArray((connection as { historicSql?: unknown }).historicSql)
|
||||
) {
|
||||
return (connection as { historicSql: Record<string, unknown> }).historicSql;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery' | 'snowflake' | null {
|
||||
const direct = historicSqlRecord(connection);
|
||||
const context =
|
||||
connection && typeof connection === 'object' && !Array.isArray(connection)
|
||||
? (connection as { context?: unknown }).context
|
||||
: null;
|
||||
const queryHistory =
|
||||
context && typeof context === 'object' && !Array.isArray(context)
|
||||
? (context as { queryHistory?: unknown }).queryHistory
|
||||
: null;
|
||||
const enabled =
|
||||
queryHistory && typeof queryHistory === 'object' && !Array.isArray(queryHistory)
|
||||
? (queryHistory as { enabled?: unknown }).enabled === true
|
||||
: direct?.enabled === true;
|
||||
if (!enabled) {
|
||||
return null;
|
||||
}
|
||||
const driver = String((connection as { driver?: unknown })?.driver ?? '').toLowerCase();
|
||||
if (driver === 'postgres' || driver === 'postgresql') return 'postgres';
|
||||
if (driver === 'bigquery') return 'bigquery';
|
||||
if (driver === 'snowflake') return 'snowflake';
|
||||
const legacyDialect = String(direct?.dialect ?? '').toLowerCase();
|
||||
return legacyDialect === 'postgres' || legacyDialect === 'bigquery' || legacyDialect === 'snowflake'
|
||||
? legacyDialect
|
||||
: null;
|
||||
}
|
||||
|
||||
function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
|
||||
const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined;
|
||||
const inputDriver = connection?.driver ?? 'unknown';
|
||||
|
|
@ -308,7 +268,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
|
|||
return undefined;
|
||||
}
|
||||
const connection = project.config.connections[connectionId];
|
||||
const dialect = enabledHistoricSqlDialect(connection);
|
||||
const dialect = queryHistoryDialectForConnection(connection);
|
||||
if (!dialect) {
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -148,6 +148,161 @@ function withPostgresQueryHistory(config: KtxProjectConfig): KtxProjectConfig {
|
|||
};
|
||||
}
|
||||
|
||||
function withSnowflakeQueryHistory(config: KtxProjectConfig): KtxProjectConfig {
|
||||
return {
|
||||
...config,
|
||||
connections: {
|
||||
...config.connections,
|
||||
warehouse: {
|
||||
driver: 'snowflake',
|
||||
account: 'EMOVRJS-CZ07756',
|
||||
warehouse: 'COMPUTE_WH',
|
||||
database: 'ANALYTICS',
|
||||
username: 'svc_ktx',
|
||||
password: 'env:SNOWFLAKE_PASSWORD', // pragma: allowlist secret
|
||||
context: { queryHistory: { enabled: true } },
|
||||
} as KtxProjectConfig['connections'][string],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function withBigQueryQueryHistory(config: KtxProjectConfig): KtxProjectConfig {
|
||||
return {
|
||||
...config,
|
||||
connections: {
|
||||
...config.connections,
|
||||
bq: {
|
||||
driver: 'bigquery',
|
||||
credentials_json: 'env:BQ_CREDENTIALS_JSON',
|
||||
context: { queryHistory: { enabled: true } },
|
||||
} as KtxProjectConfig['connections'][string],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function withMysqlQueryHistory(config: KtxProjectConfig): KtxProjectConfig {
|
||||
return {
|
||||
...config,
|
||||
connections: {
|
||||
...config.connections,
|
||||
legacy: {
|
||||
driver: 'mysql',
|
||||
host: 'db.example.com',
|
||||
database: 'analytics',
|
||||
username: 'svc',
|
||||
password: 'env:MYSQL_PASSWORD', // pragma: allowlist secret
|
||||
context: { queryHistory: { enabled: true } },
|
||||
} as KtxProjectConfig['connections'][string],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('buildProjectStatus query history dispatch', () => {
|
||||
it('runs the snowflake probe for snowflake connections, not the postgres one', async () => {
|
||||
let postgresCalls = 0;
|
||||
let snowflakeCalls = 0;
|
||||
const project = projectWithConfig(withSnowflakeQueryHistory(baseProjectConfig()));
|
||||
|
||||
const status = await buildProjectStatus(project, {
|
||||
claudeCodeAuthProbe: stubClaudeCodeAuthProbe,
|
||||
postgresQueryHistoryProbe: async () => {
|
||||
postgresCalls += 1;
|
||||
throw new Error('postgres probe should not run for snowflake');
|
||||
},
|
||||
snowflakeQueryHistoryProbe: async () => {
|
||||
snowflakeCalls += 1;
|
||||
return { warnings: [], info: [] };
|
||||
},
|
||||
});
|
||||
|
||||
expect(postgresCalls).toBe(0);
|
||||
expect(snowflakeCalls).toBe(1);
|
||||
expect(status.queryHistory).toHaveLength(1);
|
||||
expect(status.queryHistory[0]).toMatchObject({
|
||||
connection: 'warehouse',
|
||||
driver: 'snowflake',
|
||||
dialect: 'snowflake',
|
||||
status: 'ok',
|
||||
});
|
||||
expect(status.queryHistory[0].detail).toMatch(/SNOWFLAKE\.ACCOUNT_USAGE\.QUERY_HISTORY/);
|
||||
expect(status.queryHistory[0].fix).toBeUndefined();
|
||||
expect(status.verdict).not.toBe('blocked');
|
||||
});
|
||||
|
||||
it('reports snowflake probe failures with the reader-provided remediation', async () => {
|
||||
const project = projectWithConfig(withSnowflakeQueryHistory(baseProjectConfig()));
|
||||
const { HistoricSqlGrantsMissingError } = await import(
|
||||
'./context/ingest/adapters/historic-sql/errors.js'
|
||||
);
|
||||
|
||||
const status = await buildProjectStatus(project, {
|
||||
claudeCodeAuthProbe: stubClaudeCodeAuthProbe,
|
||||
snowflakeQueryHistoryProbe: async () => {
|
||||
throw new HistoricSqlGrantsMissingError({
|
||||
dialect: 'snowflake',
|
||||
message: 'role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
|
||||
remediation: 'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ktx;',
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
expect(status.queryHistory[0]).toMatchObject({
|
||||
connection: 'warehouse',
|
||||
driver: 'snowflake',
|
||||
dialect: 'snowflake',
|
||||
status: 'fail',
|
||||
fix: 'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ktx;',
|
||||
});
|
||||
expect(status.queryHistory[0].detail).not.toMatch(/Set connections.*driver to postgres/);
|
||||
});
|
||||
|
||||
it('runs the bigquery probe for bigquery connections', async () => {
|
||||
let bigqueryCalls = 0;
|
||||
const project = projectWithConfig(withBigQueryQueryHistory(baseProjectConfig()));
|
||||
|
||||
const status = await buildProjectStatus(project, {
|
||||
claudeCodeAuthProbe: stubClaudeCodeAuthProbe,
|
||||
bigqueryQueryHistoryProbe: async () => {
|
||||
bigqueryCalls += 1;
|
||||
return { warnings: [], info: [] };
|
||||
},
|
||||
});
|
||||
|
||||
expect(bigqueryCalls).toBe(1);
|
||||
expect(status.queryHistory[0]).toMatchObject({
|
||||
connection: 'bq',
|
||||
driver: 'bigquery',
|
||||
dialect: 'bigquery',
|
||||
status: 'ok',
|
||||
});
|
||||
expect(status.queryHistory[0].detail).toMatch(/INFORMATION_SCHEMA\.JOBS_BY_PROJECT/);
|
||||
});
|
||||
|
||||
it('fails with an accurate message for drivers without a query history reader', async () => {
|
||||
const project = projectWithConfig(withMysqlQueryHistory(baseProjectConfig()));
|
||||
|
||||
const status = await buildProjectStatus(project, {
|
||||
claudeCodeAuthProbe: stubClaudeCodeAuthProbe,
|
||||
postgresQueryHistoryProbe: async () => {
|
||||
throw new Error('postgres probe must not run for mysql');
|
||||
},
|
||||
});
|
||||
|
||||
expect(status.queryHistory).toHaveLength(1);
|
||||
expect(status.queryHistory[0]).toMatchObject({
|
||||
connection: 'legacy',
|
||||
driver: 'mysql',
|
||||
dialect: 'mysql',
|
||||
status: 'fail',
|
||||
detail: 'query history is not supported for driver "mysql"',
|
||||
});
|
||||
expect(status.queryHistory[0].fix).toMatch(
|
||||
/Disable connections\.legacy\.context\.queryHistory/,
|
||||
);
|
||||
expect(status.queryHistory[0].fix).not.toMatch(/driver to postgres/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildProjectStatus --fast', () => {
|
||||
it('skips claude-code probe and Postgres query-history probe', async () => {
|
||||
let claudeProbeCalls = 0;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@ import type { KtxConfigIssue, KtxProjectConfig, KtxProjectConnectionConfig, KtxP
|
|||
import type { KtxLocalProject } from './context/project/project.js';
|
||||
import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
|
||||
import type { PostgresPgssProbeResult } from './context/ingest/adapters/historic-sql/types.js';
|
||||
import {
|
||||
isQueryHistoryEnabled,
|
||||
queryHistoryDialectForConnection,
|
||||
} from './context/ingest/adapters/historic-sql/connection-dialect.js';
|
||||
import {
|
||||
formatClaudeCodePromptCachingFix,
|
||||
formatClaudeCodePromptCachingWarning,
|
||||
|
|
@ -47,7 +51,8 @@ interface ConnectionStatus extends ProjectStatusLine {
|
|||
|
||||
interface QueryHistoryStatus extends ProjectStatusLine {
|
||||
connection: string;
|
||||
dialect: 'postgres';
|
||||
driver: string;
|
||||
dialect: string;
|
||||
}
|
||||
|
||||
interface PipelineStatus {
|
||||
|
|
@ -396,45 +401,44 @@ function buildConnectionStatus(
|
|||
}
|
||||
}
|
||||
|
||||
interface PostgresQueryHistoryProbeInput {
|
||||
interface QueryHistoryProbeInput {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
connection: KtxProjectConnectionConfig;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
type PostgresQueryHistoryProbe = (
|
||||
input: PostgresQueryHistoryProbeInput,
|
||||
) => Promise<PostgresPgssProbeResult>;
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : null;
|
||||
interface GenericProbeResult {
|
||||
warnings: string[];
|
||||
info?: string[];
|
||||
}
|
||||
|
||||
function queryHistoryRecord(connection: KtxProjectConnectionConfig): Record<string, unknown> | null {
|
||||
const context = recordValue(connection.context);
|
||||
return recordValue(context?.queryHistory);
|
||||
}
|
||||
type PostgresQueryHistoryProbe = (input: QueryHistoryProbeInput) => Promise<PostgresPgssProbeResult>;
|
||||
type SnowflakeQueryHistoryProbe = (input: QueryHistoryProbeInput) => Promise<GenericProbeResult>;
|
||||
type BigQueryQueryHistoryProbe = (input: QueryHistoryProbeInput) => Promise<GenericProbeResult>;
|
||||
|
||||
function legacyHistoricSqlRecord(connection: KtxProjectConnectionConfig): Record<string, unknown> | null {
|
||||
return recordValue(connection.historicSql);
|
||||
}
|
||||
|
||||
function isEnabledPostgresQueryHistory(connection: KtxProjectConnectionConfig): boolean {
|
||||
const queryHistory = queryHistoryRecord(connection);
|
||||
if (queryHistory) {
|
||||
return queryHistory.enabled === true;
|
||||
function failureDetail(error: unknown): string {
|
||||
if (error instanceof Error && error.message.trim().length > 0) {
|
||||
return error.message.trim().split('\n')[0] ?? error.message.trim();
|
||||
}
|
||||
const legacy = legacyHistoricSqlRecord(connection);
|
||||
return legacy?.enabled === true && legacy.dialect === 'postgres';
|
||||
return String(error);
|
||||
}
|
||||
|
||||
function isPostgresDriver(connection: KtxProjectConnectionConfig): boolean {
|
||||
const driver = String(connection.driver ?? '').toLowerCase();
|
||||
return driver === 'postgres' || driver === 'postgresql';
|
||||
function postgresReadinessDetail(result: PostgresPgssProbeResult): string {
|
||||
const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : '';
|
||||
const info = result.info ?? [];
|
||||
const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : '';
|
||||
return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`;
|
||||
}
|
||||
|
||||
function queryHistoryFailureFix(error: unknown, connectionId: string, projectDir: string): string {
|
||||
function genericReadinessDetail(label: string, result: GenericProbeResult): string {
|
||||
const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : '';
|
||||
const info = result.info ?? [];
|
||||
const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : '';
|
||||
return `${label} ready${warningText}${infoText}`;
|
||||
}
|
||||
|
||||
function probeFailureFix(error: unknown, dialect: string, connectionId: string, projectDir: string): string {
|
||||
if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) {
|
||||
return String(error.remediation);
|
||||
}
|
||||
|
|
@ -444,25 +448,11 @@ function queryHistoryFailureFix(error: unknown, connectionId: string, projectDir
|
|||
if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') {
|
||||
return 'Use PostgreSQL 14 or newer, or disable query history for this connection';
|
||||
}
|
||||
return `Fix connections.${connectionId} Postgres settings, then rerun \`ktx status --project-dir ${projectDir}\``;
|
||||
}
|
||||
|
||||
function failureDetail(error: unknown): string {
|
||||
if (error instanceof Error && error.message.trim().length > 0) {
|
||||
return error.message.trim().split('\n')[0] ?? error.message.trim();
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
|
||||
function readinessDetail(result: PostgresPgssProbeResult): string {
|
||||
const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : '';
|
||||
const info = result.info ?? [];
|
||||
const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : '';
|
||||
return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`;
|
||||
return `Fix connections.${connectionId} ${dialect} settings, then rerun \`ktx status --project-dir ${projectDir}\``;
|
||||
}
|
||||
|
||||
async function defaultPostgresQueryHistoryProbe(
|
||||
input: PostgresQueryHistoryProbeInput,
|
||||
input: QueryHistoryProbeInput,
|
||||
): Promise<PostgresPgssProbeResult> {
|
||||
const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] =
|
||||
await Promise.all([
|
||||
|
|
@ -488,63 +478,225 @@ async function defaultPostgresQueryHistoryProbe(
|
|||
}
|
||||
}
|
||||
|
||||
async function defaultSnowflakeQueryHistoryProbe(
|
||||
input: QueryHistoryProbeInput,
|
||||
): Promise<GenericProbeResult> {
|
||||
const [{ SnowflakeHistoricSqlQueryHistoryReader }, { KtxSnowflakeHistoricSqlQueryClient }, { isKtxSnowflakeConnectionConfig }] =
|
||||
await Promise.all([
|
||||
import('./context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'),
|
||||
import('./connectors/snowflake/historic-sql-query-client.js'),
|
||||
import('./connectors/snowflake/connector.js'),
|
||||
]);
|
||||
|
||||
const inputDriver = input.connection.driver ?? 'unknown';
|
||||
if (!isKtxSnowflakeConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const client = new KtxSnowflakeHistoricSqlQueryClient({
|
||||
connectionId: input.connectionId,
|
||||
connection: input.connection,
|
||||
projectDir: input.projectDir,
|
||||
env: input.env,
|
||||
});
|
||||
try {
|
||||
return await new SnowflakeHistoricSqlQueryHistoryReader().probe(client);
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
async function defaultBigQueryQueryHistoryProbe(
|
||||
input: QueryHistoryProbeInput,
|
||||
): Promise<GenericProbeResult> {
|
||||
const [
|
||||
{ BigQueryHistoricSqlQueryHistoryReader },
|
||||
{ KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig },
|
||||
{ resolveKtxConfigReference },
|
||||
] = await Promise.all([
|
||||
import('./context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'),
|
||||
import('./connectors/bigquery/connector.js'),
|
||||
import('./context/core/config-reference.js'),
|
||||
]);
|
||||
|
||||
const inputDriver = input.connection.driver ?? 'unknown';
|
||||
if (!isKtxBigQueryConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const rawCredentials = typeof input.connection.credentials_json === 'string' ? input.connection.credentials_json : '';
|
||||
const resolvedCredentials = resolveKtxConfigReference(rawCredentials, input.env);
|
||||
if (!resolvedCredentials) {
|
||||
throw new Error(`Query history BigQuery connection ${input.connectionId} requires credentials_json`);
|
||||
}
|
||||
const parsed = JSON.parse(resolvedCredentials) as { project_id?: unknown };
|
||||
if (typeof parsed.project_id !== 'string' || parsed.project_id.trim().length === 0) {
|
||||
throw new Error(`Query history BigQuery connection ${input.connectionId} requires credentials_json.project_id`);
|
||||
}
|
||||
const region =
|
||||
typeof input.connection.location === 'string' && input.connection.location.trim().length > 0
|
||||
? input.connection.location.trim()
|
||||
: 'us';
|
||||
|
||||
const connector = new KtxBigQueryScanConnector({
|
||||
connectionId: input.connectionId,
|
||||
connection: input.connection,
|
||||
});
|
||||
try {
|
||||
return await new BigQueryHistoricSqlQueryHistoryReader({
|
||||
projectId: parsed.project_id,
|
||||
region,
|
||||
}).probe({
|
||||
async executeQuery(sql: string) {
|
||||
const result = await connector.executeReadOnly({ connectionId: input.connectionId, sql }, {} as never);
|
||||
return {
|
||||
headers: result.headers,
|
||||
rows: result.rows,
|
||||
totalRows: result.totalRows,
|
||||
};
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await connector.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
interface DispatchedProbe {
|
||||
label: string;
|
||||
spinnerLabel: string;
|
||||
fastSkipDetail: string;
|
||||
run: () => Promise<{ status: ProjectStatusLevel; detail: string; fix?: string }>;
|
||||
}
|
||||
|
||||
function postgresProbeDispatch(
|
||||
input: QueryHistoryProbeInput,
|
||||
probe: PostgresQueryHistoryProbe,
|
||||
): DispatchedProbe {
|
||||
return {
|
||||
label: 'postgres',
|
||||
spinnerLabel: `Probing pg_stat_statements on ${input.connectionId}`,
|
||||
fastSkipDetail: 'pg_stat_statements probe skipped (--fast)',
|
||||
run: async () => {
|
||||
const result = await probe(input);
|
||||
return {
|
||||
status: result.warnings.length > 0 ? 'warn' : 'ok',
|
||||
detail: postgresReadinessDetail(result),
|
||||
...(result.warnings.length > 0
|
||||
? {
|
||||
fix: `Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${input.projectDir}\``,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function snowflakeProbeDispatch(
|
||||
input: QueryHistoryProbeInput,
|
||||
probe: SnowflakeQueryHistoryProbe,
|
||||
): DispatchedProbe {
|
||||
return {
|
||||
label: 'snowflake',
|
||||
spinnerLabel: `Probing SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY on ${input.connectionId}`,
|
||||
fastSkipDetail: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY probe skipped (--fast)',
|
||||
run: async () => {
|
||||
const result = await probe(input);
|
||||
return {
|
||||
status: result.warnings.length > 0 ? 'warn' : 'ok',
|
||||
detail: genericReadinessDetail('SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', result),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function bigqueryProbeDispatch(
|
||||
input: QueryHistoryProbeInput,
|
||||
probe: BigQueryQueryHistoryProbe,
|
||||
): DispatchedProbe {
|
||||
return {
|
||||
label: 'bigquery',
|
||||
spinnerLabel: `Probing INFORMATION_SCHEMA.JOBS_BY_PROJECT on ${input.connectionId}`,
|
||||
fastSkipDetail: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT probe skipped (--fast)',
|
||||
run: async () => {
|
||||
const result = await probe(input);
|
||||
return {
|
||||
status: result.warnings.length > 0 ? 'warn' : 'ok',
|
||||
detail: genericReadinessDetail('INFORMATION_SCHEMA.JOBS_BY_PROJECT', result),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function buildQueryHistoryStatus(
|
||||
project: KtxLocalProject,
|
||||
options: BuildProjectStatusOptions,
|
||||
): Promise<QueryHistoryStatus[]> {
|
||||
const targets = Object.entries(project.config.connections)
|
||||
.filter(([, connection]) => isEnabledPostgresQueryHistory(connection))
|
||||
.filter(([, connection]) => isQueryHistoryEnabled(connection))
|
||||
.sort(([left], [right]) => left.localeCompare(right));
|
||||
|
||||
const probe = options.postgresQueryHistoryProbe ?? defaultPostgresQueryHistoryProbe;
|
||||
const postgresProbe = options.postgresQueryHistoryProbe ?? defaultPostgresQueryHistoryProbe;
|
||||
const snowflakeProbe = options.snowflakeQueryHistoryProbe ?? defaultSnowflakeQueryHistoryProbe;
|
||||
const bigqueryProbe = options.bigqueryQueryHistoryProbe ?? defaultBigQueryQueryHistoryProbe;
|
||||
const env = options.env ?? process.env;
|
||||
const statuses: QueryHistoryStatus[] = [];
|
||||
|
||||
for (const [connectionId, connection] of targets) {
|
||||
if (!isPostgresDriver(connection)) {
|
||||
const driver = String(connection.driver ?? 'unknown').toLowerCase();
|
||||
const dialect = queryHistoryDialectForConnection(connection);
|
||||
|
||||
if (!dialect) {
|
||||
statuses.push({
|
||||
connection: connectionId,
|
||||
dialect: 'postgres',
|
||||
driver,
|
||||
dialect: driver,
|
||||
status: 'fail',
|
||||
detail: `connections.${connectionId}.context.queryHistory is enabled but driver is ${String(connection.driver)}`,
|
||||
fix: `Set connections.${connectionId}.driver to postgres or disable query history for this connection`,
|
||||
detail: `query history is not supported for driver "${driver}"`,
|
||||
fix: `Disable connections.${connectionId}.context.queryHistory, or use a postgres, snowflake, or bigquery connection`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const probeInput: QueryHistoryProbeInput = {
|
||||
projectDir: project.projectDir,
|
||||
connectionId,
|
||||
connection,
|
||||
env,
|
||||
};
|
||||
const dispatched =
|
||||
dialect === 'postgres'
|
||||
? postgresProbeDispatch(probeInput, postgresProbe)
|
||||
: dialect === 'snowflake'
|
||||
? snowflakeProbeDispatch(probeInput, snowflakeProbe)
|
||||
: bigqueryProbeDispatch(probeInput, bigqueryProbe);
|
||||
|
||||
if (options.fast === true) {
|
||||
statuses.push({
|
||||
connection: connectionId,
|
||||
dialect: 'postgres',
|
||||
driver,
|
||||
dialect,
|
||||
status: 'skipped',
|
||||
detail: 'pg_stat_statements probe skipped (--fast)',
|
||||
detail: dispatched.fastSkipDetail,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await withSpinner(
|
||||
options.useSpinner === true,
|
||||
`Probing pg_stat_statements on ${connectionId}`,
|
||||
() => probe({ projectDir: project.projectDir, connectionId, connection, env }),
|
||||
);
|
||||
const outcome = await withSpinner(options.useSpinner === true, dispatched.spinnerLabel, dispatched.run);
|
||||
statuses.push({
|
||||
connection: connectionId,
|
||||
dialect: 'postgres',
|
||||
status: result.warnings.length > 0 ? 'warn' : 'ok',
|
||||
detail: readinessDetail(result),
|
||||
...(result.warnings.length > 0
|
||||
? {
|
||||
fix: `Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${project.projectDir}\``,
|
||||
}
|
||||
: {}),
|
||||
driver,
|
||||
dialect,
|
||||
...outcome,
|
||||
});
|
||||
} catch (error) {
|
||||
statuses.push({
|
||||
connection: connectionId,
|
||||
dialect: 'postgres',
|
||||
driver,
|
||||
dialect,
|
||||
status: 'fail',
|
||||
detail: failureDetail(error),
|
||||
fix: queryHistoryFailureFix(error, connectionId, project.projectDir),
|
||||
fix: probeFailureFix(error, dispatched.label, connectionId, project.projectDir),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -731,6 +883,8 @@ function buildVerdict(
|
|||
export interface BuildProjectStatusOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
postgresQueryHistoryProbe?: PostgresQueryHistoryProbe;
|
||||
snowflakeQueryHistoryProbe?: SnowflakeQueryHistoryProbe;
|
||||
bigqueryQueryHistoryProbe?: BigQueryQueryHistoryProbe;
|
||||
claudeCodeAuthProbe?: ClaudeCodeAuthProbe;
|
||||
configIssues?: KtxConfigIssue[];
|
||||
fast?: boolean;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue