mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-16 11:31:02 +02:00
fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match
The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.
Align the picker boundary with the canonical 3-level KtxTableRef:
- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
(resolveEnabledTables already accepts the 3-part shape) and
schemasFromEnabledTables now goes through parseDottedTableEntry so it
recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
reuse.
Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).
This commit is contained in:
parent
c526601d52
commit
cbe6a5f4b3
19 changed files with 193 additions and 51 deletions
|
|
@ -454,6 +454,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
|
||||||
params,
|
params,
|
||||||
);
|
);
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
|
catalog: this.resolved.projectId,
|
||||||
schema: row.table_schema,
|
schema: row.table_schema,
|
||||||
name: row.table_name,
|
name: row.table_name,
|
||||||
kind:
|
kind:
|
||||||
|
|
|
||||||
|
|
@ -531,6 +531,7 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
|
||||||
{ schemas: filterSchemas },
|
{ schemas: filterSchemas },
|
||||||
);
|
);
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
|
catalog: null,
|
||||||
schema: row.database,
|
schema: row.database,
|
||||||
name: row.name,
|
name: row.name,
|
||||||
kind: row.engine === 'View' || row.engine === 'MaterializedView' ? ('view' as const) : ('table' as const),
|
kind: row.engine === 'View' || row.engine === 'MaterializedView' ? ('view' as const) : ('table' as const),
|
||||||
|
|
|
||||||
|
|
@ -644,6 +644,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
||||||
filterSchemas,
|
filterSchemas,
|
||||||
);
|
);
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
|
catalog: null,
|
||||||
schema: row.TABLE_SCHEMA,
|
schema: row.TABLE_SCHEMA,
|
||||||
name: row.TABLE_NAME,
|
name: row.TABLE_NAME,
|
||||||
kind: row.TABLE_TYPE === 'VIEW' ? ('view' as const) : ('table' as const),
|
kind: row.TABLE_TYPE === 'VIEW' ? ('view' as const) : ('table' as const),
|
||||||
|
|
|
||||||
|
|
@ -607,6 +607,7 @@ export class KtxPostgresScanConnector implements KtxScanConnector {
|
||||||
[filterSchemas],
|
[filterSchemas],
|
||||||
);
|
);
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
|
catalog: null,
|
||||||
schema: row.schema_name,
|
schema: row.schema_name,
|
||||||
name: row.table_name,
|
name: row.table_name,
|
||||||
kind: row.table_kind === 'v' ? ('view' as const) : ('table' as const),
|
kind: row.table_kind === 'v' ? ('view' as const) : ('table' as const),
|
||||||
|
|
|
||||||
|
|
@ -438,6 +438,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver {
|
||||||
[this.resolved.database, ...(schemas ?? [])],
|
[this.resolved.database, ...(schemas ?? [])],
|
||||||
);
|
);
|
||||||
return result.rows.map((row) => ({
|
return result.rows.map((row) => ({
|
||||||
|
catalog: this.resolved.database,
|
||||||
schema: String(row[0]),
|
schema: String(row[0]),
|
||||||
name: String(row[1]),
|
name: String(row[1]),
|
||||||
kind: String(row[2]) === 'VIEW' ? ('view' as const) : ('table' as const),
|
kind: String(row[2]) === 'VIEW' ? ('view' as const) : ('table' as const),
|
||||||
|
|
@ -704,6 +705,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector {
|
||||||
[this.resolved.database, ...(schemas ?? [])],
|
[this.resolved.database, ...(schemas ?? [])],
|
||||||
);
|
);
|
||||||
return result.rows.map((row) => ({
|
return result.rows.map((row) => ({
|
||||||
|
catalog: this.resolved.database,
|
||||||
schema: String(row[0]),
|
schema: String(row[0]),
|
||||||
name: String(row[1]),
|
name: String(row[1]),
|
||||||
kind: String(row[2]) === 'VIEW' ? ('view' as const) : ('table' as const),
|
kind: String(row[2]) === 'VIEW' ? ('view' as const) : ('table' as const),
|
||||||
|
|
|
||||||
|
|
@ -227,6 +227,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
|
||||||
.all() as SqliteMasterRow[];
|
.all() as SqliteMasterRow[];
|
||||||
|
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
|
catalog: null,
|
||||||
schema: '',
|
schema: '',
|
||||||
name: row.name,
|
name: row.name,
|
||||||
kind: row.type === 'view' ? ('view' as const) : ('table' as const),
|
kind: row.type === 'view' ? ('view' as const) : ('table' as const),
|
||||||
|
|
|
||||||
|
|
@ -532,6 +532,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
||||||
params,
|
params,
|
||||||
);
|
);
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
|
catalog: this.poolConfig.database,
|
||||||
schema: row.schema_name,
|
schema: row.schema_name,
|
||||||
name: row.table_name,
|
name: row.table_name,
|
||||||
kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const),
|
kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const),
|
||||||
|
|
|
||||||
|
|
@ -27,12 +27,13 @@ export function resolveEnabledTables(
|
||||||
|
|
||||||
function parseEnabledTableEntry(value: unknown): KtxTableRef | null {
|
function parseEnabledTableEntry(value: unknown): KtxTableRef | null {
|
||||||
if (typeof value === 'string') {
|
if (typeof value === 'string') {
|
||||||
return parseDottedEntry(value);
|
return parseDottedTableEntry(value);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseDottedEntry(value: string): KtxTableRef | null {
|
/** @internal */
|
||||||
|
export function parseDottedTableEntry(value: string): KtxTableRef | null {
|
||||||
const trimmed = value.trim();
|
const trimmed = value.trim();
|
||||||
if (trimmed.length === 0) return null;
|
if (trimmed.length === 0) return null;
|
||||||
const parts = trimmed.split('.');
|
const parts = trimmed.split('.');
|
||||||
|
|
|
||||||
|
|
@ -297,6 +297,7 @@ export interface KtxQueryResult {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface KtxTableListEntry {
|
export interface KtxTableListEntry {
|
||||||
|
catalog: string | null;
|
||||||
schema: string;
|
schema: string;
|
||||||
name: string;
|
name: string;
|
||||||
kind: 'table' | 'view';
|
kind: 'table' | 'view';
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { parseDottedTableEntry } from './context/scan/enabled-tables.js';
|
||||||
import type { KtxTableListEntry } from './context/scan/types.js';
|
import type { KtxTableListEntry } from './context/scan/types.js';
|
||||||
import type { KtxCliIo } from './cli-runtime.js';
|
import type { KtxCliIo } from './cli-runtime.js';
|
||||||
import { profileMark } from './startup-profile.js';
|
import { profileMark } from './startup-profile.js';
|
||||||
|
|
@ -73,7 +74,9 @@ export interface PickDatabaseScopeArgs {
|
||||||
}
|
}
|
||||||
|
|
||||||
function qualifiedTableId(entry: KtxTableListEntry): string {
|
function qualifiedTableId(entry: KtxTableListEntry): string {
|
||||||
return `${entry.schema}.${entry.name}`;
|
return entry.catalog !== null
|
||||||
|
? `${entry.catalog}.${entry.schema}.${entry.name}`
|
||||||
|
: `${entry.schema}.${entry.name}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function tableTitle(entry: KtxTableListEntry): string {
|
function tableTitle(entry: KtxTableListEntry): string {
|
||||||
|
|
@ -177,7 +180,8 @@ function schemasFromEnabledTables(enabledTables: readonly string[]): string[] {
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const result: string[] = [];
|
const result: string[] = [];
|
||||||
for (const qualified of enabledTables) {
|
for (const qualified of enabledTables) {
|
||||||
const schema = qualified.split('.')[0] ?? '';
|
const ref = parseDottedTableEntry(qualified);
|
||||||
|
const schema = ref?.db ?? '';
|
||||||
if (schema.length === 0 || seen.has(schema)) continue;
|
if (schema.length === 0 || seen.has(schema)) continue;
|
||||||
seen.add(schema);
|
seen.add(schema);
|
||||||
result.push(schema);
|
result.push(schema);
|
||||||
|
|
|
||||||
|
|
@ -377,9 +377,9 @@ describe('KtxBigQueryScanConnector', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(connector.listTables(['analytics', 'mart'])).resolves.toEqual([
|
await expect(connector.listTables(['analytics', 'mart'])).resolves.toEqual([
|
||||||
{ schema: 'analytics', name: 'orders', kind: 'table' },
|
{ catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' },
|
||||||
{ schema: 'analytics', name: 'order_clone', kind: 'table' },
|
{ catalog: 'project-1', schema: 'analytics', name: 'order_clone', kind: 'table' },
|
||||||
{ schema: 'mart', name: 'orders_mv', kind: 'view' },
|
{ catalog: 'project-1', schema: 'mart', name: 'orders_mv', kind: 'view' },
|
||||||
]);
|
]);
|
||||||
|
|
||||||
expect(createQueryJob).toHaveBeenCalledTimes(1);
|
expect(createQueryJob).toHaveBeenCalledTimes(1);
|
||||||
|
|
|
||||||
|
|
@ -372,8 +372,8 @@ describe('KtxClickHouseScanConnector', () => {
|
||||||
await expect(connector.getTableRowCount('events')).resolves.toBe(2);
|
await expect(connector.getTableRowCount('events')).resolves.toBe(2);
|
||||||
await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']);
|
await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']);
|
||||||
await expect(connector.listTables(['analytics'])).resolves.toEqual([
|
await expect(connector.listTables(['analytics'])).resolves.toEqual([
|
||||||
{ schema: 'analytics', name: 'event_summary', kind: 'view' },
|
{ catalog: null, schema: 'analytics', name: 'event_summary', kind: 'view' },
|
||||||
{ schema: 'analytics', name: 'events', kind: 'table' },
|
{ catalog: null, schema: 'analytics', name: 'events', kind: 'table' },
|
||||||
]);
|
]);
|
||||||
await expect(
|
await expect(
|
||||||
connector.columnStats(
|
connector.columnStats(
|
||||||
|
|
|
||||||
|
|
@ -511,9 +511,9 @@ describe('KtxMysqlScanConnector', () => {
|
||||||
await expect(connector.getTableRowCount('orders')).resolves.toBe(2);
|
await expect(connector.getTableRowCount('orders')).resolves.toBe(2);
|
||||||
await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']);
|
await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']);
|
||||||
await expect(connector.listTables(['analytics'])).resolves.toEqual([
|
await expect(connector.listTables(['analytics'])).resolves.toEqual([
|
||||||
{ schema: 'analytics', name: 'customers', kind: 'table' },
|
{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' },
|
||||||
{ schema: 'analytics', name: 'orders', kind: 'table' },
|
{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' },
|
||||||
{ schema: 'analytics', name: 'order_summary', kind: 'view' },
|
{ catalog: null, schema: 'analytics', name: 'order_summary', kind: 'view' },
|
||||||
]);
|
]);
|
||||||
await expect(connector.columnStats(
|
await expect(connector.columnStats(
|
||||||
{ connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'orders' }, column: 'status' },
|
{ connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'orders' }, column: 'status' },
|
||||||
|
|
|
||||||
|
|
@ -390,9 +390,9 @@ describe('KtxPostgresScanConnector', () => {
|
||||||
await expect(connector.getTableRowCount({ db: 'public', name: 'orders' })).resolves.toBe(3);
|
await expect(connector.getTableRowCount({ db: 'public', name: 'orders' })).resolves.toBe(3);
|
||||||
await expect(connector.listSchemas()).resolves.toEqual(['public']);
|
await expect(connector.listSchemas()).resolves.toEqual(['public']);
|
||||||
await expect(connector.listTables(['public'])).resolves.toEqual([
|
await expect(connector.listTables(['public'])).resolves.toEqual([
|
||||||
{ schema: 'public', name: 'customers', kind: 'table' },
|
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' },
|
||||||
{ schema: 'public', name: 'orders', kind: 'table' },
|
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' },
|
||||||
{ schema: 'public', name: 'recent_orders', kind: 'view' },
|
{ catalog: null, schema: 'public', name: 'recent_orders', kind: 'view' },
|
||||||
]);
|
]);
|
||||||
await expect(connector.testConnection()).resolves.toEqual({ success: true });
|
await expect(connector.testConnection()).resolves.toEqual({ success: true });
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -64,8 +64,8 @@ function fakeDriverFactory(): KtxSnowflakeDriverFactory {
|
||||||
]),
|
]),
|
||||||
listSchemas: vi.fn(async () => ['PUBLIC', 'MART']),
|
listSchemas: vi.fn(async () => ['PUBLIC', 'MART']),
|
||||||
listTables: vi.fn(async () => [
|
listTables: vi.fn(async () => [
|
||||||
{ schema: 'PUBLIC', name: 'ORDERS', kind: 'table' as const },
|
{ catalog: 'ANALYTICS', schema: 'PUBLIC', name: 'ORDERS', kind: 'table' as const },
|
||||||
{ schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' as const },
|
{ catalog: 'ANALYTICS', schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' as const },
|
||||||
]),
|
]),
|
||||||
cleanup: vi.fn(async () => undefined),
|
cleanup: vi.fn(async () => undefined),
|
||||||
};
|
};
|
||||||
|
|
@ -572,8 +572,8 @@ describe('KtxSnowflakeScanConnector', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(connector.listTables(['MART', 'PUBLIC'])).resolves.toEqual([
|
await expect(connector.listTables(['MART', 'PUBLIC'])).resolves.toEqual([
|
||||||
{ schema: 'MART', name: 'ORDERS', kind: 'table' },
|
{ catalog: 'ANALYTICS', schema: 'MART', name: 'ORDERS', kind: 'table' },
|
||||||
{ schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' },
|
{ catalog: 'ANALYTICS', schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' },
|
||||||
]);
|
]);
|
||||||
|
|
||||||
expect(queries).toHaveLength(1);
|
expect(queries).toHaveLength(1);
|
||||||
|
|
|
||||||
|
|
@ -158,9 +158,9 @@ describe('KtxSqliteScanConnector', () => {
|
||||||
|
|
||||||
await expect(connector.listSchemas()).resolves.toEqual([]);
|
await expect(connector.listSchemas()).resolves.toEqual([]);
|
||||||
await expect(connector.listTables(['ignored'])).resolves.toEqual([
|
await expect(connector.listTables(['ignored'])).resolves.toEqual([
|
||||||
{ schema: '', name: 'customers', kind: 'table' },
|
{ catalog: null, schema: '', name: 'customers', kind: 'table' },
|
||||||
{ schema: '', name: 'orders', kind: 'table' },
|
{ catalog: null, schema: '', name: 'orders', kind: 'table' },
|
||||||
{ schema: '', name: 'recent_orders', kind: 'view' },
|
{ catalog: null, schema: '', name: 'recent_orders', kind: 'view' },
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -390,9 +390,9 @@ describe('KtxSqlServerScanConnector', () => {
|
||||||
await expect(connector.getTableRowCount('orders')).resolves.toBe(2);
|
await expect(connector.getTableRowCount('orders')).resolves.toBe(2);
|
||||||
await expect(connector.listSchemas()).resolves.toEqual(['dbo', 'sales']);
|
await expect(connector.listSchemas()).resolves.toEqual(['dbo', 'sales']);
|
||||||
await expect(connector.listTables(['dbo'])).resolves.toEqual([
|
await expect(connector.listTables(['dbo'])).resolves.toEqual([
|
||||||
{ schema: 'dbo', name: 'customers', kind: 'table' },
|
{ catalog: 'analytics', schema: 'dbo', name: 'customers', kind: 'table' },
|
||||||
{ schema: 'dbo', name: 'order_summary', kind: 'view' },
|
{ catalog: 'analytics', schema: 'dbo', name: 'order_summary', kind: 'view' },
|
||||||
{ schema: 'dbo', name: 'orders', kind: 'table' },
|
{ catalog: 'analytics', schema: 'dbo', name: 'orders', kind: 'table' },
|
||||||
]);
|
]);
|
||||||
await expect(
|
await expect(
|
||||||
connector.columnStats(
|
connector.columnStats(
|
||||||
|
|
|
||||||
|
|
@ -52,10 +52,10 @@ function captureRenderer(): {
|
||||||
}
|
}
|
||||||
|
|
||||||
const discovered = [
|
const discovered = [
|
||||||
{ schema: 'analytics', name: 'customers', kind: 'table' as const },
|
{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const },
|
||||||
{ schema: 'analytics', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'events', kind: 'view' as const },
|
{ catalog: null, schema: 'public', name: 'events', kind: 'view' as const },
|
||||||
{ schema: 'public', name: 'sessions', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'sessions', kind: 'table' as const },
|
||||||
];
|
];
|
||||||
|
|
||||||
function promptAdapter(overrides: Partial<DatabaseScopePromptAdapter> = {}): DatabaseScopePromptAdapter {
|
function promptAdapter(overrides: Partial<DatabaseScopePromptAdapter> = {}): DatabaseScopePromptAdapter {
|
||||||
|
|
@ -88,7 +88,7 @@ describe('pickDatabaseScope', () => {
|
||||||
select: vi.fn(async () => 'save'),
|
select: vi.fn(async () => 'save'),
|
||||||
});
|
});
|
||||||
const listTablesForSchemas = vi.fn(async () => [
|
const listTablesForSchemas = vi.fn(async () => [
|
||||||
{ schema: 'analytics', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const result = await pickDatabaseScope(
|
const result = await pickDatabaseScope(
|
||||||
|
|
@ -114,6 +114,58 @@ describe('pickDatabaseScope', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('emits fully-qualified catalog.schema.name ids for catalog-bearing drivers and round-trips existing selection', async () => {
|
||||||
|
const promptsSave = promptAdapter({
|
||||||
|
autocompleteMultiselect: vi.fn(async () => ['analytics']),
|
||||||
|
select: vi.fn(async () => 'save'),
|
||||||
|
});
|
||||||
|
const listTablesForSchemas = vi.fn(async () => [
|
||||||
|
{ catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' as const },
|
||||||
|
{ catalog: 'project-1', schema: 'analytics', name: 'customers', kind: 'table' as const },
|
||||||
|
]);
|
||||||
|
const saveResult = await pickDatabaseScope(
|
||||||
|
baseArgs({
|
||||||
|
schemas: ['analytics'],
|
||||||
|
schemaSuggestion: { excluded: new Set(), suggested: new Set(['analytics']) },
|
||||||
|
listTablesForSchemas,
|
||||||
|
prompts: promptsSave,
|
||||||
|
}),
|
||||||
|
makeIo().io,
|
||||||
|
captureRenderer().renderer,
|
||||||
|
);
|
||||||
|
expect(saveResult).toEqual({
|
||||||
|
kind: 'selected',
|
||||||
|
activeSchemas: ['analytics'],
|
||||||
|
enabledTables: ['project-1.analytics.orders', 'project-1.analytics.customers'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const { renderer, capture, setResult } = captureRenderer();
|
||||||
|
setResult({
|
||||||
|
kind: 'save',
|
||||||
|
selectedIds: ['project-1.analytics.orders'],
|
||||||
|
});
|
||||||
|
const refineResult = await pickDatabaseScope(
|
||||||
|
baseArgs({
|
||||||
|
schemas: ['analytics'],
|
||||||
|
schemaSuggestion: { excluded: new Set(), suggested: new Set(['analytics']) },
|
||||||
|
existing: { enabledTables: ['project-1.analytics.orders'] },
|
||||||
|
listTablesForSchemas,
|
||||||
|
prompts: promptAdapter({
|
||||||
|
autocompleteMultiselect: vi.fn(async () => ['analytics']),
|
||||||
|
select: vi.fn(async () => 'refine'),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
makeIo().io,
|
||||||
|
renderer,
|
||||||
|
);
|
||||||
|
expect(refineResult).toEqual({
|
||||||
|
kind: 'selected',
|
||||||
|
activeSchemas: ['analytics'],
|
||||||
|
enabledTables: ['project-1.analytics.orders'],
|
||||||
|
});
|
||||||
|
expect([...(capture.state?.checked ?? [])]).toContain('project-1.analytics.orders');
|
||||||
|
});
|
||||||
|
|
||||||
it('routes partial existing allowlists through Stage 2 so save preserves table selections', async () => {
|
it('routes partial existing allowlists through Stage 2 so save preserves table selections', async () => {
|
||||||
const { renderer, setResult } = captureRenderer();
|
const { renderer, setResult } = captureRenderer();
|
||||||
setResult({ kind: 'save', selectedIds: ['analytics.customers'] });
|
setResult({ kind: 'save', selectedIds: ['analytics.customers'] });
|
||||||
|
|
@ -122,8 +174,8 @@ describe('pickDatabaseScope', () => {
|
||||||
select: vi.fn(async () => 'save'),
|
select: vi.fn(async () => 'save'),
|
||||||
});
|
});
|
||||||
const listTablesForSchemas = vi.fn(async () => [
|
const listTablesForSchemas = vi.fn(async () => [
|
||||||
{ schema: 'analytics', name: 'customers', kind: 'table' as const },
|
{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const },
|
||||||
{ schema: 'analytics', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const result = await pickDatabaseScope(
|
const result = await pickDatabaseScope(
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ import type {
|
||||||
DatabaseScopePickResult,
|
DatabaseScopePickResult,
|
||||||
PickDatabaseScopeArgs,
|
PickDatabaseScopeArgs,
|
||||||
} from '../src/database-tree-picker.js';
|
} from '../src/database-tree-picker.js';
|
||||||
|
import type { KtxSetupPromptOption } from '../src/setup-prompts.js';
|
||||||
|
|
||||||
function makeIo() {
|
function makeIo() {
|
||||||
let stdout = '';
|
let stdout = '';
|
||||||
|
|
@ -1039,7 +1040,7 @@ describe('setup databases step', () => {
|
||||||
const testConnection = vi.fn(async () => 0);
|
const testConnection = vi.fn(async () => 0);
|
||||||
const scanConnection = vi.fn(async () => 0);
|
const scanConnection = vi.fn(async () => 0);
|
||||||
const listSchemas = vi.fn(async () => ['analytics', 'public']);
|
const listSchemas = vi.fn(async () => ['analytics', 'public']);
|
||||||
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]);
|
const listTables = vi.fn(async () => [{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const }]);
|
||||||
const pickers = makePickerStubs({
|
const pickers = makePickerStubs({
|
||||||
scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }],
|
scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }],
|
||||||
});
|
});
|
||||||
|
|
@ -1110,9 +1111,9 @@ describe('setup databases step', () => {
|
||||||
});
|
});
|
||||||
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
||||||
const listTables = vi.fn(async () => [
|
const listTables = vi.fn(async () => [
|
||||||
{ schema: 'public', name: 'customers', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'products', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'products', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
const pickers = makePickerStubs({
|
const pickers = makePickerStubs({
|
||||||
scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }],
|
scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }],
|
||||||
|
|
@ -1184,8 +1185,8 @@ describe('setup databases step', () => {
|
||||||
const scanConnection = vi.fn(async () => 0);
|
const scanConnection = vi.fn(async () => 0);
|
||||||
const listSchemas = vi.fn(async () => ['analytics', 'public']);
|
const listSchemas = vi.fn(async () => ['analytics', 'public']);
|
||||||
const listTables = vi.fn(async () => [
|
const listTables = vi.fn(async () => [
|
||||||
{ schema: 'analytics', name: 'customers', kind: 'table' as const },
|
{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
const pickers = makePickerStubs({ scopes: ['back'] });
|
const pickers = makePickerStubs({ scopes: ['back'] });
|
||||||
|
|
||||||
|
|
@ -1250,8 +1251,8 @@ describe('setup databases step', () => {
|
||||||
const scanConnection = vi.fn(async () => 0);
|
const scanConnection = vi.fn(async () => 0);
|
||||||
const listSchemas = vi.fn(async () => ['public']);
|
const listSchemas = vi.fn(async () => ['public']);
|
||||||
const listTables = vi.fn(async () => [
|
const listTables = vi.fn(async () => [
|
||||||
{ schema: 'public', name: 'customers', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
const pickers = makePickerStubs({ scopes: [{ schemas: ['public'], tables: 'back' }] });
|
const pickers = makePickerStubs({ scopes: [{ schemas: ['public'], tables: 'back' }] });
|
||||||
|
|
||||||
|
|
@ -1311,8 +1312,8 @@ describe('setup databases step', () => {
|
||||||
return 'back';
|
return 'back';
|
||||||
});
|
});
|
||||||
const listTables = vi.fn(async () => [
|
const listTables = vi.fn(async () => [
|
||||||
{ schema: 'public', name: 'customers', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'orders', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
const pickers = makePickerStubs({ scopes: ['enable-all'] });
|
const pickers = makePickerStubs({ scopes: ['enable-all'] });
|
||||||
|
|
||||||
|
|
@ -1610,7 +1611,7 @@ describe('setup databases step', () => {
|
||||||
});
|
});
|
||||||
const listSchemas = vi.fn(async () => ['analytics', 'mart']);
|
const listSchemas = vi.fn(async () => ['analytics', 'mart']);
|
||||||
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
|
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
|
||||||
(schemas ?? []).map((schema) => ({ schema, name: 'orders', kind: 'table' as const })),
|
(schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'orders', kind: 'table' as const })),
|
||||||
);
|
);
|
||||||
const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => {
|
const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => {
|
||||||
const scopedArgs = args as PickDatabaseScopeArgs & {
|
const scopedArgs = args as PickDatabaseScopeArgs & {
|
||||||
|
|
@ -1667,7 +1668,7 @@ describe('setup databases step', () => {
|
||||||
textValues: ['bigquery-warehouse', '/tmp/service-account.json', 'US'],
|
textValues: ['bigquery-warehouse', '/tmp/service-account.json', 'US'],
|
||||||
});
|
});
|
||||||
const listSchemas = vi.fn(async () => ['analytics']);
|
const listSchemas = vi.fn(async () => ['analytics']);
|
||||||
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'orders', kind: 'table' as const }]);
|
const listTables = vi.fn(async () => [{ catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' as const }]);
|
||||||
const pickDatabaseScope = vi.fn(async () => ({
|
const pickDatabaseScope = vi.fn(async () => ({
|
||||||
kind: 'selected' as const,
|
kind: 'selected' as const,
|
||||||
activeSchemas: ['analytics'],
|
activeSchemas: ['analytics'],
|
||||||
|
|
@ -1700,9 +1701,9 @@ describe('setup databases step', () => {
|
||||||
});
|
});
|
||||||
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
|
||||||
const listTables = vi.fn(async () => [
|
const listTables = vi.fn(async () => [
|
||||||
{ schema: 'orbit_analytics', name: 'events', kind: 'table' as const },
|
{ catalog: null, schema: 'orbit_analytics', name: 'events', kind: 'table' as const },
|
||||||
{ schema: 'orbit_raw', name: 'inputs', kind: 'table' as const },
|
{ catalog: null, schema: 'orbit_raw', name: 'inputs', kind: 'table' as const },
|
||||||
{ schema: 'public', name: 'misc', kind: 'table' as const },
|
{ catalog: null, schema: 'public', name: 'misc', kind: 'table' as const },
|
||||||
]);
|
]);
|
||||||
const pickers = makePickerStubs({
|
const pickers = makePickerStubs({
|
||||||
scopes: [
|
scopes: [
|
||||||
|
|
@ -1761,7 +1762,7 @@ describe('setup databases step', () => {
|
||||||
throw new Error('permission denied to list schemas');
|
throw new Error('permission denied to list schemas');
|
||||||
});
|
});
|
||||||
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
|
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
|
||||||
(schemas ?? []).map((schema) => ({ schema, name: 'events', kind: 'table' as const })),
|
(schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'events', kind: 'table' as const })),
|
||||||
);
|
);
|
||||||
const pickers = makePickerStubs({
|
const pickers = makePickerStubs({
|
||||||
scopes: [
|
scopes: [
|
||||||
|
|
@ -1808,18 +1809,18 @@ describe('setup databases step', () => {
|
||||||
it('passes schemas and a lazy table callback to the scope picker instead of eager table discovery', async () => {
|
it('passes schemas and a lazy table callback to the scope picker instead of eager table discovery', async () => {
|
||||||
const listSchemas = vi.fn(async () => ['analytics', 'raw']);
|
const listSchemas = vi.fn(async () => ['analytics', 'raw']);
|
||||||
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
|
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
|
||||||
(schemas ?? []).map((schema) => ({ schema, name: 'orders', kind: 'table' as const })),
|
(schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'orders', kind: 'table' as const })),
|
||||||
);
|
);
|
||||||
const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => {
|
const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => {
|
||||||
const lazyArgs = args as PickDatabaseScopeArgs & {
|
const lazyArgs = args as PickDatabaseScopeArgs & {
|
||||||
schemas: string[];
|
schemas: string[];
|
||||||
listTablesForSchemas: (schemas: string[]) => Promise<Array<{ schema: string; name: string; kind: 'table' }>>;
|
listTablesForSchemas: (schemas: string[]) => Promise<Array<{ catalog: string | null; schema: string; name: string; kind: 'table' }>>;
|
||||||
};
|
};
|
||||||
expect(lazyArgs.schemas).toEqual(['analytics', 'raw']);
|
expect(lazyArgs.schemas).toEqual(['analytics', 'raw']);
|
||||||
expect(args).not.toHaveProperty('discovered');
|
expect(args).not.toHaveProperty('discovered');
|
||||||
expect(listTables).not.toHaveBeenCalled();
|
expect(listTables).not.toHaveBeenCalled();
|
||||||
const tables = await lazyArgs.listTablesForSchemas(['analytics']);
|
const tables = await lazyArgs.listTablesForSchemas(['analytics']);
|
||||||
expect(tables).toEqual([{ schema: 'analytics', name: 'orders', kind: 'table' }]);
|
expect(tables).toEqual([{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' }]);
|
||||||
return { kind: 'selected' as const, activeSchemas: ['analytics'], enabledTables: ['analytics.orders'] };
|
return { kind: 'selected' as const, activeSchemas: ['analytics'], enabledTables: ['analytics.orders'] };
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -2557,6 +2558,81 @@ describe('setup databases step', () => {
|
||||||
expect(io.stdout()).toContain('Setup written; query history will be skipped until fixed.');
|
expect(io.stdout()).toContain('Setup written; query history will be skipped until fixed.');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('lets interactive BigQuery setup disable unavailable query history and retry after scan failure', async () => {
|
||||||
|
const io = makeIo();
|
||||||
|
const failurePromptOptions: KtxSetupPromptOption[][] = [];
|
||||||
|
let failurePromptCount = 0;
|
||||||
|
const prompts = makePromptAdapter({
|
||||||
|
textValues: ['/tmp/service-account.json', 'US'],
|
||||||
|
});
|
||||||
|
vi.mocked(prompts.select).mockImplementation(async ({ message, options }) => {
|
||||||
|
if (message.startsWith('Enable query-history ingest')) return 'yes';
|
||||||
|
if (message.includes('How much database context should KTX build?')) return 'fast';
|
||||||
|
if (message.startsWith('Database setup failed for analytics')) {
|
||||||
|
failurePromptCount += 1;
|
||||||
|
failurePromptOptions.push(options);
|
||||||
|
if (failurePromptCount === 1) return 'disable-query-history';
|
||||||
|
throw new Error('setup did not disable query history before retrying');
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected select prompt: ${message}`);
|
||||||
|
});
|
||||||
|
const runner = {
|
||||||
|
...fakeHistoricSqlRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'),
|
||||||
|
fixAdvice: () => ({
|
||||||
|
failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT',
|
||||||
|
remediation:
|
||||||
|
'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.',
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const historicSqlReadinessProbe = vi.fn(async () => ({
|
||||||
|
ok: false as const,
|
||||||
|
dialect: 'bigquery' as const,
|
||||||
|
runner,
|
||||||
|
error: new Error('access denied'),
|
||||||
|
}));
|
||||||
|
let scanAttempts = 0;
|
||||||
|
const scanConnection = vi.fn(async () => {
|
||||||
|
scanAttempts += 1;
|
||||||
|
return scanAttempts === 1 ? 1 : 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runKtxSetupDatabasesStep(
|
||||||
|
{
|
||||||
|
projectDir: tempDir,
|
||||||
|
inputMode: 'auto',
|
||||||
|
databaseDrivers: ['bigquery'],
|
||||||
|
databaseConnectionId: 'analytics',
|
||||||
|
databaseSchemas: [],
|
||||||
|
skipDatabases: false,
|
||||||
|
},
|
||||||
|
io.io,
|
||||||
|
{
|
||||||
|
prompts,
|
||||||
|
testConnection: vi.fn(async () => 0),
|
||||||
|
scanConnection,
|
||||||
|
historicSqlReadinessProbe,
|
||||||
|
listSchemas: vi.fn(async () => ['analytics']),
|
||||||
|
listTables: vi.fn(async () => [{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const }]),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.status).toBe('ready');
|
||||||
|
expect(scanConnection).toHaveBeenCalledTimes(2);
|
||||||
|
expect(historicSqlReadinessProbe).toHaveBeenCalledTimes(1);
|
||||||
|
expect(failurePromptOptions[0]).toContainEqual({
|
||||||
|
value: 'disable-query-history',
|
||||||
|
label: 'Disable query history and retry',
|
||||||
|
});
|
||||||
|
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||||
|
expect(config.connections.analytics).toMatchObject({
|
||||||
|
context: {
|
||||||
|
queryHistory: {
|
||||||
|
enabled: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('enables query history on an existing Postgres connection', async () => {
|
it('enables query history on an existing Postgres connection', async () => {
|
||||||
await writeFile(
|
await writeFile(
|
||||||
join(tempDir, 'ktx.yaml'),
|
join(tempDir, 'ktx.yaml'),
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue