apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-16 08:25:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match

Browse source 
The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.

Align the picker boundary with the canonical 3-level KtxTableRef:

- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
  resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
  (resolveEnabledTables already accepts the 3-part shape) and
  schemasFromEnabledTables now goes through parseDottedTableEntry so it
  recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
  reuse.

Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).
This commit is contained in:
Andrey Avtomonov 2026-05-25 17:23:05 +02:00
parent c526601d52
commit cbe6a5f4b3
19 changed files with 193 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

114
packages/cli/test/setup-databases.test.ts
View file

@ -16,6 +16,7 @@ import type {
DatabaseScopePickResult,
PickDatabaseScopeArgs,
} from '../src/database-tree-picker.js';
import type { KtxSetupPromptOption } from '../src/setup-prompts.js';
function makeIo() {
let stdout = '';
@ -1039,7 +1040,7 @@ describe('setup databases step', () => {
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['analytics', 'public']);
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]);
const listTables = vi.fn(async () => [{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const }]);
const pickers = makePickerStubs({
scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }],
});
@ -1110,9 +1111,9 @@ describe('setup databases step', () => {
});
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
{ schema: 'public', name: 'products', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'products', kind: 'table' as const },
]);
const pickers = makePickerStubs({
scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }],
@ -1184,8 +1185,8 @@ describe('setup databases step', () => {
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['analytics', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'analytics', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
]);
const pickers = makePickerStubs({ scopes: ['back'] });
@ -1250,8 +1251,8 @@ describe('setup databases step', () => {
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['public']);
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
]);
const pickers = makePickerStubs({ scopes: [{ schemas: ['public'], tables: 'back' }] });
@ -1311,8 +1312,8 @@ describe('setup databases step', () => {
return 'back';
});
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'customers', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const },
]);
const pickers = makePickerStubs({ scopes: ['enable-all'] });
@ -1610,7 +1611,7 @@ describe('setup databases step', () => {
});
const listSchemas = vi.fn(async () => ['analytics', 'mart']);
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
(schemas ?? []).map((schema) => ({ schema, name: 'orders', kind: 'table' as const })),
(schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'orders', kind: 'table' as const })),
);
const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => {
const scopedArgs = args as PickDatabaseScopeArgs & {
@ -1667,7 +1668,7 @@ describe('setup databases step', () => {
textValues: ['bigquery-warehouse', '/tmp/service-account.json', 'US'],
});
const listSchemas = vi.fn(async () => ['analytics']);
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'orders', kind: 'table' as const }]);
const listTables = vi.fn(async () => [{ catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' as const }]);
const pickDatabaseScope = vi.fn(async () => ({
kind: 'selected' as const,
activeSchemas: ['analytics'],
@ -1700,9 +1701,9 @@ describe('setup databases step', () => {
});
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'orbit_analytics', name: 'events', kind: 'table' as const },
{ schema: 'orbit_raw', name: 'inputs', kind: 'table' as const },
{ schema: 'public', name: 'misc', kind: 'table' as const },
{ catalog: null, schema: 'orbit_analytics', name: 'events', kind: 'table' as const },
{ catalog: null, schema: 'orbit_raw', name: 'inputs', kind: 'table' as const },
{ catalog: null, schema: 'public', name: 'misc', kind: 'table' as const },
]);
const pickers = makePickerStubs({
scopes: [
@ -1761,7 +1762,7 @@ describe('setup databases step', () => {
throw new Error('permission denied to list schemas');
});
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
(schemas ?? []).map((schema) => ({ schema, name: 'events', kind: 'table' as const })),
(schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'events', kind: 'table' as const })),
);
const pickers = makePickerStubs({
scopes: [
@ -1808,18 +1809,18 @@ describe('setup databases step', () => {
it('passes schemas and a lazy table callback to the scope picker instead of eager table discovery', async () => {
const listSchemas = vi.fn(async () => ['analytics', 'raw']);
const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) =>
(schemas ?? []).map((schema) => ({ schema, name: 'orders', kind: 'table' as const })),
(schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'orders', kind: 'table' as const })),
);
const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => {
const lazyArgs = args as PickDatabaseScopeArgs & {
schemas: string[];
listTablesForSchemas: (schemas: string[]) => Promise<Array<{ schema: string; name: string; kind: 'table' }>>;
listTablesForSchemas: (schemas: string[]) => Promise<Array<{ catalog: string | null; schema: string; name: string; kind: 'table' }>>;
};
expect(lazyArgs.schemas).toEqual(['analytics', 'raw']);
expect(args).not.toHaveProperty('discovered');
expect(listTables).not.toHaveBeenCalled();
const tables = await lazyArgs.listTablesForSchemas(['analytics']);
expect(tables).toEqual([{ schema: 'analytics', name: 'orders', kind: 'table' }]);
expect(tables).toEqual([{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' }]);
return { kind: 'selected' as const, activeSchemas: ['analytics'], enabledTables: ['analytics.orders'] };
});
@ -2557,6 +2558,81 @@ describe('setup databases step', () => {
expect(io.stdout()).toContain('Setup written; query history will be skipped until fixed.');
});
it('lets interactive BigQuery setup disable unavailable query history and retry after scan failure', async () => {
const io = makeIo();
const failurePromptOptions: KtxSetupPromptOption[][] = [];
let failurePromptCount = 0;
const prompts = makePromptAdapter({
textValues: ['/tmp/service-account.json', 'US'],
});
vi.mocked(prompts.select).mockImplementation(async ({ message, options }) => {
if (message.startsWith('Enable query-history ingest')) return 'yes';
if (message.includes('How much database context should KTX build?')) return 'fast';
if (message.startsWith('Database setup failed for analytics')) {
failurePromptCount += 1;
failurePromptOptions.push(options);
if (failurePromptCount === 1) return 'disable-query-history';
throw new Error('setup did not disable query history before retrying');
}
throw new Error(`unexpected select prompt: ${message}`);
});
const runner = {
...fakeHistoricSqlRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'),
fixAdvice: () => ({
failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT',
remediation:
'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.',
}),
};
const historicSqlReadinessProbe = vi.fn(async () => ({
ok: false as const,
dialect: 'bigquery' as const,
runner,
error: new Error('access denied'),
}));
let scanAttempts = 0;
const scanConnection = vi.fn(async () => {
scanAttempts += 1;
return scanAttempts === 1 ? 1 : 0;
});
const result = await runKtxSetupDatabasesStep(
{
projectDir: tempDir,
inputMode: 'auto',
databaseDrivers: ['bigquery'],
databaseConnectionId: 'analytics',
databaseSchemas: [],
skipDatabases: false,
},
io.io,
{
prompts,
testConnection: vi.fn(async () => 0),
scanConnection,
historicSqlReadinessProbe,
listSchemas: vi.fn(async () => ['analytics']),
listTables: vi.fn(async () => [{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const }]),
},
);
expect(result.status).toBe('ready');
expect(scanConnection).toHaveBeenCalledTimes(2);
expect(historicSqlReadinessProbe).toHaveBeenCalledTimes(1);
expect(failurePromptOptions[0]).toContainEqual({
value: 'disable-query-history',
label: 'Disable query history and retry',
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.analytics).toMatchObject({
context: {
queryHistory: {
enabled: false,
},
},
});
});
it('enables query history on an existing Postgres connection', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),