mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
feat(connections): add execute-only warehouses; stop silent full-project scans
A configured warehouse was always a scan/ingest target. The only way to use a connection purely for SQL execution (ktx sql / sql_execution) was the leaky workaround of an empty setup.database_connection_ids — which actually re-includes every warehouse via the 'fall back to all' branch — so e.g. a BigQuery connection meant only for read-only queries triggered a full-billing-project scan. - Add a per-connection scan_enabled flag (default true) to warehouse connections. scan_enabled: false registers the connection for execution only and never as a scan target. - Route every scan-target selection path through one predicate (isScanTargetWarehouse): both ingest (primaryWarehouseConnectionIds, including the all-warehouses fallback) and setup (configuredPrimaryConnectionIds) now exclude execute-only connections. Setup validates the credential but skips scope discovery and scan for them. Execution paths are untouched — the warehouse descriptor still resolves, so ktx sql / sql_execution keep working. - Scripted setup with no --database-schema no longer silently scopes the scan to every discovered schema/dataset: it warns with the count and names how to narrow (--database-schema) or opt out (scan_enabled: false).
This commit is contained in:
parent
a02fcab487
commit
ece0dfb2c8
10 changed files with 187 additions and 3 deletions
|
|
@ -1586,6 +1586,64 @@ describe('setup databases step', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('registers an execute-only connection (scan_enabled: false) without scanning it', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
['connections:', ' public_bq:', ' driver: bigquery', ' scan_enabled: false', ''].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const io = makeIo();
|
||||
const testConnection = vi.fn(async () => 0);
|
||||
const scanConnection = vi.fn(async () => 0);
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
databaseConnectionIds: ['public_bq'],
|
||||
databaseSchemas: [],
|
||||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{ testConnection, scanConnection, listSchemas: vi.fn(async () => ['a', 'b', 'c']) },
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
// The credential is validated, but the warehouse is never introspected/scanned.
|
||||
expect(testConnection).toHaveBeenCalledWith(tempDir, 'public_bq', expect.anything());
|
||||
expect(scanConnection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('warns instead of silently scanning every discovered dataset when scripted setup has no scope', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
['connections:', ' warehouse:', ' driver: bigquery', ''].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
databaseConnectionIds: ['warehouse'],
|
||||
databaseSchemas: [],
|
||||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
testConnection: vi.fn(async () => 0),
|
||||
scanConnection: vi.fn(async () => 0),
|
||||
listSchemas: vi.fn(async () => ['stripe', 'posthog', 'linear']),
|
||||
listTables: vi.fn(async () => []),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
expect(io.stderr()).toContain('No --database-schema given for warehouse');
|
||||
expect(io.stderr()).toContain('scan_enabled: false');
|
||||
});
|
||||
|
||||
it('keeps scripted database ids fail-fast even when input mode is auto', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue