mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-16 08:25:14 +02:00
* refactor(context): export and describe mapping shape schemas * feat(context): add driver-schemas module with warehouse drivers * feat(context): add metabase, looker, lookml driver schemas with mappings * feat(context): add notion, dbt, metricflow driver schemas * refactor(context): make connectionSchema a driver-discriminated union * chore(context): re-export KtxConnectionConfig from project package * docs(context): add connection driver schema plan * chore(secrets): allowlist example credentials in driver-schemas fixtures * test(cli): align metabase fixtures with required api_url field The driver-discriminated union added in this branch now requires api_url for metabase connections and a known driver for warehouses. Update slow CLI test fixtures and assertions so they exercise the new schema: - ingest.test-utils.ts: add api_url to the prod-metabase fixture. - setup.test.ts: switch metabase fixture from 'url' to 'api_url'. - local-scan-connectors.test.ts: invalid-driver/missing-driver tests now expect the schema error from loadKtxProject (parse-time rejection).
132 lines
3.9 KiB
TypeScript
132 lines
3.9 KiB
TypeScript
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { initKtxProject, loadKtxProject } from '@ktx/context/project';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
|
|
|
const bigQueryMock = vi.hoisted(() => ({
|
|
constructorInputs: [] as Array<{
|
|
connectionId: string;
|
|
connection: unknown;
|
|
}>,
|
|
}));
|
|
|
|
vi.mock('@ktx/connector-bigquery', () => ({
|
|
isKtxBigQueryConnectionConfig: (connection: { driver?: unknown } | undefined) =>
|
|
String(connection?.driver ?? '').toLowerCase() === 'bigquery',
|
|
KtxBigQueryScanConnector: class {
|
|
readonly id: string;
|
|
readonly driver = 'bigquery';
|
|
|
|
constructor(options: { connectionId: string; connection: unknown }) {
|
|
bigQueryMock.constructorInputs.push(options);
|
|
this.id = `bigquery:${options.connectionId}`;
|
|
}
|
|
},
|
|
}));
|
|
|
|
describe('createKtxCliScanConnector', () => {
|
|
let tempDir: string;
|
|
|
|
beforeEach(async () => {
|
|
bigQueryMock.constructorInputs.length = 0;
|
|
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-scan-connector-'));
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await rm(tempDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('creates a native sqlite connector from standalone config', async () => {
|
|
await initKtxProject({ projectDir: tempDir });
|
|
await writeFile(
|
|
join(tempDir, 'ktx.yaml'),
|
|
[
|
|
'connections:',
|
|
' warehouse:',
|
|
' driver: sqlite',
|
|
' path: warehouse.db',
|
|
'',
|
|
].join('\n'),
|
|
'utf-8',
|
|
);
|
|
const project = await loadKtxProject({ projectDir: tempDir });
|
|
|
|
const connector = await createKtxCliScanConnector(project, 'warehouse');
|
|
|
|
expect(connector.id).toBe('sqlite:warehouse');
|
|
expect(connector.driver).toBe('sqlite');
|
|
});
|
|
|
|
it('passes canonical BigQuery YAML scan limits through to the connector', async () => {
|
|
await initKtxProject({ projectDir: tempDir });
|
|
await writeFile(
|
|
join(tempDir, 'ktx.yaml'),
|
|
[
|
|
'connections:',
|
|
' warehouse:',
|
|
' driver: bigquery',
|
|
' dataset_id: analytics',
|
|
' max_bytes_billed: "987654321"',
|
|
' job_timeout_ms: 30000',
|
|
'',
|
|
].join('\n'),
|
|
'utf-8',
|
|
);
|
|
const project = await loadKtxProject({ projectDir: tempDir });
|
|
|
|
const connector = await createKtxCliScanConnector(project, 'warehouse');
|
|
|
|
expect(connector.id).toBe('bigquery:warehouse');
|
|
expect(connector.driver).toBe('bigquery');
|
|
expect(bigQueryMock.constructorInputs).toEqual([
|
|
expect.objectContaining({
|
|
connectionId: 'warehouse',
|
|
connection: expect.objectContaining({
|
|
max_bytes_billed: '987654321',
|
|
job_timeout_ms: 30000,
|
|
}),
|
|
}),
|
|
]);
|
|
expect(bigQueryMock.constructorInputs[0]).not.toHaveProperty('maxBytesBilled');
|
|
});
|
|
|
|
it('rejects daemon-only fallback driver configs at config parse time', async () => {
|
|
await initKtxProject({ projectDir: tempDir });
|
|
await writeFile(
|
|
join(tempDir, 'ktx.yaml'),
|
|
[
|
|
'connections:',
|
|
' warehouse:',
|
|
' driver: duckdb',
|
|
' path: warehouse.duckdb',
|
|
'',
|
|
].join('\n'),
|
|
'utf-8',
|
|
);
|
|
|
|
await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow(
|
|
/connections\.warehouse\.driver:.*Invalid discriminator value/,
|
|
);
|
|
});
|
|
|
|
it('rejects connection blocks with no driver field at config parse time', async () => {
|
|
await initKtxProject({ projectDir: tempDir });
|
|
await writeFile(
|
|
join(tempDir, 'ktx.yaml'),
|
|
[
|
|
'connections:',
|
|
' warehouse:',
|
|
' type: postgres',
|
|
' url: postgresql://example/db',
|
|
'',
|
|
].join('\n'),
|
|
'utf-8',
|
|
);
|
|
|
|
await expect(loadKtxProject({ projectDir: tempDir })).rejects.toThrow(
|
|
/connections\.warehouse\.driver:.*Invalid discriminator value/,
|
|
);
|
|
});
|
|
});
|