mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
feat: recognize duckdb in context drivers
This commit is contained in:
parent
9f095457dc
commit
22ad9dca99
12 changed files with 109 additions and 2 deletions
|
|
@ -3,6 +3,7 @@ import { z } from 'zod';
|
|||
export const connectionTypeSchema = z.enum([
|
||||
'POSTGRESQL',
|
||||
'SQLITE',
|
||||
'DUCKDB',
|
||||
'SQLSERVER',
|
||||
'BIGQUERY',
|
||||
'SNOWFLAKE',
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ describe('getDialectForDriver', () => {
|
|||
['mysql', '`public`.`orders`'],
|
||||
['clickhouse', '`public`.`orders`'],
|
||||
['sqlite', '"orders"'],
|
||||
['duckdb', '"public"."orders"'],
|
||||
['snowflake', '"analytics"."public"."orders"'],
|
||||
['bigquery', '`analytics`.`public`.`orders`'],
|
||||
['sqlserver', '[analytics].[public].[orders]'],
|
||||
|
|
@ -24,7 +25,7 @@ describe('getDialectForDriver', () => {
|
|||
|
||||
it('throws with a supported-driver list for unknown drivers', () => {
|
||||
expect(() => getDialectForDriver('oracle')).toThrow(
|
||||
'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver',
|
||||
'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, duckdb, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ export type SupportedDriver =
|
|||
| 'snowflake'
|
||||
| 'bigquery'
|
||||
| 'clickhouse'
|
||||
| 'duckdb'
|
||||
| 'sqlite'
|
||||
| 'sqlite3';
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ export interface KtxDialect {
|
|||
const supportedDrivers: SupportedDriver[] = [
|
||||
'bigquery',
|
||||
'clickhouse',
|
||||
'duckdb',
|
||||
'mysql',
|
||||
'postgres',
|
||||
'postgresql',
|
||||
|
|
@ -86,6 +88,7 @@ const dialects: Record<SupportedDriver, KtxDialect> = {
|
|||
postgresql: createDialect('postgresql', doubleQuoted),
|
||||
mysql: createDialect('mysql', backtickQuoted),
|
||||
clickhouse: createDialect('clickhouse', backtickQuoted),
|
||||
duckdb: createDialect('duckdb', doubleQuoted),
|
||||
sqlite: createDialect('sqlite', doubleQuoted, true),
|
||||
sqlite3: createDialect('sqlite3', doubleQuoted, true),
|
||||
snowflake: createDialect('snowflake', doubleQuoted),
|
||||
|
|
|
|||
|
|
@ -56,4 +56,42 @@ describe('createDefaultLocalQueryExecutor', () => {
|
|||
}),
|
||||
).rejects.toThrow('No local query executor is configured for driver "snowflake".');
|
||||
});
|
||||
|
||||
it('dispatches duckdb only when a duckdb executor slot is supplied', async () => {
|
||||
const duckdb = {
|
||||
execute: vi.fn(async () => ({
|
||||
headers: ['duckdb'],
|
||||
rows: [[3]],
|
||||
totalRows: 1,
|
||||
command: 'SELECT',
|
||||
rowCount: 1,
|
||||
})),
|
||||
};
|
||||
const executor = createDefaultLocalQueryExecutor({
|
||||
postgres: { execute: vi.fn() },
|
||||
sqlite: { execute: vi.fn() },
|
||||
duckdb,
|
||||
});
|
||||
|
||||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'duckdb' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).resolves.toMatchObject({ headers: ['duckdb'] });
|
||||
expect(duckdb.execute).toHaveBeenCalledTimes(1);
|
||||
|
||||
const missingSlot = createDefaultLocalQueryExecutor({
|
||||
postgres: { execute: vi.fn() },
|
||||
sqlite: { execute: vi.fn() },
|
||||
});
|
||||
await expect(
|
||||
missingSlot.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'duckdb' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('No local query executor is configured for driver "duckdb".');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { createSqliteQueryExecutor } from './sqlite-query-executor.js';
|
|||
export interface DefaultLocalQueryExecutorOptions {
|
||||
postgres?: KtxSqlQueryExecutorPort;
|
||||
sqlite?: KtxSqlQueryExecutorPort;
|
||||
duckdb?: KtxSqlQueryExecutorPort;
|
||||
}
|
||||
|
||||
function driverFor(input: KtxSqlQueryExecutionInput): string {
|
||||
|
|
@ -28,6 +29,12 @@ export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecut
|
|||
if (driver === 'sqlite' || driver === 'sqlite3') {
|
||||
return sqlite.execute(input);
|
||||
}
|
||||
if (driver === 'duckdb') {
|
||||
if (!options.duckdb) {
|
||||
throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
}
|
||||
return options.duckdb.execute(input);
|
||||
}
|
||||
throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -35,6 +35,15 @@ describe('localConnectionToWarehouseDescriptor', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('maps DuckDB configs to DUCKDB warehouse descriptors', () => {
|
||||
expect(localConnectionToWarehouseDescriptor('warehouse', { driver: 'duckdb', path: 'data/warehouse.duckdb' })).toMatchObject({
|
||||
id: 'warehouse',
|
||||
connection_type: 'DUCKDB',
|
||||
connection_params: { driver: 'duckdb', path: 'data/warehouse.duckdb' },
|
||||
});
|
||||
expect(localConnectionTypeForConfig('warehouse', { driver: 'duckdb', path: 'data/warehouse.duckdb' })).toBe('DUCKDB');
|
||||
});
|
||||
|
||||
it('returns null for non-warehouse adapters', () => {
|
||||
expect(
|
||||
localConnectionToWarehouseDescriptor('looker', {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ const DRIVER_TO_CONNECTION_TYPE: Record<string, ConnectionType> = {
|
|||
postgres: 'POSTGRESQL',
|
||||
postgresql: 'POSTGRESQL',
|
||||
sqlite: 'SQLITE',
|
||||
duckdb: 'DUCKDB',
|
||||
sqlserver: 'SQLSERVER',
|
||||
mssql: 'SQLSERVER',
|
||||
mysql: 'MYSQL',
|
||||
|
|
|
|||
|
|
@ -29,6 +29,13 @@ describe('connectionConfigSchema (driver discriminated union)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('accepts duckdb local file config', () => {
|
||||
expect(connectionConfigSchema.parse({ driver: 'duckdb', path: 'data/warehouse.duckdb' })).toMatchObject({
|
||||
driver: 'duckdb',
|
||||
path: 'data/warehouse.duckdb',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects an unknown driver', () => {
|
||||
expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ const warehouseDrivers = [
|
|||
'snowflake',
|
||||
'bigquery',
|
||||
'sqlite',
|
||||
'duckdb',
|
||||
'clickhouse',
|
||||
'sqlserver',
|
||||
] as const;
|
||||
|
|
@ -27,6 +28,11 @@ function warehouseConnectionSchema<const Driver extends WarehouseDriver>(driver:
|
|||
.min(1)
|
||||
.optional()
|
||||
.describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'),
|
||||
path: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.describe('Local database file path for file-backed warehouse drivers such as SQLite and DuckDB.'),
|
||||
})
|
||||
.describe(
|
||||
`${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`,
|
||||
|
|
@ -40,6 +46,7 @@ const warehouseConnectionSchemas = [
|
|||
warehouseConnectionSchema('snowflake'),
|
||||
warehouseConnectionSchema('bigquery'),
|
||||
warehouseConnectionSchema('sqlite'),
|
||||
warehouseConnectionSchema('duckdb'),
|
||||
warehouseConnectionSchema('clickhouse'),
|
||||
warehouseConnectionSchema('sqlserver'),
|
||||
] as const;
|
||||
|
|
|
|||
|
|
@ -1403,6 +1403,37 @@ describe('local scan', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('accepts duckdb as a native standalone scan driver when the host supplies a live-database adapter', async () => {
|
||||
await writeFile(
|
||||
join(project.projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: duckdb',
|
||||
' path: warehouse.duckdb',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
project = await loadKtxProject({ projectDir: project.projectDir });
|
||||
|
||||
const result = await runLocalScan({
|
||||
project,
|
||||
adapters: [fetchOnlyAdapter()],
|
||||
connectionId: 'warehouse',
|
||||
jobId: 'scan-run-duckdb',
|
||||
now: () => new Date('2026-04-29T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
expect(result.report.driver).toBe('duckdb');
|
||||
expect(result.report.artifactPaths.reportPath).toBe(
|
||||
'raw-sources/warehouse/live-database/2026-04-29-120000-scan-run-duckdb/scan-report.json',
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts mysql as a native standalone scan driver when the host supplies a live-database adapter', async () => {
|
||||
await writeFile(
|
||||
join(project.projectDir, 'ktx.yaml'),
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver {
|
|||
normalized === 'postgresql' ||
|
||||
normalized === 'sqlite' ||
|
||||
normalized === 'sqlite3' ||
|
||||
normalized === 'duckdb' ||
|
||||
normalized === 'mysql' ||
|
||||
normalized === 'clickhouse' ||
|
||||
normalized === 'sqlserver' ||
|
||||
|
|
@ -111,7 +112,7 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver {
|
|||
return normalized === 'sqlite3' ? 'sqlite' : normalized;
|
||||
}
|
||||
throw new Error(
|
||||
`Standalone ktx scan supports postgres/postgresql/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`,
|
||||
`Standalone ktx scan supports postgres/postgresql/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ export type KtxConnectionDriver =
|
|||
| 'sqlite'
|
||||
| 'postgres'
|
||||
| 'postgresql'
|
||||
| 'duckdb'
|
||||
| 'sqlserver'
|
||||
| 'bigquery'
|
||||
| 'snowflake'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue