feat: recognize duckdb in context drivers

This commit is contained in:
Andrey Avtomonov 2026-05-18 15:15:14 +02:00
parent 9f095457dc
commit 22ad9dca99
12 changed files with 109 additions and 2 deletions

View file

@ -3,6 +3,7 @@ import { z } from 'zod';
export const connectionTypeSchema = z.enum([
'POSTGRESQL',
'SQLITE',
'DUCKDB',
'SQLSERVER',
'BIGQUERY',
'SNOWFLAKE',

View file

@ -8,6 +8,7 @@ describe('getDialectForDriver', () => {
['mysql', '`public`.`orders`'],
['clickhouse', '`public`.`orders`'],
['sqlite', '"orders"'],
['duckdb', '"public"."orders"'],
['snowflake', '"analytics"."public"."orders"'],
['bigquery', '`analytics`.`public`.`orders`'],
['sqlserver', '[analytics].[public].[orders]'],
@ -24,7 +25,7 @@ describe('getDialectForDriver', () => {
it('throws with a supported-driver list for unknown drivers', () => {
expect(() => getDialectForDriver('oracle')).toThrow(
'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver',
'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, duckdb, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver',
);
});
});

View file

@ -8,6 +8,7 @@ export type SupportedDriver =
| 'snowflake'
| 'bigquery'
| 'clickhouse'
| 'duckdb'
| 'sqlite'
| 'sqlite3';
@ -21,6 +22,7 @@ export interface KtxDialect {
const supportedDrivers: SupportedDriver[] = [
'bigquery',
'clickhouse',
'duckdb',
'mysql',
'postgres',
'postgresql',
@ -86,6 +88,7 @@ const dialects: Record<SupportedDriver, KtxDialect> = {
postgresql: createDialect('postgresql', doubleQuoted),
mysql: createDialect('mysql', backtickQuoted),
clickhouse: createDialect('clickhouse', backtickQuoted),
duckdb: createDialect('duckdb', doubleQuoted),
sqlite: createDialect('sqlite', doubleQuoted, true),
sqlite3: createDialect('sqlite3', doubleQuoted, true),
snowflake: createDialect('snowflake', doubleQuoted),

View file

@ -56,4 +56,42 @@ describe('createDefaultLocalQueryExecutor', () => {
}),
).rejects.toThrow('No local query executor is configured for driver "snowflake".');
});
it('dispatches duckdb only when a duckdb executor slot is supplied', async () => {
const duckdb = {
execute: vi.fn(async () => ({
headers: ['duckdb'],
rows: [[3]],
totalRows: 1,
command: 'SELECT',
rowCount: 1,
})),
};
const executor = createDefaultLocalQueryExecutor({
postgres: { execute: vi.fn() },
sqlite: { execute: vi.fn() },
duckdb,
});
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'duckdb' },
sql: 'select 1',
}),
).resolves.toMatchObject({ headers: ['duckdb'] });
expect(duckdb.execute).toHaveBeenCalledTimes(1);
const missingSlot = createDefaultLocalQueryExecutor({
postgres: { execute: vi.fn() },
sqlite: { execute: vi.fn() },
});
await expect(
missingSlot.execute({
connectionId: 'warehouse',
connection: { driver: 'duckdb' },
sql: 'select 1',
}),
).rejects.toThrow('No local query executor is configured for driver "duckdb".');
});
});

View file

@ -9,6 +9,7 @@ import { createSqliteQueryExecutor } from './sqlite-query-executor.js';
export interface DefaultLocalQueryExecutorOptions {
postgres?: KtxSqlQueryExecutorPort;
sqlite?: KtxSqlQueryExecutorPort;
duckdb?: KtxSqlQueryExecutorPort;
}
function driverFor(input: KtxSqlQueryExecutionInput): string {
@ -28,6 +29,12 @@ export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecut
if (driver === 'sqlite' || driver === 'sqlite3') {
return sqlite.execute(input);
}
if (driver === 'duckdb') {
if (!options.duckdb) {
throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`);
}
return options.duckdb.execute(input);
}
throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`);
},
};

View file

@ -35,6 +35,15 @@ describe('localConnectionToWarehouseDescriptor', () => {
});
});
it('maps DuckDB configs to DUCKDB warehouse descriptors', () => {
expect(localConnectionToWarehouseDescriptor('warehouse', { driver: 'duckdb', path: 'data/warehouse.duckdb' })).toMatchObject({
id: 'warehouse',
connection_type: 'DUCKDB',
connection_params: { driver: 'duckdb', path: 'data/warehouse.duckdb' },
});
expect(localConnectionTypeForConfig('warehouse', { driver: 'duckdb', path: 'data/warehouse.duckdb' })).toBe('DUCKDB');
});
it('returns null for non-warehouse adapters', () => {
expect(
localConnectionToWarehouseDescriptor('looker', {

View file

@ -22,6 +22,7 @@ const DRIVER_TO_CONNECTION_TYPE: Record<string, ConnectionType> = {
postgres: 'POSTGRESQL',
postgresql: 'POSTGRESQL',
sqlite: 'SQLITE',
duckdb: 'DUCKDB',
sqlserver: 'SQLSERVER',
mssql: 'SQLSERVER',
mysql: 'MYSQL',

View file

@ -29,6 +29,13 @@ describe('connectionConfigSchema (driver discriminated union)', () => {
});
});
it('accepts duckdb local file config', () => {
expect(connectionConfigSchema.parse({ driver: 'duckdb', path: 'data/warehouse.duckdb' })).toMatchObject({
driver: 'duckdb',
path: 'data/warehouse.duckdb',
});
});
it('rejects an unknown driver', () => {
expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow();
});

View file

@ -12,6 +12,7 @@ const warehouseDrivers = [
'snowflake',
'bigquery',
'sqlite',
'duckdb',
'clickhouse',
'sqlserver',
] as const;
@ -27,6 +28,11 @@ function warehouseConnectionSchema<const Driver extends WarehouseDriver>(driver:
.min(1)
.optional()
.describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'),
path: z
.string()
.min(1)
.optional()
.describe('Local database file path for file-backed warehouse drivers such as SQLite and DuckDB.'),
})
.describe(
`${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`,
@ -40,6 +46,7 @@ const warehouseConnectionSchemas = [
warehouseConnectionSchema('snowflake'),
warehouseConnectionSchema('bigquery'),
warehouseConnectionSchema('sqlite'),
warehouseConnectionSchema('duckdb'),
warehouseConnectionSchema('clickhouse'),
warehouseConnectionSchema('sqlserver'),
] as const;

View file

@ -1403,6 +1403,37 @@ describe('local scan', () => {
);
});
it('accepts duckdb as a native standalone scan driver when the host supplies a live-database adapter', async () => {
await writeFile(
join(project.projectDir, 'ktx.yaml'),
[
'connections:',
' warehouse:',
' driver: duckdb',
' path: warehouse.duckdb',
'ingest:',
' adapters:',
' - live-database',
'',
].join('\n'),
'utf-8',
);
project = await loadKtxProject({ projectDir: project.projectDir });
const result = await runLocalScan({
project,
adapters: [fetchOnlyAdapter()],
connectionId: 'warehouse',
jobId: 'scan-run-duckdb',
now: () => new Date('2026-04-29T12:00:00.000Z'),
});
expect(result.report.driver).toBe('duckdb');
expect(result.report.artifactPaths.reportPath).toBe(
'raw-sources/warehouse/live-database/2026-04-29-120000-scan-run-duckdb/scan-report.json',
);
});
it('accepts mysql as a native standalone scan driver when the host supplies a live-database adapter', async () => {
await writeFile(
join(project.projectDir, 'ktx.yaml'),

View file

@ -102,6 +102,7 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver {
normalized === 'postgresql' ||
normalized === 'sqlite' ||
normalized === 'sqlite3' ||
normalized === 'duckdb' ||
normalized === 'mysql' ||
normalized === 'clickhouse' ||
normalized === 'sqlserver' ||
@ -111,7 +112,7 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver {
return normalized === 'sqlite3' ? 'sqlite' : normalized;
}
throw new Error(
`Standalone ktx scan supports postgres/postgresql/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`,
`Standalone ktx scan supports postgres/postgresql/sqlite/duckdb/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`,
);
}

View file

@ -2,6 +2,7 @@ export type KtxConnectionDriver =
| 'sqlite'
| 'postgres'
| 'postgresql'
| 'duckdb'
| 'sqlserver'
| 'bigquery'
| 'snowflake'