apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-19 08:28:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: stop requiring readonly connection config

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 19:21:41 +02:00
parent 754e4a9039
commit 7824b7f3b6
55 changed files with 103 additions and 292 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/cli/src/connection.test.ts
View file

@ -94,7 +94,7 @@ describe('runKtxConnection', () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
});
const io = makeIo();
@ -123,7 +123,7 @@ describe('runKtxConnection', () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
warehouse: { driver: 'sqlite' },
});
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
const createScanConnector = vi.fn(async () => connector);
@ -202,7 +202,7 @@ describe('runKtxConnection', () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
warehouse: { driver: 'sqlite' },
});
const cleanup = vi.fn(async () => undefined);
const connector: KtxScanConnector = {

1
packages/cli/src/demo-assets.ts
View file

@ -57,7 +57,6 @@ function demoConfig(databasePath: string): string {
` ${DEMO_CONNECTION_ID}:`,
' driver: sqlite',
` path: ${JSON.stringify(databasePath)}`,
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',

1
packages/cli/src/doctor.test.ts
View file

@ -275,7 +275,6 @@ describe('runKtxDoctor', () => {
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' readonly: true',
' historicSql:',
' enabled: true',
' dialect: postgres',

8
packages/cli/src/historic-sql-doctor.test.ts
View file

@ -25,7 +25,7 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
it('passes when no Postgres historic-SQL connections are enabled', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true },
warehouse: { driver: 'sqlite', path: './warehouse.db' },
}),
{
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
@ -53,7 +53,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
@ -66,7 +65,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
connection: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
env: process.env,
@ -87,7 +85,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
@ -119,7 +116,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
@ -154,7 +150,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'mysql',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
@ -180,7 +175,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
historicSql: { enabled: true, dialect: 'postgres' },
},
}),

3
packages/cli/src/historic-sql-doctor.ts
View file

@ -86,8 +86,9 @@ async function defaultPostgresHistoricSqlProbe(
const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] =
await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]);
const inputDriver = input.connection.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(input.connection)) {
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`);
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
}
const client = new KtxPostgresHistoricSqlQueryClient({

3
packages/cli/src/local-adapters.test.ts
View file

@ -45,7 +45,6 @@ describe('CLI local ingest adapters', () => {
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' readonly: true',
' historicSql:',
' enabled: true',
' dialect: postgres',
@ -76,7 +75,6 @@ describe('CLI local ingest adapters', () => {
'connections:',
' bq:',
' driver: bigquery',
' readonly: true',
' dataset_id: analytics',
' location: us',
' credentials_json: \'{"project_id":"demo-project"}\'',
@ -110,7 +108,6 @@ describe('CLI local ingest adapters', () => {
'connections:',
' sf:',
' driver: snowflake',
' readonly: true',
' account: acct',
' warehouse: wh',
' database: ANALYTICS',

20
packages/cli/src/local-adapters.ts
View file

@ -190,10 +190,9 @@ function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery'
function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined;
const inputDriver = connection?.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(connection)) {
throw new Error(
`Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Historic SQL local ingest requires a Postgres connection, got ${String(inputDriver)}`);
}
return {
async executeQuery(sql: string, params?: unknown[]) {
@ -212,10 +211,9 @@ function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, conn
function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined;
const inputDriver = connection?.driver ?? 'unknown';
if (!isKtxBigQueryConnectionConfig(connection)) {
throw new Error(
`Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`);
}
return {
async executeQuery(query: string) {
@ -243,10 +241,9 @@ async function createEphemeralSnowflakeHistoricSqlClient(
connectorModule: SnowflakeConnectorModule,
) {
const connection = project.config.connections[connectionId];
const inputDriver = connection?.driver ?? 'unknown';
if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) {
throw new Error(
`Historic SQL local ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Historic SQL local ingest requires a Snowflake connection, got ${String(inputDriver)}`);
}
return {
async executeQuery(query: string) {
@ -308,10 +305,9 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
}
if (dialect === 'bigquery') {
const inputDriver = connection?.driver ?? 'unknown';
if (!isKtxBigQueryConnectionConfig(connection)) {
throw new Error(
`Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`);
}
return {
...base,

3
packages/cli/src/local-scan-connectors.test.ts
View file

@ -49,7 +49,6 @@ describe('createKtxCliScanConnector', () => {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -72,7 +71,6 @@ describe('createKtxCliScanConnector', () => {
' warehouse:',
' driver: bigquery',
' dataset_id: analytics',
' readonly: true',
' max_bytes_billed: "987654321"',
'',
].join('\n'),
@ -123,7 +121,6 @@ describe('createKtxCliScanConnector', () => {
' warehouse:',
' type: postgres',
' url: postgresql://example/db',
' readonly: true',
'',
].join('\n'),
'utf-8',

7
packages/cli/src/scan.test.ts
View file

@ -861,7 +861,6 @@ describe('runKtxScan', () => {
' warehouse:',
' driver: mysql',
' url: env:MYSQL_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -910,7 +909,6 @@ describe('runKtxScan', () => {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -968,7 +966,6 @@ describe('runKtxScan', () => {
' database: analytics',
' username: reader',
' password: env:POSTGRES_PASSWORD',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1035,7 +1032,6 @@ describe('runKtxScan', () => {
' database: analytics',
' username: reader',
' password: env:CLICKHOUSE_PASSWORD',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1087,7 +1083,6 @@ describe('runKtxScan', () => {
' database: analytics',
' username: reader',
' schema: dbo',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1153,7 +1148,6 @@ describe('runKtxScan', () => {
' dataset_id: analytics',
' credentials_json: env:BIGQUERY_CREDENTIALS_JSON',
' location: US',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1222,7 +1216,6 @@ describe('runKtxScan', () => {
' database: ANALYTICS',
' schema_name: PUBLIC',
' username: reader',
' readonly: true',
'',
].join('\n'),
'utf-8',

13
packages/cli/src/setup-databases.test.ts
View file

@ -218,7 +218,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -281,7 +280,6 @@ describe('setup databases step', () => {
expect(config.connections['postgres-warehouse']).toEqual({
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
});
});
@ -542,7 +540,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'setup:',
' database_connection_ids:',
' - warehouse',
@ -583,7 +580,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'setup:',
' database_connection_ids:',
' - warehouse',
@ -698,7 +694,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'setup:',
' database_connection_ids:',
' - warehouse',
@ -843,7 +838,6 @@ describe('setup databases step', () => {
port: 5432,
database: 'analytics',
username: 'readonly',
readonly: true,
});
expect(connection.password).toMatch(/^file:/);
const secretPath = join(tempDir, '.ktx/secrets/postgres-warehouse-password');
@ -998,7 +992,6 @@ describe('setup databases step', () => {
expect(config.connections['postgres-warehouse']).toMatchObject({
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
});
});
@ -1115,7 +1108,6 @@ describe('setup databases step', () => {
driver: 'postgres',
url: 'env:DATABASE_URL',
schemas: ['public'],
readonly: true,
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
@ -1153,7 +1145,6 @@ describe('setup databases step', () => {
expect(config.connections.warehouse).toEqual({
driver: 'sqlite',
path: './warehouse.sqlite',
readonly: true,
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
@ -1170,7 +1161,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
' analytics:',
' driver: snowflake',
' authMethod: password',
@ -1180,7 +1170,6 @@ describe('setup databases step', () => {
' schema_name: PUBLIC',
' username: reader',
' password: env:SNOWFLAKE_PASSWORD',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1443,7 +1432,6 @@ describe('setup databases step', () => {
' driver: bigquery',
' dataset_id: analytics',
' credentials_json: env:BIGQUERY_CREDENTIALS_JSON',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1492,7 +1480,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',

10
packages/cli/src/setup-databases.ts
View file

@ -593,7 +593,6 @@ async function buildFieldsConnectionConfig(input: {
username,
...(passwordRef ? { password: passwordRef } : {}),
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -615,7 +614,6 @@ async function buildPastedUrlConnectionConfig(input: {
driver: input.driver,
url,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -629,7 +627,6 @@ async function buildPastedUrlConnectionConfig(input: {
driver: input.driver,
url: ref,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -637,7 +634,6 @@ async function buildPastedUrlConnectionConfig(input: {
driver: input.driver,
url,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -661,14 +657,12 @@ async function buildUrlConnectionConfig(input: {
driver: input.driver,
url: ref,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
return {
driver: input.driver,
url,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -706,7 +700,7 @@ async function buildConnectionConfig(input: {
'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.',
));
if (path === undefined) return 'back';
return path ? { driver: 'sqlite', path, readonly: true } : null;
return path ? { driver: 'sqlite', path } : null;
}
if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') {
return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts });
@ -728,7 +722,6 @@ async function buildConnectionConfig(input: {
dataset_id: datasetId,
credentials_json: normalizeFileReference(credentialsPath),
...(location ? { location } : {}),
readonly: true,
};
}
if (driver === 'snowflake') {
@ -767,7 +760,6 @@ async function buildConnectionConfig(input: {
username,
password: passwordRef,
...(role ? { role } : {}),
readonly: true,
};
}
throw new Error(`Unsupported database driver: ${driver}`);

3
packages/cli/src/setup-sources.test.ts
View file

@ -98,7 +98,7 @@ describe('setup sources step', () => {
...config,
connections: {
...config.connections,
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
},
setup: {
...config.setup,
@ -455,7 +455,6 @@ describe('setup sources step', () => {
driver: 'snowflake',
account: 'acme',
database: 'analytics',
readonly: true,
});
const cases: Array<{

3
packages/cli/src/setup.test.ts
View file

@ -170,7 +170,6 @@ describe('setup status', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -192,7 +191,6 @@ describe('setup status', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1373,7 +1371,6 @@ describe('setup status', () => {
' warehouse:',
' driver: postgres',
' url: env:DEMO_DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',

11
packages/cli/src/sl.test.ts
View file

@ -190,7 +190,7 @@ joins: []
it('runs sl query and prints SQL output', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -247,7 +247,7 @@ joins: []
it('runs sl query from a JSON query file', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -314,7 +314,7 @@ joins: []
it('creates default sl query compute through the managed runtime helper', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -375,7 +375,7 @@ joins: []
it('executes sl query through the injected query executor', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db', readonly: true };
project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -471,7 +471,7 @@ joins: []
`);
db.close();
project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db', readonly: true };
project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db' };
await writeFile(
join(projectDir, 'ktx.yaml'),
[
@ -480,7 +480,6 @@ joins: []
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',

1
packages/cli/src/standalone-smoke.test.ts
View file

@ -106,7 +106,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich
' warehouse:',
' driver: sqlite',
` path: ${JSON.stringify(dbPath)}`,
' readonly: true',
'ingest:',
' adapters:',
' - live-database',