apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(config): stop generating ingest adapter allow lists

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 18:08:05 +02:00
parent 3b2f9fc870
commit 9afc5c87c3
6 changed files with 57 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

36
packages/cli/src/setup-databases.test.ts
View file

@ -277,7 +277,8 @@ describe('setup databases step', () => {
});
expect(testConnection).toHaveBeenCalledWith(tempDir, 'postgres-warehouse', expect.anything());
expect(scanConnection).toHaveBeenCalledWith(tempDir, 'postgres-warehouse', expect.anything());
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
expect(config.connections['postgres-warehouse']).toEqual({
driver: 'postgres',
url: 'env:DATABASE_URL',
@ -621,7 +622,8 @@ describe('setup databases step', () => {
});
expect(testConnection).toHaveBeenCalledTimes(1);
expect(testConnection).toHaveBeenCalledWith(tempDir, 'mysql-warehouse', expect.anything());
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'mysql-warehouse']);
});
@ -835,7 +837,8 @@ describe('setup databases step', () => {
);
expect(result.status).toBe('ready');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
const connection = config.connections['postgres-warehouse'];
expect(connection).toMatchObject({
driver: 'postgres',
@ -875,7 +878,8 @@ describe('setup databases step', () => {
);
expect(result.status).toBe('ready');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
const connection = config.connections['postgres-warehouse'];
expect(connection.url).toBe(`file:${resolve(tempDir, '.ktx/secrets/postgres-warehouse-url')}`);
expect(connection.driver).toBe('postgres');
@ -1360,7 +1364,8 @@ describe('setup databases step', () => {
);
expect(result.status).toBe('ready');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
expect(config.connections.snowflake).toMatchObject({
driver: 'snowflake',
authMethod: 'password',
@ -1378,7 +1383,10 @@ describe('setup databases step', () => {
redactionPatterns: ['(?i)secret'],
},
});
expect(config.ingest.adapters).toContain('historic-sql');
expect(configText).not.toContain('live-database');
expect(configText).not.toContain('historic-sql');
expect(configText).not.toMatch(/^\s+adapters:/m);
expect(config.ingest.adapters).toEqual([]);
});
it('writes Postgres Historic SQL config with minExecutions and ignores window/redaction output', async () => {
@ -1407,7 +1415,8 @@ describe('setup databases step', () => {
);
expect(result.status).toBe('ready');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
expect(config.connections.warehouse).toMatchObject({
driver: 'postgres',
url: 'env:DATABASE_URL',
@ -1427,7 +1436,10 @@ describe('setup databases step', () => {
});
expect(config.connections.warehouse.historicSql).not.toHaveProperty('windowDays');
expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns');
expect(config.ingest.adapters).toContain('historic-sql');
expect(configText).not.toContain('live-database');
expect(configText).not.toContain('historic-sql');
expect(configText).not.toMatch(/^\s+adapters:/m);
expect(config.ingest.adapters).toEqual([]);
expect(config.ingest.workUnits.maxConcurrency).toBe(6);
expect(io.stdout()).toContain('Historic SQL probe...');
expect(io.stdout()).toContain('pg_stat_statements ready');
@ -1468,7 +1480,8 @@ describe('setup databases step', () => {
);
expect(result.status).toBe('ready');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8');
const config = parseKtxProjectConfig(configText);
expect(config.connections.analytics).toMatchObject({
historicSql: {
enabled: true,
@ -1480,7 +1493,10 @@ describe('setup databases step', () => {
redactionPatterns: [],
},
});
expect(config.ingest.adapters).toContain('historic-sql');
expect(configText).not.toContain('live-database');
expect(configText).not.toContain('historic-sql');
expect(configText).not.toMatch(/^\s+adapters:/m);
expect(config.ingest.adapters).toEqual([]);
});
it('enables Historic SQL on an existing Postgres connection', async () => {

27
packages/cli/src/setup-databases.ts
View file

@ -1369,34 +1369,25 @@ async function maybeConfigureTableScope(input: {
async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
const adapters = project.config.ingest.adapters.includes('historic-sql')
? project.config.ingest.adapters
: [...project.config.ingest.adapters, 'historic-sql'];
const maxConcurrency = Math.max(
project.config.ingest.workUnits.maxConcurrency,
HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY,
);
if (
adapters === project.config.ingest.adapters &&
maxConcurrency === project.config.ingest.workUnits.maxConcurrency
) {
if (maxConcurrency === project.config.ingest.workUnits.maxConcurrency) {
return;
}
await writeFile(
project.configPath,
serializeKtxProjectConfig(
{
...project.config,
ingest: {
...project.config.ingest,
adapters,
workUnits: {
...project.config.ingest.workUnits,
maxConcurrency,
},
serializeKtxProjectConfig({
...project.config,
ingest: {
...project.config.ingest,
workUnits: {
...project.config.ingest.workUnits,
maxConcurrency,
},
},
),
}),
'utf-8',
);
}

8
packages/cli/src/setup-sources.test.ts
View file

@ -756,7 +756,7 @@ describe('setup sources step', () => {
expect(testPrompts.text).toHaveBeenCalledTimes(4);
});
it('enables the dbt adapter when adding a dbt source connection', async () => {
it('adds a dbt source connection without adapter allow-list entries', async () => {
await addPrimarySource();
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
@ -776,7 +776,11 @@ describe('setup sources step', () => {
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
expect((await readConfig()).ingest.adapters).toContain('dbt');
const configText = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(configText).not.toContain('live-database');
expect(configText).not.toContain('historic-sql');
expect(configText).not.toMatch(/^\s+adapters:/m);
expect((await readConfig()).ingest.adapters).toEqual([]);
});
it('lets interactive setup retry or continue after initial source ingest fails', async () => {

38
packages/cli/src/setup-sources.ts
View file

@ -156,10 +156,6 @@ function sourceLabel(source: KtxSetupSourceType): string {
return SOURCE_LABELS[source];
}
function sourceAdapter(source: KtxSetupSourceType): string {
return source;
}
function connectionNamePrompt(label: string): string {
return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`;
}
@ -313,25 +309,17 @@ async function writeSourceConnection(
projectDir: string,
connectionId: string,
connection: KtxProjectConnectionConfig,
adapter: string,
): Promise<() => Promise<void>> {
assertSafeConnectionId(connectionId);
const project = await loadKtxProject({ projectDir });
const previousConnection = project.config.connections[connectionId];
const hadPreviousConnection = previousConnection !== undefined;
const shouldRemoveAdapterOnRollback = !project.config.ingest.adapters.includes(adapter);
const config = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: connection,
},
ingest: {
...project.config.ingest,
adapters: project.config.ingest.adapters.includes(adapter)
? [...project.config.ingest.adapters]
: [...project.config.ingest.adapters, adapter],
},
};
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
return async () => {
@ -345,31 +333,10 @@ async function writeSourceConnection(
await writeProjectConfig(projectDir, {
...latest.config,
connections,
ingest: {
...latest.config.ingest,
adapters: shouldRemoveAdapterOnRollback
? latest.config.ingest.adapters.filter((candidate) => candidate !== adapter)
: latest.config.ingest.adapters,
},
});
};
}
async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSourceType): Promise<void> {
const adapter = sourceAdapter(source);
const project = await loadKtxProject({ projectDir });
if (project.config.ingest.adapters.includes(adapter)) {
return;
}
await writeProjectConfig(projectDir, {
...project.config,
ingest: {
...project.config.ingest,
adapters: [...project.config.ingest.adapters, adapter],
},
});
}
async function markSourcesComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8');
@ -1519,10 +1486,7 @@ export async function runKtxSetupSourcesStep(
const rollback =
sourceChoice.kind === 'existing'
? undefined
: await writeSourceConnection(args.projectDir, connectionId, connection, sourceAdapter(source));
if (sourceChoice.kind === 'existing') {
await ensureSourceAdapterEnabled(args.projectDir, source);
}
: await writeSourceConnection(args.projectDir, connectionId, connection);
const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps);
if (!validation.ok) {

7
packages/context/src/project/config.test.ts
View file

@ -21,7 +21,7 @@ describe('KTX project config', () => {
models: {},
},
ingest: {
adapters: ['live-database', 'lookml', 'metabase', 'metricflow', 'notion'],
adapters: [],
embeddings: {
backend: 'deterministic',
model: 'deterministic',
@ -67,13 +67,12 @@ describe('KTX project config', () => {
const parsed = parseKtxProjectConfig(serialized);
expect(serialized).toContain('project: warehouse');
expect(serialized).toContain('live-database');
expect(serialized).toContain('notion');
expect(serialized).not.toContain('live-database');
expect(serialized).toContain(
' embeddings:\n backend: deterministic\n model: deterministic\n dimensions: 8',
);
expect(parsed.project).toBe('warehouse');
expect(parsed.ingest.adapters).toEqual(['live-database', 'lookml', 'metabase', 'metricflow', 'notion']);
expect(parsed.ingest.adapters).toEqual([]);
expect(parsed.ingest.embeddings).toEqual({
backend: 'deterministic',
model: 'deterministic',

14
packages/context/src/project/config.ts
View file

@ -392,7 +392,7 @@ export function buildDefaultKtxProjectConfig(projectName = 'ktx-project'): KtxPr
models: {},
},
ingest: {
adapters: ['live-database', 'lookml', 'metabase', 'metricflow', 'notion'],
adapters: [],
embeddings: {
backend: 'deterministic',
model: 'deterministic',
@ -530,5 +530,15 @@ export function parseKtxProjectConfig(raw: string): KtxProjectConfig {
}
export function serializeKtxProjectConfig(config: KtxProjectConfig): string {
return `${YAML.stringify(config, { indent: 2, lineWidth: 0 }).trimEnd()}\n`;
const serializedConfig =
config.ingest.adapters.length === 0
? {
...config,
ingest: {
embeddings: config.ingest.embeddings,
workUnits: config.ingest.workUnits,
},
}
: config;
return `${YAML.stringify(serializedConfig, { indent: 2, lineWidth: 0 }).trimEnd()}\n`;
}