diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 95d1e3fb..df8b947f 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -277,7 +277,8 @@ describe('setup databases step', () => { }); expect(testConnection).toHaveBeenCalledWith(tempDir, 'postgres-warehouse', expect.anything()); expect(scanConnection).toHaveBeenCalledWith(tempDir, 'postgres-warehouse', expect.anything()); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); expect(config.connections['postgres-warehouse']).toEqual({ driver: 'postgres', url: 'env:DATABASE_URL', @@ -621,7 +622,8 @@ describe('setup databases step', () => { }); expect(testConnection).toHaveBeenCalledTimes(1); expect(testConnection).toHaveBeenCalledWith(tempDir, 'mysql-warehouse', expect.anything()); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'mysql-warehouse']); }); @@ -835,7 +837,8 @@ describe('setup databases step', () => { ); expect(result.status).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); const connection = config.connections['postgres-warehouse']; expect(connection).toMatchObject({ driver: 'postgres', @@ -875,7 +878,8 @@ describe('setup databases step', () => { ); expect(result.status).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); const connection = config.connections['postgres-warehouse']; expect(connection.url).toBe(`file:${resolve(tempDir, '.ktx/secrets/postgres-warehouse-url')}`); expect(connection.driver).toBe('postgres'); @@ -1360,7 +1364,8 @@ describe('setup databases step', () => { ); expect(result.status).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); expect(config.connections.snowflake).toMatchObject({ driver: 'snowflake', authMethod: 'password', @@ -1378,7 +1383,10 @@ describe('setup databases step', () => { redactionPatterns: ['(?i)secret'], }, }); - expect(config.ingest.adapters).toContain('historic-sql'); + expect(configText).not.toContain('live-database'); + expect(configText).not.toContain('historic-sql'); + expect(configText).not.toMatch(/^\s+adapters:/m); + expect(config.ingest.adapters).toEqual([]); }); it('writes Postgres Historic SQL config with minExecutions and ignores window/redaction output', async () => { @@ -1407,7 +1415,8 @@ describe('setup databases step', () => { ); expect(result.status).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); expect(config.connections.warehouse).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL', @@ -1427,7 +1436,10 @@ describe('setup databases step', () => { }); expect(config.connections.warehouse.historicSql).not.toHaveProperty('windowDays'); expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns'); - expect(config.ingest.adapters).toContain('historic-sql'); + expect(configText).not.toContain('live-database'); + expect(configText).not.toContain('historic-sql'); + expect(configText).not.toMatch(/^\s+adapters:/m); + expect(config.ingest.adapters).toEqual([]); expect(config.ingest.workUnits.maxConcurrency).toBe(6); expect(io.stdout()).toContain('Historic SQL probe...'); expect(io.stdout()).toContain('pg_stat_statements ready'); @@ -1468,7 +1480,8 @@ describe('setup databases step', () => { ); expect(result.status).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); + const config = parseKtxProjectConfig(configText); expect(config.connections.analytics).toMatchObject({ historicSql: { enabled: true, @@ -1480,7 +1493,10 @@ describe('setup databases step', () => { redactionPatterns: [], }, }); - expect(config.ingest.adapters).toContain('historic-sql'); + expect(configText).not.toContain('live-database'); + expect(configText).not.toContain('historic-sql'); + expect(configText).not.toMatch(/^\s+adapters:/m); + expect(config.ingest.adapters).toEqual([]); }); it('enables Historic SQL on an existing Postgres connection', async () => { diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 85be2620..87fa1b19 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -1369,34 +1369,25 @@ async function maybeConfigureTableScope(input: { async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - const adapters = project.config.ingest.adapters.includes('historic-sql') - ? project.config.ingest.adapters - : [...project.config.ingest.adapters, 'historic-sql']; const maxConcurrency = Math.max( project.config.ingest.workUnits.maxConcurrency, HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY, ); - if ( - adapters === project.config.ingest.adapters && - maxConcurrency === project.config.ingest.workUnits.maxConcurrency - ) { + if (maxConcurrency === project.config.ingest.workUnits.maxConcurrency) { return; } await writeFile( project.configPath, - serializeKtxProjectConfig( - { - ...project.config, - ingest: { - ...project.config.ingest, - adapters, - workUnits: { - ...project.config.ingest.workUnits, - maxConcurrency, - }, + serializeKtxProjectConfig({ + ...project.config, + ingest: { + ...project.config.ingest, + workUnits: { + ...project.config.ingest.workUnits, + maxConcurrency, }, }, - ), + }), 'utf-8', ); } diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index 0ab8e4c9..2f622da5 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -756,7 +756,7 @@ describe('setup sources step', () => { expect(testPrompts.text).toHaveBeenCalledTimes(4); }); - it('enables the dbt adapter when adding a dbt source connection', async () => { + it('adds a dbt source connection without adapter allow-list entries', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); @@ -776,7 +776,11 @@ describe('setup sources step', () => { ), ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); - expect((await readConfig()).ingest.adapters).toContain('dbt'); + const configText = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); + expect(configText).not.toContain('live-database'); + expect(configText).not.toContain('historic-sql'); + expect(configText).not.toMatch(/^\s+adapters:/m); + expect((await readConfig()).ingest.adapters).toEqual([]); }); it('lets interactive setup retry or continue after initial source ingest fails', async () => { diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 066408bd..991f56cc 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -156,10 +156,6 @@ function sourceLabel(source: KtxSetupSourceType): string { return SOURCE_LABELS[source]; } -function sourceAdapter(source: KtxSetupSourceType): string { - return source; -} - function connectionNamePrompt(label: string): string { return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`; } @@ -313,25 +309,17 @@ async function writeSourceConnection( projectDir: string, connectionId: string, connection: KtxProjectConnectionConfig, - adapter: string, ): Promise<() => Promise> { assertSafeConnectionId(connectionId); const project = await loadKtxProject({ projectDir }); const previousConnection = project.config.connections[connectionId]; const hadPreviousConnection = previousConnection !== undefined; - const shouldRemoveAdapterOnRollback = !project.config.ingest.adapters.includes(adapter); const config = { ...project.config, connections: { ...project.config.connections, [connectionId]: connection, }, - ingest: { - ...project.config.ingest, - adapters: project.config.ingest.adapters.includes(adapter) - ? [...project.config.ingest.adapters] - : [...project.config.ingest.adapters, adapter], - }, }; await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); return async () => { @@ -345,31 +333,10 @@ async function writeSourceConnection( await writeProjectConfig(projectDir, { ...latest.config, connections, - ingest: { - ...latest.config.ingest, - adapters: shouldRemoveAdapterOnRollback - ? latest.config.ingest.adapters.filter((candidate) => candidate !== adapter) - : latest.config.ingest.adapters, - }, }); }; } -async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSourceType): Promise { - const adapter = sourceAdapter(source); - const project = await loadKtxProject({ projectDir }); - if (project.config.ingest.adapters.includes(adapter)) { - return; - } - await writeProjectConfig(projectDir, { - ...project.config, - ingest: { - ...project.config.ingest, - adapters: [...project.config.ingest.adapters, adapter], - }, - }); -} - async function markSourcesComplete(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8'); @@ -1519,10 +1486,7 @@ export async function runKtxSetupSourcesStep( const rollback = sourceChoice.kind === 'existing' ? undefined - : await writeSourceConnection(args.projectDir, connectionId, connection, sourceAdapter(source)); - if (sourceChoice.kind === 'existing') { - await ensureSourceAdapterEnabled(args.projectDir, source); - } + : await writeSourceConnection(args.projectDir, connectionId, connection); const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps); if (!validation.ok) { diff --git a/packages/context/src/project/config.test.ts b/packages/context/src/project/config.test.ts index ee6b8ee9..b5c22a82 100644 --- a/packages/context/src/project/config.test.ts +++ b/packages/context/src/project/config.test.ts @@ -21,7 +21,7 @@ describe('KTX project config', () => { models: {}, }, ingest: { - adapters: ['live-database', 'lookml', 'metabase', 'metricflow', 'notion'], + adapters: [], embeddings: { backend: 'deterministic', model: 'deterministic', @@ -67,13 +67,12 @@ describe('KTX project config', () => { const parsed = parseKtxProjectConfig(serialized); expect(serialized).toContain('project: warehouse'); - expect(serialized).toContain('live-database'); - expect(serialized).toContain('notion'); + expect(serialized).not.toContain('live-database'); expect(serialized).toContain( ' embeddings:\n backend: deterministic\n model: deterministic\n dimensions: 8', ); expect(parsed.project).toBe('warehouse'); - expect(parsed.ingest.adapters).toEqual(['live-database', 'lookml', 'metabase', 'metricflow', 'notion']); + expect(parsed.ingest.adapters).toEqual([]); expect(parsed.ingest.embeddings).toEqual({ backend: 'deterministic', model: 'deterministic', diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index ad5ecb8a..2dc3c2a4 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -392,7 +392,7 @@ export function buildDefaultKtxProjectConfig(projectName = 'ktx-project'): KtxPr models: {}, }, ingest: { - adapters: ['live-database', 'lookml', 'metabase', 'metricflow', 'notion'], + adapters: [], embeddings: { backend: 'deterministic', model: 'deterministic', @@ -530,5 +530,15 @@ export function parseKtxProjectConfig(raw: string): KtxProjectConfig { } export function serializeKtxProjectConfig(config: KtxProjectConfig): string { - return `${YAML.stringify(config, { indent: 2, lineWidth: 0 }).trimEnd()}\n`; + const serializedConfig = + config.ingest.adapters.length === 0 + ? { + ...config, + ingest: { + embeddings: config.ingest.embeddings, + workUnits: config.ingest.workUnits, + }, + } + : config; + return `${YAML.stringify(serializedConfig, { indent: 2, lineWidth: 0 }).trimEnd()}\n`; }