diff --git a/packages/cli/src/commands/connection-mapping.ts b/packages/cli/src/commands/connection-mapping.ts index 9389ccd1..5bae8e6e 100644 --- a/packages/cli/src/commands/connection-mapping.ts +++ b/packages/cli/src/commands/connection-mapping.ts @@ -28,7 +28,6 @@ import { loadKtxProject, parseMetabaseMappingBootstrap, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from '../index.js'; import { profileMark } from '../startup-profile.js'; @@ -170,7 +169,7 @@ async function writeMetabaseMappings( }; await project.fileStore.writeFile( 'ktx.yaml', - serializeKtxProjectConfig(stripKtxSetupCompletedSteps(nextConfig)), + serializeKtxProjectConfig(nextConfig), 'ktx', 'ktx@example.com', message, diff --git a/packages/cli/src/commands/connection-metabase-setup.ts b/packages/cli/src/commands/connection-metabase-setup.ts index 6e80200c..b0980c3b 100644 --- a/packages/cli/src/commands/connection-metabase-setup.ts +++ b/packages/cli/src/commands/connection-metabase-setup.ts @@ -32,7 +32,6 @@ import { loadKtxProject, parseMetabaseMappingBootstrap, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import { createClackSpinner, type KtxCliSpinner } from '../clack.js'; @@ -723,7 +722,7 @@ export async function runKtxConnectionMetabaseSetup( }; await project.fileStore.writeFile( 'ktx.yaml', - serializeKtxProjectConfig(stripKtxSetupCompletedSteps(finalConfig)), + serializeKtxProjectConfig(finalConfig), 'ktx', 'ktx@example.com', `Setup Metabase connection ${connectionId}`, diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 151967aa..9c5c6fd5 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -6,7 +6,6 @@ import { loadKtxProject, markKtxSetupStateStepComplete, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { withMenuOptionsSpacing, withMultiselectNavigation } from './prompt-navigation.js'; @@ -376,7 +375,7 @@ async function installTarget(input: { async function markAgentsComplete(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'agents'); } diff --git a/packages/cli/src/setup-context.test.ts b/packages/cli/src/setup-context.test.ts index 1ef044ae..9115d7a5 100644 --- a/packages/cli/src/setup-context.test.ts +++ b/packages/cli/src/setup-context.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { readKtxSetupState } from '@ktx/context/project'; +import { readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -40,12 +40,6 @@ async function writeReadyProject(projectDir: string) { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - project', - ' - llm', - ' - embeddings', - ' - databases', - ' - sources', 'connections:', ' warehouse:', ' driver: postgres', @@ -71,6 +65,9 @@ async function writeReadyProject(projectDir: string) { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(projectDir, { + completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources'], + }); } async function writeScanReport( diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index efcd35f1..8f6e5ce1 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -5,11 +5,9 @@ import { cancel, isCancel, select } from '@clack/prompts'; import { type KtxLocalProject, loadKtxProject, - ktxSetupCompletedSteps, markKtxSetupStateStepComplete, readKtxSetupState, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { buildPublicIngestPlan } from './public-ingest.js'; @@ -470,7 +468,7 @@ async function defaultVerifyContextReady(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'context'); } @@ -714,7 +712,7 @@ export async function runKtxSetupContextStep( try { const project = await loadKtxProject({ projectDir: args.projectDir }); const existingState = await readKtxSetupContextState(args.projectDir); - const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(args.projectDir)); + const completedSteps = (await readKtxSetupState(args.projectDir)).completed_steps; if (completedSteps.includes('context') && existingState.status === 'completed') { return { status: 'ready', projectDir: args.projectDir, runId: existingState.runId ?? 'setup-context-completed' }; } diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 3b6d013e..65ee191a 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupDatabaseDriver, @@ -548,12 +548,11 @@ describe('setup databases step', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - databases', '', ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const prompts = makePromptAdapter({ multiselectValues: [['back']], selectValues: ['continue'] }); const testConnection = vi.fn(async () => 0); const scanConnection = vi.fn(async () => 0); @@ -590,12 +589,11 @@ describe('setup databases step', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - databases', '', ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const prompts = makePromptAdapter({ selectValues: ['add', 'url', 'continue'], multiselectValues: [['mysql']], @@ -706,12 +704,11 @@ describe('setup databases step', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - databases', '', ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const io = makeIo(); const prompts = makePromptAdapter({ multiselectValues: [[]], @@ -1211,7 +1208,7 @@ describe('setup databases step', () => { expect(scanConnection).toHaveBeenCalledTimes(2); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'analytics']); - expect(config.setup?.completed_steps).toBeUndefined(); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases'); }); @@ -1237,7 +1234,7 @@ describe('setup databases step', () => { expect(result.status).toBe('failed'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL' }); - expect(config.setup?.completed_steps ?? []).not.toContain('databases'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(io.stderr()).toContain('Structural scan failed for warehouse.'); }); @@ -1542,7 +1539,6 @@ describe('setup databases step', () => { expect(result.status).toBe('skipped'); expect(io.stdout()).toContain('KTX cannot work until you add a primary source.'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps ?? []).not.toContain('databases'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); }); }); diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index f770c5c4..eceaf5bb 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -7,7 +7,6 @@ import { markKtxSetupStateStepComplete, serializeKtxProjectConfig, setKtxSetupDatabaseConnectionIds, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxTableListEntry } from '@ktx/context/scan'; import type { KtxCliIo } from './cli-runtime.js'; @@ -1020,7 +1019,7 @@ async function writeConnectionConfig(input: { [input.connectionId]: input.connection, }, }; - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); const historicSql = typeof input.connection.historicSql === 'object' && @@ -1314,7 +1313,7 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds)); - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'databases'); } diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 67ef83b3..e66aa05a 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js'; @@ -172,7 +172,7 @@ describe('setup embeddings step', () => { sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, }); expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings'); expect(spinnerEvents).toContainEqual( 'start:Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.', @@ -251,7 +251,7 @@ describe('setup embeddings step', () => { sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, }); expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings'); }); @@ -301,7 +301,7 @@ describe('setup embeddings step', () => { expect(result.status).toBe('failed'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps ?? []).not.toContain('embeddings'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(config.ingest.embeddings.backend).toBe('deterministic'); expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]'); expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings'); @@ -413,7 +413,7 @@ describe('setup embeddings step', () => { expect(result.status).toBe('skipped'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps ?? []).not.toContain('embeddings'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(config.ingest.embeddings.backend).toBe('deterministic'); }); @@ -450,10 +450,6 @@ describe('setup embeddings step', () => { 'project: warehouse', 'setup:', ' database_connection_ids: []', - ' completed_steps:', - ' - project', - ' - llm', - ' - embeddings', 'connections: {}', 'ingest:', ' embeddings:', @@ -466,6 +462,7 @@ describe('setup embeddings step', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'llm', 'embeddings'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); await expect( diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 1b6a2381..ba3333f1 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -4,12 +4,10 @@ import { resolveKtxConfigReference } from '@ktx/context/core'; import { type KtxProjectConfig, type KtxProjectEmbeddingConfig, - ktxSetupCompletedSteps, loadKtxProject, markKtxSetupStateStepComplete, readKtxSetupState, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm'; import type { KtxCliIo } from './cli-runtime.js'; @@ -110,7 +108,7 @@ function createPromptAdapter(): KtxSetupEmbeddingsPromptAdapter { async function hasCompletedEmbeddings(projectDir: string, config: KtxProjectConfig): Promise { return ( - ktxSetupCompletedSteps(config, await readKtxSetupState(projectDir)).includes('embeddings') && + (await readKtxSetupState(projectDir)).completed_steps.includes('embeddings') && config.ingest.embeddings.backend !== 'none' && config.ingest.embeddings.backend !== 'deterministic' && typeof config.ingest.embeddings.model === 'string' && @@ -184,22 +182,20 @@ function embeddingBackendDisplayName(backend: KtxSetupEmbeddingBackend): string async function persistEmbeddingConfig(projectDir: string, embeddings: KtxProjectEmbeddingConfig): Promise { const project = await loadKtxProject({ projectDir }); - const config = stripKtxSetupCompletedSteps( - { - ...project.config, - ingest: { - ...project.config.ingest, + const config = { + ...project.config, + ingest: { + ...project.config.ingest, + embeddings, + }, + scan: { + ...project.config.scan, + enrichment: { + ...project.config.scan.enrichment, embeddings, }, - scan: { - ...project.config.scan, - enrichment: { - ...project.config.scan.enrichment, - embeddings, - }, - }, }, - ); + }; await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'embeddings'); } diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index 82f82875..fb8acb47 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BUNDLED_ANTHROPIC_MODELS, @@ -160,7 +160,7 @@ describe('setup Anthropic model step', () => { promptCaching: { enabled: true }, }); expect(config.scan.enrichment.mode).toBe('llm'); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); @@ -199,7 +199,7 @@ describe('setup Anthropic model step', () => { }, models: { default: 'claude-sonnet-4-6' }, }); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stdout()).not.toContain('sk-ant-file'); }); @@ -516,8 +516,7 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('failed'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps ?? []).not.toContain('llm'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(io.stderr()).toContain('Anthropic model health check failed: 401 invalid x-api-key [redacted]'); expect(io.stderr()).not.toContain('sk-ant-test'); }); @@ -553,7 +552,7 @@ describe('setup Anthropic model step', () => { expect(io.stderr()).toContain('Choose a different credential source or model, or Back.'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm.models.default).toBe('claude-sonnet-4-6'); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stderr()).not.toContain('sk-ant-test'); }); @@ -565,8 +564,7 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('skipped'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps ?? []).not.toContain('llm'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); }); it('returns back without writing config when Back is selected', async () => { @@ -650,9 +648,6 @@ describe('setup Anthropic model step', () => { 'project: warehouse', 'setup:', ' database_connection_ids: []', - ' completed_steps:', - ' - project', - ' - llm', 'connections: {}', 'llm:', ' provider:', @@ -669,6 +664,7 @@ describe('setup Anthropic model step', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'llm'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); await expect( @@ -698,9 +694,6 @@ describe('setup Anthropic model step', () => { 'project: warehouse', 'setup:', ' database_connection_ids: []', - ' completed_steps:', - ' - project', - ' - llm', 'connections: {}', 'llm:', ' provider:', @@ -715,6 +708,7 @@ describe('setup Anthropic model step', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'llm'] }); const healthCheck = vi.fn(async () => ({ ok: true as const })); const io = makeIo(); diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index 6d3c6757..221dbd14 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -8,7 +8,6 @@ import { loadKtxProject, markKtxSetupStateStepComplete, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; import type { KtxCliIo } from './cli-runtime.js'; @@ -362,19 +361,17 @@ async function chooseModel( async function persistLlmConfig(projectDir: string, credentialRef: string, model: string): Promise { const project = await loadKtxProject({ projectDir }); - const config = stripKtxSetupCompletedSteps( - { - ...project.config, - llm: buildProjectLlmConfig(project.config.llm, credentialRef, model), - scan: { - ...project.config.scan, - enrichment: { + const config = { + ...project.config, + llm: buildProjectLlmConfig(project.config.llm, credentialRef, model), + scan: { + ...project.config.scan, + enrichment: { ...project.config.scan.enrichment, - mode: 'llm', + mode: 'llm' as const, }, }, - }, - ); + }; await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'llm'); } diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index f4d3a1d8..70591077 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -59,8 +59,7 @@ describe('setup project step', () => { expect(result.status).toBe('ready'); expect(result.projectDir).toBe(projectDir); - const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] }); await expect(stat(join(projectDir, '.git'))).resolves.toBeDefined(); await expect(readFile(join(projectDir, '.ktx/.gitignore'), 'utf-8')).resolves.toContain('secrets/'); @@ -68,7 +67,7 @@ describe('setup project step', () => { expect(testIo.stderr()).toBe(''); }); - it('loads an existing project with --existing and preserves existing setup metadata', async () => { + it('loads an existing project with --existing and drops config setup progress', async () => { const projectDir = join(tempDir, 'warehouse'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeFile( @@ -95,7 +94,8 @@ describe('setup project step', () => { expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], }); - expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['llm', 'project'] }); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); + expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] }); }); it('creates a missing auto-mode project only when --yes is present in no-input mode', async () => { @@ -151,8 +151,7 @@ describe('setup project step', () => { }), ); expect(prompts.text).not.toHaveBeenCalled(); - const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] }); }); diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index 18512b03..4b2f71d9 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -5,15 +5,11 @@ import { basename, join, resolve } from 'node:path'; import { cancel, isCancel, select, text } from '@clack/prompts'; import { initKtxProject, - ktxSetupCompletedSteps, type KtxLocalProject, loadKtxProject, markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, - readKtxSetupState, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, - writeKtxSetupState, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js'; @@ -170,10 +166,7 @@ async function normalizeSetupGitignore(projectDir: string): Promise { } async function persistProjectStep(project: KtxLocalProject): Promise { - const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(project.projectDir)); - const config = stripKtxSetupCompletedSteps(project.config); - await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); - await writeKtxSetupState(project.projectDir, { completed_steps: completedSteps }); + await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8'); await markKtxSetupStateStepComplete(project.projectDir, 'project'); await normalizeSetupGitignore(project.projectDir); return await loadKtxProject({ projectDir: project.projectDir }); diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index bc65c95e..27579bb3 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -102,7 +102,6 @@ describe('setup sources step', () => { }, setup: { ...config.setup, - completed_steps: config.setup?.completed_steps ?? [], database_connection_ids: ['warehouse'], }, }), @@ -137,7 +136,7 @@ describe('setup sources step', () => { projectDir, }); - expect((await readConfig()).setup?.completed_steps).toEqual(undefined); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources'); expect(io.stdout()).toContain('Context source setup skipped.'); }); @@ -171,7 +170,7 @@ describe('setup sources step', () => { source_dir: '/repo/dbt', project_name: 'analytics', }); - expect(config.setup?.completed_steps).toBeUndefined(); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources'); expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' }); }); @@ -480,7 +479,7 @@ describe('setup sources step', () => { ), ).resolves.toEqual({ status: 'failed', projectDir }); - expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources'); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(io.stderr()).toContain('No LookML files found'); }); @@ -1032,7 +1031,7 @@ describe('setup sources step', () => { expect(testPrompts.multiselect).not.toHaveBeenCalled(); expect(io.stdout()).toContain('Connect a primary source before adding context sources.'); - expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources'); + expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); }); it('auto-detects dbt_project.yml at the root of a local path', async () => { diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 6674ef75..edf83b7b 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -25,7 +25,6 @@ import { loadKtxProject, markKtxSetupStateStepComplete, serializeKtxProjectConfig, - stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnectionMapping } from './commands/connection-mapping.js'; @@ -345,7 +344,7 @@ function fileRepoUrl(sourceDir: string): string { async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); } async function writeSourceConnection( @@ -372,7 +371,7 @@ async function writeSourceConnection( : [...project.config.ingest.adapters, adapter], }, }; - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); return async () => { const latest = await loadKtxProject({ projectDir }); const connections = { ...latest.config.connections }; @@ -411,7 +410,7 @@ async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSo async function markSourcesComplete(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'sources'); } diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index bf9c381f..0cad3ebc 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -3,6 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; +import { writeKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js'; @@ -133,9 +134,6 @@ describe('setup status', () => { ' database_connection_ids:', ' - warehouse', ' - analytics', - ' completed_steps:', - ' - project', - ' - databases', 'connections:', ' warehouse:', ' driver: postgres', @@ -150,6 +148,7 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] }); await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({ databases: [ @@ -167,8 +166,6 @@ describe('setup status', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - project', 'connections:', ' warehouse:', ' driver: postgres', @@ -178,6 +175,7 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project'] }); await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({ databases: [{ connectionId: 'warehouse', ready: false }], @@ -190,9 +188,6 @@ describe('setup status', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - project', - ' - databases', 'connections:', ' warehouse:', ' driver: postgres', @@ -202,6 +197,7 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] }); await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({ databases: [{ connectionId: 'warehouse', ready: true }], @@ -215,9 +211,6 @@ describe('setup status', () => { 'project: revenue', 'setup:', ' database_connection_ids: []', - ' completed_steps:', - ' - project', - ' - sources', 'connections:', ' docs:', ' driver: notion', @@ -230,6 +223,7 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'sources'] }); await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({ sources: [{ connectionId: 'docs', type: 'notion', ready: true }], @@ -268,12 +262,6 @@ describe('setup status', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - project', - ' - llm', - ' - embeddings', - ' - databases', - ' - sources', 'connections:', ' warehouse:', ' driver: postgres', @@ -292,6 +280,9 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { + completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources'], + }); await writeKtxSetupContextState(tempDir, { runId: 'setup-context-local-abc123', status: 'running', @@ -324,10 +315,6 @@ describe('setup status', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - project', - ' - databases', - ' - sources', 'connections:', ' warehouse:', ' driver: postgres', @@ -354,6 +341,7 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases', 'sources'] }); await persistLocalBundleReport( tempDir, localFakeBundleReport('metabase-job-1', { @@ -1281,9 +1269,6 @@ describe('setup status', () => { 'setup:', ' database_connection_ids:', ' - warehouse', - ' completed_steps:', - ' - project', - ' - databases', 'connections:', ' warehouse:', ' driver: postgres', @@ -1296,6 +1281,7 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] }); await expect( runKtxSetup( @@ -1782,13 +1768,6 @@ describe('setup status', () => { [ 'project: revenue', 'setup:', - ' completed_steps:', - ' - project', - ' - llm', - ' - embeddings', - ' - sources', - ' - context', - ' - agents', ' database_connection_ids: []', 'connections: {}', 'llm:', @@ -1805,6 +1784,9 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { + completed_steps: ['project', 'llm', 'embeddings', 'sources', 'context', 'agents'], + }); await writeFile( join(tempDir, '.ktx/agents/install-manifest.json'), JSON.stringify( @@ -1893,12 +1875,6 @@ describe('setup status', () => { [ 'project: revenue', 'setup:', - ' completed_steps:', - ' - project', - ' - llm', - ' - embeddings', - ' - sources', - ' - context', ' database_connection_ids: []', 'connections: {}', 'llm:', @@ -1915,6 +1891,9 @@ describe('setup status', () => { ].join('\n'), 'utf-8', ); + await writeKtxSetupState(tempDir, { + completed_steps: ['project', 'llm', 'embeddings', 'sources', 'context'], + }); await writeKtxSetupContextState(tempDir, { runId: 'setup-context-local-ready', status: 'completed', diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index dec0f4d7..064da729 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -4,7 +4,6 @@ import { cancel, isCancel, select } from '@clack/prompts'; import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest'; import { ktxLocalStateDbPath, - ktxSetupCompletedSteps, loadKtxProject, readKtxSetupState, type KtxLocalProject, @@ -297,7 +296,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise { }), config: { ...project.config, - setup: { database_connection_ids: ['warehouse'], completed_steps: [] }, + setup: { database_connection_ids: ['warehouse'] }, connections: { warehouse: { driver: 'postgres', diff --git a/packages/context/src/project/config.test.ts b/packages/context/src/project/config.test.ts index 1be70322..cad7945c 100644 --- a/packages/context/src/project/config.test.ts +++ b/packages/context/src/project/config.test.ts @@ -81,16 +81,13 @@ describe('KTX project config', () => { }); }); - it('parses and serializes setup wizard metadata', () => { + it('parses and serializes setup warehouse metadata without setup progress', () => { const config = parseKtxProjectConfig(` project: revenue setup: database_connection_ids: - warehouse - analytics - completed_steps: - - project - - llm connections: warehouse: driver: postgres @@ -99,13 +96,12 @@ connections: expect(config.setup).toEqual({ database_connection_ids: ['warehouse', 'analytics'], - completed_steps: ['project', 'llm'], }); const serialized = serializeKtxProjectConfig(config); expect(serialized).toContain('setup:'); expect(serialized).toContain('database_connection_ids:'); - expect(serialized).toContain('completed_steps:'); + expect(serialized).not.toContain('completed_steps:'); }); it('parses global direct Anthropic LLM config', () => { diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index 2412683f..5da193f2 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -75,7 +75,6 @@ export interface KtxProjectConnectionConfig { export interface KtxProjectSetupConfig { database_connection_ids: string[]; - completed_steps?: string[]; } export interface KtxProjectConfig { @@ -505,15 +504,12 @@ export function parseKtxProjectConfig(raw: string): KtxProjectConfig { return { project: project.trim(), ...(setup - ? { - setup: { - database_connection_ids: stringArray(setup.database_connection_ids, []), - ...(setup.completed_steps !== undefined - ? { completed_steps: stringArray(setup.completed_steps, []) } - : {}), - }, - } - : {}), + ? { + setup: { + database_connection_ids: stringArray(setup.database_connection_ids, []), + }, + } + : {}), connections: isRecord(parsed.connections) ? (parsed.connections as Record) : defaults.connections, diff --git a/packages/context/src/project/index.ts b/packages/context/src/project/index.ts index 8fd171d4..8ea92bf6 100644 --- a/packages/context/src/project/index.ts +++ b/packages/context/src/project/index.ts @@ -27,12 +27,10 @@ export { initKtxProject, loadKtxProject } from './project.js'; export type { KtxSetupStep } from './setup-config.js'; export { KTX_SETUP_STEPS, - ktxSetupCompletedSteps, ktxSetupStatePath, markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, readKtxSetupState, setKtxSetupDatabaseConnectionIds, - stripKtxSetupCompletedSteps, writeKtxSetupState, } from './setup-config.js'; diff --git a/packages/context/src/project/setup-config.test.ts b/packages/context/src/project/setup-config.test.ts index 46912d43..92c02707 100644 --- a/packages/context/src/project/setup-config.test.ts +++ b/packages/context/src/project/setup-config.test.ts @@ -4,12 +4,10 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { buildDefaultKtxProjectConfig } from './config.js'; import { - ktxSetupCompletedSteps, markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, readKtxSetupState, setKtxSetupDatabaseConnectionIds, - stripKtxSetupCompletedSteps, } from './setup-config.js'; describe('KTX setup config helpers', () => { @@ -48,36 +46,6 @@ describe('KTX setup config helpers', () => { expect(config.setup).toBeUndefined(); }); - it('strips setup completed steps while preserving database connection ids', () => { - const config = { - ...buildDefaultKtxProjectConfig('warehouse'), - setup: { - database_connection_ids: ['warehouse'], - completed_steps: ['project', 'databases'], - }, - }; - - expect(stripKtxSetupCompletedSteps(config).setup).toEqual({ - database_connection_ids: ['warehouse'], - }); - }); - - it('combines legacy config setup steps with local state for reads', () => { - const config = { - ...buildDefaultKtxProjectConfig('warehouse'), - setup: { - database_connection_ids: ['warehouse'], - completed_steps: ['project', 'databases'], - }, - }; - - expect(ktxSetupCompletedSteps(config, { completed_steps: ['databases', 'sources'] })).toEqual([ - 'project', - 'databases', - 'sources', - ]); - }); - it('merges setup-local gitignore entries without removing existing lines', () => { expect(mergeKtxSetupGitignoreEntries('cache/\ndb.sqlite\n')).toBe( ['cache/', 'db.sqlite', 'db.sqlite-*', 'ingest-transcripts/', 'secrets/', 'setup/', 'agents/', ''].join('\n'), diff --git a/packages/context/src/project/setup-config.ts b/packages/context/src/project/setup-config.ts index be1e8817..b2c8e161 100644 --- a/packages/context/src/project/setup-config.ts +++ b/packages/context/src/project/setup-config.ts @@ -64,27 +64,6 @@ export async function markKtxSetupStateStepComplete(projectDir: string, step: Kt return nextState; } -export function ktxSetupCompletedSteps(config: KtxProjectConfig, state: KtxSetupState): KtxSetupStep[] { - return uniqueSetupSteps([...(config.setup?.completed_steps ?? []), ...state.completed_steps]); -} - -export function stripKtxSetupCompletedSteps(config: KtxProjectConfig): KtxProjectConfig { - if (!config.setup) { - return config; - } - const databaseConnectionIds = config.setup.database_connection_ids ?? []; - if (databaseConnectionIds.length === 0) { - const { setup: _setup, ...withoutSetup } = config; - return withoutSetup; - } - return { - ...config, - setup: { - database_connection_ids: [...databaseConnectionIds], - }, - }; -} - export function setKtxSetupDatabaseConnectionIds( config: KtxProjectConfig, connectionIds: string[],