From f70271152b4e69d834931180d1c442a3d2603fb4 Mon Sep 17 00:00:00 2001 From: Luca Martial Date: Tue, 12 May 2026 16:26:13 -0700 Subject: [PATCH 1/2] feat(context): add local .ktx/setup/state.json for setup completion tracking Move setup step completion state out of ktx.yaml into a gitignored local state file so it is not committed or shared across machines. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/context/src/project/config.ts | 2 +- packages/context/src/project/index.ts | 7 +- .../context/src/project/setup-config.test.ts | 87 +++++++++++-------- packages/context/src/project/setup-config.ts | 75 +++++++++++++--- 4 files changed, 121 insertions(+), 50 deletions(-) diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index 6d2e8941..f1aa9d71 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -75,7 +75,7 @@ export interface KtxProjectConnectionConfig { export interface KtxProjectSetupConfig { database_connection_ids: string[]; - completed_steps: string[]; + completed_steps?: string[]; } export interface KtxProjectConfig { diff --git a/packages/context/src/project/index.ts b/packages/context/src/project/index.ts index 651e9fb8..8fd171d4 100644 --- a/packages/context/src/project/index.ts +++ b/packages/context/src/project/index.ts @@ -27,7 +27,12 @@ export { initKtxProject, loadKtxProject } from './project.js'; export type { KtxSetupStep } from './setup-config.js'; export { KTX_SETUP_STEPS, - markKtxSetupStepComplete, + ktxSetupCompletedSteps, + ktxSetupStatePath, + markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, + readKtxSetupState, setKtxSetupDatabaseConnectionIds, + stripKtxSetupCompletedSteps, + writeKtxSetupState, } from './setup-config.js'; diff --git a/packages/context/src/project/setup-config.test.ts b/packages/context/src/project/setup-config.test.ts index 212f16e1..46912d43 100644 --- a/packages/context/src/project/setup-config.test.ts +++ b/packages/context/src/project/setup-config.test.ts @@ -1,43 +1,40 @@ -import { describe, expect, it } from 'vitest'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { buildDefaultKtxProjectConfig } from './config.js'; import { - markKtxSetupStepComplete, + ktxSetupCompletedSteps, + markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, + readKtxSetupState, setKtxSetupDatabaseConnectionIds, + stripKtxSetupCompletedSteps, } from './setup-config.js'; describe('KTX setup config helpers', () => { - it('marks setup steps complete without duplicating existing state', () => { - const config = buildDefaultKtxProjectConfig('warehouse'); + let tempDir: string; - const withProject = markKtxSetupStepComplete(config, 'project'); - const withProjectAgain = markKtxSetupStepComplete(withProject, 'project'); - const withLlm = markKtxSetupStepComplete(withProjectAgain, 'llm'); - const withContext = markKtxSetupStepComplete(withLlm, 'context'); - - expect(withProject.setup).toEqual({ - database_connection_ids: [], - completed_steps: ['project'], - }); - expect(withProjectAgain.setup?.completed_steps).toEqual(['project']); - expect(withLlm.setup?.completed_steps).toEqual(['project', 'llm']); - expect(withContext.setup?.completed_steps).toEqual(['project', 'llm', 'context']); - expect(config.setup).toBeUndefined(); + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-state-')); }); - it('preserves database connection ids while marking a step complete', () => { - const config = { - ...buildDefaultKtxProjectConfig('warehouse'), - setup: { - database_connection_ids: ['warehouse'], - completed_steps: ['databases'], - }, - }; + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); - expect(markKtxSetupStepComplete(config, 'project').setup).toEqual({ - database_connection_ids: ['warehouse'], - completed_steps: ['databases', 'project'], + it('marks setup steps complete in local state without duplicating existing state', async () => { + await markKtxSetupStateStepComplete(tempDir, 'project'); + await markKtxSetupStateStepComplete(tempDir, 'project'); + await markKtxSetupStateStepComplete(tempDir, 'llm'); + await markKtxSetupStateStepComplete(tempDir, 'context'); + + expect(await readKtxSetupState(tempDir)).toEqual({ + completed_steps: ['project', 'llm', 'context'], }); + await expect(readFile(join(tempDir, '.ktx', 'setup', 'state.json'), 'utf-8')).resolves.toBe( + `${JSON.stringify({ completed_steps: ['project', 'llm', 'context'] }, null, 2)}\n`, + ); }); it('sets setup database connection ids without duplicates', () => { @@ -47,22 +44,38 @@ describe('KTX setup config helpers', () => { expect(withDatabases.setup).toEqual({ database_connection_ids: ['warehouse', 'analytics'], - completed_steps: [], }); expect(config.setup).toBeUndefined(); }); - it('marks databases complete only when requested', () => { - const config = markKtxSetupStepComplete(buildDefaultKtxProjectConfig('warehouse'), 'project'); + it('strips setup completed steps while preserving database connection ids', () => { + const config = { + ...buildDefaultKtxProjectConfig('warehouse'), + setup: { + database_connection_ids: ['warehouse'], + completed_steps: ['project', 'databases'], + }, + }; - const withDatabases = setKtxSetupDatabaseConnectionIds(config, ['warehouse'], { complete: true }); - const withDatabasesAgain = setKtxSetupDatabaseConnectionIds(withDatabases, ['warehouse'], { complete: true }); - - expect(withDatabases.setup).toEqual({ + expect(stripKtxSetupCompletedSteps(config).setup).toEqual({ database_connection_ids: ['warehouse'], - completed_steps: ['project', 'databases'], }); - expect(withDatabasesAgain.setup).toEqual(withDatabases.setup); + }); + + it('combines legacy config setup steps with local state for reads', () => { + const config = { + ...buildDefaultKtxProjectConfig('warehouse'), + setup: { + database_connection_ids: ['warehouse'], + completed_steps: ['project', 'databases'], + }, + }; + + expect(ktxSetupCompletedSteps(config, { completed_steps: ['databases', 'sources'] })).toEqual([ + 'project', + 'databases', + 'sources', + ]); }); it('merges setup-local gitignore entries without removing existing lines', () => { diff --git a/packages/context/src/project/setup-config.ts b/packages/context/src/project/setup-config.ts index 76951ef6..a426caf7 100644 --- a/packages/context/src/project/setup-config.ts +++ b/packages/context/src/project/setup-config.ts @@ -1,9 +1,15 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; import type { KtxProjectConfig } from './config.js'; export const KTX_SETUP_STEPS = ['project', 'llm', 'embeddings', 'databases', 'sources', 'context', 'agents'] as const; export type KtxSetupStep = (typeof KTX_SETUP_STEPS)[number]; +export interface KtxSetupState { + completed_steps: KtxSetupStep[]; +} + const SETUP_GITIGNORE_ENTRIES = [ 'cache/', 'db.sqlite', @@ -14,14 +20,67 @@ const SETUP_GITIGNORE_ENTRIES = [ 'agents/', ] as const; -export function markKtxSetupStepComplete(config: KtxProjectConfig, step: KtxSetupStep): KtxProjectConfig { - const databaseConnectionIds = config.setup?.database_connection_ids ?? []; - const completedSteps = config.setup?.completed_steps ?? []; +function isKtxSetupStep(value: unknown): value is KtxSetupStep { + return typeof value === 'string' && (KTX_SETUP_STEPS as readonly string[]).includes(value); +} + +function uniqueSetupSteps(steps: unknown): KtxSetupStep[] { + if (!Array.isArray(steps)) { + return []; + } + return [...new Set(steps.filter(isKtxSetupStep))]; +} + +export function ktxSetupStatePath(projectDir: string): string { + return join(projectDir, '.ktx', 'setup', 'state.json'); +} + +export async function readKtxSetupState(projectDir: string): Promise { + try { + const parsed = JSON.parse(await readFile(ktxSetupStatePath(projectDir), 'utf-8')) as Record; + return { completed_steps: uniqueSetupSteps(parsed.completed_steps) }; + } catch (error) { + if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') { + return { completed_steps: [] }; + } + throw error; + } +} + +export async function writeKtxSetupState(projectDir: string, state: KtxSetupState): Promise { + await mkdir(join(projectDir, '.ktx', 'setup'), { recursive: true }); + await writeFile( + ktxSetupStatePath(projectDir), + `${JSON.stringify({ completed_steps: uniqueSetupSteps(state.completed_steps) }, null, 2)}\n`, + 'utf-8', + ); +} + +export async function markKtxSetupStateStepComplete(projectDir: string, step: KtxSetupStep): Promise { + const state = await readKtxSetupState(projectDir); + const completedSteps = state.completed_steps.includes(step) ? state.completed_steps : [...state.completed_steps, step]; + const nextState = { completed_steps: completedSteps }; + await writeKtxSetupState(projectDir, nextState); + return nextState; +} + +export function ktxSetupCompletedSteps(config: KtxProjectConfig, state: KtxSetupState): KtxSetupStep[] { + return uniqueSetupSteps([...(config.setup?.completed_steps ?? []), ...state.completed_steps]); +} + +export function stripKtxSetupCompletedSteps(config: KtxProjectConfig): KtxProjectConfig { + if (!config.setup) { + return config; + } + const databaseConnectionIds = config.setup.database_connection_ids ?? []; + if (databaseConnectionIds.length === 0) { + const { setup: _setup, ...withoutSetup } = config; + return withoutSetup; + } return { ...config, setup: { database_connection_ids: [...databaseConnectionIds], - completed_steps: completedSteps.includes(step) ? [...completedSteps] : [...completedSteps, step], }, }; } @@ -29,20 +88,14 @@ export function markKtxSetupStepComplete(config: KtxProjectConfig, step: KtxSetu export function setKtxSetupDatabaseConnectionIds( config: KtxProjectConfig, connectionIds: string[], - options: { complete?: boolean } = {}, ): KtxProjectConfig { const uniqueConnectionIds = [...new Set(connectionIds.filter((connectionId) => connectionId.trim().length > 0))]; - const completedSteps = config.setup?.completed_steps ?? []; - const nextCompletedSteps = - options.complete === true && !completedSteps.includes('databases') - ? [...completedSteps, 'databases'] - : [...completedSteps]; return { ...config, setup: { database_connection_ids: uniqueConnectionIds, - completed_steps: nextCompletedSteps, + ...(config.setup?.completed_steps ? { completed_steps: [...config.setup.completed_steps] } : {}), }, }; } From dbfee6b453e6c025784d95255260483c8d774caf Mon Sep 17 00:00:00 2001 From: Luca Martial Date: Tue, 12 May 2026 16:26:23 -0700 Subject: [PATCH 2/2] feat(cli): migrate all setup steps to use local state for completion tracking Update every setup step to write completed_steps to .ktx/setup/state.json instead of ktx.yaml, stripping legacy entries from config on write. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/setup-agents.ts | 10 ++++++-- packages/cli/src/setup-context.ts | 15 ++++++----- packages/cli/src/setup-databases.test.ts | 11 +++++--- packages/cli/src/setup-databases.ts | 31 +++++++++++++---------- packages/cli/src/setup-embeddings.test.ts | 8 +++--- packages/cli/src/setup-embeddings.ts | 15 ++++++----- packages/cli/src/setup-models.test.ts | 11 +++++--- packages/cli/src/setup-models.ts | 7 ++--- packages/cli/src/setup-project.test.ts | 11 +++++--- packages/cli/src/setup-project.ts | 11 ++++++-- packages/cli/src/setup-sources.test.ts | 7 +++-- packages/cli/src/setup-sources.ts | 14 +++++----- packages/cli/src/setup.test.ts | 5 +++- packages/cli/src/setup.ts | 10 ++++++-- 14 files changed, 105 insertions(+), 61 deletions(-) diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index dad15e7e..c227aedb 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -2,7 +2,12 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { cancel, isCancel, multiselect, select } from '@clack/prompts'; -import { loadKtxProject, markKtxSetupStepComplete, serializeKtxProjectConfig } from '@ktx/context/project'; +import { + loadKtxProject, + markKtxSetupStateStepComplete, + serializeKtxProjectConfig, + stripKtxSetupCompletedSteps, +} from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { withMenuOptionsSpacing, withMultiselectNavigation } from './prompt-navigation.js'; import { withSetupInterruptConfirmation } from './setup-interrupt.js'; @@ -401,7 +406,8 @@ async function installTarget(input: { async function markAgentsComplete(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile(project.configPath, serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'agents')), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8'); + await markKtxSetupStateStepComplete(projectDir, 'agents'); } export async function runKtxSetupAgentsStep( diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index 00928706..ec6268e8 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -5,8 +5,11 @@ import { cancel, isCancel, select } from '@clack/prompts'; import { type KtxLocalProject, loadKtxProject, - markKtxSetupStepComplete, + ktxSetupCompletedSteps, + markKtxSetupStateStepComplete, + readKtxSetupState, serializeKtxProjectConfig, + stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { buildPublicIngestPlan } from './public-ingest.js'; @@ -468,11 +471,8 @@ async function defaultVerifyContextReady(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile( - project.configPath, - serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'context')), - 'utf-8', - ); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8'); + await markKtxSetupStateStepComplete(projectDir, 'context'); } function writeBuildHeader(projectDir: string, runId: string, io: KtxCliIo): void { @@ -715,7 +715,8 @@ export async function runKtxSetupContextStep( try { const project = await loadKtxProject({ projectDir: args.projectDir }); const existingState = await readKtxSetupContextState(args.projectDir); - if (project.config.setup?.completed_steps.includes('context') === true && existingState.status === 'completed') { + const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(args.projectDir)); + if (completedSteps.includes('context') && existingState.status === 'completed') { return { status: 'ready', projectDir: args.projectDir, runId: existingState.runId ?? 'setup-context-completed' }; } diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 2f5d93c6..de91ae3f 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupDatabaseDriver, @@ -1091,8 +1091,9 @@ describe('setup databases step', () => { }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], - completed_steps: ['databases'], + completed_steps: [], }); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases'); expect(io.stdout()).toContain('Primary source ready'); expect(io.stdout()).not.toContain('DATABASE_URL='); }); @@ -1129,8 +1130,9 @@ describe('setup databases step', () => { }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], - completed_steps: ['databases'], + completed_steps: [], }); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases'); }); it('selects multiple existing connections and validates each before recording setup ids', async () => { @@ -1178,7 +1180,8 @@ describe('setup databases step', () => { expect(scanConnection).toHaveBeenCalledTimes(2); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'analytics']); - expect(config.setup?.completed_steps).toContain('databases'); + expect(config.setup?.completed_steps).toEqual([]); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases'); }); it('keeps the connection config but does not mark databases complete when scanning fails', async () => { diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 113fd048..080970cc 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -4,8 +4,10 @@ import type { HistoricSqlDialect } from '@ktx/context/ingest'; import { type KtxProjectConnectionConfig, loadKtxProject, + markKtxSetupStateStepComplete, serializeKtxProjectConfig, setKtxSetupDatabaseConnectionIds, + stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnection } from './connection.js'; @@ -923,7 +925,7 @@ async function writeConnectionConfig(input: { [input.connectionId]: input.connection, }, }; - await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); const historicSql = typeof input.connection.historicSql === 'object' && @@ -1076,25 +1078,28 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds), { complete: true }); - await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds)); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); + await markKtxSetupStateStepComplete(projectDir, 'databases'); } async function maybeRunHistoricSqlSetupProbe(input: { diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index ce3618e7..525ba733 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js'; @@ -166,7 +166,8 @@ describe('setup embeddings step', () => { sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, }); expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); - expect(config.setup?.completed_steps).toContain('embeddings'); + expect(config.setup?.completed_steps).toEqual(undefined); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings'); expect(io.stdout()).toContain( 'Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.', ); @@ -238,7 +239,8 @@ describe('setup embeddings step', () => { sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' }, }); expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings); - expect(config.setup?.completed_steps).toContain('embeddings'); + expect(config.setup?.completed_steps).toEqual(undefined); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings'); }); it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => { diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index acb97eaa..409ce28a 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -4,9 +4,12 @@ import { resolveKtxConfigReference } from '@ktx/context/core'; import { type KtxProjectConfig, type KtxProjectEmbeddingConfig, + ktxSetupCompletedSteps, loadKtxProject, - markKtxSetupStepComplete, + markKtxSetupStateStepComplete, + readKtxSetupState, serializeKtxProjectConfig, + stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm'; import type { KtxCliIo } from './cli-runtime.js'; @@ -111,9 +114,9 @@ function createPromptAdapter(): KtxSetupEmbeddingsPromptAdapter { }; } -function hasCompletedEmbeddings(config: KtxProjectConfig): boolean { +async function hasCompletedEmbeddings(projectDir: string, config: KtxProjectConfig): Promise { return ( - config.setup?.completed_steps.includes('embeddings') === true && + ktxSetupCompletedSteps(config, await readKtxSetupState(projectDir)).includes('embeddings') && config.ingest.embeddings.backend !== 'none' && config.ingest.embeddings.backend !== 'deterministic' && typeof config.ingest.embeddings.model === 'string' && @@ -187,7 +190,7 @@ function embeddingBackendDisplayName(backend: KtxSetupEmbeddingBackend): string async function persistEmbeddingConfig(projectDir: string, embeddings: KtxProjectEmbeddingConfig): Promise { const project = await loadKtxProject({ projectDir }); - const config = markKtxSetupStepComplete( + const config = stripKtxSetupCompletedSteps( { ...project.config, ingest: { @@ -202,9 +205,9 @@ async function persistEmbeddingConfig(projectDir: string, embeddings: KtxProject }, }, }, - 'embeddings', ); await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + await markKtxSetupStateStepComplete(projectDir, 'embeddings'); } async function chooseCredentialRef( @@ -400,7 +403,7 @@ export async function runKtxSetupEmbeddingsStep( const project = await loadKtxProject({ projectDir: args.projectDir }); if ( args.forcePrompt !== true && - hasCompletedEmbeddings(project.config) && + (await hasCompletedEmbeddings(args.projectDir, project.config)) && !args.embeddingBackend && !args.embeddingApiKeyEnv && !args.embeddingApiKeyFile diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index 96092b25..0c7686f7 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BUNDLED_ANTHROPIC_MODELS, @@ -160,7 +160,8 @@ describe('setup Anthropic model step', () => { promptCaching: { enabled: true }, }); expect(config.scan.enrichment.mode).toBe('llm'); - expect(config.setup?.completed_steps).toContain('llm'); + expect(config.setup?.completed_steps).toEqual(undefined); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); @@ -198,7 +199,8 @@ describe('setup Anthropic model step', () => { }, models: { default: 'claude-sonnet-4-6' }, }); - expect(config.setup?.completed_steps).toContain('llm'); + expect(config.setup?.completed_steps).toEqual(undefined); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stdout()).not.toContain('sk-ant-file'); }); @@ -551,7 +553,8 @@ describe('setup Anthropic model step', () => { expect(io.stderr()).toContain('Choose a different credential source or model, or Back.'); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.llm.models.default).toBe('claude-sonnet-4-6'); - expect(config.setup?.completed_steps).toContain('llm'); + expect(config.setup?.completed_steps).toEqual(undefined); + expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); expect(io.stderr()).not.toContain('sk-ant-test'); }); diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index 5b0dea18..843691cd 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -6,8 +6,9 @@ import { type KtxProjectConfig, type KtxProjectLlmConfig, loadKtxProject, - markKtxSetupStepComplete, + markKtxSetupStateStepComplete, serializeKtxProjectConfig, + stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; import type { KtxCliIo } from './cli-runtime.js'; @@ -361,7 +362,7 @@ async function chooseModel( async function persistLlmConfig(projectDir: string, credentialRef: string, model: string): Promise { const project = await loadKtxProject({ projectDir }); - const config = markKtxSetupStepComplete( + const config = stripKtxSetupCompletedSteps( { ...project.config, llm: buildProjectLlmConfig(project.config.llm, credentialRef, model), @@ -373,9 +374,9 @@ async function persistLlmConfig(projectDir: string, credentialRef: string, model }, }, }, - 'llm', ); await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + await markKtxSetupStateStepComplete(projectDir, 'llm'); } function buildInteractiveRetryArgs(args: KtxSetupModelArgs): KtxSetupModelArgs { diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index 6c75d554..3dd1b0cd 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project'; +import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from './setup-project.js'; @@ -60,7 +60,8 @@ describe('setup project step', () => { expect(result.status).toBe('ready'); expect(result.projectDir).toBe(projectDir); const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps).toEqual(['project']); + expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] }); await expect(stat(join(projectDir, '.git'))).resolves.toBeDefined(); await expect(readFile(join(projectDir, '.ktx/.gitignore'), 'utf-8')).resolves.toContain('secrets/'); expect(testIo.stdout()).toContain(`Project: ${projectDir}`); @@ -93,8 +94,9 @@ describe('setup project step', () => { const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], - completed_steps: ['llm', 'project'], + completed_steps: [], }); + expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['llm', 'project'] }); }); it('creates a missing auto-mode project only when --yes is present in no-input mode', async () => { @@ -150,7 +152,8 @@ describe('setup project step', () => { ); expect(prompts.text).not.toHaveBeenCalled(); const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); - expect(config.setup?.completed_steps).toEqual(['project']); + expect(config.setup?.completed_steps).toEqual(undefined); + expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] }); }); it('offers an absolute default destination for a new project folder', async () => { diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index 094f3f3f..d164e41d 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -5,11 +5,15 @@ import { basename, join, resolve } from 'node:path'; import { cancel, isCancel, select, text } from '@clack/prompts'; import { initKtxProject, + ktxSetupCompletedSteps, type KtxLocalProject, loadKtxProject, - markKtxSetupStepComplete, + markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, + readKtxSetupState, serializeKtxProjectConfig, + stripKtxSetupCompletedSteps, + writeKtxSetupState, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js'; @@ -117,8 +121,11 @@ async function normalizeSetupGitignore(projectDir: string): Promise { } async function persistProjectStep(project: KtxLocalProject): Promise { - const config = markKtxSetupStepComplete(project.config, 'project'); + const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(project.projectDir)); + const config = stripKtxSetupCompletedSteps(project.config); await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + await writeKtxSetupState(project.projectDir, { completed_steps: completedSteps }); + await markKtxSetupStateStepComplete(project.projectDir, 'project'); await normalizeSetupGitignore(project.projectDir); return await loadKtxProject({ projectDir: project.projectDir }); } diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index c74dd642..b75e2f65 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -5,6 +5,7 @@ import { initKtxProject, type KtxProjectConnectionConfig, parseKtxProjectConfig, + readKtxSetupState, serializeKtxProjectConfig, } from '@ktx/context/project'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -136,7 +137,8 @@ describe('setup sources step', () => { projectDir, }); - expect((await readConfig()).setup?.completed_steps).toContain('sources'); + expect((await readConfig()).setup?.completed_steps).toEqual(undefined); + expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources'); expect(io.stdout()).toContain('Context source setup skipped.'); }); @@ -169,7 +171,8 @@ describe('setup sources step', () => { source_dir: '/repo/dbt', project_name: 'analytics', }); - expect(config.setup?.completed_steps).toContain('sources'); + expect(config.setup?.completed_steps).toEqual([]); + expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources'); expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' }); }); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 3393b838..42b7a7a9 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -23,8 +23,9 @@ import { type KtxProjectConfig, type KtxProjectConnectionConfig, loadKtxProject, - markKtxSetupStepComplete, + markKtxSetupStateStepComplete, serializeKtxProjectConfig, + stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnectionMapping } from './commands/connection-mapping.js'; @@ -333,7 +334,7 @@ function fileRepoUrl(sourceDir: string): string { async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); } async function writeSourceConnection( @@ -360,7 +361,7 @@ async function writeSourceConnection( : [...project.config.ingest.adapters, adapter], }, }; - await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8'); return async () => { const latest = await loadKtxProject({ projectDir }); const connections = { ...latest.config.connections }; @@ -399,11 +400,8 @@ async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSo async function markSourcesComplete(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - await writeFile( - project.configPath, - serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'sources')), - 'utf-8', - ); + await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8'); + await markKtxSetupStateStepComplete(projectDir, 'sources'); } function hasPrimarySource(config: KtxProjectConfig): boolean { diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index fa9a1197..b95cf383 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -847,7 +847,10 @@ describe('setup status', () => { ).resolves.toBe(0); await expect(stat(join(tempDir, 'ktx.yaml'))).resolves.toBeDefined(); - expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).toContain('completed_steps:'); + expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); + await expect(readFile(join(tempDir, '.ktx', 'setup', 'state.json'), 'utf-8')).resolves.toBe( + `${JSON.stringify({ completed_steps: ['project', 'sources'] }, null, 2)}\n`, + ); expect(testIo.stdout()).toContain('KTX setup'); expect(testIo.stdout()).toContain(`Project: ${tempDir}`); expect(testIo.stdout()).toContain('Project ready: yes'); diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index c2ab113e..7950eec6 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -2,7 +2,13 @@ import { existsSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { cancel, isCancel, select } from '@clack/prompts'; import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest'; -import { ktxLocalStateDbPath, loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import { + ktxLocalStateDbPath, + ktxSetupCompletedSteps, + loadKtxProject, + readKtxSetupState, + type KtxLocalProject, +} from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { formatSetupNextStepLines } from './next-steps.js'; import { isKtxSetupExitError, withSetupInterruptConfirmation } from './setup-interrupt.js'; @@ -303,7 +309,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise