feat(cli): migrate all setup steps to use local state for completion tracking

Update every setup step to write completed_steps to .ktx/setup/state.json
instead of ktx.yaml, stripping legacy entries from config on write.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Luca Martial 2026-05-12 16:26:23 -07:00
parent f70271152b
commit dbfee6b453
14 changed files with 105 additions and 61 deletions

View file

@ -2,7 +2,12 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { cancel, isCancel, multiselect, select } from '@clack/prompts';
import { loadKtxProject, markKtxSetupStepComplete, serializeKtxProjectConfig } from '@ktx/context/project';
import {
loadKtxProject,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { withMenuOptionsSpacing, withMultiselectNavigation } from './prompt-navigation.js';
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
@ -401,7 +406,8 @@ async function installTarget(input: {
async function markAgentsComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'agents')), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'agents');
}
export async function runKtxSetupAgentsStep(

View file

@ -5,8 +5,11 @@ import { cancel, isCancel, select } from '@clack/prompts';
import {
type KtxLocalProject,
loadKtxProject,
markKtxSetupStepComplete,
ktxSetupCompletedSteps,
markKtxSetupStateStepComplete,
readKtxSetupState,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { buildPublicIngestPlan } from './public-ingest.js';
@ -468,11 +471,8 @@ async function defaultVerifyContextReady(projectDir: string): Promise<KtxSetupCo
async function markContextComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(
project.configPath,
serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'context')),
'utf-8',
);
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'context');
}
function writeBuildHeader(projectDir: string, runId: string, io: KtxCliIo): void {
@ -715,7 +715,8 @@ export async function runKtxSetupContextStep(
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
const existingState = await readKtxSetupContextState(args.projectDir);
if (project.config.setup?.completed_steps.includes('context') === true && existingState.status === 'completed') {
const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(args.projectDir));
if (completedSteps.includes('context') && existingState.status === 'completed') {
return { status: 'ready', projectDir: args.projectDir, runId: existingState.runId ?? 'setup-context-completed' };
}

View file

@ -1,7 +1,7 @@
import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
type KtxSetupDatabaseDriver,
@ -1091,8 +1091,9 @@ describe('setup databases step', () => {
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: ['databases'],
completed_steps: [],
});
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
expect(io.stdout()).toContain('Primary source ready');
expect(io.stdout()).not.toContain('DATABASE_URL=');
});
@ -1129,8 +1130,9 @@ describe('setup databases step', () => {
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: ['databases'],
completed_steps: [],
});
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
});
it('selects multiple existing connections and validates each before recording setup ids', async () => {
@ -1178,7 +1180,8 @@ describe('setup databases step', () => {
expect(scanConnection).toHaveBeenCalledTimes(2);
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'analytics']);
expect(config.setup?.completed_steps).toContain('databases');
expect(config.setup?.completed_steps).toEqual([]);
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
});
it('keeps the connection config but does not mark databases complete when scanning fails', async () => {

View file

@ -4,8 +4,10 @@ import type { HistoricSqlDialect } from '@ktx/context/ingest';
import {
type KtxProjectConnectionConfig,
loadKtxProject,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
setKtxSetupDatabaseConnectionIds,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { runKtxConnection } from './connection.js';
@ -923,7 +925,7 @@ async function writeConnectionConfig(input: {
[input.connectionId]: input.connection,
},
};
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
const historicSql =
typeof input.connection.historicSql === 'object' &&
@ -1076,25 +1078,28 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void
}
await writeFile(
project.configPath,
serializeKtxProjectConfig({
...project.config,
ingest: {
...project.config.ingest,
adapters,
workUnits: {
...project.config.ingest.workUnits,
maxConcurrency,
serializeKtxProjectConfig(
stripKtxSetupCompletedSteps({
...project.config,
ingest: {
...project.config.ingest,
adapters,
workUnits: {
...project.config.ingest.workUnits,
maxConcurrency,
},
},
},
}),
}),
),
'utf-8',
);
}
async function markDatabasesComplete(projectDir: string, connectionIds: string[]): Promise<void> {
const project = await loadKtxProject({ projectDir });
const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds), { complete: true });
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds));
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'databases');
}
async function maybeRunHistoricSqlSetupProbe(input: {

View file

@ -1,7 +1,7 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js';
@ -166,7 +166,8 @@ describe('setup embeddings step', () => {
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(config.setup?.completed_steps).toContain('embeddings');
expect(config.setup?.completed_steps).toEqual(undefined);
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
expect(io.stdout()).toContain(
'Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.',
);
@ -238,7 +239,8 @@ describe('setup embeddings step', () => {
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(config.setup?.completed_steps).toContain('embeddings');
expect(config.setup?.completed_steps).toEqual(undefined);
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
});
it('fails non-interactive local setup when the managed local embeddings runtime is missing', async () => {

View file

@ -4,9 +4,12 @@ import { resolveKtxConfigReference } from '@ktx/context/core';
import {
type KtxProjectConfig,
type KtxProjectEmbeddingConfig,
ktxSetupCompletedSteps,
loadKtxProject,
markKtxSetupStepComplete,
markKtxSetupStateStepComplete,
readKtxSetupState,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
@ -111,9 +114,9 @@ function createPromptAdapter(): KtxSetupEmbeddingsPromptAdapter {
};
}
function hasCompletedEmbeddings(config: KtxProjectConfig): boolean {
async function hasCompletedEmbeddings(projectDir: string, config: KtxProjectConfig): Promise<boolean> {
return (
config.setup?.completed_steps.includes('embeddings') === true &&
ktxSetupCompletedSteps(config, await readKtxSetupState(projectDir)).includes('embeddings') &&
config.ingest.embeddings.backend !== 'none' &&
config.ingest.embeddings.backend !== 'deterministic' &&
typeof config.ingest.embeddings.model === 'string' &&
@ -187,7 +190,7 @@ function embeddingBackendDisplayName(backend: KtxSetupEmbeddingBackend): string
async function persistEmbeddingConfig(projectDir: string, embeddings: KtxProjectEmbeddingConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
const config = markKtxSetupStepComplete(
const config = stripKtxSetupCompletedSteps(
{
...project.config,
ingest: {
@ -202,9 +205,9 @@ async function persistEmbeddingConfig(projectDir: string, embeddings: KtxProject
},
},
},
'embeddings',
);
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'embeddings');
}
async function chooseCredentialRef(
@ -400,7 +403,7 @@ export async function runKtxSetupEmbeddingsStep(
const project = await loadKtxProject({ projectDir: args.projectDir });
if (
args.forcePrompt !== true &&
hasCompletedEmbeddings(project.config) &&
(await hasCompletedEmbeddings(args.projectDir, project.config)) &&
!args.embeddingBackend &&
!args.embeddingApiKeyEnv &&
!args.embeddingApiKeyFile

View file

@ -1,7 +1,7 @@
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
BUNDLED_ANTHROPIC_MODELS,
@ -160,7 +160,8 @@ describe('setup Anthropic model step', () => {
promptCaching: { enabled: true },
});
expect(config.scan.enrichment.mode).toBe('llm');
expect(config.setup?.completed_steps).toContain('llm');
expect(config.setup?.completed_steps).toEqual(undefined);
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stdout()).toContain('LLM ready: yes');
expect(io.stdout()).not.toContain('sk-ant-test');
});
@ -198,7 +199,8 @@ describe('setup Anthropic model step', () => {
},
models: { default: 'claude-sonnet-4-6' },
});
expect(config.setup?.completed_steps).toContain('llm');
expect(config.setup?.completed_steps).toEqual(undefined);
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stdout()).not.toContain('sk-ant-file');
});
@ -551,7 +553,8 @@ describe('setup Anthropic model step', () => {
expect(io.stderr()).toContain('Choose a different credential source or model, or Back.');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.llm.models.default).toBe('claude-sonnet-4-6');
expect(config.setup?.completed_steps).toContain('llm');
expect(config.setup?.completed_steps).toEqual(undefined);
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stderr()).not.toContain('sk-ant-test');
});

View file

@ -6,8 +6,9 @@ import {
type KtxProjectConfig,
type KtxProjectLlmConfig,
loadKtxProject,
markKtxSetupStepComplete,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
@ -361,7 +362,7 @@ async function chooseModel(
async function persistLlmConfig(projectDir: string, credentialRef: string, model: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
const config = markKtxSetupStepComplete(
const config = stripKtxSetupCompletedSteps(
{
...project.config,
llm: buildProjectLlmConfig(project.config.llm, credentialRef, model),
@ -373,9 +374,9 @@ async function persistLlmConfig(projectDir: string, credentialRef: string, model
},
},
},
'llm',
);
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'llm');
}
function buildInteractiveRetryArgs(args: KtxSetupModelArgs): KtxSetupModelArgs {

View file

@ -1,7 +1,7 @@
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from './setup-project.js';
@ -60,7 +60,8 @@ describe('setup project step', () => {
expect(result.status).toBe('ready');
expect(result.projectDir).toBe(projectDir);
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps).toEqual(['project']);
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] });
await expect(stat(join(projectDir, '.git'))).resolves.toBeDefined();
await expect(readFile(join(projectDir, '.ktx/.gitignore'), 'utf-8')).resolves.toContain('secrets/');
expect(testIo.stdout()).toContain(`Project: ${projectDir}`);
@ -93,8 +94,9 @@ describe('setup project step', () => {
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: ['llm', 'project'],
completed_steps: [],
});
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['llm', 'project'] });
});
it('creates a missing auto-mode project only when --yes is present in no-input mode', async () => {
@ -150,7 +152,8 @@ describe('setup project step', () => {
);
expect(prompts.text).not.toHaveBeenCalled();
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps).toEqual(['project']);
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] });
});
it('offers an absolute default destination for a new project folder', async () => {

View file

@ -5,11 +5,15 @@ import { basename, join, resolve } from 'node:path';
import { cancel, isCancel, select, text } from '@clack/prompts';
import {
initKtxProject,
ktxSetupCompletedSteps,
type KtxLocalProject,
loadKtxProject,
markKtxSetupStepComplete,
markKtxSetupStateStepComplete,
mergeKtxSetupGitignoreEntries,
readKtxSetupState,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
writeKtxSetupState,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js';
@ -117,8 +121,11 @@ async function normalizeSetupGitignore(projectDir: string): Promise<void> {
}
async function persistProjectStep(project: KtxLocalProject): Promise<KtxLocalProject> {
const config = markKtxSetupStepComplete(project.config, 'project');
const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(project.projectDir));
const config = stripKtxSetupCompletedSteps(project.config);
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await writeKtxSetupState(project.projectDir, { completed_steps: completedSteps });
await markKtxSetupStateStepComplete(project.projectDir, 'project');
await normalizeSetupGitignore(project.projectDir);
return await loadKtxProject({ projectDir: project.projectDir });
}

View file

@ -5,6 +5,7 @@ import {
initKtxProject,
type KtxProjectConnectionConfig,
parseKtxProjectConfig,
readKtxSetupState,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
@ -136,7 +137,8 @@ describe('setup sources step', () => {
projectDir,
});
expect((await readConfig()).setup?.completed_steps).toContain('sources');
expect((await readConfig()).setup?.completed_steps).toEqual(undefined);
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(io.stdout()).toContain('Context source setup skipped.');
});
@ -169,7 +171,8 @@ describe('setup sources step', () => {
source_dir: '/repo/dbt',
project_name: 'analytics',
});
expect(config.setup?.completed_steps).toContain('sources');
expect(config.setup?.completed_steps).toEqual([]);
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' });
});

View file

@ -23,8 +23,9 @@ import {
type KtxProjectConfig,
type KtxProjectConnectionConfig,
loadKtxProject,
markKtxSetupStepComplete,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { runKtxConnectionMapping } from './commands/connection-mapping.js';
@ -333,7 +334,7 @@ function fileRepoUrl(sourceDir: string): string {
async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
}
async function writeSourceConnection(
@ -360,7 +361,7 @@ async function writeSourceConnection(
: [...project.config.ingest.adapters, adapter],
},
};
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
return async () => {
const latest = await loadKtxProject({ projectDir });
const connections = { ...latest.config.connections };
@ -399,11 +400,8 @@ async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSo
async function markSourcesComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(
project.configPath,
serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'sources')),
'utf-8',
);
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'sources');
}
function hasPrimarySource(config: KtxProjectConfig): boolean {

View file

@ -847,7 +847,10 @@ describe('setup status', () => {
).resolves.toBe(0);
await expect(stat(join(tempDir, 'ktx.yaml'))).resolves.toBeDefined();
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).toContain('completed_steps:');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
await expect(readFile(join(tempDir, '.ktx', 'setup', 'state.json'), 'utf-8')).resolves.toBe(
`${JSON.stringify({ completed_steps: ['project', 'sources'] }, null, 2)}\n`,
);
expect(testIo.stdout()).toContain('KTX setup');
expect(testIo.stdout()).toContain(`Project: ${tempDir}`);
expect(testIo.stdout()).toContain('Project ready: yes');

View file

@ -2,7 +2,13 @@ import { existsSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { cancel, isCancel, select } from '@clack/prompts';
import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest';
import { ktxLocalStateDbPath, loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import {
ktxLocalStateDbPath,
ktxSetupCompletedSteps,
loadKtxProject,
readKtxSetupState,
type KtxLocalProject,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { formatSetupNextStepLines } from './next-steps.js';
import { isKtxSetupExitError, withSetupInterruptConfirmation } from './setup-interrupt.js';
@ -303,7 +309,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
};
embeddings.ready = embeddingsReady(embeddings);
const completedSteps = project.config.setup?.completed_steps ?? [];
const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(resolvedProjectDir));
const contextState = await readKtxSetupContextState(resolvedProjectDir);
const setupContextStatus = setupContextStatusFromState(contextState, {
completedStep: completedSteps.includes('context'),