Merge branch 'main' into python-dependency-updates

packages/cli/src/setup-project.ts

@@ -24,17 +24,12 @@ export interface KtxSetupProjectArgs {
   allowBack?: boolean;
 }
 
-export type KtxSetupCreatedProjectCleanup =
-  | { kind: 'remove-project-dir'; projectDir: string }
-  | { kind: 'remove-ktx-scaffold'; projectDir: string };
-
 export type KtxSetupProjectResult =
   | {
       status: 'ready';
       projectDir: string;
       project: KtxLocalProject;
       confirmedCreation?: boolean;
-      createdProjectCleanup?: KtxSetupCreatedProjectCleanup;
     }
   | { status: 'back'; projectDir: string }
   | { status: 'cancelled'; projectDir: string }
@@ -59,7 +54,6 @@ type PromptProjectDirResult =
       status: 'selected';
       projectDir: string;
       confirmedCreation: boolean;
-      createdProjectCleanup?: KtxSetupCreatedProjectCleanup;
     }
   | { status: 'cancelled'; projectDir: string }
   | { status: 'missing-input'; projectDir: string }
@@ -106,26 +100,12 @@ type ConfirmProjectDirResult =
   | {
       status: 'confirmed';
       confirmedCreation: boolean;
-      createdProjectCleanup?: KtxSetupCreatedProjectCleanup;
     }
   | { status: 'choose-another' }
   | { status: 'back' }
   | { status: 'cancelled' }
   | { status: 'not-directory' };
 
-function cleanupForFolderState(
-  projectDir: string,
-  state: Awaited<ReturnType<typeof existingFolderState>>,
-): KtxSetupCreatedProjectCleanup | undefined {
-  if (state === 'missing') {
-    return { kind: 'remove-project-dir', projectDir };
-  }
-  if (state === 'empty-directory') {
-    return { kind: 'remove-ktx-scaffold', projectDir };
-  }
-  return undefined;
-}
-
 async function confirmProjectDir(
   selectedDir: string,
   io: KtxCliIo,
@@ -165,7 +145,7 @@ async function confirmProjectDir(
   if (action === 'choose-another') return { status: 'choose-another' };
   if (action === 'back') return { status: 'back' };
   if (action !== 'create') return { status: 'cancelled' };
-  return { status: 'confirmed', confirmedCreation: true, createdProjectCleanup: cleanupForFolderState(selectedDir, state) };
+  return { status: 'confirmed', confirmedCreation: true };
 }
 
 async function normalizeSetupGitignore(projectDir: string): Promise<void> {
@@ -252,24 +232,10 @@ async function promptForNewProjectDir(
       status: 'selected',
       projectDir: selectedDir,
       confirmedCreation: confirmed.confirmedCreation,
-      ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}),
     };
   }
 }
 
-async function createProjectWithCleanup(
-  projectDir: string,
-  deps: KtxSetupProjectDeps,
-): Promise<{ project: KtxLocalProject; createdProjectCleanup?: KtxSetupCreatedProjectCleanup }> {
-  const state = await existingFolderState(projectDir);
-  const project = await createProject(projectDir, deps);
-  const createdProjectCleanup = cleanupForFolderState(projectDir, state);
-  return {
-    project,
-    ...(createdProjectCleanup ? { createdProjectCleanup } : {}),
-  };
-}
-
 export async function runKtxSetupProjectStep(
   args: KtxSetupProjectArgs,
   io: KtxCliIo,
@@ -307,7 +273,6 @@ export async function runKtxSetupProjectStep(
       projectDir: selected.projectDir,
       project,
       confirmedCreation: selected.confirmedCreation,
-      ...(selected.createdProjectCleanup ? { createdProjectCleanup: selected.createdProjectCleanup } : {}),
     };
   }
 
@@ -322,13 +287,12 @@ export async function runKtxSetupProjectStep(
       io.stderr.write('Missing setup choice: pass --yes to create a project in non-interactive setup.\n');
       return { status: 'missing-input', projectDir };
     }
-    const { project, createdProjectCleanup } = await createProjectWithCleanup(projectDir, deps);
+    const project = await createProject(projectDir, deps);
     printProjectSummary(io, projectDir);
     return {
       status: 'ready',
       projectDir,
       project,
-      ...(createdProjectCleanup ? { createdProjectCleanup } : {}),
     };
   }
 
@@ -368,13 +332,12 @@ export async function runKtxSetupProjectStep(
     }
 
     if (choice === 'current') {
-      const { project, createdProjectCleanup } = await createProjectWithCleanup(projectDir, deps);
+      const project = await createProject(projectDir, deps);
       printProjectSummary(io, projectDir);
       return {
         status: 'ready',
         projectDir,
         project,
-        ...(createdProjectCleanup ? { createdProjectCleanup } : {}),
       };
     }
 
@@ -390,7 +353,6 @@ export async function runKtxSetupProjectStep(
         projectDir: defaultProjectDir,
         project,
         confirmedCreation: confirmed.confirmedCreation,
-        ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}),
       };
     }
 
@@ -419,7 +381,6 @@ export async function runKtxSetupProjectStep(
         projectDir: customDir,
         project,
         confirmedCreation: confirmed.confirmedCreation,
-        ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}),
       };
     }
 

packages/cli/src/setup.ts

@@ -1,5 +1,4 @@
 import { existsSync } from 'node:fs';
-import { rm } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
 import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js';
 import { savedMemoryCountsForReport } from './context/ingest/reports.js';
@@ -32,11 +31,7 @@ import {
   isKtxSetupLlmConfigReady,
   runKtxSetupAnthropicModelStep,
 } from './setup-models.js';
-import {
-  type KtxSetupCreatedProjectCleanup,
-  type KtxSetupProjectDeps,
-  runKtxSetupProjectStep,
-} from './setup-project.js';
+import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js';
 import {
   isKtxPreAgentSetupReady,
   isKtxSetupReady,
@@ -556,23 +551,6 @@ async function commitSetupConfigChanges(projectDir: string): Promise<void> {
   await project.git.commitFile('ktx.yaml', 'setup: update KTX project config', 'ktx setup', 'setup@ktx.local');
 }
 
-const KTX_SETUP_SCAFFOLD_PATHS = ['ktx.yaml', '.ktx', 'wiki', 'semantic-layer', 'raw-sources', '.git'];
-
-async function cleanupCreatedProjectScaffold(cleanup: KtxSetupCreatedProjectCleanup | undefined): Promise<void> {
-  if (!cleanup) {
-    return;
-  }
-  if (cleanup.kind === 'remove-project-dir') {
-    await rm(cleanup.projectDir, { recursive: true, force: true });
-    return;
-  }
-  await Promise.all(
-    KTX_SETUP_SCAFFOLD_PATHS.map((relativePath) =>
-      rm(join(cleanup.projectDir, relativePath), { recursive: true, force: true }),
-    ),
-  );
-}
-
 export async function runKtxSetup(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetupDeps = {}): Promise<number> {
   try {
     return await runKtxSetupInner(args, io, deps);
@@ -869,7 +847,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
       });
 
       if (stepResult.status === 'failed') {
-        await cleanupCreatedProjectScaffold(projectResult.createdProjectCleanup);
         return 1;
       }
       if (stepResult.status === 'missing-input') {

packages/cli/test/setup.test.ts

@@ -1,5 +1,5 @@
 import { execFile } from 'node:child_process';
-import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { promisify } from 'node:util';
@@ -602,7 +602,7 @@ describe('setup status', () => {
     expect(testIo.stderr()).toBe('');
   });
 
-  it('removes a newly created missing project directory when a later runtime step fails', async () => {
+  it('preserves a newly created missing project directory when a later setup step fails', async () => {
     const projectDir = join(tempDir, 'missing-project');
     const testIo = makeIo();
 
@@ -634,10 +634,12 @@ describe('setup status', () => {
       ),
     ).resolves.toBe(1);
 
-    await expect(stat(projectDir)).rejects.toThrow();
+    await expect(stat(projectDir)).resolves.toBeDefined();
+    await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined();
+    await expect(stat(join(projectDir, '.ktx'))).resolves.toBeDefined();
   });
 
-  it('removes KTX scaffold files from an initially empty project directory when runtime setup fails', async () => {
+  it('preserves KTX scaffold files in an initially empty project directory when setup fails', async () => {
     const testIo = makeIo();
 
     await expect(
@@ -668,8 +670,59 @@ describe('setup status', () => {
       ),
     ).resolves.toBe(1);
 
-    await expect(stat(tempDir)).resolves.toBeDefined();
-    expect(await readdir(tempDir)).toEqual([]);
+    await expect(stat(join(tempDir, 'ktx.yaml'))).resolves.toBeDefined();
+    await expect(stat(join(tempDir, '.ktx'))).resolves.toBeDefined();
+  });
+
+  it('preserves partial context-build artifacts and resume state when the context step fails', async () => {
+    const projectDir = join(tempDir, 'partial-context');
+    const testIo = makeIo();
+
+    await expect(
+      runKtxSetup(
+        {
+          command: 'run',
+          projectDir,
+          mode: 'auto',
+          agents: false,
+          skipAgents: true,
+          inputMode: 'disabled',
+          yes: true,
+          cliVersion: '0.2.0',
+          skipLlm: true,
+          skipEmbeddings: true,
+          databaseSchemas: [],
+          skipDatabases: true,
+          skipSources: true,
+        },
+        testIo.io,
+        {
+          model: async () => ({ status: 'skipped', projectDir }),
+          embeddings: async () => ({ status: 'skipped', projectDir }),
+          databases: async () => ({ status: 'skipped', projectDir }),
+          sources: async () => ({ status: 'skipped', projectDir }),
+          runtime: async () => runtimeReady(projectDir),
+          context: async () => {
+            await mkdir(join(projectDir, '.ktx', 'setup'), { recursive: true });
+            await writeFile(
+              join(projectDir, '.ktx', 'setup', 'state.json'),
+              JSON.stringify({ status: 'failed', retryableFailedTargets: [{ source: 'metabase' }] }),
+              'utf-8',
+            );
+            await mkdir(join(projectDir, 'wiki'), { recursive: true });
+            await writeFile(join(projectDir, 'wiki', 'postgres-warehouse.md'), '# warehouse\n', 'utf-8');
+            await mkdir(join(projectDir, 'semantic-layer'), { recursive: true });
+            await writeFile(join(projectDir, 'semantic-layer', 'orders.yaml'), 'name: orders\n', 'utf-8');
+            return { status: 'failed', projectDir };
+          },
+        },
+      ),
+    ).resolves.toBe(1);
+
+    await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined();
+    await expect(readFile(join(projectDir, '.ktx', 'setup', 'state.json'), 'utf-8')).resolves.toContain('"status":"failed"');
+    await expect(readFile(join(projectDir, 'wiki', 'postgres-warehouse.md'), 'utf-8')).resolves.toContain('warehouse');
+    await expect(readFile(join(projectDir, 'semantic-layer', 'orders.yaml'), 'utf-8')).resolves.toContain('orders');
   });
 
   it('preserves a pre-existing non-empty project directory when runtime setup fails', async () => {