Merge remote-tracking branch 'origin/main' into scan-during-setup

packages/cli/src/setup-agents.test.ts

@@ -1,6 +1,7 @@
 import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { readKtxSetupState } from '@ktx/context/project';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import {
   formatInstallSummary,
@@ -89,7 +90,7 @@ describe('setup agents', () => {
       projectDir: tempDir,
       installs: [{ target: 'universal', scope: 'project', mode: 'cli' }],
     });
-    expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).toContain('agents');
+    expect(await readKtxSetupState(tempDir)).toEqual({ completed_steps: ['agents'] });
     expect(io.stderr()).toBe('');
   });
 
@@ -143,7 +144,7 @@ describe('setup agents', () => {
     await expect(readKtxAgentInstallManifest(tempDir)).resolves.toEqual(null);
   });
 
-  it('uses prompts in interactive mode and supports Back', async () => {
+  it('treats cancel as skip in interactive mode', async () => {
     const io = makeIo();
     const prompts = {
       select: vi.fn(async () => 'back'),
@@ -165,7 +166,7 @@ describe('setup agents', () => {
         io.io,
         { prompts },
       ),
-    ).resolves.toEqual({ status: 'back', projectDir: tempDir });
+    ).resolves.toEqual({ status: 'skipped', projectDir: tempDir });
   });
 
   it('explains how to select multiple agent targets in interactive mode', async () => {

packages/cli/src/setup-agents.ts

@@ -1,7 +1,7 @@
 import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { dirname, join, relative, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { cancel, isCancel, multiselect, select } from '@clack/prompts';
+import { cancel, confirm, isCancel, multiselect, select } from '@clack/prompts';
 import {
   loadKtxProject,
   markKtxSetupStateStepComplete,
@@ -277,12 +277,23 @@ function createPromptAdapter(): KtxSetupAgentsPromptAdapter {
       return String(value);
     },
     async multiselect(options) {
-      const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options)));
-      if (isCancel(value)) {
-        cancel('Setup cancelled.');
-        return ['back'];
+      while (true) {
+        const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options)));
+        if (isCancel(value)) {
+          cancel('Setup cancelled.');
+          return ['back'];
+        }
+        const selected = [...value] as string[];
+        if (selected.length === 0 && !options.required) {
+          const skipConfirmed = await confirm({ message: 'Nothing selected. Skip this step?', initialValue: false });
+          if (isCancel(skipConfirmed)) {
+            cancel('Setup cancelled.');
+            return ['back'];
+          }
+          if (!skipConfirmed) continue;
+        }
+        return selected;
       }
-      return [...value] as string[];
     },
     cancel(message) {
       cancel(message);
@@ -391,10 +402,9 @@ export async function runKtxSetupAgentsStep(
           options: [
             { value: 'cli', label: 'CLI tools and skills' },
             { value: 'skip', label: 'Skip' },
-            { value: 'back', label: 'Back' },
           ],
         })) as KtxAgentInstallMode | 'skip' | 'back');
-  if (mode === 'back') return { status: 'back', projectDir: args.projectDir };
+  if (mode === 'back') return { status: 'skipped', projectDir: args.projectDir };
   if (mode === 'skip') return { status: 'skipped', projectDir: args.projectDir };
 
   const targets =

packages/cli/src/setup-databases.test.ts

@@ -571,7 +571,6 @@ describe('setup databases step', () => {
       options: [
         { value: 'continue', label: 'Continue to knowledge sources' },
         { value: 'add', label: 'Add another primary source' },
-        { value: 'back', label: 'Back' },
       ],
     });
     expect(testConnection).not.toHaveBeenCalled();
@@ -622,7 +621,6 @@ describe('setup databases step', () => {
       options: [
         { value: 'continue', label: 'Continue to knowledge sources' },
         { value: 'add', label: 'Add another primary source' },
-        { value: 'back', label: 'Back' },
       ],
     });
     expect(testConnection).toHaveBeenCalledTimes(1);
@@ -657,7 +655,6 @@ describe('setup databases step', () => {
       options: [
         { value: 'continue', label: 'Continue to knowledge sources' },
         { value: 'add', label: 'Add another primary source' },
-        { value: 'back', label: 'Back' },
       ],
     });
     const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
@@ -692,7 +689,6 @@ describe('setup databases step', () => {
       options: [
         { value: 'continue', label: 'Continue to knowledge sources' },
         { value: 'add', label: 'Add another primary source' },
-        { value: 'back', label: 'Back' },
       ],
     });
   });
@@ -735,7 +731,6 @@ describe('setup databases step', () => {
       options: [
         { value: 'continue', label: 'Continue to knowledge sources' },
         { value: 'add', label: 'Add another primary source' },
-        { value: 'back', label: 'Back' },
       ],
     });
   });

packages/cli/src/setup-databases.ts

@@ -1,5 +1,5 @@
 import { writeFile } from 'node:fs/promises';
-import { cancel, isCancel, multiselect, password, select, text } from '@clack/prompts';
+import { cancel, confirm, isCancel, multiselect, password, select, text } from '@clack/prompts';
 import type { HistoricSqlDialect } from '@ktx/context/ingest';
 import {
   type KtxProjectConnectionConfig,
@@ -9,6 +9,7 @@ import {
   setKtxSetupDatabaseConnectionIds,
   stripKtxSetupCompletedSteps,
 } from '@ktx/context/project';
+import type { KtxTableListEntry } from '@ktx/context/scan';
 import type { KtxCliIo } from './cli-runtime.js';
 import { runKtxConnection } from './connection.js';
 import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
@@ -83,6 +84,7 @@ export interface KtxSetupDatabasesDeps {
   testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
   scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
   listSchemas?: (projectDir: string, connectionId: string) => Promise<string[]>;
+  listTables?: (projectDir: string, connectionId: string) => Promise<KtxTableListEntry[]>;
   historicSqlProbe?: KtxSetupHistoricSqlProbe;
 }
 
@@ -203,12 +205,23 @@ function missingConnectionDetailsPrompt(
 function createPromptAdapter(): KtxSetupDatabasesPromptAdapter {
   return {
     async multiselect(options) {
-      const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options)));
-      if (isCancel(value)) {
-        cancel('Setup cancelled.');
-        return ['back'];
+      while (true) {
+        const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options)));
+        if (isCancel(value)) {
+          cancel('Setup cancelled.');
+          return ['back'];
+        }
+        const selected = [...value] as string[];
+        if (selected.length === 0 && !options.required) {
+          const skipConfirmed = await confirm({ message: 'Nothing selected. Skip this step?', initialValue: false });
+          if (isCancel(skipConfirmed)) {
+            cancel('Setup cancelled.');
+            return ['back'];
+          }
+          if (!skipConfirmed) continue;
+        }
+        return selected;
       }
-      return [...value] as string[];
     },
     async select(options) {
       const value = await withSetupInterruptConfirmation(() => select(withMenuOptionsSpacing(options)));
@@ -364,6 +377,89 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro
   return [];
 }
 
+function configuredSchemas(connection: KtxProjectConnectionConfig | undefined, driver: KtxSetupDatabaseDriver): string[] | undefined {
+  if (!connection) return undefined;
+  const spec = SCOPE_DISCOVERY_SPECS[driver];
+  if (!spec) return undefined;
+  const values = configuredScopeValues(connection, spec);
+  return values.length > 0 ? values : undefined;
+}
+
+async function defaultListTables(projectDir: string, connectionId: string): Promise<KtxTableListEntry[]> {
+  const project = await loadKtxProject({ projectDir });
+  const connection = project.config.connections[connectionId];
+  const driver = normalizeDriver(connection?.driver);
+  const schemas = driver ? configuredSchemas(connection, driver) : undefined;
+
+  if (driver === 'postgres') {
+    const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres');
+    if (!isKtxPostgresConnectionConfig(connection)) return [];
+    const connector = new KtxPostgresScanConnector({ connectionId, connection });
+    try {
+      return await connector.listTables(schemas);
+    } finally {
+      await connector.cleanup();
+    }
+  }
+
+  if (driver === 'mysql') {
+    const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('@ktx/connector-mysql');
+    if (!isKtxMysqlConnectionConfig(connection)) return [];
+    const connector = new KtxMysqlScanConnector({ connectionId, connection });
+    try {
+      return await connector.listTables(schemas);
+    } finally {
+      await connector.cleanup();
+    }
+  }
+
+  if (driver === 'sqlserver') {
+    const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver');
+    if (!isKtxSqlServerConnectionConfig(connection)) return [];
+    const connector = new KtxSqlServerScanConnector({ connectionId, connection });
+    try {
+      return await connector.listTables(schemas);
+    } finally {
+      await connector.cleanup();
+    }
+  }
+
+  if (driver === 'bigquery') {
+    const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery');
+    if (!isKtxBigQueryConnectionConfig(connection)) return [];
+    const connector = new KtxBigQueryScanConnector({ connectionId, connection });
+    try {
+      return await connector.listTables(schemas);
+    } finally {
+      await connector.cleanup();
+    }
+  }
+
+  if (driver === 'snowflake') {
+    const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake');
+    if (!isKtxSnowflakeConnectionConfig(connection)) return [];
+    const connector = new KtxSnowflakeScanConnector({ connectionId, connection });
+    try {
+      return await connector.listTables(schemas);
+    } finally {
+      await connector.cleanup();
+    }
+  }
+
+  if (driver === 'clickhouse') {
+    const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('@ktx/connector-clickhouse');
+    if (!isKtxClickHouseConnectionConfig(connection)) return [];
+    const connector = new KtxClickHouseScanConnector({ connectionId, connection });
+    try {
+      return await connector.listTables(schemas);
+    } finally {
+      await connector.cleanup();
+    }
+  }
+
+  return [];
+}
+
 function existingConnectionIdsByDriver(
   connections: Record<string, KtxProjectConnectionConfig>,
   driver: KtxSetupDatabaseDriver,
@@ -400,7 +496,6 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): {
     options: [
       { value: 'continue', label: 'Continue to knowledge sources' },
       { value: 'add', label: 'Add another primary source' },
-      { value: 'back', label: 'Back' },
     ],
   };
 }
@@ -975,6 +1070,22 @@ async function writeScopeConfig(input: {
   });
 }
 
+async function clearScopeConfig(projectDir: string, connectionId: string): Promise<void> {
+  const project = await loadKtxProject({ projectDir });
+  const connection = project.config.connections[connectionId];
+  if (!connection) return;
+  const driver = normalizeDriver(connection.driver);
+  if (!driver) return;
+  const spec = SCOPE_DISCOVERY_SPECS[driver];
+  if (!spec) return;
+  const cleaned = Object.fromEntries(
+    Object.entries(connection).filter(
+      ([key]) => key !== spec.configArrayField && key !== spec.configSingleField && key !== 'enabled_tables',
+    ),
+  ) as KtxProjectConnectionConfig;
+  await writeConnectionConfig({ projectDir, connectionId, connection: cleaned });
+}
+
 async function maybeConfigureSchemaScope(input: {
   projectDir: string;
   connectionId: string;
@@ -1061,6 +1172,130 @@ async function maybeConfigureSchemaScope(input: {
   return true;
 }
 
+async function maybeConfigureTableScope(input: {
+  projectDir: string;
+  connectionId: string;
+  args: KtxSetupDatabasesArgs;
+  prompts: KtxSetupDatabasesPromptAdapter;
+  io: KtxCliIo;
+  deps: KtxSetupDatabasesDeps;
+}): Promise<boolean> {
+  const project = await loadKtxProject({ projectDir: input.projectDir });
+  const connection = project.config.connections[input.connectionId];
+  const driver = normalizeDriver(connection?.driver);
+  if (!driver || driver === 'sqlite') return true;
+
+  const existingTables = connection?.enabled_tables;
+  if (Array.isArray(existingTables) && existingTables.length > 0) {
+    return true;
+  }
+
+  if (input.args.inputMode === 'disabled') {
+    return true;
+  }
+
+  writeSetupSection(input.io, 'Discovering tables', [
+    `Connecting to ${input.connectionId}…`,
+  ]);
+
+  let discovered: KtxTableListEntry[];
+  try {
+    discovered = await (input.deps.listTables ?? defaultListTables)(
+      input.projectDir,
+      input.connectionId,
+    );
+  } catch (error) {
+    input.io.stderr.write(
+      `Could not discover tables for ${input.connectionId}; continuing without table filter. ` +
+        `${error instanceof Error ? error.message : String(error)}\n`,
+    );
+    return true;
+  }
+
+  if (discovered.length === 0) {
+    return true;
+  }
+
+  const allQualified = discovered.map((t) => `${t.schema}.${t.name}`);
+
+  if (discovered.length === 1) {
+    await writeConnectionConfig({
+      projectDir: input.projectDir,
+      connectionId: input.connectionId,
+      connection: { ...connection!, enabled_tables: allQualified },
+    });
+    writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [
+      `✓ ${allQualified[0]}`,
+    ]);
+    return true;
+  }
+
+  const bySchema = new Map<string, KtxTableListEntry[]>();
+  for (const entry of discovered) {
+    const existing = bySchema.get(entry.schema) ?? [];
+    existing.push(entry);
+    bySchema.set(entry.schema, existing);
+  }
+  const schemaList = [...bySchema.keys()].sort();
+  const schemaSummary = schemaList.map((s) => `${s} (${bySchema.get(s)!.length})`).join(', ');
+
+  let selected: string[] | null = null;
+
+  while (selected === null) {
+    const action = await input.prompts.select({
+      message: `Tables found in selected schemas\n` +
+        `${discovered.length} tables across ${schemaList.length} ${schemaList.length === 1 ? 'schema' : 'schemas'}: ${schemaSummary}`,
+      options: [
+        { value: 'all', label: 'Enable all tables' },
+        { value: 'customize', label: 'Customize which tables to enable' },
+        { value: 'back', label: 'Back' },
+      ],
+    });
+
+    if (action === 'back') {
+      return false;
+    }
+
+    if (action === 'all') {
+      selected = allQualified;
+    } else {
+      const choices = await input.prompts.multiselect({
+        message: withMultiselectNavigation(
+          `Tables to enable for ${input.connectionId}\n` +
+            `Deselect any tables agents should not use.`,
+        ),
+        options: discovered.map((t) => {
+          const qualified = `${t.schema}.${t.name}`;
+          const suffix = t.kind === 'view' ? ' (view)' : '';
+          return { value: qualified, label: `${qualified}${suffix}` };
+        }),
+        initialValues: allQualified,
+        required: true,
+      });
+
+      if (choices.includes('back')) {
+        continue;
+      }
+      if (choices.length === 0) {
+        input.io.stdout.write('│  KTX needs at least one table enabled. Select a table or press Escape to go back.\n');
+        continue;
+      }
+      selected = choices;
+    }
+  }
+
+  await writeConnectionConfig({
+    projectDir: input.projectDir,
+    connectionId: input.connectionId,
+    connection: { ...connection!, enabled_tables: selected },
+  });
+
+  writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [
+    `✓ ${selected.length}/${discovered.length} tables enabled`,
+  ]);
+  return true;
+}
+
 async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void> {
   const project = await loadKtxProject({ projectDir });
   const adapters = project.config.ingest.adapters.includes('historic-sql')
@@ -1187,8 +1422,16 @@ async function validateAndScanConnection(input: {
   const testLines = ['✓ Connection test passed', `Driver: ${driverDisplay}`];
   writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines);
 
-  if (!(await maybeConfigureSchemaScope(input))) {
-    return false;
+  while (true) {
+    if (!(await maybeConfigureSchemaScope(input))) {
+      return false;
+    }
+
+    if (await maybeConfigureTableScope(input)) {
+      break;
+    }
+
+    await clearScopeConfig(input.projectDir, input.connectionId);
   }
 
   await maybeRunHistoricSqlSetupProbe({
@@ -1359,13 +1602,10 @@ export async function runKtxSetupDatabasesStep(
   while (true) {
     if (showConfiguredPrimaryMenu) {
       const action = await prompts.select(configuredPrimarySourcesPrompt(selectedConnectionIds));
-      if (action === 'continue') {
+      if (action === 'continue' || action === 'back') {
         await markDatabasesComplete(args.projectDir, selectedConnectionIds);
         return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds };
       }
-      if (action === 'back') {
-        return { status: 'back', projectDir: args.projectDir };
-      }
     }
     showConfiguredPrimaryMenu = false;
 

packages/cli/src/setup-sources.ts

@@ -2,7 +2,7 @@ import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join, relative, resolve } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
-import { cancel, isCancel, log, multiselect, password, select, text } from '@clack/prompts';
+import { cancel, confirm, isCancel, log, multiselect, password, select, text } from '@clack/prompts';
 import { localConnectionTypeForConfig, resolveNotionAuthToken } from '@ktx/context/connections';
 import { resolveKtxConfigReference } from '@ktx/context/core';
 import {
@@ -136,12 +136,23 @@ const PRIMARY_SOURCE_DRIVERS = new Set([
 function createPromptAdapter(): KtxSetupSourcesPromptAdapter {
   return {
     async multiselect(options) {
-      const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options)));
-      if (isCancel(value)) {
-        cancel('Setup cancelled.');
-        return ['back'];
+      while (true) {
+        const value = await withSetupInterruptConfirmation(() => multiselect(withMenuOptionsSpacing(options)));
+        if (isCancel(value)) {
+          cancel('Setup cancelled.');
+          return ['back'];
+        }
+        const selected = [...value] as string[];
+        if (selected.length === 0 && !options.required) {
+          const skipConfirmed = await confirm({ message: 'Nothing selected. Skip this step?', initialValue: false });
+          if (isCancel(skipConfirmed)) {
+            cancel('Setup cancelled.');
+            return ['back'];
+          }
+          if (!skipConfirmed) continue;
+        }
+        return selected;
       }
-      return [...value] as string[];
     },
     async select(options) {
       const value = await withSetupInterruptConfirmation(() => select(withMenuOptionsSpacing(options)));