Fix historic SQL ingest setup and progress

packages/cli/src/ingest.ts

@@ -8,6 +8,7 @@ import {
   ingestReportToMemoryFlowReplay,
   type LocalMetabaseFanoutResult,
   type LocalMetabaseFanoutProgress,
+  type MemoryFlowEvent,
   type MemoryFlowReplayInput,
   type RunLocalIngestOptions,
   renderMemoryFlowReplay,
@@ -170,6 +171,118 @@ function createMetabaseFanoutProgress(
   };
 }
 
+function formatDiffProgress(event: Extract<MemoryFlowEvent, { type: 'diff_computed' }>): string {
+  return `+${event.added}/~${event.modified}/-${event.deleted}/=${event.unchanged}`;
+}
+
+function completedWorkUnitCount(snapshot: MemoryFlowReplayInput): number {
+  return snapshot.events.filter((event) => event.type === 'work_unit_finished').length;
+}
+
+function plainIngestEventProgress(
+  event: MemoryFlowEvent,
+  snapshot: MemoryFlowReplayInput,
+): { percent: number; message: string } | null {
+  switch (event.type) {
+    case 'source_acquired':
+      return {
+        percent: 15,
+        message: `Fetched ${pluralize(event.fileCount, 'source file')} from ${event.adapter}`,
+      };
+    case 'raw_snapshot_written':
+      return {
+        percent: 25,
+        message: `Wrote raw snapshot ${event.syncId} with ${pluralize(event.rawFileCount, 'file')}`,
+      };
+    case 'diff_computed':
+      return { percent: 35, message: `Computed source diff ${formatDiffProgress(event)}` };
+    case 'chunks_planned':
+      return {
+        percent: 45,
+        message: `Planned ${pluralize(event.workUnitCount, 'work unit')}`,
+      };
+    case 'stage_skipped':
+      return { percent: 45, message: `Skipped ${event.stage}: ${event.reason}` };
+    case 'work_unit_started':
+      return { percent: 55, message: `Processing ${event.unitKey}` };
+    case 'work_unit_finished': {
+      const total = snapshot.plannedWorkUnits.length || completedWorkUnitCount(snapshot);
+      const completed = completedWorkUnitCount(snapshot);
+      const percent = total > 0 ? 55 + Math.round((completed / total) * 25) : 80;
+      return {
+        percent,
+        message: `Processed ${completed}/${total} work units`,
+      };
+    }
+    case 'reconciliation_finished':
+      return {
+        percent: 85,
+        message: `Reconciled results with ${pluralize(event.conflictCount, 'conflict')} and ${pluralize(
+          event.fallbackCount,
+          'fallback',
+        )}`,
+      };
+    case 'saved':
+      return {
+        percent: 90,
+        message: `Saved memory updates (${event.wikiCount} wiki, ${event.slCount} SL)`,
+      };
+    case 'provenance_recorded':
+      return { percent: 95, message: `Recorded ${pluralize(event.rowCount, 'provenance row')}` };
+    case 'report_created':
+      return { percent: 98, message: `Created ingest report ${event.reportPath ?? event.runId}` };
+    case 'scope_detected':
+    case 'work_unit_step':
+    case 'candidate_action':
+      return null;
+  }
+}
+
+function shouldWritePlainIngestProgress(
+  outputMode: KtxIngestOutputMode,
+  io: KtxIngestIo,
+  env: NodeJS.ProcessEnv,
+): boolean {
+  return outputMode === 'plain' && io.stdout.isTTY === true && env.CI !== 'true';
+}
+
+function createPlainIngestProgressRenderer(
+  args: Extract<KtxIngestArgs, { command: 'run' }>,
+  io: KtxIngestIo,
+): { start(): void; update(snapshot: MemoryFlowReplayInput): void } {
+  let printedEvents = 0;
+  let lastPercent = 0;
+  let printedCompletion = false;
+
+  const write = (percent: number, message: string) => {
+    const nextPercent = Math.max(lastPercent, Math.max(0, Math.min(100, percent)));
+    lastPercent = nextPercent;
+    io.stdout.write(`[${nextPercent}%] ${message}\n`);
+  };
+
+  return {
+    start() {
+      write(5, `Fetching source files for ${args.connectionId}/${args.adapter}`);
+    },
+    update(snapshot) {
+      while (printedEvents < snapshot.events.length) {
+        const event = snapshot.events[printedEvents++];
+        if (!event) {
+          continue;
+        }
+        const progress = plainIngestEventProgress(event, snapshot);
+        if (progress) {
+          write(progress.percent, progress.message);
+        }
+      }
+      if (!printedCompletion && snapshot.status !== 'running') {
+        printedCompletion = true;
+        write(100, snapshot.status === 'done' ? 'Ingest completed' : 'Ingest failed');
+      }
+    },
+  };
+}
+
 function writeReportJson(report: IngestReportSnapshot, io: KtxIngestIo): void {
   io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
 }
@@ -366,10 +479,14 @@ export async function runKtxIngest(
       });
       const shouldUseLiveViz =
         runOutputMode === 'viz' && (args.inputMode ?? 'auto') === 'auto' && isInteractiveTerminal(io);
-      const initialMemoryFlow = shouldUseLiveViz ? initialRunMemoryFlowInput(args, jobId ?? 'pending') : undefined;
+      const plainProgress = shouldWritePlainIngestProgress(runOutputMode, io, env)
+        ? createPlainIngestProgressRenderer(args, io)
+        : null;
+      const initialMemoryFlow =
+        shouldUseLiveViz || plainProgress ? initialRunMemoryFlowInput(args, jobId ?? 'pending') : undefined;
       let latestMemoryFlowSnapshot: MemoryFlowReplayInput | null = initialMemoryFlow ?? null;
 
-      if (initialMemoryFlow && isTuiCapableIo(io)) {
+      if (shouldUseLiveViz && initialMemoryFlow && isTuiCapableIo(io)) {
         const startLiveMemoryFlow = deps.startLiveMemoryFlow ?? startLiveMemoryFlowTui;
         liveTui = await startLiveMemoryFlow(initialMemoryFlow, io);
       }
@@ -382,13 +499,17 @@ export async function runKtxIngest(
                 liveTui.update(snapshot);
                 return;
               }
-              if (!liveTui) {
+              if (shouldUseLiveViz && !liveTui) {
                 writeMemoryFlowInput(snapshot, io, { clear: true });
+                return;
               }
+              plainProgress?.update(snapshot);
             },
           })
         : undefined;
 
+      plainProgress?.start();
+
       try {
         const result = await executeLocalIngest({
           project,
@@ -403,7 +524,7 @@ export async function runKtxIngest(
           ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
           ...(memoryFlow ? { memoryFlow } : {}),
         });
-        if (memoryFlow) {
+        if (shouldUseLiveViz && memoryFlow) {
           latestMemoryFlowSnapshot = memoryFlow.snapshot();
           liveTui?.close();
           liveTui = null;