diff --git a/packages/context/src/ingest/ingest-bundle.runner.test.ts b/packages/context/src/ingest/ingest-bundle.runner.test.ts index 59e85058..588af792 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.test.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.test.ts @@ -260,7 +260,11 @@ const buildRunner = (deps: ReturnType = makeDeps(), overrides: resolveTranscriptDir: (jobId) => `/tmp/ktx-test/run/wu-transcripts/${jobId}`, resolveTracePath: (jobId) => `/tmp/ktx-test/ingest-traces/${jobId}/trace.jsonl`, }, - settings: { probeRowCount: 1, memoryIngestionModel: 'test-model' }, + settings: { + probeRowCount: 1, + memoryIngestionModel: 'test-model', + sharedWorktreeSourceKeys: ['fake', 'notion', 'looker', 'metricflow', 'historic-sql'], + }, skillsRegistry: deps.skillsRegistry as any, promptService: deps.promptService as any, wikiService: deps.wikiService as any, diff --git a/packages/context/src/ingest/ingest-bundle.runner.ts b/packages/context/src/ingest/ingest-bundle.runner.ts index 4dab2605..fe07219d 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import pLimit from 'p-limit'; import { z } from 'zod'; @@ -78,6 +78,16 @@ import { repairWikiSlRefs, type WikiSlRefRepairResult } from './wiki-sl-ref-repa type MemoryFlowStageProgress = Extract; +async function copyTransientIngestEvidence(sourceWorkdir: string, targetWorkdir: string): Promise { + const source = join(sourceWorkdir, '.ktx/ingest-evidence'); + const target = join(targetWorkdir, '.ktx/ingest-evidence'); + await cp(source, target, { recursive: true, force: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') { + throw error; + } + }); +} + function workUnitToMemoryFlowPlannedWorkUnit(workUnit: WorkUnit): MemoryFlowPlannedWorkUnit { return { unitKey: workUnit.unitKey, @@ -1419,6 +1429,7 @@ export class IngestBundleRunner { patchDir, trace: runTrace, workUnit: wu, + afterSuccess: (child) => copyTransientIngestEvidence(child.workdir, sessionWorktree.workdir), run: async (child) => { const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir); const scopedSemanticLayerService = this.deps.semanticLayerService.forWorktree(child.workdir); diff --git a/packages/context/src/ingest/isolated-diff/work-unit-executor.ts b/packages/context/src/ingest/isolated-diff/work-unit-executor.ts index 7d64431e..ac013d5a 100644 --- a/packages/context/src/ingest/isolated-diff/work-unit-executor.ts +++ b/packages/context/src/ingest/isolated-diff/work-unit-executor.ts @@ -15,6 +15,7 @@ export interface RunIsolatedWorkUnitInput { trace: IngestTraceWriter; workUnit: WorkUnit; run(child: IngestSessionWorktree): Promise; + afterSuccess?(child: IngestSessionWorktree): Promise; } function patchFileName(unitIndex: number, unitKey: string): string { @@ -44,6 +45,7 @@ export async function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Prom return { ...outcome, childWorktreePath: child.workdir }; } + await input.afterSuccess?.(child); await mkdir(input.patchDir, { recursive: true }); const patchPath = join(input.patchDir, patchFileName(input.unitIndex, input.workUnit.unitKey)); await child.git.writeBinaryNoRenamePatch(input.ingestionBaseSha, 'HEAD', patchPath);