mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-01 08:59:39 +02:00
Preserve failed context build metadata
This commit is contained in:
parent
7c7c86c446
commit
f8aedc858b
4 changed files with 93 additions and 4 deletions
|
|
@ -587,6 +587,36 @@ describe('runContextBuild', () => {
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('returns report IDs parsed from failed source-ingest target output', async () => {
|
||||||
|
const io = makeIo();
|
||||||
|
const project = projectWithConnections({
|
||||||
|
warehouse: { driver: 'postgres' },
|
||||||
|
dbt_main: { driver: 'dbt' },
|
||||||
|
});
|
||||||
|
const executeTarget = vi.fn(async (target, _args, targetIo) => {
|
||||||
|
if (target.operation === 'scan') {
|
||||||
|
return successResult(target.connectionId, target.driver, target.operation);
|
||||||
|
}
|
||||||
|
|
||||||
|
targetIo.stdout.write('Report: report-dbt-failed\n');
|
||||||
|
targetIo.stdout.write('Work units: 3\n');
|
||||||
|
return failedResult(target.connectionId, target.driver, target.operation);
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runContextBuild(
|
||||||
|
project,
|
||||||
|
{ projectDir: '/tmp/project', inputMode: 'disabled' },
|
||||||
|
io.io,
|
||||||
|
{ executeTarget, now: () => 1000 },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
exitCode: 1,
|
||||||
|
detached: false,
|
||||||
|
reportIds: ['report-dbt-failed'],
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('viewStateFromSourceProgress', () => {
|
describe('viewStateFromSourceProgress', () => {
|
||||||
|
|
|
||||||
|
|
@ -648,10 +648,10 @@ export async function runContextBuild(
|
||||||
targetState.status = failed ? 'failed' : 'done';
|
targetState.status = failed ? 'failed' : 'done';
|
||||||
targetState.detailLine = null;
|
targetState.detailLine = null;
|
||||||
const capturedOutput = capture.captured();
|
const capturedOutput = capture.captured();
|
||||||
|
const metadata = collectOutputMetadata(capturedOutput, targetState.target.operation);
|
||||||
|
for (const reportId of metadata.reportIds) reportIds.add(reportId);
|
||||||
|
for (const artifactPath of metadata.artifactPaths) artifactPaths.add(artifactPath);
|
||||||
if (!failed) {
|
if (!failed) {
|
||||||
const metadata = collectOutputMetadata(capturedOutput, targetState.target.operation);
|
|
||||||
for (const reportId of metadata.reportIds) reportIds.add(reportId);
|
|
||||||
for (const artifactPath of metadata.artifactPaths) artifactPaths.add(artifactPath);
|
|
||||||
targetState.summaryText =
|
targetState.summaryText =
|
||||||
targetState.target.operation === 'scan'
|
targetState.target.operation === 'scan'
|
||||||
? parseScanSummary(capturedOutput)
|
? parseScanSummary(capturedOutput)
|
||||||
|
|
|
||||||
|
|
@ -215,6 +215,47 @@ describe('setup context build state', () => {
|
||||||
expect(io.stdout()).toContain('KTX context is ready for agents.');
|
expect(io.stdout()).toContain('KTX context is ready for agents.');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('records only failed sources as retryable when the context build fails', async () => {
|
||||||
|
await writeReadyProject(tempDir);
|
||||||
|
const io = makeIo();
|
||||||
|
const runContextBuildMock = vi.fn(async (_project, _args, _io, hooks) => {
|
||||||
|
hooks.onSourceProgress?.([
|
||||||
|
{ connectionId: 'warehouse', operation: 'scan', status: 'done', elapsedMs: 1000 },
|
||||||
|
{ connectionId: 'docs', operation: 'source-ingest', status: 'failed', elapsedMs: 2000 },
|
||||||
|
]);
|
||||||
|
return {
|
||||||
|
exitCode: 1,
|
||||||
|
detached: false,
|
||||||
|
reportIds: ['report-docs-failed'],
|
||||||
|
artifactPaths: ['raw-sources/docs/notion/sync-1/ingest-report.json'],
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runKtxSetupContextStep(
|
||||||
|
{ projectDir: tempDir, inputMode: 'disabled' },
|
||||||
|
io.io,
|
||||||
|
{
|
||||||
|
runIdFactory: () => 'setup-context-local-failed',
|
||||||
|
now: () => new Date('2026-05-09T10:00:00.000Z'),
|
||||||
|
runContextBuild: runContextBuildMock,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
).resolves.toEqual({ status: 'failed', projectDir: tempDir });
|
||||||
|
|
||||||
|
await expect(readKtxSetupContextState(tempDir)).resolves.toMatchObject({
|
||||||
|
runId: 'setup-context-local-failed',
|
||||||
|
status: 'failed',
|
||||||
|
reportIds: ['report-docs-failed'],
|
||||||
|
artifactPaths: ['raw-sources/docs/notion/sync-1/ingest-report.json'],
|
||||||
|
retryableFailedTargets: ['docs'],
|
||||||
|
sourceProgress: [
|
||||||
|
{ connectionId: 'warehouse', operation: 'scan', status: 'done', elapsedMs: 1000 },
|
||||||
|
{ connectionId: 'docs', operation: 'source-ingest', status: 'failed', elapsedMs: 2000 },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('marks context complete without prompting when initial source ingest already made agent context', async () => {
|
it('marks context complete without prompting when initial source ingest already made agent context', async () => {
|
||||||
await writeReadyProject(tempDir);
|
await writeReadyProject(tempDir);
|
||||||
await mkdir(join(tempDir, 'semantic-layer', 'dbt-main'), { recursive: true });
|
await mkdir(join(tempDir, 'semantic-layer', 'dbt-main'), { recursive: true });
|
||||||
|
|
|
||||||
|
|
@ -234,6 +234,24 @@ function normalizeSourceProgress(value: unknown): ContextBuildSourceProgressUpda
|
||||||
return entries.length > 0 ? entries : undefined;
|
return entries.length > 0 ? entries : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function setupContextTargetIds(targets: KtxSetupContextTargets): string[] {
|
||||||
|
return [...new Set([...targets.primarySourceConnectionIds, ...targets.contextSourceConnectionIds])];
|
||||||
|
}
|
||||||
|
|
||||||
|
function retryableFailedTargetsFromProgress(
|
||||||
|
targets: KtxSetupContextTargets,
|
||||||
|
progress: ContextBuildSourceProgressUpdate[] | undefined,
|
||||||
|
): string[] {
|
||||||
|
const targetIds = setupContextTargetIds(targets);
|
||||||
|
if (!progress || progress.length === 0) {
|
||||||
|
return targetIds;
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedIds = new Set(progress.filter((source) => source.status === 'failed').map((source) => source.connectionId));
|
||||||
|
const failedTargets = targetIds.filter((connectionId) => failedIds.has(connectionId));
|
||||||
|
return failedTargets.length > 0 ? failedTargets : targetIds;
|
||||||
|
}
|
||||||
|
|
||||||
export async function readKtxSetupContextState(projectDir: string): Promise<KtxSetupContextState> {
|
export async function readKtxSetupContextState(projectDir: string): Promise<KtxSetupContextState> {
|
||||||
const filePath = statePath(projectDir);
|
const filePath = statePath(projectDir);
|
||||||
if (!(await pathExists(filePath))) {
|
if (!(await pathExists(filePath))) {
|
||||||
|
|
@ -614,7 +632,7 @@ async function runBuild(
|
||||||
updatedAt,
|
updatedAt,
|
||||||
reportIds: completedReportIds,
|
reportIds: completedReportIds,
|
||||||
artifactPaths: completedArtifactPaths,
|
artifactPaths: completedArtifactPaths,
|
||||||
retryableFailedTargets: [...targets.primarySourceConnectionIds, ...targets.contextSourceConnectionIds],
|
retryableFailedTargets: retryableFailedTargetsFromProgress(targets, lastSourceProgress),
|
||||||
failureReason: 'Context build failed.',
|
failureReason: 'Context build failed.',
|
||||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue