Include historic SQL projection in memory counts

This commit is contained in:
Andrey Avtomonov 2026-05-11 22:52:47 +02:00
parent 1bd29c7eb1
commit b981cabdc6
5 changed files with 169 additions and 12 deletions

View file

@ -544,6 +544,63 @@ describe('runKtxIngest', () => {
expect(io.stdout()).toContain('Diff: +2/~0/-0/=0\n');
});
it('includes historic-sql projection output in saved memory counts', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => {
const result = completedLocalBundleRun(input, 'historic-sql-projection');
return {
...result,
report: localFakeBundleReport('historic-sql-projection', {
sourceKey: 'historic-sql',
body: {
workUnits: [],
postProcessor: {
sourceKey: 'historic-sql',
status: 'success',
result: {
tableUsageMerged: 56,
staleTablesMarked: 1,
patternPagesWritten: 30,
stalePatternPagesMarked: 2,
archivedPatternPages: 3,
legacyPagesDeleted: 4,
},
errors: [],
warnings: [],
touchedSources: [],
},
},
}),
};
});
const io = makeIo();
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'historic-sql',
outputMode: 'plain',
},
io.io,
{
runLocalIngest: runLocal,
createAdapters: vi.fn(() => [
{ source: 'historic-sql', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
]),
jobIdFactory: () => 'historic-sql-projection',
},
),
).resolves.toBe(0);
expect(io.stderr()).toBe('');
expect(io.stdout()).toContain('Adapter: historic-sql\n');
expect(io.stdout()).toContain('Saved memory: 39 wiki, 57 SL\n');
});
it('returns a non-zero code when local ingest reports failed work units', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);

View file

@ -14,6 +14,7 @@ import {
renderMemoryFlowReplay,
runLocalIngest,
runLocalMetabaseIngest,
savedMemoryCountsForReport,
} from '@ktx/context/ingest';
import { loadKtxProject } from '@ktx/context/project';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
@ -89,16 +90,8 @@ function reportStatus(report: IngestReportSnapshot): 'done' | 'error' {
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
}
function reportActionCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
return {
wikiCount: actions.filter((action) => action.target === 'wiki').length,
slCount: actions.filter((action) => action.target === 'sl').length,
};
}
function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void {
const counts = reportActionCounts(report);
const counts = savedMemoryCountsForReport(report);
io.stdout.write(`Report: ${report.id}\n`);
io.stdout.write(`Run: ${report.runId}\n`);
io.stdout.write(`Job: ${report.jobId}\n`);
@ -117,7 +110,7 @@ function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void
function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIngestIo): void {
const counts = result.children.reduce(
(acc, child) => {
const childCounts = reportActionCounts(child.report);
const childCounts = savedMemoryCountsForReport(child.report);
return {
wikiCount: acc.wikiCount + childCounts.wikiCount,
slCount: acc.slCount + childCounts.slCount,