mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-16 08:25:14 +02:00
Include historic SQL projection in memory counts
This commit is contained in:
parent
1bd29c7eb1
commit
b981cabdc6
5 changed files with 169 additions and 12 deletions
|
|
@ -544,6 +544,63 @@ describe('runKtxIngest', () => {
|
|||
expect(io.stdout()).toContain('Diff: +2/~0/-0/=0\n');
|
||||
});
|
||||
|
||||
it('includes historic-sql projection output in saved memory counts', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions) => {
|
||||
const result = completedLocalBundleRun(input, 'historic-sql-projection');
|
||||
return {
|
||||
...result,
|
||||
report: localFakeBundleReport('historic-sql-projection', {
|
||||
sourceKey: 'historic-sql',
|
||||
body: {
|
||||
workUnits: [],
|
||||
postProcessor: {
|
||||
sourceKey: 'historic-sql',
|
||||
status: 'success',
|
||||
result: {
|
||||
tableUsageMerged: 56,
|
||||
staleTablesMarked: 1,
|
||||
patternPagesWritten: 30,
|
||||
stalePatternPagesMarked: 2,
|
||||
archivedPatternPages: 3,
|
||||
legacyPagesDeleted: 4,
|
||||
},
|
||||
errors: [],
|
||||
warnings: [],
|
||||
touchedSources: [],
|
||||
},
|
||||
},
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'historic-sql',
|
||||
outputMode: 'plain',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
createAdapters: vi.fn(() => [
|
||||
{ source: 'historic-sql', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
|
||||
]),
|
||||
jobIdFactory: () => 'historic-sql-projection',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stderr()).toBe('');
|
||||
expect(io.stdout()).toContain('Adapter: historic-sql\n');
|
||||
expect(io.stdout()).toContain('Saved memory: 39 wiki, 57 SL\n');
|
||||
});
|
||||
|
||||
it('returns a non-zero code when local ingest reports failed work units', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
renderMemoryFlowReplay,
|
||||
runLocalIngest,
|
||||
runLocalMetabaseIngest,
|
||||
savedMemoryCountsForReport,
|
||||
} from '@ktx/context/ingest';
|
||||
import { loadKtxProject } from '@ktx/context/project';
|
||||
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
|
||||
|
|
@ -89,16 +90,8 @@ function reportStatus(report: IngestReportSnapshot): 'done' | 'error' {
|
|||
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
||||
}
|
||||
|
||||
function reportActionCounts(report: IngestReportSnapshot): { wikiCount: number; slCount: number } {
|
||||
const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
|
||||
return {
|
||||
wikiCount: actions.filter((action) => action.target === 'wiki').length,
|
||||
slCount: actions.filter((action) => action.target === 'sl').length,
|
||||
};
|
||||
}
|
||||
|
||||
function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void {
|
||||
const counts = reportActionCounts(report);
|
||||
const counts = savedMemoryCountsForReport(report);
|
||||
io.stdout.write(`Report: ${report.id}\n`);
|
||||
io.stdout.write(`Run: ${report.runId}\n`);
|
||||
io.stdout.write(`Job: ${report.jobId}\n`);
|
||||
|
|
@ -117,7 +110,7 @@ function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void
|
|||
function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIngestIo): void {
|
||||
const counts = result.children.reduce(
|
||||
(acc, child) => {
|
||||
const childCounts = reportActionCounts(child.report);
|
||||
const childCounts = savedMemoryCountsForReport(child.report);
|
||||
return {
|
||||
wikiCount: acc.wikiCount + childCounts.wikiCount,
|
||||
slCount: acc.slCount + childCounts.slCount,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue