ktx/packages/cli/test/context/ingest/finalization-scope.test.ts

132 lines
3.8 KiB
TypeScript
Raw Permalink Normal View History

feat(ingest): adapter-owned finalization replaces post-processor escape hatch (#136) * Refine adapter-owned ingest finalization design after adversarial review iteration 1 * Refine adapter-owned ingest finalization design after adversarial review iteration 2 * Refine adapter-owned ingest finalization design after adversarial review iteration 3 * Implement adapter-owned ingest finalization v1 Moves finalization from runner-owned post-processors into typed SourceAdapter.finalize() contracts. Adds finalization report schema, scope derivation, override replay context, and migrates historic-SQL projection. Removes IngestBundlePostProcessorPort wiring and HistoricSqlProjectionPostProcessor. * feat(ingest): export finalization adapter contract types * test(ingest): exercise historic sql finalization locally * docs(plans): add adapter-owned finalization v1 closure plan * fix(setup): unblock clean Linux installs and add enabled_tables allowlist - Pin managed Python runtime to 3.13 via `uv venv --python 3.13` so installs don't pick the system 3.12 on Ubuntu 24.04 and fail at wheel install. - Sanitize NO_PROXY/no_proxy for the daemon child process — drop IPv6 CIDR entries that httpx rejects with InvalidURL (OrbStack injects these by default). - Add `enabled_tables` allowlist on warehouse connections (zod schema + live-database introspection filter) to scope ingest to specific tables. - Add `getting-started/troubleshooting-linux` docs page covering the Python 3.13 prerequisite, IPv6 proxy gotcha, and a minimal working recipe; link it from the quickstart troubleshooting table and the llms-docs map. - Make docs-site origin overridable via `KTX_DOCS_ORIGIN` so local builds can serve under host.docker.internal. * Move docs changes to specs repo * fix(cli): keep managed runtime python version private * Deduplicate enabled tables filtering
2026-05-20 14:17:10 +02:00
import { describe, expect, it } from 'vitest';
import {
compareFinalizationDeclarations,
deriveFinalizationTouchedSources,
deriveFinalizationWikiPageKeys,
2026-05-25 13:17:46 +02:00
} from '../../../src/context/ingest/finalization-scope.js';
feat(ingest): adapter-owned finalization replaces post-processor escape hatch (#136) * Refine adapter-owned ingest finalization design after adversarial review iteration 1 * Refine adapter-owned ingest finalization design after adversarial review iteration 2 * Refine adapter-owned ingest finalization design after adversarial review iteration 3 * Implement adapter-owned ingest finalization v1 Moves finalization from runner-owned post-processors into typed SourceAdapter.finalize() contracts. Adds finalization report schema, scope derivation, override replay context, and migrates historic-SQL projection. Removes IngestBundlePostProcessorPort wiring and HistoricSqlProjectionPostProcessor. * feat(ingest): export finalization adapter contract types * test(ingest): exercise historic sql finalization locally * docs(plans): add adapter-owned finalization v1 closure plan * fix(setup): unblock clean Linux installs and add enabled_tables allowlist - Pin managed Python runtime to 3.13 via `uv venv --python 3.13` so installs don't pick the system 3.12 on Ubuntu 24.04 and fail at wheel install. - Sanitize NO_PROXY/no_proxy for the daemon child process — drop IPv6 CIDR entries that httpx rejects with InvalidURL (OrbStack injects these by default). - Add `enabled_tables` allowlist on warehouse connections (zod schema + live-database introspection filter) to scope ingest to specific tables. - Add `getting-started/troubleshooting-linux` docs page covering the Python 3.13 prerequisite, IPv6 proxy gotcha, and a minimal working recipe; link it from the quickstart troubleshooting table and the llms-docs map. - Make docs-site origin overridable via `KTX_DOCS_ORIGIN` so local builds can serve under host.docker.internal. * Move docs changes to specs repo * fix(cli): keep managed runtime python version private * Deduplicate enabled tables filtering
2026-05-20 14:17:10 +02:00
describe('deriveFinalizationWikiPageKeys', () => {
it('maps changed global wiki markdown paths to page keys', () => {
expect(
deriveFinalizationWikiPageKeys([
'wiki/global/historic-sql-orders.md',
'wiki/global/nested/page.md',
'README.md',
]),
).toEqual(['historic-sql-orders']);
});
});
describe('deriveFinalizationTouchedSources', () => {
it('maps standalone semantic-layer files directly', async () => {
const result = await deriveFinalizationTouchedSources({
changedPaths: ['semantic-layer/warehouse/orders.yaml'],
beforeSourcesByConnection: new Map(),
afterSourcesByConnection: new Map(),
});
expect(result).toEqual({
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
unresolvedPaths: [],
});
});
it('resolves aggregate _schema changes by comparing loaded source snapshots', async () => {
const beforeSourcesByConnection = new Map([
[
'warehouse',
[
{
name: 'orders',
grain: ['order_id'],
columns: [{ name: 'order_id', type: 'string' }],
joins: [],
measures: [],
usage: {
narrative: 'old',
frequencyTier: 'low' as const,
commonFilters: [],
commonJoins: [],
},
},
],
],
]);
const afterSourcesByConnection = new Map([
[
'warehouse',
[
{
name: 'orders',
grain: ['order_id'],
columns: [{ name: 'order_id', type: 'string' }],
joins: [],
measures: [],
usage: {
narrative: 'new',
frequencyTier: 'high' as const,
commonFilters: [],
commonJoins: [],
},
},
],
],
]);
const result = await deriveFinalizationTouchedSources({
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
beforeSourcesByConnection,
afterSourcesByConnection,
});
expect(result).toEqual({
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
unresolvedPaths: [],
});
});
it('flags aggregate _schema changes that cannot be resolved to logical sources', async () => {
const beforeSourcesByConnection = new Map([['warehouse', []]]);
const afterSourcesByConnection = new Map([['warehouse', []]]);
const result = await deriveFinalizationTouchedSources({
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
beforeSourcesByConnection,
afterSourcesByConnection,
});
expect(result).toEqual({
touchedSources: [],
unresolvedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
});
});
});
describe('compareFinalizationDeclarations', () => {
it('reports missing and extra adapter declarations', () => {
expect(
compareFinalizationDeclarations({
declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'customers' }],
declaredChangedWikiPageKeys: ['orders'],
derivedChangedWikiPageKeys: ['orders', 'patterns'],
}),
).toEqual([
{
artifactKind: 'sl',
key: 'warehouse:customers',
direction: 'missing_from_adapter_declaration',
},
{
artifactKind: 'sl',
key: 'warehouse:orders',
direction: 'extra_in_adapter_declaration',
},
{
artifactKind: 'wiki',
key: 'patterns',
direction: 'missing_from_adapter_declaration',
},
]);
});
});