mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-13 08:15:14 +02:00
* Refine adapter-owned ingest finalization design after adversarial review iteration 1 * Refine adapter-owned ingest finalization design after adversarial review iteration 2 * Refine adapter-owned ingest finalization design after adversarial review iteration 3 * Implement adapter-owned ingest finalization v1 Moves finalization from runner-owned post-processors into typed SourceAdapter.finalize() contracts. Adds finalization report schema, scope derivation, override replay context, and migrates historic-SQL projection. Removes IngestBundlePostProcessorPort wiring and HistoricSqlProjectionPostProcessor. * feat(ingest): export finalization adapter contract types * test(ingest): exercise historic sql finalization locally * docs(plans): add adapter-owned finalization v1 closure plan * fix(setup): unblock clean Linux installs and add enabled_tables allowlist - Pin managed Python runtime to 3.13 via `uv venv --python 3.13` so installs don't pick the system 3.12 on Ubuntu 24.04 and fail at wheel install. - Sanitize NO_PROXY/no_proxy for the daemon child process — drop IPv6 CIDR entries that httpx rejects with InvalidURL (OrbStack injects these by default). - Add `enabled_tables` allowlist on warehouse connections (zod schema + live-database introspection filter) to scope ingest to specific tables. - Add `getting-started/troubleshooting-linux` docs page covering the Python 3.13 prerequisite, IPv6 proxy gotcha, and a minimal working recipe; link it from the quickstart troubleshooting table and the llms-docs map. - Make docs-site origin overridable via `KTX_DOCS_ORIGIN` so local builds can serve under host.docker.internal. * Move docs changes to specs repo * fix(cli): keep managed runtime python version private * Deduplicate enabled tables filtering
131 lines
3.8 KiB
TypeScript
131 lines
3.8 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import {
|
|
compareFinalizationDeclarations,
|
|
deriveFinalizationTouchedSources,
|
|
deriveFinalizationWikiPageKeys,
|
|
} from './finalization-scope.js';
|
|
|
|
describe('deriveFinalizationWikiPageKeys', () => {
|
|
it('maps changed global wiki markdown paths to page keys', () => {
|
|
expect(
|
|
deriveFinalizationWikiPageKeys([
|
|
'wiki/global/historic-sql-orders.md',
|
|
'wiki/global/nested/page.md',
|
|
'README.md',
|
|
]),
|
|
).toEqual(['historic-sql-orders']);
|
|
});
|
|
});
|
|
|
|
describe('deriveFinalizationTouchedSources', () => {
|
|
it('maps standalone semantic-layer files directly', async () => {
|
|
const result = await deriveFinalizationTouchedSources({
|
|
changedPaths: ['semantic-layer/warehouse/orders.yaml'],
|
|
beforeSourcesByConnection: new Map(),
|
|
afterSourcesByConnection: new Map(),
|
|
});
|
|
expect(result).toEqual({
|
|
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
unresolvedPaths: [],
|
|
});
|
|
});
|
|
|
|
it('resolves aggregate _schema changes by comparing loaded source snapshots', async () => {
|
|
const beforeSourcesByConnection = new Map([
|
|
[
|
|
'warehouse',
|
|
[
|
|
{
|
|
name: 'orders',
|
|
grain: ['order_id'],
|
|
columns: [{ name: 'order_id', type: 'string' }],
|
|
joins: [],
|
|
measures: [],
|
|
usage: {
|
|
narrative: 'old',
|
|
frequencyTier: 'low' as const,
|
|
commonFilters: [],
|
|
commonJoins: [],
|
|
},
|
|
},
|
|
],
|
|
],
|
|
]);
|
|
const afterSourcesByConnection = new Map([
|
|
[
|
|
'warehouse',
|
|
[
|
|
{
|
|
name: 'orders',
|
|
grain: ['order_id'],
|
|
columns: [{ name: 'order_id', type: 'string' }],
|
|
joins: [],
|
|
measures: [],
|
|
usage: {
|
|
narrative: 'new',
|
|
frequencyTier: 'high' as const,
|
|
commonFilters: [],
|
|
commonJoins: [],
|
|
},
|
|
},
|
|
],
|
|
],
|
|
]);
|
|
|
|
const result = await deriveFinalizationTouchedSources({
|
|
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
|
beforeSourcesByConnection,
|
|
afterSourcesByConnection,
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
unresolvedPaths: [],
|
|
});
|
|
});
|
|
|
|
it('flags aggregate _schema changes that cannot be resolved to logical sources', async () => {
|
|
const beforeSourcesByConnection = new Map([['warehouse', []]]);
|
|
const afterSourcesByConnection = new Map([['warehouse', []]]);
|
|
|
|
const result = await deriveFinalizationTouchedSources({
|
|
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
|
beforeSourcesByConnection,
|
|
afterSourcesByConnection,
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
touchedSources: [],
|
|
unresolvedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('compareFinalizationDeclarations', () => {
|
|
it('reports missing and extra adapter declarations', () => {
|
|
expect(
|
|
compareFinalizationDeclarations({
|
|
declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'customers' }],
|
|
declaredChangedWikiPageKeys: ['orders'],
|
|
derivedChangedWikiPageKeys: ['orders', 'patterns'],
|
|
}),
|
|
).toEqual([
|
|
{
|
|
artifactKind: 'sl',
|
|
key: 'warehouse:customers',
|
|
direction: 'missing_from_adapter_declaration',
|
|
},
|
|
{
|
|
artifactKind: 'sl',
|
|
key: 'warehouse:orders',
|
|
direction: 'extra_in_adapter_declaration',
|
|
},
|
|
{
|
|
artifactKind: 'wiki',
|
|
key: 'patterns',
|
|
direction: 'missing_from_adapter_declaration',
|
|
},
|
|
]);
|
|
});
|
|
});
|