mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-04 10:52:13 +02:00
feat(ingest): adapter-owned finalization replaces post-processor escape hatch (#136)
* Refine adapter-owned ingest finalization design after adversarial review iteration 1 * Refine adapter-owned ingest finalization design after adversarial review iteration 2 * Refine adapter-owned ingest finalization design after adversarial review iteration 3 * Implement adapter-owned ingest finalization v1 Moves finalization from runner-owned post-processors into typed SourceAdapter.finalize() contracts. Adds finalization report schema, scope derivation, override replay context, and migrates historic-SQL projection. Removes IngestBundlePostProcessorPort wiring and HistoricSqlProjectionPostProcessor. * feat(ingest): export finalization adapter contract types * test(ingest): exercise historic sql finalization locally * docs(plans): add adapter-owned finalization v1 closure plan * fix(setup): unblock clean Linux installs and add enabled_tables allowlist - Pin managed Python runtime to 3.13 via `uv venv --python 3.13` so installs don't pick the system 3.12 on Ubuntu 24.04 and fail at wheel install. - Sanitize NO_PROXY/no_proxy for the daemon child process — drop IPv6 CIDR entries that httpx rejects with InvalidURL (OrbStack injects these by default). - Add `enabled_tables` allowlist on warehouse connections (zod schema + live-database introspection filter) to scope ingest to specific tables. - Add `getting-started/troubleshooting-linux` docs page covering the Python 3.13 prerequisite, IPv6 proxy gotcha, and a minimal working recipe; link it from the quickstart troubleshooting table and the llms-docs map. - Make docs-site origin overridable via `KTX_DOCS_ORIGIN` so local builds can serve under host.docker.internal. * Move docs changes to specs repo * fix(cli): keep managed runtime python version private * Deduplicate enabled tables filtering
This commit is contained in:
parent
fb82993ce1
commit
4ec5903aa5
29 changed files with 1423 additions and 314 deletions
131
packages/context/src/ingest/finalization-scope.test.ts
Normal file
131
packages/context/src/ingest/finalization-scope.test.ts
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
compareFinalizationDeclarations,
|
||||
deriveFinalizationTouchedSources,
|
||||
deriveFinalizationWikiPageKeys,
|
||||
} from './finalization-scope.js';
|
||||
|
||||
describe('deriveFinalizationWikiPageKeys', () => {
|
||||
it('maps changed global wiki markdown paths to page keys', () => {
|
||||
expect(
|
||||
deriveFinalizationWikiPageKeys([
|
||||
'wiki/global/historic-sql-orders.md',
|
||||
'wiki/global/nested/page.md',
|
||||
'README.md',
|
||||
]),
|
||||
).toEqual(['historic-sql-orders']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deriveFinalizationTouchedSources', () => {
|
||||
it('maps standalone semantic-layer files directly', async () => {
|
||||
const result = await deriveFinalizationTouchedSources({
|
||||
changedPaths: ['semantic-layer/warehouse/orders.yaml'],
|
||||
beforeSourcesByConnection: new Map(),
|
||||
afterSourcesByConnection: new Map(),
|
||||
});
|
||||
expect(result).toEqual({
|
||||
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
||||
unresolvedPaths: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('resolves aggregate _schema changes by comparing loaded source snapshots', async () => {
|
||||
const beforeSourcesByConnection = new Map([
|
||||
[
|
||||
'warehouse',
|
||||
[
|
||||
{
|
||||
name: 'orders',
|
||||
grain: ['order_id'],
|
||||
columns: [{ name: 'order_id', type: 'string' }],
|
||||
joins: [],
|
||||
measures: [],
|
||||
usage: {
|
||||
narrative: 'old',
|
||||
frequencyTier: 'low' as const,
|
||||
commonFilters: [],
|
||||
commonJoins: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
]);
|
||||
const afterSourcesByConnection = new Map([
|
||||
[
|
||||
'warehouse',
|
||||
[
|
||||
{
|
||||
name: 'orders',
|
||||
grain: ['order_id'],
|
||||
columns: [{ name: 'order_id', type: 'string' }],
|
||||
joins: [],
|
||||
measures: [],
|
||||
usage: {
|
||||
narrative: 'new',
|
||||
frequencyTier: 'high' as const,
|
||||
commonFilters: [],
|
||||
commonJoins: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
const result = await deriveFinalizationTouchedSources({
|
||||
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
||||
beforeSourcesByConnection,
|
||||
afterSourcesByConnection,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
||||
unresolvedPaths: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('flags aggregate _schema changes that cannot be resolved to logical sources', async () => {
|
||||
const beforeSourcesByConnection = new Map([['warehouse', []]]);
|
||||
const afterSourcesByConnection = new Map([['warehouse', []]]);
|
||||
|
||||
const result = await deriveFinalizationTouchedSources({
|
||||
changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
||||
beforeSourcesByConnection,
|
||||
afterSourcesByConnection,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
touchedSources: [],
|
||||
unresolvedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('compareFinalizationDeclarations', () => {
|
||||
it('reports missing and extra adapter declarations', () => {
|
||||
expect(
|
||||
compareFinalizationDeclarations({
|
||||
declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
||||
derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'customers' }],
|
||||
declaredChangedWikiPageKeys: ['orders'],
|
||||
derivedChangedWikiPageKeys: ['orders', 'patterns'],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
artifactKind: 'sl',
|
||||
key: 'warehouse:customers',
|
||||
direction: 'missing_from_adapter_declaration',
|
||||
},
|
||||
{
|
||||
artifactKind: 'sl',
|
||||
key: 'warehouse:orders',
|
||||
direction: 'extra_in_adapter_declaration',
|
||||
},
|
||||
{
|
||||
artifactKind: 'wiki',
|
||||
key: 'patterns',
|
||||
direction: 'missing_from_adapter_declaration',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue