apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-28 08:49:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(ingest): adapter-owned finalization replaces post-processor escape hatch (#136)

Browse source 
* Refine adapter-owned ingest finalization design after adversarial review iteration 1

* Refine adapter-owned ingest finalization design after adversarial review iteration 2

* Refine adapter-owned ingest finalization design after adversarial review iteration 3

* Implement adapter-owned ingest finalization v1

Moves finalization from runner-owned post-processors into typed
SourceAdapter.finalize() contracts. Adds finalization report schema,
scope derivation, override replay context, and migrates historic-SQL
projection. Removes IngestBundlePostProcessorPort wiring and
HistoricSqlProjectionPostProcessor.

* feat(ingest): export finalization adapter contract types

* test(ingest): exercise historic sql finalization locally

* docs(plans): add adapter-owned finalization v1 closure plan

* fix(setup): unblock clean Linux installs and add enabled_tables allowlist

- Pin managed Python runtime to 3.13 via `uv venv --python 3.13` so installs
  don't pick the system 3.12 on Ubuntu 24.04 and fail at wheel install.
- Sanitize NO_PROXY/no_proxy for the daemon child process — drop IPv6 CIDR
  entries that httpx rejects with InvalidURL (OrbStack injects these by
  default).
- Add `enabled_tables` allowlist on warehouse connections (zod schema +
  live-database introspection filter) to scope ingest to specific tables.
- Add `getting-started/troubleshooting-linux` docs page covering the Python
  3.13 prerequisite, IPv6 proxy gotcha, and a minimal working recipe; link
  it from the quickstart troubleshooting table and the llms-docs map.
- Make docs-site origin overridable via `KTX_DOCS_ORIGIN` so local builds
  can serve under host.docker.internal.

* Move docs changes to specs repo

* fix(cli): keep managed runtime python version private

* Deduplicate enabled tables filtering
This commit is contained in:
Andrey Avtomonov 2026-05-20 14:17:10 +02:00 • committed by GitHub
parent fb82993ce1
commit 4ec5903aa5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 1423 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

17
packages/context/src/scan/enabled-tables.ts Normal file
View file

@ -0,0 +1,17 @@
import type { KtxSchemaSnapshot } from './types.js';
export function resolveEnabledTables(connection: Record<string, unknown> | undefined): Set<string> | null {
const raw = connection?.enabled_tables;
if (!Array.isArray(raw) || raw.length === 0) return null;
return new Set(raw.filter((v): v is string => typeof v === 'string'));
}
export function filterSnapshotTables(snapshot: KtxSchemaSnapshot, enabledTables: Set<string>): KtxSchemaSnapshot {
return {
...snapshot,
tables: snapshot.tables.filter((table) => {
const key = table.db ? `${table.db}.${table.name}` : table.name;
return enabledTables.has(key);
}),
};
}

19
packages/context/src/scan/local-scan.ts
View file

@ -15,6 +15,7 @@ import type { KtxProjectLlmConfig, KtxScanEnrichmentConfig, KtxScanRelationshipC
import type { KtxLocalProject } from '../project/index.js';
import { ktxLocalStateDbPath } from '../project/local-state-db.js';
import { redactKtxScanReport } from './credentials.js';
import { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js';
import { completedKtxScanEnrichmentStateSummary } from './enrichment-state.js';
import { failedKtxScanEnrichmentSummary, ktxScanErrorMessage } from './enrichment-summary.js';
import {
@ -320,22 +321,6 @@ async function readScanReport(
}
}
export function resolveEnabledTables(connection: Record<string, unknown> | undefined): Set<string> | null {
const raw = connection?.enabled_tables;
if (!Array.isArray(raw) || raw.length === 0) return null;
return new Set(raw.filter((v): v is string => typeof v === 'string'));
}
export function filterSnapshotTables(snapshot: KtxSchemaSnapshot, enabledTables: Set<string>): KtxSchemaSnapshot {
return {
...snapshot,
tables: snapshot.tables.filter((table) => {
const key = table.db ? `${table.db}.${table.name}` : table.name;
return enabledTables.has(key);
}),
};
}
function createFilteredConnector(connector: KtxScanConnector, enabledTables: Set<string>): KtxScanConnector {
return {
...connector,
@ -346,6 +331,8 @@ function createFilteredConnector(connector: KtxScanConnector, enabledTables: Set
};
}
export { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js';
function withInternalLiveDatabaseAdapter(project: KtxLocalProject): KtxLocalProject {
if (project.config.ingest.adapters.includes(LIVE_DATABASE_ADAPTER)) {
return project;