apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-07-01 08:59:39 +02:00
Code Issues Projects Releases Packages Wiki Activity

rename klo to ktx

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-10 23:51:24 +02:00
parent 1a42152e6f
commit 3ce510b55b
704 changed files with 10205 additions and 10255 deletions
Download patch file Download diff file Expand all files Collapse all files

56
packages/context/src/scan/credentials.test.ts
View file

@ -1,26 +1,26 @@
import { describe, expect, it } from 'vitest';
import {
REDACTED_KLO_CREDENTIAL_VALUE,
redactKloCredentialEnvelope,
redactKloCredentialValue,
redactKloScanMetadata,
redactKloScanReport,
redactKloScanWarning,
REDACTED_KTX_CREDENTIAL_VALUE,
redactKtxCredentialEnvelope,
redactKtxCredentialValue,
redactKtxScanMetadata,
redactKtxScanReport,
redactKtxScanWarning,
} from './credentials.js';
import type { KloCredentialEnvelope, KloScanReport, KloScanWarning } from './types.js';
import type { KtxCredentialEnvelope, KtxScanReport, KtxScanWarning } from './types.js';
describe('KLO scan credential redaction', () => {
describe('KTX scan credential redaction', () => {
it('keeps credential references inspectable', () => {
const envReference: KloCredentialEnvelope = { kind: 'env', name: 'DATABASE_URL' };
const fileReference: KloCredentialEnvelope = { kind: 'file', path: '~/.config/klo/warehouse' };
const envReference: KtxCredentialEnvelope = { kind: 'env', name: 'DATABASE_URL' };
const fileReference: KtxCredentialEnvelope = { kind: 'file', path: '~/.config/ktx/warehouse' };
expect(redactKloCredentialEnvelope(envReference)).toEqual(envReference);
expect(redactKloCredentialEnvelope(fileReference)).toEqual(fileReference);
expect(redactKtxCredentialEnvelope(envReference)).toEqual(envReference);
expect(redactKtxCredentialEnvelope(fileReference)).toEqual(fileReference);
});
it('redacts resolved credential envelope values recursively', () => {
expect(
redactKloCredentialEnvelope({
redactKtxCredentialEnvelope({
kind: 'resolved',
source: 'host',
values: {
@ -39,19 +39,19 @@ describe('KLO scan credential redaction', () => {
redacted: true,
values: {
username: 'readonly',
password: REDACTED_KLO_CREDENTIAL_VALUE,
password: REDACTED_KTX_CREDENTIAL_VALUE,
nested: {
api_key: REDACTED_KLO_CREDENTIAL_VALUE,
api_key: REDACTED_KTX_CREDENTIAL_VALUE,
warehouse: 'compute_wh',
},
headers: [{ authorizationToken: REDACTED_KLO_CREDENTIAL_VALUE }, { label: 'safe' }],
headers: [{ authorizationToken: REDACTED_KTX_CREDENTIAL_VALUE }, { label: 'safe' }],
},
});
});
it('redacts scan metadata fields that commonly contain secrets', () => {
expect(
redactKloScanMetadata({
redactKtxScanMetadata({
driver: 'postgres',
url: 'postgres://user:pass@example.test/db', // pragma: allowlist secret
serviceAccountJson: {
@ -62,17 +62,17 @@ describe('KLO scan credential redaction', () => {
}),
).toEqual({
driver: 'postgres',
url: REDACTED_KLO_CREDENTIAL_VALUE,
url: REDACTED_KTX_CREDENTIAL_VALUE,
serviceAccountJson: {
client_email: 'reader@example.test',
private_key: REDACTED_KLO_CREDENTIAL_VALUE,
private_key: REDACTED_KTX_CREDENTIAL_VALUE,
},
safeCount: 3,
});
});
it('redacts scan warning messages and metadata without hiding safe context', () => {
const warning: KloScanWarning = {
const warning: KtxScanWarning = {
code: 'sampling_failed',
message: 'sample failed for postgres://reader:secret@example.test/db', // pragma: allowlist secret
recoverable: true,
@ -86,15 +86,15 @@ describe('KLO scan credential redaction', () => {
},
};
expect(redactKloScanWarning(warning)).toEqual({
expect(redactKtxScanWarning(warning)).toEqual({
code: 'sampling_failed',
message: 'sample failed for postgres://reader:<redacted>@example.test/db',
recoverable: true,
metadata: {
table: 'orders',
url: REDACTED_KLO_CREDENTIAL_VALUE,
url: REDACTED_KTX_CREDENTIAL_VALUE,
nested: {
api_key: REDACTED_KLO_CREDENTIAL_VALUE,
api_key: REDACTED_KTX_CREDENTIAL_VALUE,
schema: 'public',
},
},
@ -102,7 +102,7 @@ describe('KLO scan credential redaction', () => {
});
it('redacts scan report warning metadata recursively', () => {
const report: KloScanReport = {
const report: KtxScanReport = {
connectionId: 'warehouse',
driver: 'postgres',
syncId: 'sync-1',
@ -164,10 +164,10 @@ describe('KLO scan credential redaction', () => {
createdAt: '2026-04-29T00:00:00.000Z',
};
const redacted = redactKloScanReport(report);
const redacted = redactKtxScanReport(report);
expect(redacted.warnings[0]?.metadata).toEqual({
credentials_json: REDACTED_KLO_CREDENTIAL_VALUE,
credentials_json: REDACTED_KTX_CREDENTIAL_VALUE,
safeCount: 2,
});
expect(report.warnings[0]?.metadata).toEqual({
@ -177,7 +177,7 @@ describe('KLO scan credential redaction', () => {
});
it('redacts standalone primitive credential values only when the field key is sensitive', () => {
expect(redactKloCredentialValue('password', 'abc')).toBe(REDACTED_KLO_CREDENTIAL_VALUE);
expect(redactKloCredentialValue('schema', 'public')).toBe('public');
expect(redactKtxCredentialValue('password', 'abc')).toBe(REDACTED_KTX_CREDENTIAL_VALUE);
expect(redactKtxCredentialValue('schema', 'public')).toBe('public');
});
});

36
packages/context/src/scan/credentials.ts
View file

@ -1,22 +1,22 @@
import {
redactKloSensitiveMetadata,
redactKloSensitiveText,
redactKloSensitiveValue,
REDACTED_KLO_CREDENTIAL_VALUE,
redactKtxSensitiveMetadata,
redactKtxSensitiveText,
redactKtxSensitiveValue,
REDACTED_KTX_CREDENTIAL_VALUE,
} from '../core/redaction.js';
import type { KloCredentialEnvelope, KloScanReport, KloScanWarning } from './types.js';
import type { KtxCredentialEnvelope, KtxScanReport, KtxScanWarning } from './types.js';
export { REDACTED_KLO_CREDENTIAL_VALUE };
export { REDACTED_KTX_CREDENTIAL_VALUE };
export function redactKloCredentialValue(key: string, value: unknown): unknown {
return redactKloSensitiveValue(key, value);
export function redactKtxCredentialValue(key: string, value: unknown): unknown {
return redactKtxSensitiveValue(key, value);
}
export function redactKloScanMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
return redactKloSensitiveMetadata(metadata);
export function redactKtxScanMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
return redactKtxSensitiveMetadata(metadata);
}
export function redactKloCredentialEnvelope(envelope: KloCredentialEnvelope): KloCredentialEnvelope {
export function redactKtxCredentialEnvelope(envelope: KtxCredentialEnvelope): KtxCredentialEnvelope {
if (envelope.kind !== 'resolved') {
return envelope;
}
@ -24,27 +24,27 @@ export function redactKloCredentialEnvelope(envelope: KloCredentialEnvelope): Kl
kind: 'resolved',
source: envelope.source,
redacted: true,
values: redactKloScanMetadata(envelope.values),
values: redactKtxScanMetadata(envelope.values),
};
}
export function redactKloScanWarning(warning: KloScanWarning): KloScanWarning {
export function redactKtxScanWarning(warning: KtxScanWarning): KtxScanWarning {
if (!warning.metadata) {
return {
...warning,
message: redactKloSensitiveText(warning.message),
message: redactKtxSensitiveText(warning.message),
};
}
return {
...warning,
message: redactKloSensitiveText(warning.message),
metadata: redactKloScanMetadata(warning.metadata),
message: redactKtxSensitiveText(warning.message),
metadata: redactKtxScanMetadata(warning.metadata),
};
}
export function redactKloScanReport(report: KloScanReport): KloScanReport {
export function redactKtxScanReport(report: KtxScanReport): KtxScanReport {
return {
...report,
warnings: report.warnings.map((warning) => redactKloScanWarning(warning)),
warnings: report.warnings.map((warning) => redactKtxScanWarning(warning)),
};
}

120
packages/context/src/scan/data-dictionary.test.ts
View file

@ -1,113 +1,113 @@
import { describe, expect, it } from 'vitest';
import {
defaultKloDataDictionarySettings,
isKloDataDictionaryCandidate,
shouldKloSampleColumnForDictionary,
defaultKtxDataDictionarySettings,
isKtxDataDictionaryCandidate,
shouldKtxSampleColumnForDictionary,
} from './data-dictionary.js';
const defaultPatterns = defaultKloDataDictionarySettings.excludePatterns;
const defaultPatterns = defaultKtxDataDictionarySettings.excludePatterns;
describe('KLO scan data dictionary policy', () => {
describe('KTX scan data dictionary policy', () => {
it('includes text-like and boolean categorical types', () => {
expect(isKloDataDictionaryCandidate('varchar(50)', 'status', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('VARCHAR', 'category', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('text', 'region', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('string', 'payment_method', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('nvarchar(100)', 'tier', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('enum', 'status', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('boolean', 'active', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('bool', 'verified', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('character varying(50)', 'region', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('character(1)', 'flag', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('ntext', 'category', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar(50)', 'status', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('VARCHAR', 'category', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('text', 'region', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('string', 'payment_method', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('nvarchar(100)', 'tier', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('enum', 'status', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('boolean', 'active', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('bool', 'verified', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('character varying(50)', 'region', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('character(1)', 'flag', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('ntext', 'category', defaultPatterns)).toBe(true);
});
it('excludes non-categorical primitive types', () => {
expect(isKloDataDictionaryCandidate('integer', 'count', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('bigint', 'total', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('timestamp', 'created', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('date', 'birth', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('numeric', 'amount', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('decimal(10,2)', 'price', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('float', 'rate', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('integer', 'count', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('bigint', 'total', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('timestamp', 'created', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('date', 'birth', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('numeric', 'amount', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('decimal(10,2)', 'price', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('float', 'rate', defaultPatterns)).toBe(false);
});
it('excludes configured high-cardinality or sensitive name patterns', () => {
expect(isKloDataDictionaryCandidate('varchar', 'user_id', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'session_uuid', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'api_key', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'password_hash', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'auth_token', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'id', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'created_at', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'birth_date', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('text', 'description', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('text', 'email_body', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'image_url', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'email', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'phone_number', defaultPatterns)).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'street_address', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'user_id', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'session_uuid', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'api_key', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'password_hash', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'auth_token', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'id', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'created_at', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'birth_date', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('text', 'description', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('text', 'email_body', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'image_url', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'email', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'phone_number', defaultPatterns)).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'street_address', defaultPatterns)).toBe(false);
});
it('keeps business categorical names eligible', () => {
expect(isKloDataDictionaryCandidate('varchar', 'status', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'region', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'country', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'payment_method', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'currency', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'plan', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'category', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'tier', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'gender', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'language', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'order_type', defaultPatterns)).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'order_status', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'status', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'region', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'country', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'payment_method', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'currency', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'plan', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'category', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'tier', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'gender', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'language', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'order_type', defaultPatterns)).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'order_status', defaultPatterns)).toBe(true);
});
it('respects host-provided exclusion patterns and skips invalid regex patterns', () => {
expect(isKloDataDictionaryCandidate('varchar', 'company_size', ['company'])).toBe(false);
expect(isKloDataDictionaryCandidate('varchar', 'status', ['company'])).toBe(true);
expect(isKloDataDictionaryCandidate('varchar', 'status', ['[invalid', '(unclosed'])).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'company_size', ['company'])).toBe(false);
expect(isKtxDataDictionaryCandidate('varchar', 'status', ['company'])).toBe(true);
expect(isKtxDataDictionaryCandidate('varchar', 'status', ['[invalid', '(unclosed'])).toBe(true);
});
it('skips columns that already have persisted dictionary state', () => {
expect(
shouldKloSampleColumnForDictionary({
shouldKtxSampleColumnForDictionary({
columnType: 'varchar',
columnName: 'status',
sampleValues: ['paid'],
cardinality: null,
settings: defaultKloDataDictionarySettings,
settings: defaultKtxDataDictionarySettings,
}),
).toEqual({ sample: false, reason: 'already_populated' });
expect(
shouldKloSampleColumnForDictionary({
shouldKtxSampleColumnForDictionary({
columnType: 'varchar',
columnName: 'empty_status',
sampleValues: null,
cardinality: 0,
settings: defaultKloDataDictionarySettings,
settings: defaultKtxDataDictionarySettings,
}),
).toEqual({ sample: false, reason: 'empty_column' });
expect(
shouldKloSampleColumnForDictionary({
shouldKtxSampleColumnForDictionary({
columnType: 'varchar',
columnName: 'customer_name',
sampleValues: null,
cardinality: 300,
settings: defaultKloDataDictionarySettings,
settings: defaultKtxDataDictionarySettings,
}),
).toEqual({ sample: false, reason: 'high_cardinality' });
expect(
shouldKloSampleColumnForDictionary({
shouldKtxSampleColumnForDictionary({
columnType: 'varchar',
columnName: 'status',
sampleValues: null,
cardinality: null,
settings: defaultKloDataDictionarySettings,
settings: defaultKtxDataDictionarySettings,
}),
).toEqual({ sample: true });
});

26
packages/context/src/scan/data-dictionary.ts
View file

@ -1,4 +1,4 @@
export interface KloDataDictionarySettings {
export interface KtxDataDictionarySettings {
cardinalityThreshold: number;
maxValuesToStore: number;
sampleSize: number;
@ -6,7 +6,7 @@ export interface KloDataDictionarySettings {
excludePatterns: string[];
}
export const defaultKloDataDictionarySettings: KloDataDictionarySettings = {
export const defaultKtxDataDictionarySettings: KtxDataDictionarySettings = {
cardinalityThreshold: 200,
maxValuesToStore: 100,
sampleSize: 10000,
@ -36,31 +36,31 @@ export const defaultKloDataDictionarySettings: KloDataDictionarySettings = {
],
};
export type KloDataDictionarySkipReason =
export type KtxDataDictionarySkipReason =
| 'not_candidate'
| 'already_populated'
| 'empty_column'
| 'high_cardinality';
export interface KloDataDictionarySampleDecision {
export interface KtxDataDictionarySampleDecision {
sample: boolean;
reason?: KloDataDictionarySkipReason;
reason?: KtxDataDictionarySkipReason;
}
export interface KloDataDictionaryColumnState {
export interface KtxDataDictionaryColumnState {
columnType: string;
columnName: string;
sampleValues?: readonly string[] | null;
cardinality?: number | null;
settings: KloDataDictionarySettings;
settings: KtxDataDictionarySettings;
}
const categoricalCandidateTypes = /^(n?varchar|n?char|n?text|string|character|enum|bool(ean)?)/i;
export function isKloDataDictionaryCandidate(
export function isKtxDataDictionaryCandidate(
columnType: string,
columnName: string,
excludePatterns: readonly string[] = defaultKloDataDictionarySettings.excludePatterns,
excludePatterns: readonly string[] = defaultKtxDataDictionarySettings.excludePatterns,
): boolean {
const typeLower = columnType.toLowerCase();
const nameLower = columnName.toLowerCase();
@ -83,9 +83,9 @@ export function isKloDataDictionaryCandidate(
return true;
}
export function shouldKloSampleColumnForDictionary(
input: KloDataDictionaryColumnState,
): KloDataDictionarySampleDecision {
export function shouldKtxSampleColumnForDictionary(
input: KtxDataDictionaryColumnState,
): KtxDataDictionarySampleDecision {
const sampleValues = input.sampleValues ?? null;
const cardinality = input.cardinality ?? null;
@ -101,7 +101,7 @@ export function shouldKloSampleColumnForDictionary(
return { sample: false, reason: 'high_cardinality' };
}
if (!isKloDataDictionaryCandidate(input.columnType, input.columnName, input.settings.excludePatterns)) {
if (!isKtxDataDictionaryCandidate(input.columnType, input.columnName, input.settings.excludePatterns)) {
return { sample: false, reason: 'not_candidate' };
}

36
packages/context/src/scan/description-generation.test.ts
View file

@ -7,15 +7,15 @@ vi.mock('ai', async (importOriginal) => {
import { generateText } from 'ai';
import {
buildKloColumnDescriptionPrompt,
buildKloDataSourceDescriptionPrompt,
buildKloTableDescriptionPrompt,
type KloDescriptionCachePort,
KloDescriptionGenerator,
buildKtxColumnDescriptionPrompt,
buildKtxDataSourceDescriptionPrompt,
buildKtxTableDescriptionPrompt,
type KtxDescriptionCachePort,
KtxDescriptionGenerator,
} from './description-generation.js';
import { createKloConnectorCapabilities, type KloScanConnector } from './types.js';
import { createKtxConnectorCapabilities, type KtxScanConnector } from './types.js';
function createCache(initial: Record<string, string> = {}): KloDescriptionCachePort {
function createCache(initial: Record<string, string> = {}): KtxDescriptionCachePort {
const data = new Map(Object.entries(initial));
return {
buildTableKey: (table) => [table.catalog, table.db, table.name].filter(Boolean).join('.'),
@ -51,11 +51,11 @@ function createLlmProvider(text = 'generated description') {
} as any;
}
function createConnector(): KloScanConnector {
function createConnector(): KtxScanConnector {
return {
id: 'test-connector',
driver: 'postgres',
capabilities: createKloConnectorCapabilities({
capabilities: createKtxConnectorCapabilities({
tableSampling: true,
columnSampling: true,
nestedAnalysis: true,
@ -79,9 +79,9 @@ function createConnector(): KloScanConnector {
};
}
describe('KLO description prompt builders', () => {
describe('KTX description prompt builders', () => {
it('builds column prompts with sample values, source descriptions, and nested BigQuery guidance', () => {
const prompt = buildKloColumnDescriptionPrompt({
const prompt = buildKtxColumnDescriptionPrompt({
columnName: 'payload',
columnValues: [{ nested: true }, '[1,2]'],
tableContext: 'Table: events | Columns: payload | Data source: BIGQUERY',
@ -112,7 +112,7 @@ describe('KLO description prompt builders', () => {
};
expect(
buildKloTableDescriptionPrompt({
buildKtxTableDescriptionPrompt({
tableName: 'orders',
sampleData: sample,
dataSourceType: 'POSTGRESQL',
@ -121,7 +121,7 @@ describe('KLO description prompt builders', () => {
).toContain('status: paid, refunded');
expect(
buildKloDataSourceDescriptionPrompt({
buildKtxDataSourceDescriptionPrompt({
tableSamples: [['orders', sample]],
dataSourceType: 'POSTGRESQL',
}),
@ -129,12 +129,12 @@ describe('KLO description prompt builders', () => {
});
});
describe('KloDescriptionGenerator', () => {
describe('KtxDescriptionGenerator', () => {
it('generates column descriptions with pre-fetched values, cache hits, and word-limit metadata', async () => {
const cache = createCache({ 'warehouse.public.orders.cached_status': 'Cached status description' });
const llmProvider = createLlmProvider('Payment state');
const connector = createConnector();
const generator = new KloDescriptionGenerator({
const generator = new KtxDescriptionGenerator({
llmProvider,
cache,
settings: {
@ -189,7 +189,7 @@ describe('KloDescriptionGenerator', () => {
it('samples through the connector when column values are not pre-fetched', async () => {
const connector = createConnector();
const generator = new KloDescriptionGenerator({
const generator = new KtxDescriptionGenerator({
llmProvider: createLlmProvider('Current order state'),
settings: {
columnMaxWords: 12,
@ -238,7 +238,7 @@ describe('KloDescriptionGenerator', () => {
totalRows: 1,
})),
};
const generator = new KloDescriptionGenerator({
const generator = new KtxDescriptionGenerator({
llmProvider: createLlmProvider('Generated through sampler'),
settings: {
columnMaxWords: 12,
@ -277,7 +277,7 @@ describe('KloDescriptionGenerator', () => {
it('generates and caches table and data-source descriptions', async () => {
const cache = createCache();
const connector = createConnector();
const generator = new KloDescriptionGenerator({
const generator = new KtxDescriptionGenerator({
llmProvider: createLlmProvider('Commerce orders'),
cache,
settings: {

146
packages/context/src/scan/description-generation.ts
View file

@ -1,30 +1,30 @@
import type { KloLlmProvider } from '@klo/llm';
import { generateKloText } from '../llm/index.js';
import type { KtxLlmProvider } from '@ktx/llm';
import { generateKtxText } from '../llm/index.js';
import type {
KloColumnSampleInput,
KloColumnSampleResult,
KloScanContext,
KloScanLoggerPort,
KloTableRef,
KloTableSampleInput,
KloTableSampleResult,
KtxColumnSampleInput,
KtxColumnSampleResult,
KtxScanContext,
KtxScanLoggerPort,
KtxTableRef,
KtxTableSampleInput,
KtxTableSampleResult,
} from './types.js';
export interface KloDescriptionCachePort {
buildTableKey(table: KloTableRef): string;
buildColumnKey(table: KloTableRef, columnName: string): string;
export interface KtxDescriptionCachePort {
buildTableKey(table: KtxTableRef): string;
buildColumnKey(table: KtxTableRef, columnName: string): string;
buildConnectionKey(connectionName: string): string;
get(key: string): Promise<string | null>;
set(key: string, value: string): Promise<void>;
}
export interface KloDescriptionSamplingPort {
export interface KtxDescriptionSamplingPort {
id: string;
sampleColumn?(input: KloColumnSampleInput, ctx: KloScanContext): Promise<KloColumnSampleResult>;
sampleTable?(input: KloTableSampleInput, ctx: KloScanContext): Promise<KloTableSampleResult>;
sampleColumn?(input: KtxColumnSampleInput, ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
sampleTable?(input: KtxTableSampleInput, ctx: KtxScanContext): Promise<KtxTableSampleResult>;
}
export interface KloDescriptionGenerationSettings {
export interface KtxDescriptionGenerationSettings {
columnMaxWords: number;
tableMaxWords: number;
dataSourceMaxWords: number;
@ -32,7 +32,7 @@ export interface KloDescriptionGenerationSettings {
concurrencyLimit?: number;
}
interface ResolvedKloDescriptionGenerationSettings {
interface ResolvedKtxDescriptionGenerationSettings {
columnMaxWords: number;
tableMaxWords: number;
dataSourceMaxWords: number;
@ -40,28 +40,28 @@ interface ResolvedKloDescriptionGenerationSettings {
concurrencyLimit: number;
}
export interface KloDescriptionColumn {
export interface KtxDescriptionColumn {
name: string;
type?: string;
rawDescriptions?: Record<string, string>;
sampleValues?: unknown[];
}
export interface KloDescriptionColumnTable extends KloTableRef {
columns: KloDescriptionColumn[];
export interface KtxDescriptionColumnTable extends KtxTableRef {
columns: KtxDescriptionColumn[];
}
export interface KloDescriptionTableInput extends KloTableRef {
export interface KtxDescriptionTableInput extends KtxTableRef {
rawDescriptions?: Record<string, string>;
}
export interface KloColumnAnalysisResult {
export interface KtxColumnAnalysisResult {
columnDescriptions: Array<[string, string | null]>;
processedColumns: string[];
skippedColumns: string[];
}
export interface KloColumnDescriptionPromptInput {
export interface KtxColumnDescriptionPromptInput {
columnName: string;
columnValues: unknown[];
tableContext: string;
@ -70,51 +70,51 @@ export interface KloColumnDescriptionPromptInput {
rawDescriptions?: Record<string, string>;
}
export interface KloTableDescriptionPromptInput {
export interface KtxTableDescriptionPromptInput {
tableName: string;
sampleData: KloTableSampleResult;
sampleData: KtxTableSampleResult;
dataSourceType: string;
rawDescriptions?: Record<string, string>;
}
export interface KloDataSourceDescriptionPromptInput {
tableSamples: Array<[string, KloTableSampleResult]>;
export interface KtxDataSourceDescriptionPromptInput {
tableSamples: Array<[string, KtxTableSampleResult]>;
dataSourceType: string;
}
export interface KloGenerateColumnDescriptionsInput {
export interface KtxGenerateColumnDescriptionsInput {
connectionId: string;
connector: KloDescriptionSamplingPort;
context: KloScanContext;
connector: KtxDescriptionSamplingPort;
context: KtxScanContext;
dataSourceType: string;
supportsNestedAnalysis: boolean;
table: KloDescriptionColumnTable;
table: KtxDescriptionColumnTable;
skipExisting?: boolean;
existingDescriptions?: Record<string, string | null>;
}
export interface KloGenerateTableDescriptionInput {
export interface KtxGenerateTableDescriptionInput {
connectionId: string;
connector: KloDescriptionSamplingPort;
context: KloScanContext;
connector: KtxDescriptionSamplingPort;
context: KtxScanContext;
dataSourceType: string;
table: KloDescriptionTableInput;
table: KtxDescriptionTableInput;
}
export interface KloGenerateDataSourceDescriptionInput {
export interface KtxGenerateDataSourceDescriptionInput {
connectionId: string;
connector: KloDescriptionSamplingPort;
context: KloScanContext;
connector: KtxDescriptionSamplingPort;
context: KtxScanContext;
dataSourceType: string;
tables: KloTableRef[];
tables: KtxTableRef[];
connectionName?: string;
}
export interface KloDescriptionGeneratorOptions {
llmProvider: KloLlmProvider;
cache?: KloDescriptionCachePort;
logger?: KloScanLoggerPort;
settings: KloDescriptionGenerationSettings;
export interface KtxDescriptionGeneratorOptions {
llmProvider: KtxLlmProvider;
cache?: KtxDescriptionCachePort;
logger?: KtxScanLoggerPort;
settings: KtxDescriptionGenerationSettings;
}
interface ColumnTaskResult {
@ -136,7 +136,7 @@ function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function toTableRef(table: KloTableRef): KloTableRef {
function toTableRef(table: KtxTableRef): KtxTableRef {
return {
catalog: table.catalog,
db: table.db,
@ -169,11 +169,11 @@ async function runWithConcurrency<TInput, TOutput>(
return results;
}
export function appendKloWordLimitInstruction(prompt: string, maxWords: number): string {
export function appendKtxWordLimitInstruction(prompt: string, maxWords: number): string {
return `${prompt}\n\nPlease provide a concise description in ${maxWords} words or less.`;
}
export function buildKloColumnDescriptionPrompt(input: KloColumnDescriptionPromptInput): string {
export function buildKtxColumnDescriptionPrompt(input: KtxColumnDescriptionPromptInput): string {
const sampleValues = input.columnValues.slice(0, 5);
const valuesStr = sampleValues
.filter((value) => value !== null && value !== undefined)
@ -221,7 +221,7 @@ Example:
return prompt.trim();
}
export function buildKloTableDescriptionPrompt(input: KloTableDescriptionPromptInput): string {
export function buildKtxTableDescriptionPrompt(input: KtxTableDescriptionPromptInput): string {
const columnInfo: string[] = [];
for (let index = 0; index < Math.min(input.sampleData.headers.length, 10); index += 1) {
const header = input.sampleData.headers[index];
@ -268,7 +268,7 @@ export function buildKloTableDescriptionPrompt(input: KloTableDescriptionPromptI
return prompt.trim();
}
export function buildKloDataSourceDescriptionPrompt(input: KloDataSourceDescriptionPromptInput): string {
export function buildKtxDataSourceDescriptionPrompt(input: KtxDataSourceDescriptionPromptInput): string {
const tablesText = input.tableSamples
.map(
([tableName, sampleData]) =>
@ -301,13 +301,13 @@ export function buildKloDataSourceDescriptionPrompt(input: KloDataSourceDescript
return prompt.trim();
}
export class KloDescriptionGenerator {
private readonly llmProvider: KloLlmProvider;
private readonly cache?: KloDescriptionCachePort;
private readonly logger?: KloScanLoggerPort;
private readonly settings: ResolvedKloDescriptionGenerationSettings;
export class KtxDescriptionGenerator {
private readonly llmProvider: KtxLlmProvider;
private readonly cache?: KtxDescriptionCachePort;
private readonly logger?: KtxScanLoggerPort;
private readonly settings: ResolvedKtxDescriptionGenerationSettings;
constructor(options: KloDescriptionGeneratorOptions) {
constructor(options: KtxDescriptionGeneratorOptions) {
this.llmProvider = options.llmProvider;
this.cache = options.cache;
this.logger = options.logger;
@ -320,7 +320,7 @@ export class KloDescriptionGenerator {
};
}
async generateColumnDescriptions(input: KloGenerateColumnDescriptionsInput): Promise<KloColumnAnalysisResult> {
async generateColumnDescriptions(input: KtxGenerateColumnDescriptionsInput): Promise<KtxColumnAnalysisResult> {
const columnsToProcess = input.table.columns;
const tableContext = `Table: ${input.table.name} | Columns: ${columnsToProcess.map((column) => column.name).join(', ')} | Data source: ${input.dataSourceType}`;
@ -348,7 +348,7 @@ export class KloDescriptionGenerator {
};
}
async generateTableDescription(input: KloGenerateTableDescriptionInput): Promise<string> {
async generateTableDescription(input: KtxGenerateTableDescriptionInput): Promise<string> {
const tableRef = toTableRef(input.table);
const cacheKey = this.cache?.buildTableKey(tableRef);
if (cacheKey) {
@ -359,7 +359,7 @@ export class KloDescriptionGenerator {
}
if (!input.connector.sampleTable) {
this.logger?.warn('KLO scan connector does not support table sampling for table description generation', {
this.logger?.warn('KTX scan connector does not support table sampling for table description generation', {
connectorId: input.connector.id,
table: input.table.name,
});
@ -375,7 +375,7 @@ export class KloDescriptionGenerator {
},
input.context,
);
const prompt = buildKloTableDescriptionPrompt({
const prompt = buildKtxTableDescriptionPrompt({
tableName: input.table.name,
sampleData,
dataSourceType: input.dataSourceType,
@ -384,7 +384,7 @@ export class KloDescriptionGenerator {
const description = await this.generateAiDescription(
prompt,
this.settings.tableMaxWords,
'klo-table-description',
'ktx-table-description',
);
if (cacheKey) {
await this.cache?.set(cacheKey, description);
@ -396,7 +396,7 @@ export class KloDescriptionGenerator {
}
}
async generateDataSourceDescription(input: KloGenerateDataSourceDescriptionInput): Promise<string> {
async generateDataSourceDescription(input: KtxGenerateDataSourceDescriptionInput): Promise<string> {
if (input.tables.length === 0) {
return 'No tables found in database';
}
@ -410,7 +410,7 @@ export class KloDescriptionGenerator {
}
if (!input.connector.sampleTable) {
this.logger?.warn('KLO scan connector does not support table sampling for data-source description generation', {
this.logger?.warn('KTX scan connector does not support table sampling for data-source description generation', {
connectorId: input.connector.id,
});
return 'No accessible tables found in database';
@ -427,7 +427,7 @@ export class KloDescriptionGenerator {
},
input.context,
);
return [table.name, sampleData] as [string, KloTableSampleResult];
return [table.name, sampleData] as [string, KtxTableSampleResult];
} catch (error) {
this.logger?.warn(`Failed to sample table '${table.name}' for data source analysis - ${errorMessage(error)}`);
return null;
@ -435,21 +435,21 @@ export class KloDescriptionGenerator {
});
const accessibleSamples = tableSamples.filter(
(sample): sample is [string, KloTableSampleResult] => sample !== null,
(sample): sample is [string, KtxTableSampleResult] => sample !== null,
);
if (accessibleSamples.length === 0) {
return 'No accessible tables found in database';
}
try {
const prompt = buildKloDataSourceDescriptionPrompt({
const prompt = buildKtxDataSourceDescriptionPrompt({
tableSamples: accessibleSamples,
dataSourceType: input.dataSourceType,
});
const description = await this.generateAiDescription(
prompt,
this.settings.dataSourceMaxWords,
'klo-data-source-description',
'ktx-data-source-description',
);
if (cacheKey) {
await this.cache?.set(cacheKey, description);
@ -462,8 +462,8 @@ export class KloDescriptionGenerator {
}
private async generateOneColumnDescription(
input: KloGenerateColumnDescriptionsInput,
column: KloDescriptionColumn,
input: KtxGenerateColumnDescriptionsInput,
column: KtxDescriptionColumn,
tableContext: string,
): Promise<ColumnTaskResult> {
const existingDescription = input.existingDescriptions?.[column.name];
@ -494,7 +494,7 @@ export class KloDescriptionGenerator {
let columnValues = column.sampleValues;
if (!columnValues || columnValues.length === 0) {
if (!input.connector.sampleColumn) {
this.logger?.warn('KLO scan connector does not support column sampling for column description generation', {
this.logger?.warn('KTX scan connector does not support column sampling for column description generation', {
connectorId: input.connector.id,
table: input.table.name,
column: column.name,
@ -529,7 +529,7 @@ export class KloDescriptionGenerator {
};
}
const prompt = buildKloColumnDescriptionPrompt({
const prompt = buildKtxColumnDescriptionPrompt({
columnName: column.name,
columnValues: nonNullValues,
tableContext,
@ -540,7 +540,7 @@ export class KloDescriptionGenerator {
const description = await this.generateAiDescription(
prompt,
this.settings.columnMaxWords,
'klo-column-description',
'ktx-column-description',
);
if (cacheKey) {
@ -566,10 +566,10 @@ export class KloDescriptionGenerator {
private async generateAiDescription(prompt: string, maxWords: number, _operationName: string): Promise<string> {
try {
const text = await generateKloText({
const text = await generateKtxText({
llmProvider: this.llmProvider,
role: 'candidateExtraction',
prompt: appendKloWordLimitInstruction(prompt, maxWords),
prompt: appendKtxWordLimitInstruction(prompt, maxWords),
temperature: this.settings.temperature,
});
const description = text.trim();

10
packages/context/src/scan/embedding-text.test.ts
View file

@ -1,10 +1,10 @@
import { describe, expect, it } from 'vitest';
import { buildKloColumnEmbeddingText } from './embedding-text.js';
import { buildKtxColumnEmbeddingText } from './embedding-text.js';
describe('KLO scan embedding text', () => {
describe('KTX scan embedding text', () => {
it('builds column embedding text with table, description, FK, and sample-value context', () => {
expect(
buildKloColumnEmbeddingText({
buildKtxColumnEmbeddingText({
tableName: 'orders',
columnName: 'status',
columnType: 'varchar',
@ -24,7 +24,7 @@ describe('KLO scan embedding text', () => {
it('omits optional sections when the scan has no enrichment context yet', () => {
expect(
buildKloColumnEmbeddingText({
buildKtxColumnEmbeddingText({
tableName: 'orders',
columnName: 'id',
columnType: 'integer',
@ -35,7 +35,7 @@ describe('KLO scan embedding text', () => {
it('keeps all available sample values when no explicit max is supplied', () => {
expect(
buildKloColumnEmbeddingText({
buildKtxColumnEmbeddingText({
tableName: 'orders',
columnName: 'status',
columnType: 'varchar',

8
packages/context/src/scan/embedding-text.ts
View file

@ -1,20 +1,20 @@
export interface KloColumnEmbeddingForeignKeys {
export interface KtxColumnEmbeddingForeignKeys {
outgoing: Array<{ toTable: string; toColumn: string }>;
incoming: Array<{ fromTable: string; fromColumn: string }>;
}
export interface KloColumnEmbeddingTextInput {
export interface KtxColumnEmbeddingTextInput {
tableName: string;
columnName: string;
columnType: string;
resolvedDescription: string | null;
sampleValues?: readonly string[] | null;
resolvedTableDescription?: string | null;
foreignKeys?: KloColumnEmbeddingForeignKeys | null;
foreignKeys?: KtxColumnEmbeddingForeignKeys | null;
maxSampleValues?: number;
}
export function buildKloColumnEmbeddingText(input: KloColumnEmbeddingTextInput): string {
export function buildKtxColumnEmbeddingText(input: KtxColumnEmbeddingTextInput): string {
const parts: string[] = [];
parts.push(`${input.tableName}.${input.columnName} (${input.columnType})`);

24
packages/context/src/scan/enrichment-state.test.ts
View file

@ -3,14 +3,14 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
completedKloScanEnrichmentStateSummary,
computeKloScanEnrichmentInputHash,
summarizeKloScanEnrichmentState,
completedKtxScanEnrichmentStateSummary,
computeKtxScanEnrichmentInputHash,
summarizeKtxScanEnrichmentState,
} from './enrichment-state.js';
import { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js';
import type { KloSchemaSnapshot } from './types.js';
import type { KtxSchemaSnapshot } from './types.js';
const snapshot: KloSchemaSnapshot = {
const snapshot: KtxSchemaSnapshot = {
connectionId: 'warehouse',
driver: 'postgres',
extractedAt: '2026-04-29T12:00:00.000Z',
@ -45,7 +45,7 @@ describe('scan enrichment state', () => {
let store: SqliteLocalScanEnrichmentStateStore;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-scan-enrichment-state-'));
tempDir = await mkdtemp(join(tmpdir(), 'ktx-scan-enrichment-state-'));
store = new SqliteLocalScanEnrichmentStateStore({ dbPath: join(tempDir, 'db.sqlite') });
});
@ -54,13 +54,13 @@ describe('scan enrichment state', () => {
});
it('computes stable input hashes without depending on object key order', () => {
const first = computeKloScanEnrichmentInputHash({
const first = computeKtxScanEnrichmentInputHash({
snapshot,
mode: 'enriched',
detectRelationships: true,
providerIdentity: { provider: 'deterministic', embeddingDimensions: 8, llmModel: 'a' },
});
const second = computeKloScanEnrichmentInputHash({
const second = computeKtxScanEnrichmentInputHash({
snapshot: { ...snapshot, metadata: {} },
mode: 'enriched',
detectRelationships: true,
@ -70,7 +70,7 @@ describe('scan enrichment state', () => {
if (!firstTable) {
throw new Error('Expected test snapshot table');
}
const changed = computeKloScanEnrichmentInputHash({
const changed = computeKtxScanEnrichmentInputHash({
snapshot: { ...snapshot, tables: [{ ...firstTable, name: 'orders_v2' }] },
mode: 'enriched',
detectRelationships: true,
@ -83,7 +83,7 @@ describe('scan enrichment state', () => {
});
it('persists completed stages and ignores stale hashes', async () => {
const inputHash = computeKloScanEnrichmentInputHash({
const inputHash = computeKtxScanEnrichmentInputHash({
snapshot,
mode: 'enriched',
detectRelationships: true,
@ -155,7 +155,7 @@ describe('scan enrichment state', () => {
it('summarizes resumed, completed, and failed stages for reports', () => {
expect(
summarizeKloScanEnrichmentState({
summarizeKtxScanEnrichmentState({
resumedStages: ['descriptions'],
completedStages: ['descriptions', 'embeddings'],
failedStages: ['relationships'],
@ -166,7 +166,7 @@ describe('scan enrichment state', () => {
failedStages: ['relationships'],
});
expect(completedKloScanEnrichmentStateSummary()).toEqual({
expect(completedKtxScanEnrichmentStateSummary()).toEqual({
resumedStages: [],
completedStages: [],
failedStages: [],

58
packages/context/src/scan/enrichment-state.ts
View file

@ -1,24 +1,24 @@
import { createHash } from 'node:crypto';
import type { KloScanEnrichmentStage, KloScanEnrichmentStateSummary, KloScanMode, KloSchemaSnapshot } from './types.js';
import type { KtxScanEnrichmentStage, KtxScanEnrichmentStateSummary, KtxScanMode, KtxSchemaSnapshot } from './types.js';
export const KLO_SCAN_ENRICHMENT_STAGES: readonly KloScanEnrichmentStage[] = [
export const KTX_SCAN_ENRICHMENT_STAGES: readonly KtxScanEnrichmentStage[] = [
'descriptions',
'embeddings',
'relationships',
] as const;
export interface KloScanEnrichmentStageLookup {
export interface KtxScanEnrichmentStageLookup {
runId: string;
stage: KloScanEnrichmentStage;
stage: KtxScanEnrichmentStage;
inputHash: string;
}
export interface KloScanEnrichmentCompletedStage<TOutput = unknown> {
export interface KtxScanEnrichmentCompletedStage<TOutput = unknown> {
runId: string;
connectionId: string;
syncId: string;
mode: KloScanMode;
stage: KloScanEnrichmentStage;
mode: KtxScanMode;
stage: KtxScanEnrichmentStage;
inputHash: string;
status: 'completed';
output: TOutput;
@ -26,12 +26,12 @@ export interface KloScanEnrichmentCompletedStage<TOutput = unknown> {
updatedAt: string;
}
export interface KloScanEnrichmentFailedStage {
export interface KtxScanEnrichmentFailedStage {
runId: string;
connectionId: string;
syncId: string;
mode: KloScanMode;
stage: KloScanEnrichmentStage;
mode: KtxScanMode;
stage: KtxScanEnrichmentStage;
inputHash: string;
status: 'failed';
output: null;
@ -39,24 +39,24 @@ export interface KloScanEnrichmentFailedStage {
updatedAt: string;
}
export type KloScanEnrichmentStageRecord<TOutput = unknown> =
| KloScanEnrichmentCompletedStage<TOutput>
| KloScanEnrichmentFailedStage;
export type KtxScanEnrichmentStageRecord<TOutput = unknown> =
| KtxScanEnrichmentCompletedStage<TOutput>
| KtxScanEnrichmentFailedStage;
export interface KloScanEnrichmentStateStore {
export interface KtxScanEnrichmentStateStore {
findCompletedStage<TOutput = unknown>(
input: KloScanEnrichmentStageLookup,
): Promise<KloScanEnrichmentCompletedStage<TOutput> | null>;
input: KtxScanEnrichmentStageLookup,
): Promise<KtxScanEnrichmentCompletedStage<TOutput> | null>;
saveCompletedStage<TOutput = unknown>(
input: Omit<KloScanEnrichmentCompletedStage<TOutput>, 'status' | 'errorMessage'>,
input: Omit<KtxScanEnrichmentCompletedStage<TOutput>, 'status' | 'errorMessage'>,
): Promise<void>;
saveFailedStage(input: Omit<KloScanEnrichmentFailedStage, 'status' | 'output'>): Promise<void>;
listRunStages(runId: string): Promise<KloScanEnrichmentStageRecord[]>;
saveFailedStage(input: Omit<KtxScanEnrichmentFailedStage, 'status' | 'output'>): Promise<void>;
listRunStages(runId: string): Promise<KtxScanEnrichmentStageRecord[]>;
}
export interface ComputeKloScanEnrichmentInputHashInput {
snapshot: KloSchemaSnapshot;
mode: KloScanMode;
export interface ComputeKtxScanEnrichmentInputHashInput {
snapshot: KtxSchemaSnapshot;
mode: KtxScanMode;
detectRelationships: boolean;
providerIdentity: Record<string, unknown>;
relationshipSettings?: unknown;
@ -75,14 +75,14 @@ function stableJson(value: unknown): string {
return JSON.stringify(value);
}
export function computeKloScanEnrichmentInputHash(input: ComputeKloScanEnrichmentInputHashInput): string {
export function computeKtxScanEnrichmentInputHash(input: ComputeKtxScanEnrichmentInputHashInput): string {
return createHash('sha256').update(stableJson(input)).digest('hex');
}
function uniqueStages(stages: KloScanEnrichmentStage[]): KloScanEnrichmentStage[] {
const seen = new Set<KloScanEnrichmentStage>();
const ordered: KloScanEnrichmentStage[] = [];
for (const stage of KLO_SCAN_ENRICHMENT_STAGES) {
function uniqueStages(stages: KtxScanEnrichmentStage[]): KtxScanEnrichmentStage[] {
const seen = new Set<KtxScanEnrichmentStage>();
const ordered: KtxScanEnrichmentStage[] = [];
for (const stage of KTX_SCAN_ENRICHMENT_STAGES) {
if (stages.includes(stage) && !seen.has(stage)) {
seen.add(stage);
ordered.push(stage);
@ -91,7 +91,7 @@ function uniqueStages(stages: KloScanEnrichmentStage[]): KloScanEnrichmentStage[
return ordered;
}
export function completedKloScanEnrichmentStateSummary(): KloScanEnrichmentStateSummary {
export function completedKtxScanEnrichmentStateSummary(): KtxScanEnrichmentStateSummary {
return {
resumedStages: [],
completedStages: [],
@ -99,7 +99,7 @@ export function completedKloScanEnrichmentStateSummary(): KloScanEnrichmentState
};
}
export function summarizeKloScanEnrichmentState(input: KloScanEnrichmentStateSummary): KloScanEnrichmentStateSummary {
export function summarizeKtxScanEnrichmentState(input: KtxScanEnrichmentStateSummary): KtxScanEnrichmentStateSummary {
return {
resumedStages: uniqueStages(input.resumedStages),
completedStages: uniqueStages(input.completedStages),

20
packages/context/src/scan/enrichment-summary.test.ts
View file

@ -1,17 +1,17 @@
import { describe, expect, it } from 'vitest';
import {
failedKloScanEnrichmentSummary,
kloScanErrorMessage,
skippedKloScanEnrichmentSummary,
failedKtxScanEnrichmentSummary,
ktxScanErrorMessage,
skippedKtxScanEnrichmentSummary,
} from './enrichment-summary.js';
describe('KLO scan enrichment summaries', () => {
describe('KTX scan enrichment summaries', () => {
it('keeps structural scans skipped when no enrichment was requested', () => {
expect(failedKloScanEnrichmentSummary('structural', false)).toEqual(skippedKloScanEnrichmentSummary);
expect(failedKtxScanEnrichmentSummary('structural', false)).toEqual(skippedKtxScanEnrichmentSummary);
});
it('marks relationship stages failed when relationship detection fails', () => {
expect(failedKloScanEnrichmentSummary('relationships', true)).toEqual({
expect(failedKtxScanEnrichmentSummary('relationships', true)).toEqual({
dataDictionary: 'skipped',
tableDescriptions: 'skipped',
columnDescriptions: 'skipped',
@ -23,7 +23,7 @@ describe('KLO scan enrichment summaries', () => {
});
it('marks every enriched-only stage failed when full enrichment fails', () => {
expect(failedKloScanEnrichmentSummary('enriched', true)).toEqual({
expect(failedKtxScanEnrichmentSummary('enriched', true)).toEqual({
dataDictionary: 'failed',
tableDescriptions: 'failed',
columnDescriptions: 'failed',
@ -35,8 +35,8 @@ describe('KLO scan enrichment summaries', () => {
});
it('formats unknown thrown values for scan warnings', () => {
expect(kloScanErrorMessage(new Error('gateway timeout'))).toBe('gateway timeout');
expect(kloScanErrorMessage('plain failure')).toBe('plain failure');
expect(kloScanErrorMessage({ code: 'E_SCAN' })).toBe('{"code":"E_SCAN"}');
expect(ktxScanErrorMessage(new Error('gateway timeout'))).toBe('gateway timeout');
expect(ktxScanErrorMessage('plain failure')).toBe('plain failure');
expect(ktxScanErrorMessage({ code: 'E_SCAN' })).toBe('{"code":"E_SCAN"}');
});
});

16
packages/context/src/scan/enrichment-summary.ts
View file

@ -1,6 +1,6 @@
import type { KloScanEnrichmentSummary, KloScanMode } from './types.js';
import type { KtxScanEnrichmentSummary, KtxScanMode } from './types.js';
export const skippedKloScanEnrichmentSummary: KloScanEnrichmentSummary = {
export const skippedKtxScanEnrichmentSummary: KtxScanEnrichmentSummary = {
dataDictionary: 'skipped',
tableDescriptions: 'skipped',
columnDescriptions: 'skipped',
@ -10,10 +10,10 @@ export const skippedKloScanEnrichmentSummary: KloScanEnrichmentSummary = {
statisticalValidation: 'skipped',
};
export function failedKloScanEnrichmentSummary(
mode: KloScanMode,
export function failedKtxScanEnrichmentSummary(
mode: KtxScanMode,
detectRelationships = false,
): KloScanEnrichmentSummary {
): KtxScanEnrichmentSummary {
if (mode === 'enriched') {
return {
dataDictionary: 'failed',
@ -28,16 +28,16 @@ export function failedKloScanEnrichmentSummary(
if (mode === 'relationships' || detectRelationships) {
return {
...skippedKloScanEnrichmentSummary,
...skippedKtxScanEnrichmentSummary,
deterministicRelationships: 'failed',
statisticalValidation: 'failed',
};
}
return skippedKloScanEnrichmentSummary;
return skippedKtxScanEnrichmentSummary;
}
export function kloScanErrorMessage(error: unknown): string {
export function ktxScanErrorMessage(error: unknown): string {
if (error instanceof Error) {
return error.message;
}

42
packages/context/src/scan/enrichment-types.test.ts
View file

@ -1,19 +1,19 @@
import { describe, expect, it } from 'vitest';
import type {
KloColumnSampleUpdate,
KloDescriptionUpdate,
KloEmbeddingUpdate,
KloEnrichedSchema,
KloJoinUpdate,
KloRelationshipEndpoint,
KloRelationshipUpdate,
KloScanMetadataStore,
KloStructuralSyncPlan,
KtxColumnSampleUpdate,
KtxDescriptionUpdate,
KtxEmbeddingUpdate,
KtxEnrichedSchema,
KtxJoinUpdate,
KtxRelationshipEndpoint,
KtxRelationshipUpdate,
KtxScanMetadataStore,
KtxStructuralSyncPlan,
} from './enrichment-types.js';
describe('KLO scan enrichment contracts', () => {
describe('KTX scan enrichment contracts', () => {
it('models an enriched schema with reusable table, column, and relationship metadata', () => {
const schema: KloEnrichedSchema = {
const schema: KtxEnrichedSchema = {
connectionId: 'warehouse',
tables: [
{
@ -69,36 +69,36 @@ describe('KLO scan enrichment contracts', () => {
});
it('models metadata-store updates without requiring a concrete store implementation', async () => {
const structuralPlan: KloStructuralSyncPlan = {
const structuralPlan: KtxStructuralSyncPlan = {
connectionId: 'warehouse',
snapshotId: 'snapshot-1',
operations: [{ kind: 'create_table', table: 'orders' }],
};
const descriptionUpdate: KloDescriptionUpdate = {
const descriptionUpdate: KtxDescriptionUpdate = {
connectionId: 'warehouse',
table: { catalog: 'analytics', db: 'public', name: 'orders' },
source: 'ai',
tableDescription: 'Customer orders',
columnDescriptions: { status: 'Payment lifecycle state' },
};
const sampleUpdate: KloColumnSampleUpdate = {
const sampleUpdate: KtxColumnSampleUpdate = {
columnId: 'column-orders-status',
sampleValues: ['paid', 'refunded'],
cardinality: 2,
};
const embeddingUpdate: KloEmbeddingUpdate = {
const embeddingUpdate: KtxEmbeddingUpdate = {
columnId: 'column-orders-status',
text: 'orders.status (varchar). Values: paid, refunded',
embedding: [0.25, 0.75],
};
const relationshipUpdate: KloRelationshipUpdate = {
const relationshipUpdate: KtxRelationshipUpdate = {
connectionId: 'warehouse',
accepted: [],
rejected: [],
skipped: [{ reason: 'missing parent table', relationshipId: 'candidate-1' }],
};
const store: KloScanMetadataStore = {
const store: KtxScanMetadataStore = {
loadSchema: async () => null,
applyStructuralPlan: async (plan) => ({
connectionId: plan.connectionId,
@ -134,21 +134,21 @@ describe('KLO scan enrichment contracts', () => {
describe('relationship tuple contracts', () => {
it('represents relationship endpoints and join updates as ordered column tuples', () => {
const endpoint: KloRelationshipEndpoint = {
const endpoint: KtxRelationshipEndpoint = {
tableId: 'public.order_lines',
columnIds: ['public.order_lines.order_id', 'public.order_lines.line_number'],
table: { catalog: null, db: 'public', name: 'order_lines' },
columns: ['order_id', 'line_number'],
};
const update: KloJoinUpdate = {
const update: KtxJoinUpdate = {
connectionId: 'warehouse',
fromTable: 'order_line_allocations',
fromColumns: ['order_id', 'line_number'],
toTable: 'order_lines',
toColumns: ['order_id', 'line_number'],
relationship: 'many_to_one',
author: 'klo',
authorEmail: 'klo@example.com',
author: 'ktx',
authorEmail: 'ktx@example.com',
};
expect(endpoint.columns).toEqual(['order_id', 'line_number']);

90
packages/context/src/scan/enrichment-types.ts
View file

@ -1,69 +1,69 @@
import type { KloSchemaDimensionType, KloTableRef } from './types.js';
import type { KtxSchemaDimensionType, KtxTableRef } from './types.js';
export type KloDescriptionSource = 'ai' | 'db' | 'dbt' | 'user' | (string & {});
export type KtxDescriptionSource = 'ai' | 'db' | 'dbt' | 'user' | (string & {});
export type KloRelationshipSource = 'formal' | 'inferred' | 'manual';
export type KtxRelationshipSource = 'formal' | 'inferred' | 'manual';
export type KloRelationshipType = 'many_to_one' | 'one_to_many' | 'one_to_one';
export type KtxRelationshipType = 'many_to_one' | 'one_to_many' | 'one_to_one';
export interface KloEnrichedColumn {
export interface KtxEnrichedColumn {
id: string;
tableId: string;
tableRef: KloTableRef;
tableRef: KtxTableRef;
name: string;
nativeType: string;
normalizedType: string;
dimensionType: KloSchemaDimensionType;
dimensionType: KtxSchemaDimensionType;
nullable: boolean;
primaryKey: boolean;
parentColumnId: string | null;
descriptions: Partial<Record<KloDescriptionSource, string>>;
descriptions: Partial<Record<KtxDescriptionSource, string>>;
embedding: number[] | null;
sampleValues: string[] | null;
cardinality: number | null;
}
export interface KloEnrichedTable {
export interface KtxEnrichedTable {
id: string;
ref: KloTableRef;
ref: KtxTableRef;
enabled: boolean;
descriptions: Partial<Record<KloDescriptionSource, string>>;
columns: KloEnrichedColumn[];
descriptions: Partial<Record<KtxDescriptionSource, string>>;
columns: KtxEnrichedColumn[];
}
export interface KloRelationshipEndpoint {
export interface KtxRelationshipEndpoint {
tableId: string;
columnIds: string[];
table: KloTableRef;
table: KtxTableRef;
columns: string[];
}
export interface KloEnrichedRelationship {
export interface KtxEnrichedRelationship {
id: string;
source: KloRelationshipSource;
from: KloRelationshipEndpoint;
to: KloRelationshipEndpoint;
relationshipType: KloRelationshipType;
source: KtxRelationshipSource;
from: KtxRelationshipEndpoint;
to: KtxRelationshipEndpoint;
relationshipType: KtxRelationshipType;
confidence: number;
isPrimaryKeyReference: boolean;
}
export interface KloEnrichedSchema {
export interface KtxEnrichedSchema {
connectionId: string;
tables: KloEnrichedTable[];
relationships: KloEnrichedRelationship[];
tables: KtxEnrichedTable[];
relationships: KtxEnrichedRelationship[];
}
export interface KloStructuralSyncPlan {
export interface KtxStructuralSyncPlan {
connectionId: string;
snapshotId: string;
operations: Array<Record<string, unknown>>;
}
export interface KloDescriptionUpdate {
export interface KtxDescriptionUpdate {
connectionId: string;
table: KloTableRef;
source: KloDescriptionSource;
table: KtxTableRef;
source: KtxDescriptionSource;
tableDescription?: string;
columnDescriptions?: Record<string, string | null>;
}
@ -77,54 +77,54 @@ const PREFERRED_METADATA_FIELD_NAMES = [
'lineage',
] as const;
export interface KloMetadataUpdate {
export interface KtxMetadataUpdate {
connectionId: string;
table: KloTableRef;
source: KloDescriptionSource;
table: KtxTableRef;
source: KtxDescriptionSource;
tableFields?: Record<string, unknown>;
columnFields?: Record<string, Record<string, unknown>>;
}
export interface KloJoinUpdate {
export interface KtxJoinUpdate {
connectionId: string;
fromTable: string;
fromColumns: string[];
toTable: string;
toColumns: string[];
relationship: KloRelationshipType;
relationship: KtxRelationshipType;
author: string;
authorEmail: string;
}
export interface KloColumnSampleUpdate {
export interface KtxColumnSampleUpdate {
columnId: string;
sampleValues: string[] | null;
cardinality: number | null;
}
export interface KloEmbeddingUpdate {
export interface KtxEmbeddingUpdate {
columnId: string;
text: string;
embedding: number[];
}
export interface KloSkippedRelationship {
export interface KtxSkippedRelationship {
relationshipId: string;
reason: string;
}
export interface KloRelationshipUpdate {
export interface KtxRelationshipUpdate {
connectionId: string;
accepted: KloEnrichedRelationship[];
rejected: KloEnrichedRelationship[];
skipped: KloSkippedRelationship[];
accepted: KtxEnrichedRelationship[];
rejected: KtxEnrichedRelationship[];
skipped: KtxSkippedRelationship[];
}
export interface KloScanMetadataStore {
loadSchema(connectionId: string): Promise<KloEnrichedSchema | null>;
applyStructuralPlan(plan: KloStructuralSyncPlan): Promise<KloEnrichedSchema>;
updateDescriptions(input: KloDescriptionUpdate): Promise<void>;
updateColumnSamples(input: KloColumnSampleUpdate[]): Promise<void>;
updateColumnEmbeddings(input: KloEmbeddingUpdate[]): Promise<void>;
updateInferredRelationships(input: KloRelationshipUpdate): Promise<void>;
export interface KtxScanMetadataStore {
loadSchema(connectionId: string): Promise<KtxEnrichedSchema | null>;
applyStructuralPlan(plan: KtxStructuralSyncPlan): Promise<KtxEnrichedSchema>;
updateDescriptions(input: KtxDescriptionUpdate): Promise<void>;
updateColumnSamples(input: KtxColumnSampleUpdate[]): Promise<void>;
updateColumnEmbeddings(input: KtxEmbeddingUpdate[]): Promise<void>;
updateInferredRelationships(input: KtxRelationshipUpdate): Promise<void>;
}

530
packages/context/src/scan/index.ts
View file

@ -1,93 +1,93 @@
export {
REDACTED_KLO_CREDENTIAL_VALUE,
redactKloCredentialEnvelope,
redactKloCredentialValue,
redactKloScanMetadata,
redactKloScanReport,
redactKloScanWarning,
REDACTED_KTX_CREDENTIAL_VALUE,
redactKtxCredentialEnvelope,
redactKtxCredentialValue,
redactKtxScanMetadata,
redactKtxScanReport,
redactKtxScanWarning,
} from './credentials.js';
export type {
KloDataDictionaryColumnState,
KloDataDictionarySampleDecision,
KloDataDictionarySettings,
KloDataDictionarySkipReason,
KtxDataDictionaryColumnState,
KtxDataDictionarySampleDecision,
KtxDataDictionarySettings,
KtxDataDictionarySkipReason,
} from './data-dictionary.js';
export {
defaultKloDataDictionarySettings,
isKloDataDictionaryCandidate,
shouldKloSampleColumnForDictionary,
defaultKtxDataDictionarySettings,
isKtxDataDictionaryCandidate,
shouldKtxSampleColumnForDictionary,
} from './data-dictionary.js';
export type {
KloColumnAnalysisResult,
KloColumnDescriptionPromptInput,
KloDataSourceDescriptionPromptInput,
KloDescriptionCachePort,
KloDescriptionColumn,
KloDescriptionColumnTable,
KloDescriptionGenerationSettings,
KloDescriptionGeneratorOptions,
KloDescriptionSamplingPort,
KloDescriptionTableInput,
KloGenerateColumnDescriptionsInput,
KloGenerateDataSourceDescriptionInput,
KloGenerateTableDescriptionInput,
KloTableDescriptionPromptInput,
KtxColumnAnalysisResult,
KtxColumnDescriptionPromptInput,
KtxDataSourceDescriptionPromptInput,
KtxDescriptionCachePort,
KtxDescriptionColumn,
KtxDescriptionColumnTable,
KtxDescriptionGenerationSettings,
KtxDescriptionGeneratorOptions,
KtxDescriptionSamplingPort,
KtxDescriptionTableInput,
KtxGenerateColumnDescriptionsInput,
KtxGenerateDataSourceDescriptionInput,
KtxGenerateTableDescriptionInput,
KtxTableDescriptionPromptInput,
} from './description-generation.js';
export {
appendKloWordLimitInstruction,
buildKloColumnDescriptionPrompt,
buildKloDataSourceDescriptionPrompt,
buildKloTableDescriptionPrompt,
KloDescriptionGenerator,
appendKtxWordLimitInstruction,
buildKtxColumnDescriptionPrompt,
buildKtxDataSourceDescriptionPrompt,
buildKtxTableDescriptionPrompt,
KtxDescriptionGenerator,
} from './description-generation.js';
export type { KloColumnEmbeddingForeignKeys, KloColumnEmbeddingTextInput } from './embedding-text.js';
export { buildKloColumnEmbeddingText } from './embedding-text.js';
export type { KtxColumnEmbeddingForeignKeys, KtxColumnEmbeddingTextInput } from './embedding-text.js';
export { buildKtxColumnEmbeddingText } from './embedding-text.js';
export type {
ComputeKloScanEnrichmentInputHashInput,
KloScanEnrichmentCompletedStage,
KloScanEnrichmentFailedStage,
KloScanEnrichmentStageLookup,
KloScanEnrichmentStageRecord,
KloScanEnrichmentStateStore,
ComputeKtxScanEnrichmentInputHashInput,
KtxScanEnrichmentCompletedStage,
KtxScanEnrichmentFailedStage,
KtxScanEnrichmentStageLookup,
KtxScanEnrichmentStageRecord,
KtxScanEnrichmentStateStore,
} from './enrichment-state.js';
export {
completedKloScanEnrichmentStateSummary,
computeKloScanEnrichmentInputHash,
KLO_SCAN_ENRICHMENT_STAGES,
summarizeKloScanEnrichmentState,
completedKtxScanEnrichmentStateSummary,
computeKtxScanEnrichmentInputHash,
KTX_SCAN_ENRICHMENT_STAGES,
summarizeKtxScanEnrichmentState,
} from './enrichment-state.js';
export {
failedKloScanEnrichmentSummary,
kloScanErrorMessage,
skippedKloScanEnrichmentSummary,
failedKtxScanEnrichmentSummary,
ktxScanErrorMessage,
skippedKtxScanEnrichmentSummary,
} from './enrichment-summary.js';
export type {
KloColumnSampleUpdate,
KloDescriptionSource,
KloDescriptionUpdate,
KloEmbeddingUpdate,
KloEnrichedColumn,
KloEnrichedRelationship,
KloEnrichedSchema,
KloEnrichedTable,
KloRelationshipEndpoint,
KloRelationshipSource,
KloRelationshipType,
KloRelationshipUpdate,
KloScanMetadataStore,
KloSkippedRelationship,
KloStructuralSyncPlan,
KtxColumnSampleUpdate,
KtxDescriptionSource,
KtxDescriptionUpdate,
KtxEmbeddingUpdate,
KtxEnrichedColumn,
KtxEnrichedRelationship,
KtxEnrichedSchema,
KtxEnrichedTable,
KtxRelationshipEndpoint,
KtxRelationshipSource,
KtxRelationshipType,
KtxRelationshipUpdate,
KtxScanMetadataStore,
KtxSkippedRelationship,
KtxStructuralSyncPlan,
} from './enrichment-types.js';
export type {
DeterministicLocalScanEnrichmentProviderOptions,
KloLocalScanEnrichmentInput,
KloLocalScanEnrichmentProviders,
KloLocalScanEnrichmentResult,
KtxLocalScanEnrichmentInput,
KtxLocalScanEnrichmentProviders,
KtxLocalScanEnrichmentResult,
} from './local-enrichment.js';
export {
createDeterministicLocalScanEnrichmentProviders,
runLocalScanEnrichment,
snapshotToKloEnrichedSchema,
snapshotToKtxEnrichedSchema,
} from './local-enrichment.js';
export type {
WriteLocalScanEnrichmentArtifactsInput,
@ -109,182 +109,182 @@ export { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-sc
export type { ReadLocalScanStructuralSnapshotInput } from './local-structural-artifacts.js';
export { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js';
export type {
KloEnrichmentScanPhaseResult,
KloScanOrchestratorOptions,
KloScanOrchestratorRunInput,
KloScanOrchestratorRunResult,
KloStructuralScanPhaseResult,
KtxEnrichmentScanPhaseResult,
KtxScanOrchestratorOptions,
KtxScanOrchestratorRunInput,
KtxScanOrchestratorRunResult,
KtxStructuralScanPhaseResult,
} from './orchestrator.js';
export { KloScanOrchestrator } from './orchestrator.js';
export { KtxScanOrchestrator } from './orchestrator.js';
export type {
KloRelationshipArtifactStatus,
KtxRelationshipArtifactStatus,
ReadLocalScanRelationshipArtifactsResult,
} from './relationship-artifacts.js';
export { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js';
export type {
KloRelationshipBenchmarkReport,
KloRelationshipBenchmarkReportCase,
KloRelationshipBenchmarkReportCaseStatus,
KtxRelationshipBenchmarkReport,
KtxRelationshipBenchmarkReportCase,
KtxRelationshipBenchmarkReportCaseStatus,
} from './relationship-benchmark-report.js';
export {
buildKloRelationshipBenchmarkReport,
formatKloRelationshipBenchmarkReportMarkdown,
buildKtxRelationshipBenchmarkReport,
formatKtxRelationshipBenchmarkReportMarkdown,
} from './relationship-benchmark-report.js';
export type {
KloRelationshipBenchmarkCaseResult,
KloRelationshipBenchmarkDetectedLink,
KloRelationshipBenchmarkDetectedPk,
KloRelationshipBenchmarkDetector,
KloRelationshipBenchmarkDetectorInput,
KloRelationshipBenchmarkDetectorResult,
KloRelationshipBenchmarkExpectedLink,
KloRelationshipBenchmarkExpectedLinks,
KloRelationshipBenchmarkExpectedPk,
KloRelationshipBenchmarkFixture,
KloRelationshipBenchmarkMetrics,
KloRelationshipBenchmarkMode,
KloRelationshipBenchmarkStatus,
KloRelationshipBenchmarkSuiteResult,
KloRelationshipBenchmarkTier,
KtxRelationshipBenchmarkCaseResult,
KtxRelationshipBenchmarkDetectedLink,
KtxRelationshipBenchmarkDetectedPk,
KtxRelationshipBenchmarkDetector,
KtxRelationshipBenchmarkDetectorInput,
KtxRelationshipBenchmarkDetectorResult,
KtxRelationshipBenchmarkExpectedLink,
KtxRelationshipBenchmarkExpectedLinks,
KtxRelationshipBenchmarkExpectedPk,
KtxRelationshipBenchmarkFixture,
KtxRelationshipBenchmarkMetrics,
KtxRelationshipBenchmarkMode,
KtxRelationshipBenchmarkStatus,
KtxRelationshipBenchmarkSuiteResult,
KtxRelationshipBenchmarkTier,
} from './relationship-benchmarks.js';
export {
currentKloRelationshipBenchmarkDetector,
kloRelationshipBenchmarkDetectorWithLlm,
KLO_RELATIONSHIP_BENCHMARK_MODES,
KLO_RELATIONSHIP_BENCHMARK_TIERS,
loadKloRelationshipBenchmarkFixture,
loadKloRelationshipBenchmarkFixtures,
maskKloRelationshipBenchmarkSnapshot,
runKloRelationshipBenchmarkCase,
runKloRelationshipBenchmarkSuite,
currentKtxRelationshipBenchmarkDetector,
ktxRelationshipBenchmarkDetectorWithLlm,
KTX_RELATIONSHIP_BENCHMARK_MODES,
KTX_RELATIONSHIP_BENCHMARK_TIERS,
loadKtxRelationshipBenchmarkFixture,
loadKtxRelationshipBenchmarkFixtures,
maskKtxRelationshipBenchmarkSnapshot,
runKtxRelationshipBenchmarkCase,
runKtxRelationshipBenchmarkSuite,
} from './relationship-benchmarks.js';
export type {
ApplyKloRelationshipValidationBudgetInput,
KloRelationshipBudgetedCandidate,
KloRelationshipValidationBudget,
KloRelationshipValidationBudgetResult,
ApplyKtxRelationshipValidationBudgetInput,
KtxRelationshipBudgetedCandidate,
KtxRelationshipValidationBudget,
KtxRelationshipValidationBudgetResult,
} from './relationship-budget.js';
export {
applyKloRelationshipValidationBudget,
defaultKloRelationshipValidationBudget,
applyKtxRelationshipValidationBudget,
defaultKtxRelationshipValidationBudget,
} from './relationship-budget.js';
export type {
KloRelationshipDiscoveryCandidate,
KloRelationshipDiscoveryCandidateEvidence,
KloRelationshipDiscoveryCandidateOptions,
KloRelationshipDiscoveryCandidateSource,
KloRelationshipDiscoveryCandidateStatus,
KloRelationshipInferredTargetPk,
KtxRelationshipDiscoveryCandidate,
KtxRelationshipDiscoveryCandidateEvidence,
KtxRelationshipDiscoveryCandidateOptions,
KtxRelationshipDiscoveryCandidateSource,
KtxRelationshipDiscoveryCandidateStatus,
KtxRelationshipInferredTargetPk,
} from './relationship-candidates.js';
export {
generateKloRelationshipDiscoveryCandidates,
inferKloRelationshipTargetPks,
mergeKloRelationshipDiscoveryCandidates,
generateKtxRelationshipDiscoveryCandidates,
inferKtxRelationshipTargetPks,
mergeKtxRelationshipDiscoveryCandidates,
} from './relationship-candidates.js';
export type {
DiscoverKloCompositeRelationshipsInput,
DiscoverKloCompositeRelationshipsResult,
KloCompositePrimaryKeyCandidate,
KloCompositeRelationshipCandidate,
KloCompositeRelationshipStatus,
KloCompositeRelationshipTupleEndpoint,
KloCompositeRelationshipValidationEvidence,
DiscoverKtxCompositeRelationshipsInput,
DiscoverKtxCompositeRelationshipsResult,
KtxCompositePrimaryKeyCandidate,
KtxCompositeRelationshipCandidate,
KtxCompositeRelationshipStatus,
KtxCompositeRelationshipTupleEndpoint,
KtxCompositeRelationshipValidationEvidence,
} from './relationship-composite-candidates.js';
export { discoverKloCompositeRelationships } from './relationship-composite-candidates.js';
export { discoverKtxCompositeRelationships } from './relationship-composite-candidates.js';
export type {
BuildKloRelationshipArtifactsInput,
BuildKloRelationshipDiagnosticsInput,
EmptyKloRelationshipProfileArtifactInput,
KloRelationshipArtifact,
KloRelationshipArtifactEdge,
KloRelationshipArtifactEndpoint,
KloRelationshipDiagnosticsArtifact,
KloRelationshipDiagnosticsSummary,
KloRelationshipDiagnosticsThresholds,
KloRelationshipDiagnosticsValidation,
BuildKtxRelationshipArtifactsInput,
BuildKtxRelationshipDiagnosticsInput,
EmptyKtxRelationshipProfileArtifactInput,
KtxRelationshipArtifact,
KtxRelationshipArtifactEdge,
KtxRelationshipArtifactEndpoint,
KtxRelationshipDiagnosticsArtifact,
KtxRelationshipDiagnosticsSummary,
KtxRelationshipDiagnosticsThresholds,
KtxRelationshipDiagnosticsValidation,
} from './relationship-diagnostics.js';
export {
buildKloRelationshipArtifacts,
buildKloRelationshipDiagnostics,
emptyKloRelationshipProfileArtifact,
buildKtxRelationshipArtifacts,
buildKtxRelationshipDiagnostics,
emptyKtxRelationshipProfileArtifact,
} from './relationship-diagnostics.js';
export type {
BuildKloRelationshipFeedbackCalibrationReportInput,
BuildKtxRelationshipFeedbackCalibrationReportInput,
CalibrateLocalRelationshipFeedbackLabelsInput,
KloRelationshipFeedbackCalibrationBucket,
KloRelationshipFeedbackCalibrationLabel,
KloRelationshipFeedbackCalibrationReport,
KtxRelationshipFeedbackCalibrationBucket,
KtxRelationshipFeedbackCalibrationLabel,
KtxRelationshipFeedbackCalibrationReport,
} from './relationship-feedback-calibration.js';
export {
buildKloRelationshipFeedbackCalibrationReport,
buildKtxRelationshipFeedbackCalibrationReport,
calibrateLocalRelationshipFeedbackLabels,
formatKloRelationshipFeedbackCalibrationMarkdown,
formatKtxRelationshipFeedbackCalibrationMarkdown,
} from './relationship-feedback-calibration.js';
export type {
ExportLocalRelationshipFeedbackLabelsInput,
ExportLocalRelationshipFeedbackLabelsResult,
KloRelationshipFeedbackDecisionFilter,
KloRelationshipFeedbackExportWarning,
KloRelationshipFeedbackLabel,
KtxRelationshipFeedbackDecisionFilter,
KtxRelationshipFeedbackExportWarning,
KtxRelationshipFeedbackLabel,
} from './relationship-feedback-export.js';
export {
exportLocalRelationshipFeedbackLabels,
formatKloRelationshipFeedbackLabelsJsonl,
formatKtxRelationshipFeedbackLabelsJsonl,
} from './relationship-feedback-export.js';
export {
collectKloFormalMetadataRelationships,
type KloFormalMetadataRelationshipCollection,
collectKtxFormalMetadataRelationships,
type KtxFormalMetadataRelationshipCollection,
} from './relationship-formal-metadata.js';
export type {
KloRelationshipGraphResolutionResult,
KloRelationshipGraphResolverSettings,
KloResolvedRelationshipDiscoveryCandidate,
KloResolvedRelationshipGraphEvidence,
KloResolvedRelationshipPk,
KloResolvedRelationshipPkEvidence,
KloResolvedRelationshipStatus,
ResolveKloRelationshipGraphInput,
KtxRelationshipGraphResolutionResult,
KtxRelationshipGraphResolverSettings,
KtxResolvedRelationshipDiscoveryCandidate,
KtxResolvedRelationshipGraphEvidence,
KtxResolvedRelationshipPk,
KtxResolvedRelationshipPkEvidence,
KtxResolvedRelationshipStatus,
ResolveKtxRelationshipGraphInput,
} from './relationship-graph-resolver.js';
export { resolveKloRelationshipGraph } from './relationship-graph-resolver.js';
export { resolveKtxRelationshipGraph } from './relationship-graph-resolver.js';
export type {
KloRelationshipLlmProposalGenerateText,
KloRelationshipLlmProposalResult,
KloRelationshipLlmProposalSettings,
ProposeKloRelationshipCandidatesWithLlmInput,
KtxRelationshipLlmProposalGenerateText,
KtxRelationshipLlmProposalResult,
KtxRelationshipLlmProposalSettings,
ProposeKtxRelationshipCandidatesWithLlmInput,
} from './relationship-llm-proposal.js';
export { proposeKloRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
export { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
export type {
KloRelationshipLocalityCandidateTable,
LocalKloRelationshipCandidateTablesInput,
KtxRelationshipLocalityCandidateTable,
LocalKtxRelationshipCandidateTablesInput,
} from './relationship-locality.js';
export { localCandidateTables } from './relationship-locality.js';
export type {
KloRelationshipNormalizedName,
KloRelationshipTokenInput,
KtxRelationshipNormalizedName,
KtxRelationshipTokenInput,
} from './relationship-name-similarity.js';
export {
normalizeKloRelationshipName,
pluralizeKloRelationshipToken,
singularizeKloRelationshipToken,
tokenizeKloRelationshipName,
normalizeKtxRelationshipName,
pluralizeKtxRelationshipToken,
singularizeKtxRelationshipToken,
tokenizeKtxRelationshipName,
tokenSimilarity,
} from './relationship-name-similarity.js';
export type {
DiscoverKloRelationshipsInput,
DiscoverKloRelationshipsResult,
DiscoverKtxRelationshipsInput,
DiscoverKtxRelationshipsResult,
} from './relationship-discovery.js';
export { discoverKloRelationships } from './relationship-discovery.js';
export { discoverKtxRelationships } from './relationship-discovery.js';
export type {
KloRelationshipColumnProfile,
KloRelationshipProfileArtifact,
KloRelationshipReadOnlyExecutor,
KloRelationshipTableProfile,
ProfileKloRelationshipSchemaInput,
KtxRelationshipColumnProfile,
KtxRelationshipProfileArtifact,
KtxRelationshipReadOnlyExecutor,
KtxRelationshipTableProfile,
ProfileKtxRelationshipSchemaInput,
} from './relationship-profiling.js';
export {
formatKloRelationshipTableRef,
profileKloRelationshipSchema,
quoteKloRelationshipIdentifier,
formatKtxRelationshipTableRef,
profileKtxRelationshipSchema,
quoteKtxRelationshipIdentifier,
} from './relationship-profiling.js';
export type {
AppliedRelationshipReviewDecision,
@ -293,108 +293,108 @@ export type {
} from './relationship-review-apply.js';
export { applyLocalScanRelationshipReviewDecisions } from './relationship-review-apply.js';
export type {
KloRelationshipReviewDecisionArtifact,
KloRelationshipReviewDecisionEntry,
KloRelationshipReviewDecisionValue,
KtxRelationshipReviewDecisionArtifact,
KtxRelationshipReviewDecisionEntry,
KtxRelationshipReviewDecisionValue,
WriteLocalScanRelationshipReviewDecisionInput,
WriteLocalScanRelationshipReviewDecisionResult,
} from './relationship-review-decisions.js';
export { writeLocalScanRelationshipReviewDecision } from './relationship-review-decisions.js';
export type {
KloRelationshipFixtureOrigin,
KloRelationshipScoreBreakdown,
KloRelationshipScoreSignal,
KloRelationshipScoreWeights,
KloRelationshipScoringCalibrationObservation,
KloRelationshipSignalVector,
KtxRelationshipFixtureOrigin,
KtxRelationshipScoreBreakdown,
KtxRelationshipScoreSignal,
KtxRelationshipScoreWeights,
KtxRelationshipScoringCalibrationObservation,
KtxRelationshipSignalVector,
} from './relationship-scoring.js';
export {
calibrateWeightsFromSyntheticFixtures,
defaultKloRelationshipScoreWeights,
KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS,
normalizeKloRelationshipScoreWeights,
scoreKloRelationshipCandidate,
defaultKtxRelationshipScoreWeights,
KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS,
normalizeKtxRelationshipScoreWeights,
scoreKtxRelationshipCandidate,
} from './relationship-scoring.js';
export type {
AdviseLocalRelationshipFeedbackThresholdsInput,
BuildKloRelationshipThresholdAdviceReportInput,
KloRelationshipThresholdAdviceCandidate,
KloRelationshipThresholdAdviceReport,
KloRelationshipThresholdAdviceStatus,
BuildKtxRelationshipThresholdAdviceReportInput,
KtxRelationshipThresholdAdviceCandidate,
KtxRelationshipThresholdAdviceReport,
KtxRelationshipThresholdAdviceStatus,
} from './relationship-threshold-advice.js';
export {
adviseLocalRelationshipFeedbackThresholds,
buildKloRelationshipThresholdAdviceReport,
formatKloRelationshipThresholdAdviceMarkdown,
buildKtxRelationshipThresholdAdviceReport,
formatKtxRelationshipThresholdAdviceMarkdown,
} from './relationship-threshold-advice.js';
export type {
KloRelationshipValidationEvidence,
KloRelationshipValidationSettings,
KloValidatedRelationshipDiscoveryCandidate,
KloValidatedRelationshipStatus,
ValidateKloRelationshipDiscoveryCandidatesInput,
KtxRelationshipValidationEvidence,
KtxRelationshipValidationSettings,
KtxValidatedRelationshipDiscoveryCandidate,
KtxValidatedRelationshipStatus,
ValidateKtxRelationshipDiscoveryCandidatesInput,
} from './relationship-validation.js';
export { validateKloRelationshipDiscoveryCandidates } from './relationship-validation.js';
export { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js';
export type { SqliteLocalScanEnrichmentStateStoreOptions } from './sqlite-local-enrichment-state-store.js';
export { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js';
export type { KloColumnTypeMapping } from './type-normalization.js';
export type { KtxColumnTypeMapping } from './type-normalization.js';
export {
inferKloDimensionType,
kloColumnTypeMappingFromNative,
normalizeKloNativeType,
inferKtxDimensionType,
ktxColumnTypeMappingFromNative,
normalizeKtxNativeType,
} from './type-normalization.js';
export type {
KloColumnSampleInput,
KloColumnSampleResult,
KloColumnStatsInput,
KloColumnStatsResult,
KloConnectionDriver,
KloConnectorCapabilities,
KloCredentialEnvelope,
KloCredentialEnvReference,
KloCredentialFileReference,
KloEmbeddingPort,
KloEventPropertyDiscovery,
KloEventPropertyDiscoveryInput,
KloEventPropertyValuesInput,
KloEventPropertyValuesResult,
KloEventStreamDiscoveryPort,
KloEventTypeDiscovery,
KloEventTypeDiscoveryInput,
KloNetworkEndpoint,
KloNetworkTunnelPort,
KloNetworkTunnelRequest,
KloOptionalConnectorCapabilities,
KloProgressPort,
KloProgressUpdateOptions,
KloQueryResult,
KloReadOnlyQueryInput,
KloResolvedCredentialEnvelope,
KloScanArtifactPaths,
KloScanConnector,
KloScanContext,
KloScanDiffSummary,
KloScanEnrichmentStage,
KloScanEnrichmentStateSummary,
KloScanEnrichmentSummary,
KloScanInput,
KloScanLoggerPort,
KloScanMode,
KloScanRelationshipSummary,
KloScanReport,
KloScanTrigger,
KloScanWarning,
KloScanWarningCode,
KloSchemaColumn,
KloSchemaDimensionType,
KloSchemaForeignKey,
KloSchemaScope,
KloSchemaSnapshot,
KloSchemaTable,
KloSchemaTableKind,
KloStructuralSyncStats,
KloTableRef,
KloTableSampleInput,
KloTableSampleResult,
KtxColumnSampleInput,
KtxColumnSampleResult,
KtxColumnStatsInput,
KtxColumnStatsResult,
KtxConnectionDriver,
KtxConnectorCapabilities,
KtxCredentialEnvelope,
KtxCredentialEnvReference,
KtxCredentialFileReference,
KtxEmbeddingPort,
KtxEventPropertyDiscovery,
KtxEventPropertyDiscoveryInput,
KtxEventPropertyValuesInput,
KtxEventPropertyValuesResult,
KtxEventStreamDiscoveryPort,
KtxEventTypeDiscovery,
KtxEventTypeDiscoveryInput,
KtxNetworkEndpoint,
KtxNetworkTunnelPort,
KtxNetworkTunnelRequest,
KtxOptionalConnectorCapabilities,
KtxProgressPort,
KtxProgressUpdateOptions,
KtxQueryResult,
KtxReadOnlyQueryInput,
KtxResolvedCredentialEnvelope,
KtxScanArtifactPaths,
KtxScanConnector,
KtxScanContext,
KtxScanDiffSummary,
KtxScanEnrichmentStage,
KtxScanEnrichmentStateSummary,
KtxScanEnrichmentSummary,
KtxScanInput,
KtxScanLoggerPort,
KtxScanMode,
KtxScanRelationshipSummary,
KtxScanReport,
KtxScanTrigger,
KtxScanWarning,
KtxScanWarningCode,
KtxSchemaColumn,
KtxSchemaDimensionType,
KtxSchemaForeignKey,
KtxSchemaScope,
KtxSchemaSnapshot,
KtxSchemaTable,
KtxSchemaTableKind,
KtxStructuralSyncStats,
KtxTableRef,
KtxTableSampleInput,
KtxTableSampleResult,
} from './types.js';
export { createKloConnectorCapabilities } from './types.js';
export { createKtxConnectorCapabilities } from './types.js';

30
packages/context/src/scan/local-enrichment-artifacts.test.ts
View file

@ -3,12 +3,12 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import YAML from 'yaml';
import { initKloProject, type KloLocalProject } from '../project/index.js';
import type { KloLocalScanEnrichmentResult } from './local-enrichment.js';
import { initKtxProject, type KtxLocalProject } from '../project/index.js';
import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js';
import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from './local-enrichment-artifacts.js';
import type { KloSchemaSnapshot } from './types.js';
import type { KtxSchemaSnapshot } from './types.js';
const snapshot: KloSchemaSnapshot = {
const snapshot: KtxSchemaSnapshot = {
connectionId: 'warehouse',
driver: 'postgres',
extractedAt: '2026-04-29T12:00:00.000Z',
@ -76,7 +76,7 @@ const snapshot: KloSchemaSnapshot = {
],
};
function enrichment(): KloLocalScanEnrichmentResult {
function enrichment(): KtxLocalScanEnrichmentResult {
return {
snapshot,
summary: {
@ -225,11 +225,11 @@ function enrichment(): KloLocalScanEnrichmentResult {
describe('writeLocalScanEnrichmentArtifacts', () => {
let tempDir: string;
let project: KloLocalProject;
let project: KtxLocalProject;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-enrichment-artifacts-'));
project = await initKloProject({
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-enrichment-artifacts-'));
project = await initKtxProject({
projectDir: join(tempDir, 'project'),
projectName: 'warehouse',
});
@ -269,8 +269,8 @@ describe('writeLocalScanEnrichmentArtifacts', () => {
},
{ indent: 2, lineWidth: 0 },
),
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed manifest shard',
);
@ -426,7 +426,7 @@ describe('writeLocalScanEnrichmentArtifacts', () => {
it('writes formal accepted relationships into relationship artifacts and manifest shards', async () => {
const source = enrichment();
const formalEnrichment: KloLocalScanEnrichmentResult = {
const formalEnrichment: KtxLocalScanEnrichmentResult = {
...source,
relationshipUpdate: {
connectionId: 'warehouse',
@ -554,7 +554,7 @@ describe('writeLocalScanEnrichmentArtifacts', () => {
});
it('writes accepted composite relationships to relationship artifacts and manifest shards', async () => {
const compositeSnapshot: KloSchemaSnapshot = {
const compositeSnapshot: KtxSchemaSnapshot = {
connectionId: 'warehouse',
driver: 'postgres',
extractedAt: '2026-05-07T12:00:00.000Z',
@ -621,7 +621,7 @@ describe('writeLocalScanEnrichmentArtifacts', () => {
},
],
};
const compositeEnrichment: KloLocalScanEnrichmentResult = Object.assign(enrichment(), {
const compositeEnrichment: KtxLocalScanEnrichmentResult = Object.assign(enrichment(), {
snapshot: compositeSnapshot,
relationships: { accepted: 1, review: 0, rejected: 0, skipped: 0 },
descriptionUpdates: [],
@ -763,8 +763,8 @@ describe('writeLocalScanEnrichmentArtifacts', () => {
},
{ indent: 2, lineWidth: 0 },
),
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed structural manifest shard',
);

68
packages/context/src/scan/local-enrichment-artifacts.ts
View file

@ -7,31 +7,31 @@ import {
type LiveDatabaseManifestShard,
type LiveDatabaseManifestTableData,
} from '../ingest/index.js';
import type { KloScanRelationshipConfig } from '../project/config.js';
import type { KloLocalProject } from '../project/index.js';
import type { KloLocalScanEnrichmentResult } from './local-enrichment.js';
import type { KtxScanRelationshipConfig } from '../project/config.js';
import type { KtxLocalProject } from '../project/index.js';
import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js';
import {
buildKloRelationshipArtifacts,
buildKloRelationshipDiagnostics,
emptyKloRelationshipProfileArtifact,
buildKtxRelationshipArtifacts,
buildKtxRelationshipDiagnostics,
emptyKtxRelationshipProfileArtifact,
} from './relationship-diagnostics.js';
import type { KloConnectionDriver, KloSchemaColumn, KloSchemaSnapshot, KloSchemaTable } from './types.js';
import type { KtxConnectionDriver, KtxSchemaColumn, KtxSchemaSnapshot, KtxSchemaTable } from './types.js';
const LIVE_DATABASE_ADAPTER = 'live-database';
const LOCAL_AUTHOR = 'klo';
const LOCAL_AUTHOR_EMAIL = 'klo@example.com';
const LOCAL_AUTHOR = 'ktx';
const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
const SCHEMA_DIR = '_schema';
const SL_DIR_PREFIX = 'semantic-layer';
export interface WriteLocalScanManifestShardsInput {
project: KloLocalProject;
project: KtxLocalProject;
connectionId: string;
syncId: string;
driver: KloConnectionDriver;
snapshot: KloSchemaSnapshot;
driver: KtxConnectionDriver;
snapshot: KtxSchemaSnapshot;
dryRun: boolean;
descriptionUpdates?: KloLocalScanEnrichmentResult['descriptionUpdates'];
relationshipUpdate?: KloLocalScanEnrichmentResult['relationshipUpdate'];
descriptionUpdates?: KtxLocalScanEnrichmentResult['descriptionUpdates'];
relationshipUpdate?: KtxLocalScanEnrichmentResult['relationshipUpdate'];
}
export interface WriteLocalScanManifestShardsResult {
@ -40,13 +40,13 @@ export interface WriteLocalScanManifestShardsResult {
}
export interface WriteLocalScanEnrichmentArtifactsInput {
project: KloLocalProject;
project: KtxLocalProject;
connectionId: string;
syncId: string;
driver: KloConnectionDriver;
enrichment: KloLocalScanEnrichmentResult;
driver: KtxConnectionDriver;
enrichment: KtxLocalScanEnrichmentResult;
dryRun: boolean;
relationshipSettings?: KloScanRelationshipConfig;
relationshipSettings?: KtxScanRelationshipConfig;
}
export interface WriteLocalScanEnrichmentArtifactsResult extends WriteLocalScanManifestShardsResult {
@ -58,7 +58,7 @@ interface ExistingManifestState {
preservedJoins: Map<string, LiveDatabaseManifestJoinEntry[]>;
}
type LocalDescriptionUpdates = KloLocalScanEnrichmentResult['descriptionUpdates'];
type LocalDescriptionUpdates = KtxLocalScanEnrichmentResult['descriptionUpdates'];
function artifactDir(connectionId: string, syncId: string): string {
return `raw-sources/${connectionId}/${LIVE_DATABASE_ADAPTER}/${syncId}/enrichment`;
@ -69,7 +69,7 @@ function schemaDir(connectionId: string): string {
}
function tableDescription(
table: KloSchemaTable,
table: KtxSchemaTable,
descriptionUpdates: LocalDescriptionUpdates = [],
): Record<string, string> | undefined {
const update = descriptionUpdates.find((candidate) => candidate.table.name === table.name);
@ -84,8 +84,8 @@ function tableDescription(
}
function columnDescription(
table: KloSchemaTable,
column: KloSchemaColumn,
table: KtxSchemaTable,
column: KtxSchemaColumn,
descriptionUpdates: LocalDescriptionUpdates = [],
): Record<string, string> | undefined {
const update = descriptionUpdates.find((candidate) => candidate.table.name === table.name);
@ -101,7 +101,7 @@ function columnDescription(
}
function snapshotTablesToManifestData(
snapshot: KloSchemaSnapshot,
snapshot: KtxSchemaSnapshot,
descriptionUpdates: LocalDescriptionUpdates = [],
): LiveDatabaseManifestTableData[] {
return snapshot.tables.map((table) => ({
@ -119,7 +119,7 @@ function snapshotTablesToManifestData(
}));
}
function formalJoins(snapshot: KloSchemaSnapshot): LiveDatabaseManifestJoinData[] {
function formalJoins(snapshot: KtxSchemaSnapshot): LiveDatabaseManifestJoinData[] {
const joins: LiveDatabaseManifestJoinData[] = [];
for (const table of snapshot.tables) {
for (const foreignKey of table.foreignKeys) {
@ -137,7 +137,7 @@ function formalJoins(snapshot: KloSchemaSnapshot): LiveDatabaseManifestJoinData[
}
function acceptedRelationshipJoins(
relationshipUpdate: KloLocalScanEnrichmentResult['relationshipUpdate'] | undefined,
relationshipUpdate: KtxLocalScanEnrichmentResult['relationshipUpdate'] | undefined,
): LiveDatabaseManifestJoinData[] {
return (relationshipUpdate?.accepted ?? []).map((relationship) => ({
fromTable: relationship.from.table.name,
@ -150,8 +150,8 @@ function acceptedRelationshipJoins(
}
function relationshipJoins(
snapshot: KloSchemaSnapshot,
relationshipUpdate: KloLocalScanEnrichmentResult['relationshipUpdate'] | undefined,
snapshot: KtxSchemaSnapshot,
relationshipUpdate: KtxLocalScanEnrichmentResult['relationshipUpdate'] | undefined,
): LiveDatabaseManifestJoinData[] {
const accepted = acceptedRelationshipJoins(relationshipUpdate);
const manual = accepted.filter((relationship) => relationship.source === 'manual');
@ -159,7 +159,7 @@ function relationshipJoins(
return [...manual, ...formalJoins(snapshot), ...generated];
}
function validColumns(snapshot: KloSchemaSnapshot): Map<string, Set<string>> {
function validColumns(snapshot: KtxSchemaSnapshot): Map<string, Set<string>> {
return new Map(snapshot.tables.map((table) => [table.name, new Set(table.columns.map((column) => column.name))]));
}
@ -190,9 +190,9 @@ function joinReferencesExistingColumns(
}
async function loadExistingManifestState(
project: KloLocalProject,
project: KtxLocalProject,
connectionId: string,
snapshot: KloSchemaSnapshot,
snapshot: KtxSchemaSnapshot,
): Promise<ExistingManifestState> {
const descriptions = new Map<string, LiveDatabaseManifestExistingDescriptions>();
const preservedJoins = new Map<string, LiveDatabaseManifestJoinEntry[]>();
@ -245,7 +245,7 @@ async function loadExistingManifestState(
}
async function writeJsonArtifact(
project: KloLocalProject,
project: KtxLocalProject,
path: string,
value: unknown,
commitMessage: string,
@ -340,7 +340,7 @@ export async function writeLocalScanEnrichmentArtifacts(
}
enrichmentArtifacts.push(relationshipsArtifact, relationshipProfileArtifact, relationshipDiagnosticsArtifact);
const hasResolvedRelationships = input.enrichment.resolvedRelationships !== null;
const relationshipArtifacts = buildKloRelationshipArtifacts({
const relationshipArtifacts = buildKtxRelationshipArtifacts({
connectionId: input.connectionId,
resolvedRelationships: hasResolvedRelationships ? (input.enrichment.resolvedRelationships ?? []) : undefined,
compositeRelationships: input.enrichment.compositeRelationships ?? undefined,
@ -353,12 +353,12 @@ export async function writeLocalScanEnrichmentArtifacts(
});
const relationshipProfile =
input.enrichment.relationshipProfile ??
emptyKloRelationshipProfileArtifact({
emptyKtxRelationshipProfileArtifact({
connectionId: input.connectionId,
driver: input.driver,
reason: 'relationship_profiling_not_run',
});
const relationshipDiagnostics = buildKloRelationshipDiagnostics({
const relationshipDiagnostics = buildKtxRelationshipDiagnostics({
connectionId: input.connectionId,
artifacts: relationshipArtifacts,
profile: relationshipProfile,

72
packages/context/src/scan/local-enrichment.test.ts
View file

@ -1,28 +1,28 @@
import Database from 'better-sqlite3';
import { describe, expect, it, vi } from 'vitest';
import { buildDefaultKloProjectConfig } from '../project/config.js';
import { buildDefaultKtxProjectConfig } from '../project/config.js';
import type {
KloScanEnrichmentCompletedStage,
KloScanEnrichmentFailedStage,
KloScanEnrichmentStageLookup,
KloScanEnrichmentStateStore,
KtxScanEnrichmentCompletedStage,
KtxScanEnrichmentFailedStage,
KtxScanEnrichmentStageLookup,
KtxScanEnrichmentStateStore,
} from './enrichment-state.js';
import {
createDeterministicLocalScanEnrichmentProviders,
runLocalScanEnrichment,
snapshotToKloEnrichedSchema,
snapshotToKtxEnrichedSchema,
} from './local-enrichment.js';
import { createLocalScanEnrichmentProvidersFromConfig } from './local-scan.js';
import {
createKloConnectorCapabilities,
type KloQueryResult,
type KloReadOnlyQueryInput,
type KloScanConnector,
type KloScanContext,
type KloSchemaSnapshot,
createKtxConnectorCapabilities,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxSchemaSnapshot,
} from './types.js';
const snapshot: KloSchemaSnapshot = {
const snapshot: KtxSchemaSnapshot = {
connectionId: 'warehouse',
driver: 'postgres',
extractedAt: '2026-04-29T12:00:00.000Z',
@ -81,11 +81,11 @@ const snapshot: KloSchemaSnapshot = {
],
};
function connector(): KloScanConnector {
function connector(): KtxScanConnector {
return {
id: 'test:warehouse',
driver: 'postgres',
capabilities: createKloConnectorCapabilities({
capabilities: createKtxConnectorCapabilities({
tableSampling: true,
columnSampling: true,
readOnlySql: true,
@ -108,7 +108,7 @@ function connector(): KloScanConnector {
class InMemorySqliteExecutor {
readonly db = new Database(':memory:');
executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
return Promise.resolve({
@ -124,7 +124,7 @@ class InMemorySqliteExecutor {
}
}
function noDeclaredRelationshipSnapshot(): KloSchemaSnapshot {
function noDeclaredRelationshipSnapshot(): KtxSchemaSnapshot {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -185,16 +185,16 @@ function noDeclaredRelationshipSnapshot(): KloSchemaSnapshot {
};
}
function memoryEnrichmentStateStore(): KloScanEnrichmentStateStore {
const records = new Map<string, KloScanEnrichmentCompletedStage | KloScanEnrichmentFailedStage>();
const key = (input: Pick<KloScanEnrichmentStageLookup, 'runId' | 'stage'>) => `${input.runId}:${input.stage}`;
function memoryEnrichmentStateStore(): KtxScanEnrichmentStateStore {
const records = new Map<string, KtxScanEnrichmentCompletedStage | KtxScanEnrichmentFailedStage>();
const key = (input: Pick<KtxScanEnrichmentStageLookup, 'runId' | 'stage'>) => `${input.runId}:${input.stage}`;
return {
async findCompletedStage<TOutput>(input: KloScanEnrichmentStageLookup) {
async findCompletedStage<TOutput>(input: KtxScanEnrichmentStageLookup) {
const record = records.get(key(input));
if (!record || record.status !== 'completed' || record.inputHash !== input.inputHash) {
return null;
}
return record as KloScanEnrichmentCompletedStage<TOutput>;
return record as KtxScanEnrichmentCompletedStage<TOutput>;
},
async saveCompletedStage(input) {
records.set(key(input), {
@ -218,7 +218,7 @@ function memoryEnrichmentStateStore(): KloScanEnrichmentStateStore {
describe('local scan enrichment', () => {
it('maps a scan snapshot into relationship detector schema', () => {
const schema = snapshotToKloEnrichedSchema(snapshot);
const schema = snapshotToKtxEnrichedSchema(snapshot);
expect(schema.connectionId).toBe('warehouse');
expect(schema.tables).toHaveLength(2);
@ -262,7 +262,7 @@ describe('local scan enrichment', () => {
),
};
const schema = snapshotToKloEnrichedSchema(snapshotWithForeignKey);
const schema = snapshotToKtxEnrichedSchema(snapshotWithForeignKey);
expect(schema.relationships).toEqual([
{
@ -306,7 +306,7 @@ describe('local scan enrichment', () => {
expect(result.summary.statisticalValidation).toBe('skipped');
expect(result.warnings).toContainEqual({
code: 'relationship_validation_failed',
message: 'KLO scan connector advertises readOnlySql but does not expose executeReadOnly',
message: 'KTX scan connector advertises readOnlySql but does not expose executeReadOnly',
recoverable: true,
metadata: { capability: 'readOnlySql' },
});
@ -324,7 +324,7 @@ describe('local scan enrichment', () => {
const scanConnector = {
...connector(),
driver: 'sqlite' as const,
capabilities: createKloConnectorCapabilities({ readOnlySql: true, columnStats: true }),
capabilities: createKtxConnectorCapabilities({ readOnlySql: true, columnStats: true }),
introspect: vi.fn(async () => noDeclaredRelationshipSnapshot()),
executeReadOnly: executor.executeReadOnly.bind(executor),
};
@ -371,7 +371,7 @@ describe('local scan enrichment', () => {
},
},
relationshipSettings: {
...buildDefaultKloProjectConfig('warehouse').scan.relationships,
...buildDefaultKtxProjectConfig('warehouse').scan.relationships,
llmProposals: false,
maxLlmTablesPerBatch: 40,
},
@ -383,7 +383,7 @@ describe('local scan enrichment', () => {
it('skips relationship detection when scan relationships are disabled', async () => {
const settings = {
...buildDefaultKloProjectConfig('warehouse').scan.relationships,
...buildDefaultKtxProjectConfig('warehouse').scan.relationships,
enabled: false,
};
const result = await runLocalScanEnrichment({
@ -488,7 +488,7 @@ describe('local scan enrichment', () => {
});
it('splits enrichment embedding requests by provider batch size', async () => {
const manyColumnSnapshot: KloSchemaSnapshot = {
const manyColumnSnapshot: KtxSchemaSnapshot = {
...snapshot,
tables: [
{
@ -644,7 +644,7 @@ describe('local scan enrichment', () => {
const scanConnector = {
...connector(),
driver: 'sqlite' as const,
capabilities: createKloConnectorCapabilities({ readOnlySql: true, columnStats: true }),
capabilities: createKtxConnectorCapabilities({ readOnlySql: true, columnStats: true }),
introspect: vi.fn(async () => noDeclaredRelationshipSnapshot()),
executeReadOnly: executor.executeReadOnly.bind(executor),
};
@ -695,10 +695,10 @@ describe('local scan enrichment', () => {
});
it('resolves gateway LLM providers and OpenAI embeddings from local scan config', () => {
const createKloLlmProvider = vi.fn(() => ({
const createKtxLlmProvider = vi.fn(() => ({
getModel: vi.fn().mockReturnValue({ modelId: 'provider/language-model', provider: 'gateway' }),
}));
const createKloEmbeddingProvider = vi.fn(() => ({
const createKtxEmbeddingProvider = vi.fn(() => ({
dimensions: 1536,
maxBatchSize: 8,
embed: vi.fn(),
@ -724,18 +724,18 @@ describe('local scan enrichment', () => {
models: { default: 'provider/language-model' },
},
{
createKloLlmProvider: createKloLlmProvider as any,
createKloEmbeddingProvider: createKloEmbeddingProvider as any,
createKtxLlmProvider: createKtxLlmProvider as any,
createKtxEmbeddingProvider: createKtxEmbeddingProvider as any,
env: { OPENAI_API_KEY: 'openai-key' },
},
);
expect(providers?.embedding.dimensions).toBe(1536);
expect(providers?.embedding.maxBatchSize).toBe(8);
expect(createKloLlmProvider).toHaveBeenCalledWith(
expect(createKtxLlmProvider).toHaveBeenCalledWith(
expect.objectContaining({ backend: 'gateway', modelSlots: { default: 'provider/language-model' } }),
);
expect(createKloEmbeddingProvider).toHaveBeenCalledWith(
expect(createKtxEmbeddingProvider).toHaveBeenCalledWith(
expect.objectContaining({ backend: 'openai', model: 'provider/embedding-model' }),
);
});

234
packages/context/src/scan/local-enrichment.ts
View file

@ -1,43 +1,43 @@
import type { KloLlmProvider } from '@klo/llm';
import { buildDefaultKloProjectConfig, type KloScanRelationshipConfig } from '../project/config.js';
import { type KloDescriptionColumnTable, KloDescriptionGenerator } from './description-generation.js';
import { buildKloColumnEmbeddingText } from './embedding-text.js';
import type { KtxLlmProvider } from '@ktx/llm';
import { buildDefaultKtxProjectConfig, type KtxScanRelationshipConfig } from '../project/config.js';
import { type KtxDescriptionColumnTable, KtxDescriptionGenerator } from './description-generation.js';
import { buildKtxColumnEmbeddingText } from './embedding-text.js';
import {
completedKloScanEnrichmentStateSummary,
computeKloScanEnrichmentInputHash,
type KloScanEnrichmentStateStore,
summarizeKloScanEnrichmentState,
completedKtxScanEnrichmentStateSummary,
computeKtxScanEnrichmentInputHash,
type KtxScanEnrichmentStateStore,
summarizeKtxScanEnrichmentState,
} from './enrichment-state.js';
import { skippedKloScanEnrichmentSummary } from './enrichment-summary.js';
import { skippedKtxScanEnrichmentSummary } from './enrichment-summary.js';
import type {
KloEmbeddingUpdate,
KloEnrichedColumn,
KloEnrichedRelationship,
KloEnrichedSchema,
KloEnrichedTable,
KloRelationshipEndpoint,
KloRelationshipUpdate,
KtxEmbeddingUpdate,
KtxEnrichedColumn,
KtxEnrichedRelationship,
KtxEnrichedSchema,
KtxEnrichedTable,
KtxRelationshipEndpoint,
KtxRelationshipUpdate,
} from './enrichment-types.js';
import type { KloCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
import type { KloResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
import { discoverKloRelationships } from './relationship-discovery.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
import type { KtxResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
import { discoverKtxRelationships } from './relationship-discovery.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type {
KloEmbeddingPort,
KloProgressPort,
KloScanConnector,
KloScanContext,
KloScanEnrichmentStage,
KloScanEnrichmentStateSummary,
KloScanEnrichmentSummary,
KloScanMode,
KloScanRelationshipSummary,
KloScanWarning,
KloSchemaColumn,
KloSchemaForeignKey,
KloSchemaSnapshot,
KloSchemaTable,
KloTableRef,
KtxEmbeddingPort,
KtxProgressPort,
KtxScanConnector,
KtxScanContext,
KtxScanEnrichmentStage,
KtxScanEnrichmentStateSummary,
KtxScanEnrichmentSummary,
KtxScanMode,
KtxScanRelationshipSummary,
KtxScanWarning,
KtxSchemaColumn,
KtxSchemaForeignKey,
KtxSchemaSnapshot,
KtxSchemaTable,
KtxTableRef,
} from './types.js';
export interface DeterministicLocalScanEnrichmentProviderOptions {
@ -45,52 +45,52 @@ export interface DeterministicLocalScanEnrichmentProviderOptions {
maxBatchSize?: number;
}
export interface KloLocalScanEnrichmentProviders {
llm: KloLlmProvider;
embedding: KloEmbeddingPort;
export interface KtxLocalScanEnrichmentProviders {
llm: KtxLlmProvider;
embedding: KtxEmbeddingPort;
}
export interface KloLocalScanEnrichmentInput {
export interface KtxLocalScanEnrichmentInput {
connectionId: string;
mode: KloScanMode;
mode: KtxScanMode;
detectRelationships?: boolean;
connector: KloScanConnector;
context: KloScanContext;
providers: KloLocalScanEnrichmentProviders | null;
stateStore?: KloScanEnrichmentStateStore | null;
connector: KtxScanConnector;
context: KtxScanContext;
providers: KtxLocalScanEnrichmentProviders | null;
stateStore?: KtxScanEnrichmentStateStore | null;
syncId?: string;
providerIdentity?: Record<string, unknown>;
relationshipSettings?: KloScanRelationshipConfig;
relationshipSettings?: KtxScanRelationshipConfig;
now?: () => Date;
}
export interface KloLocalScanEnrichmentResult {
snapshot: KloSchemaSnapshot;
summary: KloScanEnrichmentSummary;
relationships: KloScanRelationshipSummary;
state: KloScanEnrichmentStateSummary;
warnings: KloScanWarning[];
export interface KtxLocalScanEnrichmentResult {
snapshot: KtxSchemaSnapshot;
summary: KtxScanEnrichmentSummary;
relationships: KtxScanRelationshipSummary;
state: KtxScanEnrichmentStateSummary;
warnings: KtxScanWarning[];
descriptionUpdates: Array<{
table: KloTableRef;
table: KtxTableRef;
tableDescription: string | null;
columnDescriptions: Record<string, string | null>;
}>;
embeddingUpdates: KloEmbeddingUpdate[];
relationshipUpdate: KloRelationshipUpdate | null;
relationshipProfile: KloRelationshipProfileArtifact | null;
resolvedRelationships: KloResolvedRelationshipDiscoveryCandidate[] | null;
compositeRelationships: KloCompositeRelationshipCandidate[] | null;
embeddingUpdates: KtxEmbeddingUpdate[];
relationshipUpdate: KtxRelationshipUpdate | null;
relationshipProfile: KtxRelationshipProfileArtifact | null;
resolvedRelationships: KtxResolvedRelationshipDiscoveryCandidate[] | null;
compositeRelationships: KtxCompositeRelationshipCandidate[] | null;
}
function tableId(table: KloSchemaTable): string {
function tableId(table: KtxSchemaTable): string {
return [table.catalog, table.db, table.name].filter((value): value is string => Boolean(value)).join('.');
}
function columnId(table: KloSchemaTable, column: KloSchemaColumn): string {
function columnId(table: KtxSchemaTable, column: KtxSchemaColumn): string {
return `${tableId(table)}.${column.name}`;
}
function tableRef(table: KloSchemaTable): KloTableRef {
function tableRef(table: KtxSchemaTable): KtxTableRef {
return {
catalog: table.catalog,
db: table.db,
@ -98,7 +98,7 @@ function tableRef(table: KloSchemaTable): KloTableRef {
};
}
function endpoint(table: KloEnrichedTable, column: KloEnrichedColumn): KloRelationshipEndpoint {
function endpoint(table: KtxEnrichedTable, column: KtxEnrichedColumn): KtxRelationshipEndpoint {
return {
tableId: table.id,
columnIds: [column.id],
@ -107,11 +107,11 @@ function endpoint(table: KloEnrichedTable, column: KloEnrichedColumn): KloRelati
};
}
function relationshipId(from: KloRelationshipEndpoint, to: KloRelationshipEndpoint): string {
function relationshipId(from: KtxRelationshipEndpoint, to: KtxRelationshipEndpoint): string {
return `${from.tableId}:(${from.columnIds.join(',')})->${to.tableId}:(${to.columnIds.join(',')})`;
}
function targetMatchesForeignKey(table: KloEnrichedTable, foreignKey: KloSchemaForeignKey): boolean {
function targetMatchesForeignKey(table: KtxEnrichedTable, foreignKey: KtxSchemaForeignKey): boolean {
return (
table.ref.name === foreignKey.toTable &&
(foreignKey.toCatalog === null || table.ref.catalog === foreignKey.toCatalog) &&
@ -120,11 +120,11 @@ function targetMatchesForeignKey(table: KloEnrichedTable, foreignKey: KloSchemaF
}
function formalRelationshipsFromSnapshot(
snapshot: KloSchemaSnapshot,
tables: readonly KloEnrichedTable[],
): KloEnrichedRelationship[] {
snapshot: KtxSchemaSnapshot,
tables: readonly KtxEnrichedTable[],
): KtxEnrichedRelationship[] {
const tableById = new Map(tables.map((table) => [table.id, table]));
const relationships: KloEnrichedRelationship[] = [];
const relationships: KtxEnrichedRelationship[] = [];
for (const sourceTableSnapshot of snapshot.tables) {
const sourceTable = tableById.get(tableId(sourceTableSnapshot));
@ -157,7 +157,7 @@ function formalRelationshipsFromSnapshot(
return relationships.sort((left, right) => left.id.localeCompare(right.id));
}
function providerlessEnrichedWarning(relationshipDetection: boolean): KloScanWarning {
function providerlessEnrichedWarning(relationshipDetection: boolean): KtxScanWarning {
return {
code: 'scan_enrichment_backend_not_configured',
message:
@ -183,7 +183,7 @@ function hashEmbedding(text: string, dimensions: number): number[] {
export function createDeterministicLocalScanEnrichmentProviders(
options: DeterministicLocalScanEnrichmentProviderOptions = {},
): KloLocalScanEnrichmentProviders {
): KtxLocalScanEnrichmentProviders {
const dimensions = options.embeddingDimensions ?? 8;
const maxBatchSize = options.maxBatchSize ?? 64;
return {
@ -198,14 +198,14 @@ export function createDeterministicLocalScanEnrichmentProviders(
};
}
function deterministicLlmProvider(): KloLlmProvider {
function deterministicLlmProvider(): KtxLlmProvider {
const model = { modelId: 'deterministic-scan', provider: 'deterministic' };
return {
getModel() {
return model as ReturnType<KloLlmProvider['getModel']>;
return model as ReturnType<KtxLlmProvider['getModel']>;
},
getModelByName() {
return model as ReturnType<KloLlmProvider['getModelByName']>;
return model as ReturnType<KtxLlmProvider['getModelByName']>;
},
cacheMarker() {
return undefined;
@ -237,14 +237,14 @@ function deterministicLlmProvider(): KloLlmProvider {
};
}
export function snapshotToKloEnrichedSchema(
snapshot: KloSchemaSnapshot,
export function snapshotToKtxEnrichedSchema(
snapshot: KtxSchemaSnapshot,
embeddingsByColumnId: ReadonlyMap<string, number[]> = new Map(),
): KloEnrichedSchema {
const tables: KloEnrichedTable[] = snapshot.tables.map((table) => {
): KtxEnrichedSchema {
const tables: KtxEnrichedTable[] = snapshot.tables.map((table) => {
const id = tableId(table);
const ref = tableRef(table);
const columns: KloEnrichedColumn[] = table.columns.map((column) => {
const columns: KtxEnrichedColumn[] = table.columns.map((column) => {
const idForColumn = columnId(table, column);
return {
id: idForColumn,
@ -283,7 +283,7 @@ export function snapshotToKloEnrichedSchema(
};
}
function descriptionTable(table: KloSchemaTable): KloDescriptionColumnTable {
function descriptionTable(table: KtxSchemaTable): KtxDescriptionColumnTable {
return {
catalog: table.catalog,
db: table.db,
@ -300,13 +300,13 @@ function embeddingBatchSize(maxBatchSize: number): number {
}
async function generateDescriptions(input: {
snapshot: KloSchemaSnapshot;
connector: KloScanConnector;
context: KloScanContext;
providers: KloLocalScanEnrichmentProviders;
progress?: KloProgressPort;
}): Promise<KloLocalScanEnrichmentResult['descriptionUpdates']> {
const generator = new KloDescriptionGenerator({
snapshot: KtxSchemaSnapshot;
connector: KtxScanConnector;
context: KtxScanContext;
providers: KtxLocalScanEnrichmentProviders;
progress?: KtxProgressPort;
}): Promise<KtxLocalScanEnrichmentResult['descriptionUpdates']> {
const generator = new KtxDescriptionGenerator({
llmProvider: input.providers.llm,
settings: {
columnMaxWords: 16,
@ -316,7 +316,7 @@ async function generateDescriptions(input: {
},
});
const updates: KloLocalScanEnrichmentResult['descriptionUpdates'] = [];
const updates: KtxLocalScanEnrichmentResult['descriptionUpdates'] = [];
const totalTables = input.snapshot.tables.length;
if (totalTables === 0) {
await input.progress?.update(1, 'No tables to describe');
@ -362,11 +362,11 @@ async function generateDescriptions(input: {
}
async function buildEmbeddings(input: {
snapshot: KloSchemaSnapshot;
providers: KloLocalScanEnrichmentProviders;
descriptions: KloLocalScanEnrichmentResult['descriptionUpdates'];
progress?: KloProgressPort;
}): Promise<{ updates: KloEmbeddingUpdate[]; byColumnId: Map<string, number[]> }> {
snapshot: KtxSchemaSnapshot;
providers: KtxLocalScanEnrichmentProviders;
descriptions: KtxLocalScanEnrichmentResult['descriptionUpdates'];
progress?: KtxProgressPort;
}): Promise<{ updates: KtxEmbeddingUpdate[]; byColumnId: Map<string, number[]> }> {
const descriptionByTable = new Map(input.descriptions.map((item) => [item.table.name, item]));
const texts: Array<{ columnId: string; text: string }> = [];
@ -374,7 +374,7 @@ async function buildEmbeddings(input: {
const tableDescriptions = descriptionByTable.get(table.name);
for (const column of table.columns) {
const id = columnId(table, column);
const text = buildKloColumnEmbeddingText({
const text = buildKtxColumnEmbeddingText({
tableName: table.name,
columnName: column.name,
columnType: column.nativeType,
@ -429,17 +429,17 @@ async function buildEmbeddings(input: {
}
async function runEnrichmentStage<TOutput>(input: {
stateStore: KloScanEnrichmentStateStore | null | undefined;
stateStore: KtxScanEnrichmentStateStore | null | undefined;
runId: string;
connectionId: string;
syncId: string;
mode: KloScanMode;
stage: KloScanEnrichmentStage;
mode: KtxScanMode;
stage: KtxScanEnrichmentStage;
inputHash: string;
now: () => Date;
resumedStages: KloScanEnrichmentStage[];
completedStages: KloScanEnrichmentStage[];
failedStages: KloScanEnrichmentStage[];
resumedStages: KtxScanEnrichmentStage[];
completedStages: KtxScanEnrichmentStage[];
failedStages: KtxScanEnrichmentStage[];
compute: () => Promise<TOutput>;
}): Promise<TOutput> {
const existing = await input.stateStore?.findCompletedStage<TOutput>({
@ -483,13 +483,13 @@ async function runEnrichmentStage<TOutput>(input: {
}
}
function embeddingsByColumnId(updates: KloEmbeddingUpdate[]): Map<string, number[]> {
function embeddingsByColumnId(updates: KtxEmbeddingUpdate[]): Map<string, number[]> {
return new Map(updates.map((update) => [update.columnId, update.embedding]));
}
export async function runLocalScanEnrichment(
input: KloLocalScanEnrichmentInput,
): Promise<KloLocalScanEnrichmentResult> {
input: KtxLocalScanEnrichmentInput,
): Promise<KtxLocalScanEnrichmentResult> {
const progress = input.context.progress;
await progress?.update(0, 'Loading enrichment schema snapshot');
const snapshot = await input.connector.introspect(
@ -504,22 +504,22 @@ export async function runLocalScanEnrichment(
await progress?.update(0.05, `Loaded schema snapshot with ${snapshot.tables.length} tables`);
const now = input.now ?? (() => new Date());
const state = completedKloScanEnrichmentStateSummary();
const state = completedKtxScanEnrichmentStateSummary();
const syncId = input.syncId ?? input.context.runId;
const relationshipSettings =
input.relationshipSettings ?? buildDefaultKloProjectConfig(input.connectionId).scan.relationships;
const inputHash = computeKloScanEnrichmentInputHash({
input.relationshipSettings ?? buildDefaultKtxProjectConfig(input.connectionId).scan.relationships;
const inputHash = computeKtxScanEnrichmentInputHash({
snapshot,
mode: input.mode,
detectRelationships: input.detectRelationships ?? false,
providerIdentity: input.providerIdentity ?? {},
relationshipSettings,
});
const warnings: KloScanWarning[] = [];
let descriptions: KloLocalScanEnrichmentResult['descriptionUpdates'] = [];
let embeddingUpdates: KloEmbeddingUpdate[] = [];
let schema = snapshotToKloEnrichedSchema(snapshot);
const summary: KloScanEnrichmentSummary = { ...skippedKloScanEnrichmentSummary };
const warnings: KtxScanWarning[] = [];
let descriptions: KtxLocalScanEnrichmentResult['descriptionUpdates'] = [];
let embeddingUpdates: KtxEmbeddingUpdate[] = [];
let schema = snapshotToKtxEnrichedSchema(snapshot);
const summary: KtxScanEnrichmentSummary = { ...skippedKtxScanEnrichmentSummary };
const relationshipDetectionEnabled = relationshipSettings.enabled;
const shouldDetectRelationships =
relationshipDetectionEnabled &&
@ -576,18 +576,18 @@ export async function runLocalScanEnrichment(
return embeddings.updates;
},
});
schema = snapshotToKloEnrichedSchema(snapshot, embeddingsByColumnId(embeddingUpdates));
schema = snapshotToKtxEnrichedSchema(snapshot, embeddingsByColumnId(embeddingUpdates));
summary.dataDictionary = input.connector.sampleColumn ? 'completed' : 'skipped';
summary.tableDescriptions = 'completed';
summary.columnDescriptions = 'completed';
summary.embeddings = 'completed';
}
let relationshipUpdate: KloRelationshipUpdate | null = null;
let relationshipProfile: KloRelationshipProfileArtifact | null = null;
let resolvedRelationships: KloResolvedRelationshipDiscoveryCandidate[] | null = null;
let compositeRelationships: KloCompositeRelationshipCandidate[] | null = null;
let relationships: KloScanRelationshipSummary = { accepted: 0, review: 0, rejected: 0, skipped: 0 };
let relationshipUpdate: KtxRelationshipUpdate | null = null;
let relationshipProfile: KtxRelationshipProfileArtifact | null = null;
let resolvedRelationships: KtxResolvedRelationshipDiscoveryCandidate[] | null = null;
let compositeRelationships: KtxCompositeRelationshipCandidate[] | null = null;
let relationships: KtxScanRelationshipSummary = { accepted: 0, review: 0, rejected: 0, skipped: 0 };
if (shouldDetectRelationships) {
const relationshipProgress = progress?.startPhase(0.25);
const relationshipStage = await runEnrichmentStage({
@ -604,7 +604,7 @@ export async function runLocalScanEnrichment(
failedStages: state.failedStages,
compute: async () => {
await relationshipProgress?.update(0, 'Detecting relationships');
const detection = await discoverKloRelationships({
const detection = await discoverKtxRelationships({
connectionId: input.connectionId,
driver: snapshot.driver,
connector: input.connector,
@ -647,7 +647,7 @@ export async function runLocalScanEnrichment(
snapshot,
summary,
relationships,
state: summarizeKloScanEnrichmentState(state),
state: summarizeKtxScanEnrichmentState(state),
warnings,
descriptionUpdates: descriptions,
embeddingUpdates,

54
packages/context/src/scan/local-scan.test.ts
View file

@ -1,18 +1,18 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { KloLlmProvider } from '@klo/llm';
import type { KtxLlmProvider } from '@ktx/llm';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import YAML from 'yaml';
import type { SourceAdapter } from '../ingest/index.js';
import { initKloProject, type KloLocalProject, loadKloProject } from '../project/index.js';
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
import { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-scan.js';
import type { KloQueryResult, KloReadOnlyQueryInput } from './types.js';
import type { KtxQueryResult, KtxReadOnlyQueryInput } from './types.js';
function relationshipSqlResult(
input: KloReadOnlyQueryInput,
input: KtxReadOnlyQueryInput,
options: { throwOnCoverage?: boolean } = {},
): KloQueryResult {
): KtxQueryResult {
if (input.sql.includes('child_values')) {
if (options.throwOnCoverage) {
throw new Error('validation failed for postgres://reader:secret@example.test/db'); // pragma: allowlist secret
@ -79,7 +79,7 @@ function relationshipSqlResult(
throw new Error(`Unexpected relationship SQL: ${input.sql}`);
}
function deterministicLlmProvider(): KloLlmProvider {
function deterministicLlmProvider(): KtxLlmProvider {
return {
getModel: () => ({ provider: 'deterministic', modelId: 'deterministic' }) as never,
getModelByName: () => ({ provider: 'deterministic', modelId: 'deterministic' }) as never,
@ -103,7 +103,7 @@ function deterministicLlmProvider(): KloLlmProvider {
async function writeLiveDatabaseConfig(projectDir: string): Promise<void> {
await writeFile(
join(projectDir, 'klo.yaml'),
join(projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -164,14 +164,14 @@ function fetchOnlyAdapter(options: { extractedAt?: () => string } = {}): SourceA
describe('local scan', () => {
let tempDir: string;
let project: KloLocalProject;
let project: KtxLocalProject;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-scan-'));
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-scan-'));
const projectDir = join(tempDir, 'project');
await initKloProject({ projectDir, projectName: 'warehouse' });
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeLiveDatabaseConfig(projectDir);
project = await loadKloProject({ projectDir });
project = await loadKtxProject({ projectDir });
});
afterEach(async () => {
@ -509,7 +509,7 @@ describe('local scan', () => {
],
};
},
async executeReadOnly(input: KloReadOnlyQueryInput) {
async executeReadOnly(input: KtxReadOnlyQueryInput) {
return relationshipSqlResult(input);
},
};
@ -603,7 +603,7 @@ describe('local scan', () => {
],
};
},
async executeReadOnly(input: KloReadOnlyQueryInput) {
async executeReadOnly(input: KtxReadOnlyQueryInput) {
return relationshipSqlResult(input);
},
};
@ -712,7 +712,7 @@ describe('local scan', () => {
],
};
},
async executeReadOnly(input: KloReadOnlyQueryInput) {
async executeReadOnly(input: KtxReadOnlyQueryInput) {
return relationshipSqlResult(input);
},
};
@ -838,7 +838,7 @@ describe('local scan', () => {
],
};
},
async executeReadOnly(input: KloReadOnlyQueryInput) {
async executeReadOnly(input: KtxReadOnlyQueryInput) {
return relationshipSqlResult(input);
},
};
@ -968,7 +968,7 @@ describe('local scan', () => {
],
};
},
async executeReadOnly(input: KloReadOnlyQueryInput) {
async executeReadOnly(input: KtxReadOnlyQueryInput) {
return relationshipSqlResult(input, { throwOnCoverage: true });
},
};
@ -999,7 +999,7 @@ describe('local scan', () => {
it('runs enriched scans when deterministic standalone enrichment is configured', async () => {
await writeFile(
join(project.projectDir, 'klo.yaml'),
join(project.projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -1017,7 +1017,7 @@ describe('local scan', () => {
].join('\n'),
'utf-8',
);
project = await loadKloProject({ projectDir: project.projectDir });
project = await loadKtxProject({ projectDir: project.projectDir });
const connector = {
id: 'test:warehouse',
@ -1200,7 +1200,7 @@ describe('local scan', () => {
expect(result.report.warnings).toEqual([
{
code: 'enrichment_failed',
message: 'KLO scan enrichment failed after structural scan completed: embedding service timed out',
message: 'KTX scan enrichment failed after structural scan completed: embedding service timed out',
recoverable: true,
metadata: {
mode: 'enriched',
@ -1356,7 +1356,7 @@ describe('local scan', () => {
it('accepts sqlite as a native standalone scan driver when the host supplies a live-database adapter', async () => {
await writeFile(
join(project.projectDir, 'klo.yaml'),
join(project.projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -1371,7 +1371,7 @@ describe('local scan', () => {
].join('\n'),
'utf-8',
);
project = await loadKloProject({ projectDir: project.projectDir });
project = await loadKtxProject({ projectDir: project.projectDir });
const result = await runLocalScan({
project,
@ -1389,7 +1389,7 @@ describe('local scan', () => {
it('accepts mysql as a native standalone scan driver when the host supplies a live-database adapter', async () => {
await writeFile(
join(project.projectDir, 'klo.yaml'),
join(project.projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -1404,7 +1404,7 @@ describe('local scan', () => {
].join('\n'),
'utf-8',
);
project = await loadKloProject({ projectDir: project.projectDir });
project = await loadKtxProject({ projectDir: project.projectDir });
const result = await runLocalScan({
project,
@ -1422,7 +1422,7 @@ describe('local scan', () => {
it('accepts clickhouse as a native standalone scan driver when the host supplies a live-database adapter', async () => {
await writeFile(
join(project.projectDir, 'klo.yaml'),
join(project.projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -1440,7 +1440,7 @@ describe('local scan', () => {
].join('\n'),
'utf-8',
);
project = await loadKloProject({ projectDir: project.projectDir });
project = await loadKtxProject({ projectDir: project.projectDir });
const result = await runLocalScan({
project,
@ -1458,7 +1458,7 @@ describe('local scan', () => {
it('accepts sqlserver as a native standalone scan driver when the host supplies a live-database adapter', async () => {
await writeFile(
join(project.projectDir, 'klo.yaml'),
join(project.projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -1476,7 +1476,7 @@ describe('local scan', () => {
].join('\n'),
'utf-8',
);
project = await loadKloProject({ projectDir: project.projectDir });
project = await loadKtxProject({ projectDir: project.projectDir });
const result = await runLocalScan({
project,

140
packages/context/src/scan/local-scan.ts
View file

@ -1,4 +1,4 @@
import type { createKloEmbeddingProvider, createKloLlmProvider } from '@klo/llm';
import type { createKtxEmbeddingProvider, createKtxLlmProvider } from '@ktx/llm';
import {
createDefaultLocalIngestAdapters,
getLocalStageOnlyIngestStatus,
@ -7,50 +7,50 @@ import {
type SourceAdapter,
} from '../ingest/index.js';
import {
createLocalKloEmbeddingProviderFromConfig,
createLocalKloLlmProviderFromConfig,
KloScanEmbeddingPortAdapter,
createLocalKtxEmbeddingProviderFromConfig,
createLocalKtxLlmProviderFromConfig,
KtxScanEmbeddingPortAdapter,
} from '../llm/index.js';
import type { KloProjectLlmConfig, KloScanEnrichmentConfig, KloScanRelationshipConfig } from '../project/config.js';
import type { KloLocalProject } from '../project/index.js';
import { kloLocalStateDbPath } from '../project/local-state-db.js';
import { redactKloScanReport } from './credentials.js';
import { completedKloScanEnrichmentStateSummary } from './enrichment-state.js';
import { failedKloScanEnrichmentSummary, kloScanErrorMessage } from './enrichment-summary.js';
import type { KtxProjectLlmConfig, KtxScanEnrichmentConfig, KtxScanRelationshipConfig } from '../project/config.js';
import type { KtxLocalProject } from '../project/index.js';
import { ktxLocalStateDbPath } from '../project/local-state-db.js';
import { redactKtxScanReport } from './credentials.js';
import { completedKtxScanEnrichmentStateSummary } from './enrichment-state.js';
import { failedKtxScanEnrichmentSummary, ktxScanErrorMessage } from './enrichment-summary.js';
import {
createDeterministicLocalScanEnrichmentProviders,
type KloLocalScanEnrichmentProviders,
type KtxLocalScanEnrichmentProviders,
runLocalScanEnrichment,
} from './local-enrichment.js';
import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from './local-enrichment-artifacts.js';
import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js';
import { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js';
import type {
KloConnectionDriver,
KloProgressPort,
KloScanConnector,
KloScanEnrichmentStateSummary,
KloScanMode,
KloScanReport,
KloScanTrigger,
KtxConnectionDriver,
KtxProgressPort,
KtxScanConnector,
KtxScanEnrichmentStateSummary,
KtxScanMode,
KtxScanReport,
KtxScanTrigger,
} from './types.js';
export interface RunLocalScanOptions {
project: KloLocalProject;
project: KtxLocalProject;
connectionId: string;
mode?: KloScanMode;
mode?: KtxScanMode;
detectRelationships?: boolean;
dryRun?: boolean;
trigger?: KloScanTrigger;
trigger?: KtxScanTrigger;
databaseIntrospectionUrl?: string;
adapters?: SourceAdapter[];
jobId?: string;
now?: () => Date;
connector?: KloScanConnector;
createConnector?: (connectionId: string) => KloScanConnector | Promise<KloScanConnector>;
enrichmentProviders?: KloLocalScanEnrichmentProviders | null;
connector?: KtxScanConnector;
createConnector?: (connectionId: string) => KtxScanConnector | Promise<KtxScanConnector>;
enrichmentProviders?: KtxLocalScanEnrichmentProviders | null;
enrichmentStateStore?: SqliteLocalScanEnrichmentStateStore | null;
progress?: KloProgressPort;
progress?: KtxProgressPort;
}
export interface LocalScanRunResult {
@ -58,10 +58,10 @@ export interface LocalScanRunResult {
status: 'done';
done: true;
connectionId: string;
mode: KloScanMode;
mode: KtxScanMode;
dryRun: boolean;
syncId: string;
report: KloScanReport;
report: KtxScanReport;
}
export interface LocalScanStatusResponse {
@ -69,14 +69,14 @@ export interface LocalScanStatusResponse {
status: LocalIngestRunRecord['status'];
done: boolean;
connectionId: string;
mode: KloScanMode;
mode: KtxScanMode;
dryRun: boolean;
syncId: string;
progress: number;
startedAt: string;
completedAt: string;
reportPath: string | null;
warnings: KloScanReport['warnings'];
warnings: KtxScanReport['warnings'];
}
export interface LocalScanMcpOptions {
@ -84,15 +84,15 @@ export interface LocalScanMcpOptions {
databaseIntrospectionUrl?: string;
jobIdFactory?: () => string;
now?: () => Date;
createConnector?: (connectionId: string) => KloScanConnector | Promise<KloScanConnector>;
createConnector?: (connectionId: string) => KtxScanConnector | Promise<KtxScanConnector>;
}
const LIVE_DATABASE_ADAPTER = 'live-database';
const SCAN_REPORT_FILE = 'scan-report.json';
const LOCAL_AUTHOR = 'klo';
const LOCAL_AUTHOR_EMAIL = 'klo@example.com';
const LOCAL_AUTHOR = 'ktx';
const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
function normalizeDriver(driver: string | undefined): KloConnectionDriver {
function normalizeDriver(driver: string | undefined): KtxConnectionDriver {
const normalized = (driver ?? '').toLowerCase();
if (
normalized === 'postgres' ||
@ -109,7 +109,7 @@ function normalizeDriver(driver: string | undefined): KloConnectionDriver {
return normalized === 'sqlite3' ? 'sqlite' : normalized;
}
throw new Error(
`Standalone klo scan supports postgres/postgresql/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake/posthog in this phase, received "${driver ?? 'unknown'}"`,
`Standalone ktx scan supports postgres/postgresql/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake/posthog in this phase, received "${driver ?? 'unknown'}"`,
);
}
@ -125,13 +125,13 @@ function scanReportPath(connectionId: string, syncId: string): string {
return `${rawSourcesDir(connectionId, syncId)}/${SCAN_REPORT_FILE}`;
}
function assertSupportedMode(mode: KloScanMode): void {
function assertSupportedMode(mode: KtxScanMode): void {
if (mode !== 'structural' && mode !== 'relationships' && mode !== 'enriched') {
throw new Error(`Unsupported KLO scan mode: ${mode}`);
throw new Error(`Unsupported KTX scan mode: ${mode}`);
}
}
async function resolveScanConnector(options: RunLocalScanOptions, mode: KloScanMode): Promise<KloScanConnector | null> {
async function resolveScanConnector(options: RunLocalScanOptions, mode: KtxScanMode): Promise<KtxScanConnector | null> {
if (mode === 'structural' && !options.detectRelationships) {
return null;
}
@ -141,20 +141,20 @@ async function resolveScanConnector(options: RunLocalScanOptions, mode: KloScanM
if (options.createConnector) {
return options.createConnector(options.connectionId);
}
throw new Error('klo scan --enrich and --detect-relationships require a native standalone scan connector');
throw new Error('ktx scan --enrich and --detect-relationships require a native standalone scan connector');
}
interface LocalScanEnrichmentProviderDeps {
createKloLlmProvider?: typeof createKloLlmProvider;
createKloEmbeddingProvider?: typeof createKloEmbeddingProvider;
createKtxLlmProvider?: typeof createKtxLlmProvider;
createKtxEmbeddingProvider?: typeof createKtxEmbeddingProvider;
env?: NodeJS.ProcessEnv;
}
export function createLocalScanEnrichmentProvidersFromConfig(
config: KloScanEnrichmentConfig,
llmConfig: KloProjectLlmConfig,
config: KtxScanEnrichmentConfig,
llmConfig: KtxProjectLlmConfig,
deps: LocalScanEnrichmentProviderDeps = {},
): KloLocalScanEnrichmentProviders | null {
): KtxLocalScanEnrichmentProviders | null {
if (config.mode === 'deterministic') {
return createDeterministicLocalScanEnrichmentProviders();
}
@ -163,15 +163,15 @@ export function createLocalScanEnrichmentProvidersFromConfig(
return null;
}
const llm = createLocalKloLlmProviderFromConfig(llmConfig, deps);
const embeddingProvider = createLocalKloEmbeddingProviderFromConfig(config.embeddings, deps);
const llm = createLocalKtxLlmProviderFromConfig(llmConfig, deps);
const embeddingProvider = createLocalKtxEmbeddingProviderFromConfig(config.embeddings, deps);
if (!llm || !embeddingProvider) {
return null;
}
return {
llm,
embedding: new KloScanEmbeddingPortAdapter(embeddingProvider),
embedding: new KtxScanEmbeddingPortAdapter(embeddingProvider),
};
}
@ -182,13 +182,13 @@ function createLocalScanEnrichmentStateStore(options: RunLocalScanOptions): Sqli
if (options.enrichmentStateStore !== undefined) {
return options.enrichmentStateStore;
}
return new SqliteLocalScanEnrichmentStateStore({ dbPath: kloLocalStateDbPath(options.project) });
return new SqliteLocalScanEnrichmentStateStore({ dbPath: ktxLocalStateDbPath(options.project) });
}
function localScanProviderIdentity(
config: KloScanEnrichmentConfig,
llmConfig: KloProjectLlmConfig,
relationships: KloScanRelationshipConfig,
config: KtxScanEnrichmentConfig,
llmConfig: KtxProjectLlmConfig,
relationships: KtxScanRelationshipConfig,
): Record<string, unknown> {
return {
mode: config.mode,
@ -203,12 +203,12 @@ function localScanProviderIdentity(
function reportFromIngest(input: {
record: LocalIngestRunRecord;
driver: KloConnectionDriver;
mode: KloScanMode;
driver: KtxConnectionDriver;
mode: KtxScanMode;
dryRun: boolean;
trigger: KloScanTrigger;
trigger: KtxScanTrigger;
createdAt: string;
}): KloScanReport {
}): KtxScanReport {
const reportPath = input.dryRun ? null : scanReportPath(input.record.connectionId, input.record.syncId);
return {
connectionId: input.record.connectionId,
@ -254,12 +254,12 @@ function reportFromIngest(input: {
capabilityGaps: [],
warnings: [],
relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 },
enrichmentState: completedKloScanEnrichmentStateSummary(),
enrichmentState: completedKtxScanEnrichmentStateSummary(),
createdAt: input.createdAt,
};
}
async function writeScanReport(project: KloLocalProject, report: KloScanReport): Promise<void> {
async function writeScanReport(project: KtxLocalProject, report: KtxScanReport): Promise<void> {
if (!report.artifactPaths.reportPath) {
return;
}
@ -272,7 +272,7 @@ async function writeScanReport(project: KloLocalProject, report: KloScanReport):
);
}
function scanDiffSummaryFromRecord(record: LocalIngestRunRecord): KloScanReport['diffSummary'] {
function scanDiffSummaryFromRecord(record: LocalIngestRunRecord): KtxScanReport['diffSummary'] {
return {
tablesAdded: tablePathCount(record.diffPaths.added),
tablesModified: tablePathCount(record.diffPaths.modified),
@ -293,7 +293,7 @@ function hasNoContentChanges(record: LocalIngestRunRecord): boolean {
);
}
function scanChangeSummary(diffSummary: KloScanReport['diffSummary']): string {
function scanChangeSummary(diffSummary: KtxScanReport['diffSummary']): string {
const changedTables = diffSummary.tablesAdded + diffSummary.tablesModified + diffSummary.tablesDeleted;
const totalTables = changedTables + diffSummary.tablesUnchanged;
const changeNoun = changedTables === 1 ? 'change' : 'changes';
@ -302,13 +302,13 @@ function scanChangeSummary(diffSummary: KloScanReport['diffSummary']): string {
}
async function readScanReport(
project: KloLocalProject,
project: KtxLocalProject,
connectionId: string,
syncId: string,
): Promise<KloScanReport | null> {
): Promise<KtxScanReport | null> {
try {
const raw = await project.fileStore.readFile(scanReportPath(connectionId, syncId));
return JSON.parse(raw.content) as KloScanReport;
return JSON.parse(raw.content) as KtxScanReport;
} catch {
return null;
}
@ -322,7 +322,7 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
const connection = options.project.config.connections[options.connectionId];
if (!connection) {
throw new Error(`Connection "${options.connectionId}" is not configured in klo.yaml`);
throw new Error(`Connection "${options.connectionId}" is not configured in ktx.yaml`);
}
const driver = normalizeDriver(connection.driver);
const adapters =
@ -370,7 +370,7 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
reusedExistingScanArtifacts = true;
}
const enrichmentStateStore = connector ? createLocalScanEnrichmentStateStore(options) : null;
let enrichmentState: KloScanEnrichmentStateSummary = completedKloScanEnrichmentStateSummary();
let enrichmentState: KtxScanEnrichmentStateSummary = completedKtxScanEnrichmentStateSummary();
if (!reusedExistingScanArtifacts && !report.dryRun && report.artifactPaths.rawSourcesDir) {
await options.progress?.update(0.7, 'Writing schema artifacts');
const structuralSnapshot = await readLocalScanStructuralSnapshot({
@ -434,11 +434,11 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
report.artifactPaths.manifestShards = artifacts.manifestShards;
report.manifestShardsWritten = artifacts.manifestShardsWritten;
} catch (error) {
const message = kloScanErrorMessage(error);
report.enrichment = failedKloScanEnrichmentSummary(mode, options.detectRelationships ?? false);
const message = ktxScanErrorMessage(error);
report.enrichment = failedKtxScanEnrichmentSummary(mode, options.detectRelationships ?? false);
const stages = await enrichmentStateStore?.listRunStages(record.runId);
if (stages) {
enrichmentState = completedKloScanEnrichmentStateSummary();
enrichmentState = completedKtxScanEnrichmentStateSummary();
for (const stage of stages) {
if (stage.status === 'completed') {
enrichmentState.completedStages.push(stage.stage);
@ -450,13 +450,13 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
}
report.warnings.push({
code: 'enrichment_failed',
message: `KLO scan enrichment failed after structural scan completed: ${message}`,
message: `KTX scan enrichment failed after structural scan completed: ${message}`,
recoverable: true,
metadata: { mode, detectRelationships: options.detectRelationships ?? false },
});
}
}
report = redactKloScanReport(report);
report = redactKtxScanReport(report);
if (!reusedExistingScanArtifacts) {
await writeScanReport(options.project, report);
}
@ -473,7 +473,7 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
};
}
export async function getLocalScanReport(project: KloLocalProject, runId: string): Promise<KloScanReport | null> {
export async function getLocalScanReport(project: KtxLocalProject, runId: string): Promise<KtxScanReport | null> {
const status = await getLocalStageOnlyIngestStatus(project, runId);
if (!status || status.adapter !== LIVE_DATABASE_ADAPTER) {
return null;
@ -491,7 +491,7 @@ export async function getLocalScanReport(project: KloLocalProject, runId: string
}
export async function getLocalScanStatus(
project: KloLocalProject,
project: KtxLocalProject,
runId: string,
): Promise<LocalScanStatusResponse | null> {
const status = await getLocalStageOnlyIngestStatus(project, runId);

28
packages/context/src/scan/local-structural-artifacts.test.ts
View file

@ -2,16 +2,16 @@ import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { initKloProject, type KloLocalProject } from '../project/index.js';
import { initKtxProject, type KtxLocalProject } from '../project/index.js';
import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js';
describe('readLocalScanStructuralSnapshot', () => {
let tempDir: string;
let project: KloLocalProject;
let project: KtxLocalProject;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-structural-artifacts-'));
project = await initKloProject({
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-structural-artifacts-'));
project = await initKtxProject({
projectDir: join(tempDir, 'project'),
projectName: 'warehouse',
});
@ -35,8 +35,8 @@ describe('readLocalScanStructuralSnapshot', () => {
null,
2,
)}\n`,
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed connection artifact',
);
await project.fileStore.writeFile(
@ -65,8 +65,8 @@ describe('readLocalScanStructuralSnapshot', () => {
null,
2,
)}\n`,
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed customers artifact',
);
await project.fileStore.writeFile(
@ -113,8 +113,8 @@ describe('readLocalScanStructuralSnapshot', () => {
null,
2,
)}\n`,
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed orders artifact',
);
@ -171,15 +171,15 @@ describe('readLocalScanStructuralSnapshot', () => {
await project.fileStore.writeFile(
`${rawRoot}/connection.json`,
'{"connectionId":"warehouse","metadata":{}}\n',
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed connection artifact without extractedAt',
);
await project.fileStore.writeFile(
`${rawRoot}/tables/orders.json`,
'{"name":"orders","catalog":null,"db":null,"kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":true,"comment":null}],"foreignKeys":[]}\n',
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed orders artifact',
);

32
packages/context/src/scan/local-structural-artifacts.ts
View file

@ -1,16 +1,16 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import type {
KloConnectionDriver,
KloSchemaColumn,
KloSchemaForeignKey,
KloSchemaSnapshot,
KloSchemaTable,
KtxConnectionDriver,
KtxSchemaColumn,
KtxSchemaForeignKey,
KtxSchemaSnapshot,
KtxSchemaTable,
} from './types.js';
export interface ReadLocalScanStructuralSnapshotInput {
project: KloLocalProject;
project: KtxLocalProject;
connectionId: string;
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
rawSourcesDir: string;
extractedAtFallback: string;
}
@ -37,7 +37,7 @@ function optionalStringOrNull(value: unknown): string | null | undefined {
return typeof value === 'string' ? value : null;
}
function parseColumn(rawColumn: unknown, path: string): KloSchemaColumn {
function parseColumn(rawColumn: unknown, path: string): KtxSchemaColumn {
if (
!isRecord(rawColumn) ||
typeof rawColumn.name !== 'string' ||
@ -48,7 +48,7 @@ function parseColumn(rawColumn: unknown, path: string): KloSchemaColumn {
rawColumn.dimensionType !== 'number' &&
rawColumn.dimensionType !== 'boolean')
) {
throw new Error(`Invalid KLO schema column artifact: ${path}`);
throw new Error(`Invalid KTX schema column artifact: ${path}`);
}
return {
name: rawColumn.name,
@ -61,14 +61,14 @@ function parseColumn(rawColumn: unknown, path: string): KloSchemaColumn {
};
}
function parseForeignKey(rawForeignKey: unknown, path: string): KloSchemaForeignKey {
function parseForeignKey(rawForeignKey: unknown, path: string): KtxSchemaForeignKey {
if (
!isRecord(rawForeignKey) ||
typeof rawForeignKey.fromColumn !== 'string' ||
typeof rawForeignKey.toTable !== 'string' ||
typeof rawForeignKey.toColumn !== 'string'
) {
throw new Error(`Invalid KLO schema foreign key artifact: ${path}`);
throw new Error(`Invalid KTX schema foreign key artifact: ${path}`);
}
return {
fromColumn: rawForeignKey.fromColumn,
@ -80,10 +80,10 @@ function parseForeignKey(rawForeignKey: unknown, path: string): KloSchemaForeign
};
}
function parseTable(raw: string, path: string): KloSchemaTable {
function parseTable(raw: string, path: string): KtxSchemaTable {
const parsed = JSON.parse(raw) as unknown;
if (!isRecord(parsed) || typeof parsed.name !== 'string' || !Array.isArray(parsed.columns)) {
throw new Error(`Invalid KLO schema table artifact: ${path}`);
throw new Error(`Invalid KTX schema table artifact: ${path}`);
}
return {
catalog: optionalStringOrNull(parsed.catalog) ?? null,
@ -102,13 +102,13 @@ function parseTable(raw: string, path: string): KloSchemaTable {
export async function readLocalScanStructuralSnapshot(
input: ReadLocalScanStructuralSnapshotInput,
): Promise<KloSchemaSnapshot> {
): Promise<KtxSchemaSnapshot> {
const connectionRaw = await input.project.fileStore.readFile(`${input.rawSourcesDir}/connection.json`);
const connection = JSON.parse(connectionRaw.content) as LiveDatabaseConnectionArtifact;
const listedTables = await input.project.fileStore.listFiles(`${input.rawSourcesDir}/tables`);
const tablePaths = listedTables.files.filter((path) => path.endsWith('.json')).sort();
const tables: KloSchemaTable[] = [];
const tables: KtxSchemaTable[] = [];
for (const path of tablePaths) {
const tableRaw = await input.project.fileStore.readFile(path);
tables.push(parseTable(tableRaw.content, path));

56
packages/context/src/scan/orchestrator.test.ts
View file

@ -1,15 +1,15 @@
import { describe, expect, it, vi } from 'vitest';
import {
createKloConnectorCapabilities,
type KloScanConnector,
type KloScanContext,
type KloScanEnrichmentStateSummary,
type KloScanInput,
KloScanOrchestrator,
type KloSchemaSnapshot,
createKtxConnectorCapabilities,
type KtxScanConnector,
type KtxScanContext,
type KtxScanEnrichmentStateSummary,
type KtxScanInput,
KtxScanOrchestrator,
type KtxSchemaSnapshot,
} from './index.js';
function snapshot(): KloSchemaSnapshot {
function snapshot(): KtxSchemaSnapshot {
return {
connectionId: 'warehouse',
driver: 'postgres',
@ -42,8 +42,8 @@ function snapshot(): KloSchemaSnapshot {
}
function connector(
capabilities = createKloConnectorCapabilities({ tableSampling: true, columnSampling: true }),
): KloScanConnector {
capabilities = createKtxConnectorCapabilities({ tableSampling: true, columnSampling: true }),
): KtxScanConnector {
return {
id: 'connector-1',
driver: 'postgres',
@ -52,7 +52,7 @@ function connector(
};
}
function context(): KloScanContext {
function context(): KtxScanContext {
return {
runId: 'scan-run-1',
logger: {
@ -64,24 +64,24 @@ function context(): KloScanContext {
};
}
const input: KloScanInput = {
const input: KtxScanInput = {
connectionId: 'warehouse',
driver: 'postgres',
mode: 'structural',
};
describe('KloScanOrchestrator', () => {
describe('KtxScanOrchestrator', () => {
it('runs structural scans through connector introspection and structural host callback', async () => {
const scanConnector = connector();
const scanContext = context();
const runStructural = vi.fn(async (scanSnapshot: KloSchemaSnapshot) => ({
const runStructural = vi.fn(async (scanSnapshot: KtxSchemaSnapshot) => ({
result: { synced: true },
diffSummary: { tablesAdded: scanSnapshot.tables.length, columnsAdded: 1 },
structuralSyncStats: { tablesCreated: 1, columnsCreated: 1 },
artifactPaths: { manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'] },
}));
const result = await new KloScanOrchestrator({
const result = await new KtxScanOrchestrator({
now: () => new Date('2026-04-29T00:10:00.000Z'),
syncIdFactory: () => 'sync-1',
}).run({
@ -137,7 +137,7 @@ describe('KloScanOrchestrator', () => {
it('runs enriched scans through structural and enrichment host callbacks', async () => {
const scanConnector = connector(
createKloConnectorCapabilities({
createKtxConnectorCapabilities({
tableSampling: true,
columnSampling: true,
columnStats: true,
@ -146,7 +146,7 @@ describe('KloScanOrchestrator', () => {
);
const scanContext = context();
const result = await new KloScanOrchestrator({ syncIdFactory: () => 'sync-2' }).run({
const result = await new KtxScanOrchestrator({ syncIdFactory: () => 'sync-2' }).run({
connector: scanConnector,
input: { ...input, mode: 'enriched', detectRelationships: true },
trigger: 'schema_scan',
@ -178,20 +178,20 @@ describe('KloScanOrchestrator', () => {
it('reports host enrichment state summaries from enriched scan phases', async () => {
const scanConnector = connector(
createKloConnectorCapabilities({
createKtxConnectorCapabilities({
tableSampling: true,
columnSampling: true,
columnStats: true,
readOnlySql: true,
}),
);
const enrichmentState: Partial<KloScanEnrichmentStateSummary> = {
const enrichmentState: Partial<KtxScanEnrichmentStateSummary> = {
resumedStages: ['relationships', 'descriptions', 'descriptions'],
completedStages: ['embeddings', 'descriptions', 'relationships'],
failedStages: [],
};
const result = await new KloScanOrchestrator({ syncIdFactory: () => 'sync-state' }).run({
const result = await new KtxScanOrchestrator({ syncIdFactory: () => 'sync-state' }).run({
connector: scanConnector,
input: { ...input, mode: 'enriched', detectRelationships: true },
trigger: 'schema_scan',
@ -211,8 +211,8 @@ describe('KloScanOrchestrator', () => {
});
it('records recoverable warnings for missing optional capabilities during enriched scans', async () => {
const result = await new KloScanOrchestrator({ syncIdFactory: () => 'sync-3' }).run({
connector: connector(createKloConnectorCapabilities()),
const result = await new KtxScanOrchestrator({ syncIdFactory: () => 'sync-3' }).run({
connector: connector(createKtxConnectorCapabilities()),
input: { ...input, mode: 'enriched', detectRelationships: true },
trigger: 'schema_scan',
context: context(),
@ -231,9 +231,9 @@ describe('KloScanOrchestrator', () => {
});
it('redacts structural and enrichment warning metadata before returning reports', async () => {
const result = await new KloScanOrchestrator({ syncIdFactory: () => 'sync-redacted' }).run({
const result = await new KtxScanOrchestrator({ syncIdFactory: () => 'sync-redacted' }).run({
connector: connector(
createKloConnectorCapabilities({
createKtxConnectorCapabilities({
tableSampling: true,
columnSampling: true,
columnStats: true,
@ -301,7 +301,7 @@ describe('KloScanOrchestrator', () => {
it('keeps structural results when the enrichment phase fails after structural sync', async () => {
const scanConnector = connector(
createKloConnectorCapabilities({
createKtxConnectorCapabilities({
tableSampling: true,
columnSampling: true,
columnStats: true,
@ -320,7 +320,7 @@ describe('KloScanOrchestrator', () => {
throw new Error('AI Gateway timed out');
});
const result = await new KloScanOrchestrator({
const result = await new KtxScanOrchestrator({
now: () => new Date('2026-04-29T18:00:00.000Z'),
syncIdFactory: () => 'sync-failed-enrichment',
}).run({
@ -348,7 +348,7 @@ describe('KloScanOrchestrator', () => {
expect(result.report.warnings).toEqual([
{
code: 'enrichment_failed',
message: 'KLO scan enrichment failed after structural scan completed: AI Gateway timed out',
message: 'KTX scan enrichment failed after structural scan completed: AI Gateway timed out',
recoverable: true,
metadata: {
mode: 'enriched',
@ -361,7 +361,7 @@ describe('KloScanOrchestrator', () => {
it('marks dry-run reports without changing host callback behavior', async () => {
const runStructural = vi.fn(async () => ({ result: { planned: true }, manifestShardsWritten: 0 }));
const result = await new KloScanOrchestrator({ syncIdFactory: () => 'sync-4' }).run({
const result = await new KtxScanOrchestrator({ syncIdFactory: () => 'sync-4' }).run({
connector: connector(),
input: { ...input, dryRun: true },
trigger: 'cli',

180
packages/context/src/scan/orchestrator.ts
View file

@ -1,79 +1,79 @@
import { redactKloScanReport } from './credentials.js';
import { completedKloScanEnrichmentStateSummary, summarizeKloScanEnrichmentState } from './enrichment-state.js';
import { redactKtxScanReport } from './credentials.js';
import { completedKtxScanEnrichmentStateSummary, summarizeKtxScanEnrichmentState } from './enrichment-state.js';
import {
failedKloScanEnrichmentSummary,
kloScanErrorMessage,
skippedKloScanEnrichmentSummary,
failedKtxScanEnrichmentSummary,
ktxScanErrorMessage,
skippedKtxScanEnrichmentSummary,
} from './enrichment-summary.js';
import type {
KloConnectorCapabilities,
KloScanArtifactPaths,
KloScanConnector,
KloScanContext,
KloScanDiffSummary,
KloScanEnrichmentSummary,
KloScanEnrichmentStateSummary,
KloScanInput,
KloScanRelationshipSummary,
KloScanReport,
KloScanTrigger,
KloScanWarning,
KloSchemaSnapshot,
KloStructuralSyncStats,
KtxConnectorCapabilities,
KtxScanArtifactPaths,
KtxScanConnector,
KtxScanContext,
KtxScanDiffSummary,
KtxScanEnrichmentSummary,
KtxScanEnrichmentStateSummary,
KtxScanInput,
KtxScanRelationshipSummary,
KtxScanReport,
KtxScanTrigger,
KtxScanWarning,
KtxSchemaSnapshot,
KtxStructuralSyncStats,
} from './types.js';
type CapabilityGap = keyof Omit<KloConnectorCapabilities, 'structuralIntrospection'>;
type CapabilityGap = keyof Omit<KtxConnectorCapabilities, 'structuralIntrospection'>;
export interface KloStructuralScanPhaseResult<TResult = unknown> {
export interface KtxStructuralScanPhaseResult<TResult = unknown> {
result: TResult;
diffSummary?: Partial<KloScanDiffSummary>;
structuralSyncStats?: Partial<KloStructuralSyncStats>;
diffSummary?: Partial<KtxScanDiffSummary>;
structuralSyncStats?: Partial<KtxStructuralSyncStats>;
manifestShardsWritten?: number;
artifactPaths?: Partial<KloScanArtifactPaths>;
relationships?: Partial<KloScanRelationshipSummary>;
warnings?: KloScanWarning[];
artifactPaths?: Partial<KtxScanArtifactPaths>;
relationships?: Partial<KtxScanRelationshipSummary>;
warnings?: KtxScanWarning[];
}
export interface KloEnrichmentScanPhaseResult<TResult = unknown> {
export interface KtxEnrichmentScanPhaseResult<TResult = unknown> {
result: TResult;
enrichment?: Partial<KloScanEnrichmentSummary>;
enrichmentState?: Partial<KloScanEnrichmentStateSummary>;
enrichment?: Partial<KtxScanEnrichmentSummary>;
enrichmentState?: Partial<KtxScanEnrichmentStateSummary>;
manifestShardsWritten?: number;
artifactPaths?: Partial<KloScanArtifactPaths>;
relationships?: Partial<KloScanRelationshipSummary>;
warnings?: KloScanWarning[];
artifactPaths?: Partial<KtxScanArtifactPaths>;
relationships?: Partial<KtxScanRelationshipSummary>;
warnings?: KtxScanWarning[];
}
export interface KloScanOrchestratorRunInput<TStructuralResult = unknown, TEnrichmentResult = unknown> {
connector: KloScanConnector;
input: KloScanInput;
trigger: KloScanTrigger;
context: KloScanContext;
export interface KtxScanOrchestratorRunInput<TStructuralResult = unknown, TEnrichmentResult = unknown> {
connector: KtxScanConnector;
input: KtxScanInput;
trigger: KtxScanTrigger;
context: KtxScanContext;
syncId?: string;
runStructural: (
snapshot: KloSchemaSnapshot,
context: KloScanContext,
) => Promise<KloStructuralScanPhaseResult<TStructuralResult>>;
snapshot: KtxSchemaSnapshot,
context: KtxScanContext,
) => Promise<KtxStructuralScanPhaseResult<TStructuralResult>>;
runEnrichment?: (
snapshot: KloSchemaSnapshot,
structural: KloStructuralScanPhaseResult<TStructuralResult>,
context: KloScanContext,
) => Promise<KloEnrichmentScanPhaseResult<TEnrichmentResult>>;
snapshot: KtxSchemaSnapshot,
structural: KtxStructuralScanPhaseResult<TStructuralResult>,
context: KtxScanContext,
) => Promise<KtxEnrichmentScanPhaseResult<TEnrichmentResult>>;
}
export interface KloScanOrchestratorRunResult<TStructuralResult = unknown, TEnrichmentResult = unknown> {
snapshot: KloSchemaSnapshot;
structural: KloStructuralScanPhaseResult<TStructuralResult>;
enrichment: KloEnrichmentScanPhaseResult<TEnrichmentResult> | null;
report: KloScanReport;
export interface KtxScanOrchestratorRunResult<TStructuralResult = unknown, TEnrichmentResult = unknown> {
snapshot: KtxSchemaSnapshot;
structural: KtxStructuralScanPhaseResult<TStructuralResult>;
enrichment: KtxEnrichmentScanPhaseResult<TEnrichmentResult> | null;
report: KtxScanReport;
}
export interface KloScanOrchestratorOptions {
export interface KtxScanOrchestratorOptions {
now?: () => Date;
syncIdFactory?: (input: KloScanInput, context: KloScanContext) => string;
syncIdFactory?: (input: KtxScanInput, context: KtxScanContext) => string;
}
const emptyDiffSummary: KloScanDiffSummary = {
const emptyDiffSummary: KtxScanDiffSummary = {
tablesAdded: 0,
tablesModified: 0,
tablesDeleted: 0,
@ -83,7 +83,7 @@ const emptyDiffSummary: KloScanDiffSummary = {
columnsDeleted: 0,
};
const emptyStructuralSyncStats: KloStructuralSyncStats = {
const emptyStructuralSyncStats: KtxStructuralSyncStats = {
tablesCreated: 0,
tablesUpdated: 0,
tablesDeleted: 0,
@ -92,31 +92,31 @@ const emptyStructuralSyncStats: KloStructuralSyncStats = {
columnsDeleted: 0,
};
const emptyArtifactPaths: KloScanArtifactPaths = {
const emptyArtifactPaths: KtxScanArtifactPaths = {
rawSourcesDir: null,
reportPath: null,
manifestShards: [],
enrichmentArtifacts: [],
};
function mergeDiffSummary(input?: Partial<KloScanDiffSummary>): KloScanDiffSummary {
function mergeDiffSummary(input?: Partial<KtxScanDiffSummary>): KtxScanDiffSummary {
return { ...emptyDiffSummary, ...input };
}
function mergeStructuralSyncStats(input?: Partial<KloStructuralSyncStats>): KloStructuralSyncStats {
function mergeStructuralSyncStats(input?: Partial<KtxStructuralSyncStats>): KtxStructuralSyncStats {
return { ...emptyStructuralSyncStats, ...input };
}
function mergeEnrichmentSummary(input?: Partial<KloScanEnrichmentSummary>): KloScanEnrichmentSummary {
return { ...skippedKloScanEnrichmentSummary, ...input };
function mergeEnrichmentSummary(input?: Partial<KtxScanEnrichmentSummary>): KtxScanEnrichmentSummary {
return { ...skippedKtxScanEnrichmentSummary, ...input };
}
function mergeEnrichmentState(input?: Partial<KloScanEnrichmentStateSummary>): KloScanEnrichmentStateSummary {
function mergeEnrichmentState(input?: Partial<KtxScanEnrichmentStateSummary>): KtxScanEnrichmentStateSummary {
if (!input) {
return completedKloScanEnrichmentStateSummary();
return completedKtxScanEnrichmentStateSummary();
}
return summarizeKloScanEnrichmentState({
return summarizeKtxScanEnrichmentState({
resumedStages: input.resumedStages ?? [],
completedStages: input.completedStages ?? [],
failedStages: input.failedStages ?? [],
@ -124,9 +124,9 @@ function mergeEnrichmentState(input?: Partial<KloScanEnrichmentStateSummary>): K
}
function mergeArtifactPaths(
structural?: Partial<KloScanArtifactPaths>,
enrichment?: Partial<KloScanArtifactPaths>,
): KloScanArtifactPaths {
structural?: Partial<KtxScanArtifactPaths>,
enrichment?: Partial<KtxScanArtifactPaths>,
): KtxScanArtifactPaths {
return {
...emptyArtifactPaths,
...structural,
@ -137,9 +137,9 @@ function mergeArtifactPaths(
}
function mergeRelationshipSummary(
structural?: Partial<KloScanRelationshipSummary>,
enrichment?: Partial<KloScanRelationshipSummary>,
): KloScanRelationshipSummary {
structural?: Partial<KtxScanRelationshipSummary>,
enrichment?: Partial<KtxScanRelationshipSummary>,
): KtxScanRelationshipSummary {
return {
accepted: (structural?.accepted ?? 0) + (enrichment?.accepted ?? 0),
review: (structural?.review ?? 0) + (enrichment?.review ?? 0),
@ -150,12 +150,12 @@ function mergeRelationshipSummary(
function manifestShardsWritten(phase: {
manifestShardsWritten?: number;
artifactPaths?: Partial<KloScanArtifactPaths>;
artifactPaths?: Partial<KtxScanArtifactPaths>;
}): number {
return phase.manifestShardsWritten ?? phase.artifactPaths?.manifestShards?.length ?? 0;
}
function requiredCapabilities(mode: KloScanInput['mode'], detectRelationships: boolean | undefined): CapabilityGap[] {
function requiredCapabilities(mode: KtxScanInput['mode'], detectRelationships: boolean | undefined): CapabilityGap[] {
const required = new Set<CapabilityGap>();
if (mode === 'enriched') {
@ -173,45 +173,45 @@ function requiredCapabilities(mode: KloScanInput['mode'], detectRelationships: b
return [...required];
}
function capabilityGaps(capabilities: KloConnectorCapabilities, input: KloScanInput): CapabilityGap[] {
function capabilityGaps(capabilities: KtxConnectorCapabilities, input: KtxScanInput): CapabilityGap[] {
return requiredCapabilities(input.mode ?? 'structural', input.detectRelationships).filter(
(capability) => !capabilities[capability],
);
}
function warningsForCapabilityGaps(gaps: CapabilityGap[]): KloScanWarning[] {
function warningsForCapabilityGaps(gaps: CapabilityGap[]): KtxScanWarning[] {
return gaps.map((gap) => ({
code: 'connector_capability_missing',
message: `KLO scan connector is missing optional capability: ${gap}`,
message: `KTX scan connector is missing optional capability: ${gap}`,
recoverable: true,
metadata: { capability: gap },
}));
}
function assertNotAborted(context: KloScanContext): void {
function assertNotAborted(context: KtxScanContext): void {
if (context.signal?.aborted) {
throw new Error('KLO scan aborted');
throw new Error('KTX scan aborted');
}
}
export class KloScanOrchestrator {
export class KtxScanOrchestrator {
private readonly now: () => Date;
private readonly syncIdFactory: (input: KloScanInput, context: KloScanContext) => string;
private readonly syncIdFactory: (input: KtxScanInput, context: KtxScanContext) => string;
constructor(options: KloScanOrchestratorOptions = {}) {
constructor(options: KtxScanOrchestratorOptions = {}) {
this.now = options.now ?? (() => new Date());
this.syncIdFactory = options.syncIdFactory ?? ((_, context) => context.runId);
}
async run<TStructuralResult = unknown, TEnrichmentResult = unknown>(
input: KloScanOrchestratorRunInput<TStructuralResult, TEnrichmentResult>,
): Promise<KloScanOrchestratorRunResult<TStructuralResult, TEnrichmentResult>> {
input: KtxScanOrchestratorRunInput<TStructuralResult, TEnrichmentResult>,
): Promise<KtxScanOrchestratorRunResult<TStructuralResult, TEnrichmentResult>> {
const mode = input.input.mode ?? 'structural';
const syncId = input.syncId ?? this.syncIdFactory(input.input, input.context);
const gaps = capabilityGaps(input.connector.capabilities, input.input);
const warnings = warningsForCapabilityGaps(gaps);
input.context.logger?.info('Starting KLO scan', {
input.context.logger?.info('Starting KTX scan', {
connectionId: input.input.connectionId,
connectorId: input.connector.id,
mode,
@ -224,23 +224,23 @@ export class KloScanOrchestrator {
assertNotAborted(input.context);
const structural = await input.runStructural(snapshot, input.context);
let enrichment: KloEnrichmentScanPhaseResult<TEnrichmentResult> | null = null;
let failedEnrichment: KloScanEnrichmentSummary | null = null;
let enrichment: KtxEnrichmentScanPhaseResult<TEnrichmentResult> | null = null;
let failedEnrichment: KtxScanEnrichmentSummary | null = null;
if (mode !== 'structural' || input.input.detectRelationships) {
if (input.runEnrichment) {
assertNotAborted(input.context);
try {
enrichment = await input.runEnrichment(snapshot, structural, input.context);
} catch (error) {
const message = kloScanErrorMessage(error);
failedEnrichment = failedKloScanEnrichmentSummary(mode, input.input.detectRelationships ?? false);
const message = ktxScanErrorMessage(error);
failedEnrichment = failedKtxScanEnrichmentSummary(mode, input.input.detectRelationships ?? false);
warnings.push({
code: 'enrichment_failed',
message: `KLO scan enrichment failed after structural scan completed: ${message}`,
message: `KTX scan enrichment failed after structural scan completed: ${message}`,
recoverable: true,
metadata: { mode, detectRelationships: input.input.detectRelationships ?? false },
});
input.context.logger?.warn('KLO scan enrichment failed after structural scan completed', {
input.context.logger?.warn('KTX scan enrichment failed after structural scan completed', {
connectionId: input.input.connectionId,
runId: input.context.runId,
mode,
@ -248,10 +248,10 @@ export class KloScanOrchestrator {
});
}
} else {
failedEnrichment = failedKloScanEnrichmentSummary(mode, input.input.detectRelationships ?? false);
failedEnrichment = failedKtxScanEnrichmentSummary(mode, input.input.detectRelationships ?? false);
warnings.push({
code: 'connector_capability_missing',
message: 'KLO scan requested enrichment or relationship detection, but no enrichment phase was provided',
message: 'KTX scan requested enrichment or relationship detection, but no enrichment phase was provided',
recoverable: true,
metadata: { mode, detectRelationships: input.input.detectRelationships ?? false },
});
@ -260,7 +260,7 @@ export class KloScanOrchestrator {
const manifestShardCount = manifestShardsWritten(structural) + (enrichment ? manifestShardsWritten(enrichment) : 0);
const report: KloScanReport = redactKloScanReport({
const report: KtxScanReport = redactKtxScanReport({
connectionId: input.input.connectionId,
driver: input.input.driver,
syncId,
@ -280,7 +280,7 @@ export class KloScanOrchestrator {
createdAt: this.now().toISOString(),
});
input.context.logger?.info('Completed KLO scan', {
input.context.logger?.info('Completed KTX scan', {
connectionId: report.connectionId,
runId: report.runId,
syncId: report.syncId,

36
packages/context/src/scan/relationship-artifacts.test.ts
View file

@ -2,12 +2,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { runLocalStageOnlyIngest, type SourceAdapter } from '../ingest/index.js';
import { initKloProject, loadKloProject } from '../project/index.js';
import { initKtxProject, loadKtxProject } from '../project/index.js';
import { describe, expect, it } from 'vitest';
import { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js';
import type { KloRelationshipArtifact, KloRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KloScanReport } from './types.js';
import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxScanReport } from './types.js';
async function writeProjectFile(projectDir: string, relativePath: string, content: string): Promise<void> {
const absolutePath = join(projectDir, relativePath);
@ -17,7 +17,7 @@ async function writeProjectFile(projectDir: string, relativePath: string, conten
async function writeWarehouseConfig(projectDir: string): Promise<void> {
await writeFile(
join(projectDir, 'klo.yaml'),
join(projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -68,9 +68,9 @@ function liveDatabaseAdapter(): SourceAdapter {
}
async function createLiveDatabaseRun(projectDir: string, runId: string) {
await initKloProject({ projectDir, projectName: 'warehouse' });
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeWarehouseConfig(projectDir);
const project = await loadKloProject({ projectDir });
const project = await loadKtxProject({ projectDir });
await runLocalStageOnlyIngest({
project,
adapters: [liveDatabaseAdapter()],
@ -82,7 +82,7 @@ async function createLiveDatabaseRun(projectDir: string, runId: string) {
return project;
}
function scanReport(enrichmentArtifacts: string[], syncId = '2026-05-07-100000-scan-run-review'): KloScanReport {
function scanReport(enrichmentArtifacts: string[], syncId = '2026-05-07-100000-scan-run-review'): KtxScanReport {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -136,7 +136,7 @@ function scanReport(enrichmentArtifacts: string[], syncId = '2026-05-07-100000-s
};
}
const relationshipArtifact: KloRelationshipArtifact = {
const relationshipArtifact: KtxRelationshipArtifact = {
connectionId: 'warehouse',
accepted: [],
review: [
@ -198,7 +198,7 @@ const relationshipArtifact: KloRelationshipArtifact = {
skipped: [],
};
const diagnosticsArtifact: KloRelationshipDiagnosticsArtifact = {
const diagnosticsArtifact: KtxRelationshipDiagnosticsArtifact = {
connectionId: 'warehouse',
generatedAt: '2026-05-07T10:00:00.000Z',
summary: { accepted: 0, review: 1, rejected: 1, skipped: 0 },
@ -213,22 +213,22 @@ const diagnosticsArtifact: KloRelationshipDiagnosticsArtifact = {
validationConcurrency: 4,
},
warnings: [],
profileWarnings: ['KLO scan connector cannot run read-only SQL relationship validation'],
profileWarnings: ['KTX scan connector cannot run read-only SQL relationship validation'],
};
const profileArtifact: KloRelationshipProfileArtifact = {
const profileArtifact: KtxRelationshipProfileArtifact = {
connectionId: 'warehouse',
driver: 'sqlite',
sqlAvailable: false,
tables: [],
columns: {},
queryCount: 0,
warnings: ['KLO scan connector cannot run read-only SQL relationship validation'],
warnings: ['KTX scan connector cannot run read-only SQL relationship validation'],
};
describe('local scan relationship artifact reader', () => {
it('loads relationship, diagnostics, and profile artifacts for a scan run', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-artifacts-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-'));
try {
const project = await createLiveDatabaseRun(projectDir, 'scan-run-review');
const syncId = '2026-05-07-100000-scan-run-review';
@ -282,10 +282,10 @@ describe('local scan relationship artifact reader', () => {
});
it('returns null when the scan run has no report', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-artifacts-missing-run-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-run-'));
try {
await initKloProject({ projectDir, projectName: 'warehouse' });
const project = await loadKloProject({ projectDir });
await initKtxProject({ projectDir, projectName: 'warehouse' });
const project = await loadKtxProject({ projectDir });
await expect(readLocalScanRelationshipArtifacts(project, 'missing-run')).resolves.toBeNull();
} finally {
@ -294,7 +294,7 @@ describe('local scan relationship artifact reader', () => {
});
it('throws a focused error when a scan report does not reference relationships.json', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-artifacts-missing-artifact-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-artifact-'));
try {
const project = await createLiveDatabaseRun(projectDir, 'scan-run-review');
const report = scanReport([]);

32
packages/context/src/scan/relationship-artifacts.ts
View file

@ -1,19 +1,19 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import { getLocalScanReport } from './local-scan.js';
import type { KloRelationshipArtifact, KloRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KloScanReport } from './types.js';
import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxScanReport } from './types.js';
export type KloRelationshipArtifactStatus = 'accepted' | 'review' | 'rejected' | 'skipped' | 'all';
export type KtxRelationshipArtifactStatus = 'accepted' | 'review' | 'rejected' | 'skipped' | 'all';
export interface ReadLocalScanRelationshipArtifactsResult {
runId: string;
connectionId: string;
syncId: string;
report: KloScanReport;
relationships: KloRelationshipArtifact;
diagnostics: KloRelationshipDiagnosticsArtifact | null;
profile: KloRelationshipProfileArtifact | null;
report: KtxScanReport;
relationships: KtxRelationshipArtifact;
diagnostics: KtxRelationshipDiagnosticsArtifact | null;
profile: KtxRelationshipProfileArtifact | null;
paths: {
relationships: string;
diagnostics: string | null;
@ -21,16 +21,16 @@ export interface ReadLocalScanRelationshipArtifactsResult {
};
}
function findArtifactPath(report: KloScanReport, fileName: string): string | null {
function findArtifactPath(report: KtxScanReport, fileName: string): string | null {
return report.artifactPaths.enrichmentArtifacts.find((path) => path.endsWith(`/enrichment/${fileName}`)) ?? null;
}
async function readJsonArtifact<T>(project: KloLocalProject, path: string): Promise<T> {
async function readJsonArtifact<T>(project: KtxLocalProject, path: string): Promise<T> {
const raw = await project.fileStore.readFile(path);
return JSON.parse(raw.content) as T;
}
async function readOptionalJsonArtifact<T>(project: KloLocalProject, path: string | null): Promise<T | null> {
async function readOptionalJsonArtifact<T>(project: KtxLocalProject, path: string | null): Promise<T | null> {
if (!path) {
return null;
}
@ -42,7 +42,7 @@ async function readOptionalJsonArtifact<T>(project: KloLocalProject, path: strin
}
export async function readLocalScanRelationshipArtifacts(
project: KloLocalProject,
project: KtxLocalProject,
runId: string,
): Promise<ReadLocalScanRelationshipArtifactsResult | null> {
const report = await getLocalScanReport(project, runId);
@ -63,9 +63,9 @@ export async function readLocalScanRelationshipArtifacts(
connectionId: report.connectionId,
syncId: report.syncId,
report,
relationships: await readJsonArtifact<KloRelationshipArtifact>(project, relationshipsPath),
diagnostics: await readOptionalJsonArtifact<KloRelationshipDiagnosticsArtifact>(project, diagnosticsPath),
profile: await readOptionalJsonArtifact<KloRelationshipProfileArtifact>(project, profilePath),
relationships: await readJsonArtifact<KtxRelationshipArtifact>(project, relationshipsPath),
diagnostics: await readOptionalJsonArtifact<KtxRelationshipDiagnosticsArtifact>(project, diagnosticsPath),
profile: await readOptionalJsonArtifact<KtxRelationshipProfileArtifact>(project, profilePath),
paths: {
relationships: relationshipsPath,
diagnostics: diagnosticsPath,

60
packages/context/src/scan/relationship-benchmark-report.test.ts
View file

@ -1,19 +1,19 @@
import { describe, expect, it } from 'vitest';
import {
buildKloRelationshipBenchmarkReport,
formatKloRelationshipBenchmarkReportMarkdown,
buildKtxRelationshipBenchmarkReport,
formatKtxRelationshipBenchmarkReportMarkdown,
} from './relationship-benchmark-report.js';
import type {
KloRelationshipBenchmarkCaseResult,
KloRelationshipBenchmarkFixture,
KloRelationshipBenchmarkSuiteResult,
KtxRelationshipBenchmarkCaseResult,
KtxRelationshipBenchmarkFixture,
KtxRelationshipBenchmarkSuiteResult,
} from './relationship-benchmarks.js';
type CaseResultOverrides = Omit<Partial<KloRelationshipBenchmarkCaseResult>, 'metrics'> & {
metrics?: Partial<KloRelationshipBenchmarkCaseResult['metrics']>;
type CaseResultOverrides = Omit<Partial<KtxRelationshipBenchmarkCaseResult>, 'metrics'> & {
metrics?: Partial<KtxRelationshipBenchmarkCaseResult['metrics']>;
};
function caseResult(overrides: CaseResultOverrides = {}): KloRelationshipBenchmarkCaseResult {
function caseResult(overrides: CaseResultOverrides = {}): KtxRelationshipBenchmarkCaseResult {
return {
fixtureId: overrides.fixtureId ?? 'demo_b2b_no_declared_constraints',
mode: overrides.mode ?? 'declared_pks_and_declared_fks_removed',
@ -49,7 +49,7 @@ function caseResult(overrides: CaseResultOverrides = {}): KloRelationshipBenchma
};
}
function fixture(overrides: Partial<KloRelationshipBenchmarkFixture> = {}): KloRelationshipBenchmarkFixture {
function fixture(overrides: Partial<KtxRelationshipBenchmarkFixture> = {}): KtxRelationshipBenchmarkFixture {
return {
id: overrides.id ?? 'demo_b2b_no_declared_constraints',
name: overrides.name ?? 'Packaged B2B demo with declared PK and FK metadata masked',
@ -74,7 +74,7 @@ function fixture(overrides: Partial<KloRelationshipBenchmarkFixture> = {}): KloR
describe('relationship benchmark report', () => {
it('classifies run, validation-blocked, and not-run benchmark cases', () => {
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [
caseResult(),
caseResult({
@ -102,7 +102,7 @@ describe('relationship benchmark report', () => {
},
};
const report = buildKloRelationshipBenchmarkReport({
const report = buildKtxRelationshipBenchmarkReport({
fixtures: [fixture()],
suite,
modes: ['declared_pks_and_declared_fks_removed', 'validation_disabled', 'profiling_disabled'],
@ -126,7 +126,7 @@ describe('relationship benchmark report', () => {
});
it('surfaces validation budget review candidates in the report reason', () => {
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [
caseResult({
fixtureId: 'scale_stress_no_declared_constraints',
@ -155,7 +155,7 @@ describe('relationship benchmark report', () => {
},
};
const report = buildKloRelationshipBenchmarkReport({
const report = buildKtxRelationshipBenchmarkReport({
fixtures: [
fixture({
id: 'scale_stress_no_declared_constraints',
@ -170,7 +170,7 @@ describe('relationship benchmark report', () => {
});
expect(report.cases[0]?.reason).toBe('review candidate validation reasons: validation_unattempted (1)');
expect(formatKloRelationshipBenchmarkReportMarkdown(report)).toContain('validation_unattempted');
expect(formatKtxRelationshipBenchmarkReportMarkdown(report)).toContain('validation_unattempted');
});
it('uses benchmark suite eligibility for product and smoke report rows', () => {
@ -182,7 +182,7 @@ describe('relationship benchmark report', () => {
metrics: { fkRecall: 0, acceptedOrReviewRecall: 1, sqlQueries: 0 },
});
const smokeCase = caseResult({ fixtureId: 'smoke_even_if_marked' });
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [productCase, productBlocked, smokeCase],
validationBlockedCases: ['product_curated:validation_disabled'],
aggregate: {
@ -197,7 +197,7 @@ describe('relationship benchmark report', () => {
},
};
const report = buildKloRelationshipBenchmarkReport({
const report = buildKtxRelationshipBenchmarkReport({
fixtures: [
fixture({
id: 'product_curated',
@ -224,13 +224,13 @@ describe('relationship benchmark report', () => {
'smoke_even_if_marked:declared_pks_and_declared_fks_removed:false',
'smoke_even_if_marked:validation_disabled:false',
]);
expect(formatKloRelationshipBenchmarkReportMarkdown(report)).toContain(
expect(formatKtxRelationshipBenchmarkReportMarkdown(report)).toContain(
'| product_curated | product | declared_pks_and_declared_fks_removed | run | yes |',
);
});
it('formats a compact Markdown report with false negatives and blocked modes', () => {
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [
caseResult({
metrics: { fkRecall: 0, acceptedOrReviewRecall: 0 },
@ -250,15 +250,15 @@ describe('relationship benchmark report', () => {
},
};
const markdown = formatKloRelationshipBenchmarkReportMarkdown(
buildKloRelationshipBenchmarkReport({
const markdown = formatKtxRelationshipBenchmarkReportMarkdown(
buildKtxRelationshipBenchmarkReport({
fixtures: [fixture()],
suite,
modes: ['declared_pks_and_declared_fks_removed'],
}),
);
expect(markdown).toContain('# KLO Relationship Discovery Benchmark Evidence');
expect(markdown).toContain('# KTX Relationship Discovery Benchmark Evidence');
expect(markdown).toContain(
'| demo_b2b_no_declared_constraints | smoke | declared_pks_and_declared_fks_removed | run | no | 0.500 | 0.000 | 0.000 | 0 |',
);
@ -271,7 +271,7 @@ describe('relationship benchmark report', () => {
});
it('keeps headline failures separate from non-headline failure details', () => {
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [
caseResult({
fixtureId: 'product_curated',
@ -301,8 +301,8 @@ describe('relationship benchmark report', () => {
},
};
const markdown = formatKloRelationshipBenchmarkReportMarkdown(
buildKloRelationshipBenchmarkReport({
const markdown = formatKtxRelationshipBenchmarkReportMarkdown(
buildKtxRelationshipBenchmarkReport({
fixtures: [
fixture({
id: 'product_curated',
@ -326,7 +326,7 @@ describe('relationship benchmark report', () => {
});
it('formats headline failure context from remaining headline false negatives', () => {
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [
caseResult({
fixtureId: 'public_headline_fixture',
@ -350,8 +350,8 @@ describe('relationship benchmark report', () => {
},
};
const markdown = formatKloRelationshipBenchmarkReportMarkdown(
buildKloRelationshipBenchmarkReport({
const markdown = formatKtxRelationshipBenchmarkReportMarkdown(
buildKtxRelationshipBenchmarkReport({
fixtures: [
fixture({
id: 'public_headline_fixture',
@ -380,7 +380,7 @@ describe('relationship benchmark report', () => {
it('formats skipped composite ground truth separately from false-negative details', () => {
const compositePk = 'order_lines.(order_id,line_number)';
const compositeFk = 'order_line_allocations.(order_id,line_number)->order_lines.(order_id,line_number)';
const suite: KloRelationshipBenchmarkSuiteResult = {
const suite: KtxRelationshipBenchmarkSuiteResult = {
cases: [
caseResult({
fixtureId: 'composite_keys_no_declared_constraints',
@ -418,7 +418,7 @@ describe('relationship benchmark report', () => {
},
};
const report = buildKloRelationshipBenchmarkReport({
const report = buildKtxRelationshipBenchmarkReport({
fixtures: [
fixture({
id: 'composite_keys_no_declared_constraints',
@ -436,7 +436,7 @@ describe('relationship benchmark report', () => {
fk: [compositeFk],
});
const markdown = formatKloRelationshipBenchmarkReportMarkdown(report);
const markdown = formatKtxRelationshipBenchmarkReportMarkdown(report);
expect(markdown).toContain('## Composite Ground Truth Skips');
expect(markdown).toContain(
'### Skipped Composite PKs\n\n- `composite_keys_no_declared_constraints` / `declared_pks_and_declared_fks_removed` / `run`: order_lines.(order_id,line_number)',

78
packages/context/src/scan/relationship-benchmark-report.ts
View file

@ -1,19 +1,19 @@
import { isKloRelationshipBenchmarkTuningEligible } from './relationship-benchmarks.js';
import { isKtxRelationshipBenchmarkTuningEligible } from './relationship-benchmarks.js';
import type {
KloRelationshipBenchmarkCaseResult,
KloRelationshipBenchmarkFixture,
KloRelationshipBenchmarkMode,
KloRelationshipBenchmarkSuiteResult,
KtxRelationshipBenchmarkCaseResult,
KtxRelationshipBenchmarkFixture,
KtxRelationshipBenchmarkMode,
KtxRelationshipBenchmarkSuiteResult,
} from './relationship-benchmarks.js';
export type KloRelationshipBenchmarkReportCaseStatus = 'run' | 'validation_blocked' | 'not_run';
export type KtxRelationshipBenchmarkReportCaseStatus = 'run' | 'validation_blocked' | 'not_run';
export interface KloRelationshipBenchmarkReportCase {
export interface KtxRelationshipBenchmarkReportCase {
fixtureId: string;
fixtureName: string;
tier: string;
mode: KloRelationshipBenchmarkMode;
status: KloRelationshipBenchmarkReportCaseStatus;
mode: KtxRelationshipBenchmarkMode;
status: KtxRelationshipBenchmarkReportCaseStatus;
reason: string | null;
tuningEligible: boolean;
metrics: {
@ -39,7 +39,7 @@ export interface KloRelationshipBenchmarkReportCase {
};
}
export interface KloRelationshipBenchmarkReport {
export interface KtxRelationshipBenchmarkReport {
generatedAt: string;
headline: {
caseCount: number;
@ -50,10 +50,10 @@ export interface KloRelationshipBenchmarkReport {
acceptedFalsePositiveCount: number;
validationBlockedCount: number;
};
cases: KloRelationshipBenchmarkReportCase[];
cases: KtxRelationshipBenchmarkReportCase[];
}
function key(fixtureId: string, mode: KloRelationshipBenchmarkMode): string {
function key(fixtureId: string, mode: KtxRelationshipBenchmarkMode): string {
return `${fixtureId}:${mode}`;
}
@ -62,8 +62,8 @@ function fixed(value: number | null): string {
}
function reportCaseReason(input: {
fixture: KloRelationshipBenchmarkFixture;
result: KloRelationshipBenchmarkCaseResult;
fixture: KtxRelationshipBenchmarkFixture;
result: KtxRelationshipBenchmarkCaseResult;
}): string | null {
if (input.result.validationBlocked) {
return 'validation unavailable for this benchmark mode';
@ -77,10 +77,10 @@ function reportCaseReason(input: {
}
function reportCaseFromResult(input: {
fixture: KloRelationshipBenchmarkFixture;
mode: KloRelationshipBenchmarkMode;
result: KloRelationshipBenchmarkCaseResult;
}): KloRelationshipBenchmarkReportCase {
fixture: KtxRelationshipBenchmarkFixture;
mode: KtxRelationshipBenchmarkMode;
result: KtxRelationshipBenchmarkCaseResult;
}): KtxRelationshipBenchmarkReportCase {
const status = input.result.validationBlocked ? 'validation_blocked' : 'run';
return {
fixtureId: input.fixture.id,
@ -89,7 +89,7 @@ function reportCaseFromResult(input: {
mode: input.mode,
status,
reason: reportCaseReason({ fixture: input.fixture, result: input.result }),
tuningEligible: isKloRelationshipBenchmarkTuningEligible({
tuningEligible: isKtxRelationshipBenchmarkTuningEligible({
fixture: input.fixture,
mode: input.mode,
validationBlocked: input.result.validationBlocked,
@ -110,10 +110,10 @@ function reportCaseFromResult(input: {
}
function notRunCase(input: {
fixture: KloRelationshipBenchmarkFixture;
mode: KloRelationshipBenchmarkMode;
fixture: KtxRelationshipBenchmarkFixture;
mode: KtxRelationshipBenchmarkMode;
reason: string;
}): KloRelationshipBenchmarkReportCase {
}): KtxRelationshipBenchmarkReportCase {
return {
fixtureId: input.fixture.id,
fixtureName: input.fixture.name,
@ -137,14 +137,14 @@ function notRunCase(input: {
};
}
export function buildKloRelationshipBenchmarkReport(input: {
fixtures: readonly KloRelationshipBenchmarkFixture[];
suite: KloRelationshipBenchmarkSuiteResult;
modes: readonly KloRelationshipBenchmarkMode[];
export function buildKtxRelationshipBenchmarkReport(input: {
fixtures: readonly KtxRelationshipBenchmarkFixture[];
suite: KtxRelationshipBenchmarkSuiteResult;
modes: readonly KtxRelationshipBenchmarkMode[];
generatedAt?: string;
}): KloRelationshipBenchmarkReport {
}): KtxRelationshipBenchmarkReport {
const resultsByKey = new Map(input.suite.cases.map((result) => [key(result.fixtureId, result.mode), result]));
const cases: KloRelationshipBenchmarkReportCase[] = [];
const cases: KtxRelationshipBenchmarkReportCase[] = [];
for (const fixture of input.fixtures) {
const selectedModes = new Set(fixture.defaultModes);
@ -182,13 +182,13 @@ export function buildKloRelationshipBenchmarkReport(input: {
};
}
type KloRelationshipBenchmarkFailureSelector = (
item: KloRelationshipBenchmarkReportCase,
type KtxRelationshipBenchmarkFailureSelector = (
item: KtxRelationshipBenchmarkReportCase,
) => readonly string[];
function sortedFailureLines(input: {
cases: readonly KloRelationshipBenchmarkReportCase[];
select: KloRelationshipBenchmarkFailureSelector;
cases: readonly KtxRelationshipBenchmarkReportCase[];
select: KtxRelationshipBenchmarkFailureSelector;
}): string[] {
return input.cases
.flatMap((item) =>
@ -209,14 +209,14 @@ function sortedFailureLines(input: {
function failureBlock(input: {
title: string;
cases: readonly KloRelationshipBenchmarkReportCase[];
select: KloRelationshipBenchmarkFailureSelector;
cases: readonly KtxRelationshipBenchmarkReportCase[];
select: KtxRelationshipBenchmarkFailureSelector;
}): string[] {
const values = sortedFailureLines({ cases: input.cases, select: input.select });
return ['', `### ${input.title}`, '', ...(values.length > 0 ? values : ['- none'])];
}
function headlineFailureContextBlocks(report: KloRelationshipBenchmarkReport): string[] {
function headlineFailureContextBlocks(report: KtxRelationshipBenchmarkReport): string[] {
const headlineCases = report.cases.filter((item) => item.tuningEligible);
const remainingPkMisses = sortedFailureLines({
cases: headlineCases,
@ -246,7 +246,7 @@ function headlineFailureContextBlocks(report: KloRelationshipBenchmarkReport): s
];
}
function failureDetailBlocks(report: KloRelationshipBenchmarkReport): string[] {
function failureDetailBlocks(report: KtxRelationshipBenchmarkReport): string[] {
const headlineCases = report.cases.filter((item) => item.tuningEligible);
const otherCases = report.cases.filter((item) => !item.tuningEligible);
@ -296,7 +296,7 @@ function failureDetailBlocks(report: KloRelationshipBenchmarkReport): string[] {
];
}
function compositeSkipBlocks(report: KloRelationshipBenchmarkReport): string[] {
function compositeSkipBlocks(report: KtxRelationshipBenchmarkReport): string[] {
const headlineCases = report.cases.filter((item) => item.tuningEligible);
return [
@ -315,9 +315,9 @@ function compositeSkipBlocks(report: KloRelationshipBenchmarkReport): string[] {
];
}
export function formatKloRelationshipBenchmarkReportMarkdown(report: KloRelationshipBenchmarkReport): string {
export function formatKtxRelationshipBenchmarkReportMarkdown(report: KtxRelationshipBenchmarkReport): string {
const lines = [
'# KLO Relationship Discovery Benchmark Evidence',
'# KTX Relationship Discovery Benchmark Evidence',
'',
`Generated: ${report.generatedAt}`,
'',

136
packages/context/src/scan/relationship-benchmarks.test.ts
View file

@ -3,20 +3,20 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import type {
KloRelationshipBenchmarkExpectedLinks,
KloRelationshipBenchmarkFixture,
KtxRelationshipBenchmarkExpectedLinks,
KtxRelationshipBenchmarkFixture,
} from './relationship-benchmarks.js';
import {
currentKloRelationshipBenchmarkDetector,
loadKloRelationshipBenchmarkFixture,
loadKloRelationshipBenchmarkFixtures,
maskKloRelationshipBenchmarkSnapshot,
runKloRelationshipBenchmarkCase,
runKloRelationshipBenchmarkSuite,
currentKtxRelationshipBenchmarkDetector,
loadKtxRelationshipBenchmarkFixture,
loadKtxRelationshipBenchmarkFixtures,
maskKtxRelationshipBenchmarkSnapshot,
runKtxRelationshipBenchmarkCase,
runKtxRelationshipBenchmarkSuite,
} from './relationship-benchmarks.js';
import type { KloSchemaSnapshot } from './types.js';
import type { KtxSchemaSnapshot } from './types.js';
const EXPECTED_LINKS: KloRelationshipBenchmarkExpectedLinks = {
const EXPECTED_LINKS: KtxRelationshipBenchmarkExpectedLinks = {
expectedPks: [
{ table: 'accounts', columns: ['id'] },
{ table: 'users', columns: ['id'] },
@ -53,7 +53,7 @@ const CHECKED_IN_FIXTURE_ORIGINS = {
semantic_embedding_aliases_no_declared_constraints: 'synthetic',
} as const;
function snapshot(): KloSchemaSnapshot {
function snapshot(): KtxSchemaSnapshot {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -136,16 +136,16 @@ describe('relationship benchmarks', () => {
it('keeps the current benchmark detector on the relationship-discovery path only', async () => {
const source = await readFile(new URL('relationship-benchmarks.ts', import.meta.url), 'utf-8');
expect(source).not.toMatch(/KloRelationshipDetector/);
expect(source).not.toMatch(/KtxRelationshipDetector/);
expect(source).not.toMatch(/relationship-detection\.js/);
expect(source).not.toMatch(/\bacceptedLinks\b/);
expect(source).toMatch(/generateKloRelationshipDiscoveryCandidates/);
expect(source).toMatch(/validateKloRelationshipDiscoveryCandidates/);
expect(source).toMatch(/resolveKloRelationshipGraph/);
expect(source).toMatch(/generateKtxRelationshipDiscoveryCandidates/);
expect(source).toMatch(/validateKtxRelationshipDiscoveryCandidates/);
expect(source).toMatch(/resolveKtxRelationshipGraph/);
});
it('scores the current detector with declared metadata present', async () => {
const result = await runKloRelationshipBenchmarkCase({
const result = await runKtxRelationshipBenchmarkCase({
fixture: {
id: 'mini_declared',
name: 'Mini declared fixture',
@ -158,7 +158,7 @@ describe('relationship benchmarks', () => {
columnEmbeddings: {},
},
mode: 'metadata_present',
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
expect(result.metrics.pkRecall).toBe(1);
@ -170,7 +170,7 @@ describe('relationship benchmarks', () => {
});
it('keeps no-declared-constraint misses in benchmark metrics', async () => {
const result = await runKloRelationshipBenchmarkCase({
const result = await runKtxRelationshipBenchmarkCase({
fixture: {
id: 'mini_no_declared',
name: 'Mini no declared fixture',
@ -183,7 +183,7 @@ describe('relationship benchmarks', () => {
columnEmbeddings: {},
},
mode: 'declared_pks_and_declared_fks_removed',
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
expect(result.metrics.pkRecall).toBe(0.5);
@ -197,7 +197,7 @@ describe('relationship benchmarks', () => {
});
it('keeps composite ground truth in recall denominators and skipped-composite buckets', async () => {
const compositeExpected: KloRelationshipBenchmarkExpectedLinks = {
const compositeExpected: KtxRelationshipBenchmarkExpectedLinks = {
expectedPks: [{ table: 'order_lines', columns: ['order_id', 'line_number'] }],
expectedLinks: [
{
@ -222,7 +222,7 @@ describe('relationship benchmarks', () => {
},
};
const result = await runKloRelationshipBenchmarkCase({
const result = await runKtxRelationshipBenchmarkCase({
fixture: {
id: 'composite_no_declared',
name: 'Composite relationship fixture without declared constraints',
@ -256,7 +256,7 @@ describe('relationship benchmarks', () => {
it('loads the composite-key fixture and accepts composite ground truth as headline evidence', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'composite_keys_no_declared_constraints'),
);
@ -270,9 +270,9 @@ describe('relationship benchmarks', () => {
]);
expect(fixture.dataPath).toMatch(/composite_keys_no_declared_constraints\/data\.sqlite$/);
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [fixture],
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
const headline = suite.cases.find(
(item) =>
@ -319,7 +319,7 @@ describe('relationship benchmarks', () => {
it('counts formal metadata links in metadata-present mode without SQL validation', async () => {
const source = snapshot();
const fixture: KloRelationshipBenchmarkFixture = {
const fixture: KtxRelationshipBenchmarkFixture = {
id: 'declared_without_sql',
name: 'Declared relationships without SQL validation',
tier: 'unit',
@ -357,7 +357,7 @@ describe('relationship benchmarks', () => {
columnEmbeddings: {},
};
const result = await runKloRelationshipBenchmarkCase({
const result = await runKtxRelationshipBenchmarkCase({
fixture,
mode: 'metadata_present',
});
@ -369,8 +369,8 @@ describe('relationship benchmarks', () => {
});
it('masks primary keys and foreign keys independently', () => {
const pksRemoved = maskKloRelationshipBenchmarkSnapshot(snapshot(), 'declared_pks_removed');
const fksRemoved = maskKloRelationshipBenchmarkSnapshot(snapshot(), 'declared_fks_removed');
const pksRemoved = maskKtxRelationshipBenchmarkSnapshot(snapshot(), 'declared_pks_removed');
const fksRemoved = maskKtxRelationshipBenchmarkSnapshot(snapshot(), 'declared_fks_removed');
expect(pksRemoved.tables.flatMap((table) => table.columns.filter((column) => column.primaryKey))).toEqual([]);
expect(pksRemoved.tables.find((table) => table.name === 'users')?.foreignKeys).toHaveLength(1);
@ -379,7 +379,7 @@ describe('relationship benchmarks', () => {
});
it('loads fixture.yaml, snapshot.json, and expected-links.yaml from a fixture directory', async () => {
const fixtureDir = await mkdtemp(join(tmpdir(), 'klo-relationship-fixture-'));
const fixtureDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-fixture-'));
try {
await writeFile(
join(fixtureDir, 'fixture.yaml'),
@ -425,7 +425,7 @@ describe('relationship benchmarks', () => {
].join('\n'),
);
await expect(loadKloRelationshipBenchmarkFixture(fixtureDir)).resolves.toMatchObject({
await expect(loadKtxRelationshipBenchmarkFixture(fixtureDir)).resolves.toMatchObject({
id: 'mini_loaded',
origin: 'synthetic',
validationBudget: 3,
@ -470,7 +470,7 @@ describe('relationship benchmarks', () => {
},
};
await runKloRelationshipBenchmarkSuite({
await runKtxRelationshipBenchmarkSuite({
fixtures: [
{
id: 'budgeted_fixture',
@ -503,7 +503,7 @@ describe('relationship benchmarks', () => {
});
it('requires relationship benchmark fixture origin provenance', async () => {
const fixtureDir = await mkdtemp(join(tmpdir(), 'klo-relationship-missing-origin-'));
const fixtureDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-missing-origin-'));
try {
await writeFile(
join(fixtureDir, 'fixture.yaml'),
@ -522,14 +522,14 @@ describe('relationship benchmarks', () => {
['expectedPks:', ' - table: accounts', ' columns: [id]', 'expectedLinks: []', ''].join('\n'),
);
await expect(loadKloRelationshipBenchmarkFixture(fixtureDir)).rejects.toThrow(/origin/);
await expect(loadKtxRelationshipBenchmarkFixture(fixtureDir)).rejects.toThrow(/origin/);
} finally {
await rm(fixtureDir, { recursive: true, force: true });
}
});
it('loads all benchmark fixture directories in stable order', async () => {
const fixtureRoot = await mkdtemp(join(tmpdir(), 'klo-relationship-fixture-root-'));
const fixtureRoot = await mkdtemp(join(tmpdir(), 'ktx-relationship-fixture-root-'));
async function writeFixtureDir(dirName: string, fixtureId: string): Promise<void> {
const fixtureDir = join(fixtureRoot, dirName);
@ -570,7 +570,7 @@ describe('relationship benchmarks', () => {
await writeFixtureDir('z_fixture', 'z_fixture');
await writeFixtureDir('a_fixture', 'a_fixture');
await expect(loadKloRelationshipBenchmarkFixtures(fixtureRoot)).resolves.toMatchObject([
await expect(loadKtxRelationshipBenchmarkFixtures(fixtureRoot)).resolves.toMatchObject([
{ id: 'a_fixture', origin: 'synthetic' },
{ id: 'z_fixture', origin: 'synthetic' },
]);
@ -588,7 +588,7 @@ describe('relationship benchmarks', () => {
expect(fixtureDirs).toEqual(Object.keys(CHECKED_IN_FIXTURE_ORIGINS).sort());
const fixtures = await loadKloRelationshipBenchmarkFixtures(fixtureRoot.pathname);
const fixtures = await loadKtxRelationshipBenchmarkFixtures(fixtureRoot.pathname);
expect(Object.fromEntries(fixtures.map((fixture) => [fixture.id, fixture.origin]))).toEqual(
CHECKED_IN_FIXTURE_ORIGINS,
);
@ -596,7 +596,7 @@ describe('relationship benchmarks', () => {
it('loads May 8 evidence-fusion adversarial fixtures as reported synthetic evidence', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixtures = await loadKloRelationshipBenchmarkFixtures(fixtureRoot.pathname);
const fixtures = await loadKtxRelationshipBenchmarkFixtures(fixtureRoot.pathname);
const byId = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
const adversarialIds = [
'non_english_naming_no_declared_constraints',
@ -629,7 +629,7 @@ describe('relationship benchmarks', () => {
it('loads the May 8 scale stress fixture with bounded benchmark validation', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'),
);
@ -646,14 +646,14 @@ describe('relationship benchmarks', () => {
it('runs the scale stress fixture inside the benchmark validation budget', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'),
);
const result = await runKloRelationshipBenchmarkCase({
const result = await runKtxRelationshipBenchmarkCase({
fixture,
mode: 'declared_pks_and_declared_fks_removed',
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
expect(result.metrics.runtimeSeconds).toBeLessThan(60);
@ -662,7 +662,7 @@ describe('relationship benchmarks', () => {
}, 60_000);
it('aggregates suite metrics without hiding validation-blocked cases', async () => {
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [
{
id: 'mini_declared',
@ -687,7 +687,7 @@ describe('relationship benchmarks', () => {
columnEmbeddings: {},
},
],
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
expect(suite.cases.map((item) => `${item.fixtureId}:${item.mode}`)).toEqual([
@ -730,7 +730,7 @@ describe('relationship benchmarks', () => {
},
};
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [
{
id: 'smoke_no_declared',
@ -792,7 +792,7 @@ describe('relationship benchmarks', () => {
},
};
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [
{
id: 'product_not_curated',
@ -841,10 +841,10 @@ describe('relationship benchmarks', () => {
it('loads the packaged B2B demo fixtures and records the current relationship-discovery baseline', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const declared = await loadKloRelationshipBenchmarkFixture(
const declared = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'demo_b2b_declared_metadata'),
);
const noDeclared = await loadKloRelationshipBenchmarkFixture(
const noDeclared = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'demo_b2b_no_declared_constraints'),
);
@ -868,9 +868,9 @@ describe('relationship benchmarks', () => {
'embeddings_disabled',
]);
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [declared, noDeclared],
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
const declaredCase = suite.cases.find(
@ -922,7 +922,7 @@ describe('relationship benchmarks', () => {
it('loads the public Chinook benchmark fixture with declared metadata', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'chinook_with_declared_metadata'),
);
expect(fixture.tier).toBe('row_bearing');
@ -940,7 +940,7 @@ describe('relationship benchmarks', () => {
it('loads the public Northwind benchmark fixture with declared metadata', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'northwind_with_declared_metadata'),
);
expect(fixture.tier).toBe('row_bearing');
@ -956,7 +956,7 @@ describe('relationship benchmarks', () => {
it('loads the public Sakila benchmark fixture with declared metadata', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'sakila_with_declared_metadata'),
);
expect(fixture.tier).toBe('row_bearing');
@ -972,7 +972,7 @@ describe('relationship benchmarks', () => {
it('loads the public AdventureWorksLT benchmark fixture with declared metadata', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'adventureworkslt_with_declared_metadata'),
);
@ -1032,7 +1032,7 @@ describe('relationship benchmarks', () => {
it('loads the full AdventureWorks OLTP benchmark fixture with declared metadata', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'adventureworks_oltp_with_declared_metadata'),
);
@ -1092,7 +1092,7 @@ describe('relationship benchmarks', () => {
it('loads the row-bearing natural-key fixture and counts it as headline evidence', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const naturalKeys = await loadKloRelationshipBenchmarkFixture(
const naturalKeys = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'natural_keys_no_declared_constraints'),
);
@ -1105,9 +1105,9 @@ describe('relationship benchmarks', () => {
'embeddings_disabled',
]);
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [naturalKeys],
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
const headline = suite.cases.find(
(item) =>
@ -1126,7 +1126,7 @@ describe('relationship benchmarks', () => {
it('accepts plan-code suffix relationships only when validation is available', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'plan_code_no_declared_constraints'),
);
@ -1139,9 +1139,9 @@ describe('relationship benchmarks', () => {
'embeddings_disabled',
]);
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [fixture],
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
const expectedAccepted = [
'mart_account_segments.(current_plan_code)->stg_plans.(plan_code)',
@ -1187,7 +1187,7 @@ describe('relationship benchmarks', () => {
it('uses embedding fixtures for semantic alias relationship benchmark cases', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'semantic_embedding_aliases_no_declared_constraints'),
);
@ -1196,15 +1196,15 @@ describe('relationship benchmarks', () => {
'orders.buyer_ref': [0.995, 0.005, 0],
});
const withEmbeddings = await runKloRelationshipBenchmarkCase({
const withEmbeddings = await runKtxRelationshipBenchmarkCase({
fixture,
mode: 'declared_pks_and_declared_fks_removed',
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
const withoutEmbeddings = await runKloRelationshipBenchmarkCase({
const withoutEmbeddings = await runKtxRelationshipBenchmarkCase({
fixture,
mode: 'embeddings_disabled',
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
expect(withEmbeddings.predicted.acceptedFk).toEqual(['orders.(buyer_ref)->customers.(id)']);
@ -1218,7 +1218,7 @@ describe('relationship benchmarks', () => {
it('loads the Orbit-style product fixture as curated relationship-discovery benchmark evidence', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'orbit_style_product_no_declared_constraints'),
);
@ -1232,9 +1232,9 @@ describe('relationship benchmarks', () => {
'embeddings_disabled',
]);
const suite = await runKloRelationshipBenchmarkSuite({
const suite = await runKtxRelationshipBenchmarkSuite({
fixtures: [fixture],
detector: currentKloRelationshipBenchmarkDetector(),
detector: currentKtxRelationshipBenchmarkDetector(),
});
const headline = suite.cases.find(
(item) =>

258
packages/context/src/scan/relationship-benchmarks.ts
View file

@ -6,30 +6,30 @@ import { gunzipSync } from 'node:zlib';
import Database from 'better-sqlite3';
import YAML from 'yaml';
import { z } from 'zod';
import type { KloEnrichedRelationship, KloEnrichedSchema, KloRelationshipType } from './enrichment-types.js';
import { snapshotToKloEnrichedSchema } from './local-enrichment.js';
import type { KloRelationshipDiscoveryCandidate } from './relationship-candidates.js';
import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipType } from './enrichment-types.js';
import { snapshotToKtxEnrichedSchema } from './local-enrichment.js';
import type { KtxRelationshipDiscoveryCandidate } from './relationship-candidates.js';
import {
generateKloRelationshipDiscoveryCandidates,
mergeKloRelationshipDiscoveryCandidates,
generateKtxRelationshipDiscoveryCandidates,
mergeKtxRelationshipDiscoveryCandidates,
} from './relationship-candidates.js';
import type { KloLlmProvider } from '@klo/llm';
import { proposeKloRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
import type { KtxLlmProvider } from '@ktx/llm';
import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
import {
discoverKloCompositeRelationships,
type KloCompositePrimaryKeyCandidate,
type KloCompositeRelationshipCandidate,
discoverKtxCompositeRelationships,
type KtxCompositePrimaryKeyCandidate,
type KtxCompositeRelationshipCandidate,
} from './relationship-composite-candidates.js';
import { emptyKloRelationshipProfileArtifact } from './relationship-diagnostics.js';
import { collectKloFormalMetadataRelationships } from './relationship-formal-metadata.js';
import { resolveKloRelationshipGraph } from './relationship-graph-resolver.js';
import { type KloRelationshipReadOnlyExecutor, profileKloRelationshipSchema } from './relationship-profiling.js';
import type { KloRelationshipValidationBudget } from './relationship-budget.js';
import type { KloRelationshipFixtureOrigin } from './relationship-scoring.js';
import { validateKloRelationshipDiscoveryCandidates } from './relationship-validation.js';
import type { KloQueryResult, KloReadOnlyQueryInput, KloScanContext, KloSchemaSnapshot } from './types.js';
import { emptyKtxRelationshipProfileArtifact } from './relationship-diagnostics.js';
import { collectKtxFormalMetadataRelationships } from './relationship-formal-metadata.js';
import { resolveKtxRelationshipGraph } from './relationship-graph-resolver.js';
import { type KtxRelationshipReadOnlyExecutor, profileKtxRelationshipSchema } from './relationship-profiling.js';
import type { KtxRelationshipValidationBudget } from './relationship-budget.js';
import type { KtxRelationshipFixtureOrigin } from './relationship-scoring.js';
import { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js';
import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext, KtxSchemaSnapshot } from './types.js';
export const KLO_RELATIONSHIP_BENCHMARK_MODES = [
export const KTX_RELATIONSHIP_BENCHMARK_MODES = [
'metadata_present',
'declared_fks_removed',
'declared_pks_removed',
@ -40,87 +40,87 @@ export const KLO_RELATIONSHIP_BENCHMARK_MODES = [
'embeddings_disabled',
] as const;
export type KloRelationshipBenchmarkMode = (typeof KLO_RELATIONSHIP_BENCHMARK_MODES)[number];
export type KtxRelationshipBenchmarkMode = (typeof KTX_RELATIONSHIP_BENCHMARK_MODES)[number];
export const KLO_RELATIONSHIP_BENCHMARK_TIERS = ['unit', 'row_bearing', 'schema_only', 'smoke', 'product'] as const;
export const KTX_RELATIONSHIP_BENCHMARK_TIERS = ['unit', 'row_bearing', 'schema_only', 'smoke', 'product'] as const;
export type KloRelationshipBenchmarkTier = (typeof KLO_RELATIONSHIP_BENCHMARK_TIERS)[number];
export type KtxRelationshipBenchmarkTier = (typeof KTX_RELATIONSHIP_BENCHMARK_TIERS)[number];
export type KloRelationshipBenchmarkStatus = 'accepted' | 'review' | 'rejected';
export type KtxRelationshipBenchmarkStatus = 'accepted' | 'review' | 'rejected';
export interface KloRelationshipBenchmarkExpectedPk {
export interface KtxRelationshipBenchmarkExpectedPk {
table: string;
columns: string[];
}
export interface KloRelationshipBenchmarkExpectedLink {
export interface KtxRelationshipBenchmarkExpectedLink {
fromTable: string;
fromColumns: string[];
toTable: string;
toColumns: string[];
relationship: KloRelationshipType;
relationship: KtxRelationshipType;
}
export interface KloRelationshipBenchmarkExpectedLinks {
expectedPks: KloRelationshipBenchmarkExpectedPk[];
expectedLinks: KloRelationshipBenchmarkExpectedLink[];
export interface KtxRelationshipBenchmarkExpectedLinks {
expectedPks: KtxRelationshipBenchmarkExpectedPk[];
expectedLinks: KtxRelationshipBenchmarkExpectedLink[];
}
export interface KloRelationshipBenchmarkFixture {
export interface KtxRelationshipBenchmarkFixture {
id: string;
name: string;
tier: KloRelationshipBenchmarkTier;
origin: KloRelationshipFixtureOrigin;
tier: KtxRelationshipBenchmarkTier;
origin: KtxRelationshipFixtureOrigin;
thresholdEligible?: boolean;
validationBudget?: KloRelationshipValidationBudget;
snapshot: KloSchemaSnapshot;
expected: KloRelationshipBenchmarkExpectedLinks;
defaultModes: KloRelationshipBenchmarkMode[];
validationBudget?: KtxRelationshipValidationBudget;
snapshot: KtxSchemaSnapshot;
expected: KtxRelationshipBenchmarkExpectedLinks;
defaultModes: KtxRelationshipBenchmarkMode[];
dataPath: string | null;
columnEmbeddings: Record<string, number[]>;
}
export interface KloRelationshipBenchmarkDetectedPk {
export interface KtxRelationshipBenchmarkDetectedPk {
table: string;
columns: string[];
score: number;
status: KloRelationshipBenchmarkStatus;
status: KtxRelationshipBenchmarkStatus;
}
export interface KloRelationshipBenchmarkDetectedLink {
export interface KtxRelationshipBenchmarkDetectedLink {
fromTable: string;
fromColumns: string[];
toTable: string;
toColumns: string[];
relationship: KloRelationshipType;
relationship: KtxRelationshipType;
score: number;
status: KloRelationshipBenchmarkStatus;
status: KtxRelationshipBenchmarkStatus;
source: string;
}
export interface KloRelationshipBenchmarkDetectorResult {
pks: KloRelationshipBenchmarkDetectedPk[];
links: KloRelationshipBenchmarkDetectedLink[];
export interface KtxRelationshipBenchmarkDetectorResult {
pks: KtxRelationshipBenchmarkDetectedPk[];
links: KtxRelationshipBenchmarkDetectedLink[];
validationBlocked: boolean;
sqlQueries: number;
llmCalls: number;
runtimeSeconds: number;
}
export interface KloRelationshipBenchmarkDetectorInput {
export interface KtxRelationshipBenchmarkDetectorInput {
fixtureId: string;
mode: KloRelationshipBenchmarkMode;
snapshot: KloSchemaSnapshot;
schema: KloEnrichedSchema;
mode: KtxRelationshipBenchmarkMode;
snapshot: KtxSchemaSnapshot;
schema: KtxEnrichedSchema;
dataPath: string | null;
validationBudget?: KloRelationshipValidationBudget;
validationBudget?: KtxRelationshipValidationBudget;
}
export interface KloRelationshipBenchmarkDetector {
detect(input: KloRelationshipBenchmarkDetectorInput): Promise<KloRelationshipBenchmarkDetectorResult>;
export interface KtxRelationshipBenchmarkDetector {
detect(input: KtxRelationshipBenchmarkDetectorInput): Promise<KtxRelationshipBenchmarkDetectorResult>;
}
export interface KloRelationshipBenchmarkMetrics {
export interface KtxRelationshipBenchmarkMetrics {
pkPrecision: number;
pkRecall: number;
pkF1: number;
@ -135,10 +135,10 @@ export interface KloRelationshipBenchmarkMetrics {
llmCalls: number;
}
export interface KloRelationshipBenchmarkCaseResult {
export interface KtxRelationshipBenchmarkCaseResult {
fixtureId: string;
mode: KloRelationshipBenchmarkMode;
metrics: KloRelationshipBenchmarkMetrics;
mode: KtxRelationshipBenchmarkMode;
metrics: KtxRelationshipBenchmarkMetrics;
expected: {
pk: string[];
fk: string[];
@ -164,8 +164,8 @@ export interface KloRelationshipBenchmarkCaseResult {
validationBlocked: boolean;
}
export interface KloRelationshipBenchmarkSuiteResult {
cases: KloRelationshipBenchmarkCaseResult[];
export interface KtxRelationshipBenchmarkSuiteResult {
cases: KtxRelationshipBenchmarkCaseResult[];
validationBlockedCases: string[];
aggregate: {
caseCount: number;
@ -179,7 +179,7 @@ export interface KloRelationshipBenchmarkSuiteResult {
};
}
class KloRelationshipBenchmarkSqliteExecutor implements KloRelationshipReadOnlyExecutor {
class KtxRelationshipBenchmarkSqliteExecutor implements KtxRelationshipReadOnlyExecutor {
private readonly db: Database.Database;
queryCount = 0;
@ -187,7 +187,7 @@ class KloRelationshipBenchmarkSqliteExecutor implements KloRelationshipReadOnlyE
this.db = new Database(dataPath, { readonly: true, fileMustExist: true });
}
async executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
this.queryCount += 1;
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
@ -236,7 +236,7 @@ async function fixtureDataPath(fixtureDir: string): Promise<string | null> {
return null;
}
const digest = createHash('sha256').update(fixtureDir).digest('hex').slice(0, 16);
const tempRoot = await mkdtemp(join(tmpdir(), `klo-relationship-benchmark-${digest}-`));
const tempRoot = await mkdtemp(join(tmpdir(), `ktx-relationship-benchmark-${digest}-`));
const extractedPath = join(tempRoot, 'data.sqlite');
await writeFile(extractedPath, gunzipSync(await readFile(compressedPath)));
return extractedPath;
@ -266,8 +266,8 @@ async function fixtureColumnEmbeddings(fixtureDir: string): Promise<Record<strin
}
}
const modeSchema = z.enum(KLO_RELATIONSHIP_BENCHMARK_MODES);
const tierSchema = z.enum(KLO_RELATIONSHIP_BENCHMARK_TIERS);
const modeSchema = z.enum(KTX_RELATIONSHIP_BENCHMARK_MODES);
const tierSchema = z.enum(KTX_RELATIONSHIP_BENCHMARK_TIERS);
const originSchema = z.enum(['synthetic', 'public', 'customer']);
const validationBudgetSchema = z.union([z.literal('all'), z.number().int().nonnegative()]);
@ -307,21 +307,21 @@ function tupleKey(columns: readonly string[]): string {
return `(${columns.join(',')})`;
}
function pkKey(pk: Pick<KloRelationshipBenchmarkExpectedPk, 'table' | 'columns'>): string {
function pkKey(pk: Pick<KtxRelationshipBenchmarkExpectedPk, 'table' | 'columns'>): string {
return `${pk.table}.${tupleKey(pk.columns)}`;
}
function fkKey(
link: Pick<KloRelationshipBenchmarkExpectedLink, 'fromTable' | 'fromColumns' | 'toTable' | 'toColumns'>,
link: Pick<KtxRelationshipBenchmarkExpectedLink, 'fromTable' | 'fromColumns' | 'toTable' | 'toColumns'>,
): string {
return `${link.fromTable}.${tupleKey(link.fromColumns)}->${link.toTable}.${tupleKey(link.toColumns)}`;
}
function relationshipKey(link: KloRelationshipBenchmarkDetectedLink): string {
function relationshipKey(link: KtxRelationshipBenchmarkDetectedLink): string {
return fkKey(link);
}
function relationshipToBenchmarkLink(candidate: KloEnrichedRelationship): KloRelationshipBenchmarkDetectedLink {
function relationshipToBenchmarkLink(candidate: KtxEnrichedRelationship): KtxRelationshipBenchmarkDetectedLink {
return {
fromTable: candidate.from.table.name,
fromColumns: candidate.from.columns,
@ -335,8 +335,8 @@ function relationshipToBenchmarkLink(candidate: KloEnrichedRelationship): KloRel
}
function broadCandidateToBenchmarkLink(
candidate: Pick<KloRelationshipDiscoveryCandidate, 'confidence' | 'from' | 'relationshipType' | 'source' | 'to'>,
): KloRelationshipBenchmarkDetectedLink {
candidate: Pick<KtxRelationshipDiscoveryCandidate, 'confidence' | 'from' | 'relationshipType' | 'source' | 'to'>,
): KtxRelationshipBenchmarkDetectedLink {
return {
fromTable: candidate.from.table.name,
fromColumns: candidate.from.columns,
@ -349,7 +349,7 @@ function broadCandidateToBenchmarkLink(
};
}
function compositePkToBenchmarkPk(candidate: KloCompositePrimaryKeyCandidate): KloRelationshipBenchmarkDetectedPk {
function compositePkToBenchmarkPk(candidate: KtxCompositePrimaryKeyCandidate): KtxRelationshipBenchmarkDetectedPk {
return {
table: candidate.table.name,
columns: candidate.columns,
@ -359,8 +359,8 @@ function compositePkToBenchmarkPk(candidate: KloCompositePrimaryKeyCandidate): K
}
function compositeRelationshipToBenchmarkLink(
candidate: KloCompositeRelationshipCandidate,
): KloRelationshipBenchmarkDetectedLink {
candidate: KtxCompositeRelationshipCandidate,
): KtxRelationshipBenchmarkDetectedLink {
return {
fromTable: candidate.from.table.name,
fromColumns: candidate.from.columns,
@ -391,30 +391,30 @@ function intersectionSize(left: readonly string[], right: readonly string[]): nu
return left.filter((item) => rightSet.has(item)).length;
}
function compositePkKeys(expected: KloRelationshipBenchmarkExpectedLinks): string[] {
function compositePkKeys(expected: KtxRelationshipBenchmarkExpectedLinks): string[] {
return sortedUnique(expected.expectedPks.filter((pk) => pk.columns.length > 1).map(pkKey));
}
function compositeFkKeys(expected: KloRelationshipBenchmarkExpectedLinks): string[] {
function compositeFkKeys(expected: KtxRelationshipBenchmarkExpectedLinks): string[] {
return sortedUnique(
expected.expectedLinks.filter((link) => link.fromColumns.length > 1 || link.toColumns.length > 1).map(fkKey),
);
}
function scalarExpectedPkKeys(expected: KloRelationshipBenchmarkExpectedLinks): string[] {
function scalarExpectedPkKeys(expected: KtxRelationshipBenchmarkExpectedLinks): string[] {
return sortedUnique(expected.expectedPks.map(pkKey));
}
function scalarExpectedFkKeys(expected: KloRelationshipBenchmarkExpectedLinks): string[] {
function scalarExpectedFkKeys(expected: KtxRelationshipBenchmarkExpectedLinks): string[] {
return sortedUnique(expected.expectedLinks.map(fkKey));
}
function scoreBenchmarkCase(input: {
fixtureId: string;
mode: KloRelationshipBenchmarkMode;
expected: KloRelationshipBenchmarkExpectedLinks;
detected: KloRelationshipBenchmarkDetectorResult;
}): KloRelationshipBenchmarkCaseResult {
mode: KtxRelationshipBenchmarkMode;
expected: KtxRelationshipBenchmarkExpectedLinks;
detected: KtxRelationshipBenchmarkDetectorResult;
}): KtxRelationshipBenchmarkCaseResult {
const expectedPk = scalarExpectedPkKeys(input.expected);
const expectedFk = scalarExpectedFkKeys(input.expected);
const predictedPk = sortedUnique(input.detected.pks.map(pkKey));
@ -478,10 +478,10 @@ function scoreBenchmarkCase(input: {
};
}
export function maskKloRelationshipBenchmarkSnapshot(
snapshot: KloSchemaSnapshot,
mode: KloRelationshipBenchmarkMode,
): KloSchemaSnapshot {
export function maskKtxRelationshipBenchmarkSnapshot(
snapshot: KtxSchemaSnapshot,
mode: KtxRelationshipBenchmarkMode,
): KtxSchemaSnapshot {
const relationshipDiscoveryMode =
mode === 'declared_pks_and_declared_fks_removed' ||
mode === 'llm_disabled' ||
@ -506,9 +506,9 @@ export function maskKloRelationshipBenchmarkSnapshot(
};
}
export function isKloRelationshipBenchmarkTuningEligible(input: {
fixture: Pick<KloRelationshipBenchmarkFixture, 'tier' | 'thresholdEligible'>;
mode: KloRelationshipBenchmarkMode;
export function isKtxRelationshipBenchmarkTuningEligible(input: {
fixture: Pick<KtxRelationshipBenchmarkFixture, 'tier' | 'thresholdEligible'>;
mode: KtxRelationshipBenchmarkMode;
validationBlocked: boolean;
}): boolean {
if (input.validationBlocked || input.mode !== 'declared_pks_and_declared_fks_removed') {
@ -526,49 +526,49 @@ export function isKloRelationshipBenchmarkTuningEligible(input: {
return input.fixture.tier === 'unit' || input.fixture.tier === 'row_bearing';
}
export function kloRelationshipBenchmarkDetectorWithLlm(
llmProvider: KloLlmProvider,
): KloRelationshipBenchmarkDetector {
export function ktxRelationshipBenchmarkDetectorWithLlm(
llmProvider: KtxLlmProvider,
): KtxRelationshipBenchmarkDetector {
return {
async detect(input) {
const startedAt = performance.now();
const formalMetadata = collectKloFormalMetadataRelationships(input.schema);
const formalMetadata = collectKtxFormalMetadataRelationships(input.schema);
const formalLinks = formalMetadata.accepted.map((relationship) => relationshipToBenchmarkLink(relationship));
const acceptedKeys = new Set(formalLinks.map(fkKey));
const sqliteDataAvailable = Boolean(input.dataPath && input.snapshot.driver === 'sqlite');
const profilingExecutor =
sqliteDataAvailable && input.mode !== 'profiling_disabled'
? new KloRelationshipBenchmarkSqliteExecutor(input.dataPath as string)
? new KtxRelationshipBenchmarkSqliteExecutor(input.dataPath as string)
: null;
const validationExecutor = profilingExecutor && input.mode !== 'validation_disabled' ? profilingExecutor : null;
const profiles =
input.mode === 'profiling_disabled'
? emptyKloRelationshipProfileArtifact({
? emptyKtxRelationshipProfileArtifact({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
reason: 'relationship_benchmark_profiling_disabled',
})
: await profileKloRelationshipSchema({
: await profileKtxRelationshipSchema({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
schema: input.schema,
executor: profilingExecutor,
ctx: { runId: `relationship-benchmark:${input.fixtureId}:${input.mode}:profile` },
});
const broadRelationshipCandidates = generateKloRelationshipDiscoveryCandidates(input.schema, {
const broadRelationshipCandidates = generateKtxRelationshipDiscoveryCandidates(input.schema, {
profiles,
useEmbeddings: input.mode !== 'embeddings_disabled',
});
const llmProposalResult =
input.mode === 'llm_disabled'
? { candidates: [], warnings: [], llmCalls: 0, summary: 'skipped' as const }
: await proposeKloRelationshipCandidatesWithLlm({
: await proposeKtxRelationshipCandidatesWithLlm({
connectionId: input.snapshot.connectionId,
schema: input.schema,
profile: profiles,
llmProvider,
});
const candidates = mergeKloRelationshipDiscoveryCandidates([
const candidates = mergeKtxRelationshipDiscoveryCandidates([
...broadRelationshipCandidates,
...llmProposalResult.candidates,
]);
@ -578,7 +578,7 @@ export function kloRelationshipBenchmarkDetectorWithLlm(
: input.validationBudget === undefined
? 'all'
: Math.max(0, input.validationBudget - profiles.queryCount);
const validatedBroadCandidates = await validateKloRelationshipDiscoveryCandidates({
const validatedBroadCandidates = await validateKtxRelationshipDiscoveryCandidates({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
candidates,
@ -595,7 +595,7 @@ export function kloRelationshipBenchmarkDetectorWithLlm(
validationExecutor &&
input.mode !== 'profiling_disabled' &&
input.mode !== 'validation_disabled'
? await discoverKloCompositeRelationships({
? await discoverKtxCompositeRelationships({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
schema: input.schema,
@ -605,7 +605,7 @@ export function kloRelationshipBenchmarkDetectorWithLlm(
})
: { primaryKeys: [], relationships: [], queryCount: 0, warnings: [] };
profilingExecutor?.close();
const graph = resolveKloRelationshipGraph({
const graph = resolveKtxRelationshipGraph({
schema: input.schema,
profiles,
candidates: validatedBroadCandidates,
@ -663,34 +663,34 @@ export function kloRelationshipBenchmarkDetectorWithLlm(
};
}
export function currentKloRelationshipBenchmarkDetector(): KloRelationshipBenchmarkDetector {
export function currentKtxRelationshipBenchmarkDetector(): KtxRelationshipBenchmarkDetector {
return {
async detect(input) {
const startedAt = performance.now();
const formalMetadata = collectKloFormalMetadataRelationships(input.schema);
const formalMetadata = collectKtxFormalMetadataRelationships(input.schema);
const formalLinks = formalMetadata.accepted.map((relationship) => relationshipToBenchmarkLink(relationship));
const acceptedKeys = new Set(formalLinks.map(fkKey));
const sqliteDataAvailable = Boolean(input.dataPath && input.snapshot.driver === 'sqlite');
const profilingExecutor =
sqliteDataAvailable && input.mode !== 'profiling_disabled'
? new KloRelationshipBenchmarkSqliteExecutor(input.dataPath as string)
? new KtxRelationshipBenchmarkSqliteExecutor(input.dataPath as string)
: null;
const validationExecutor = profilingExecutor && input.mode !== 'validation_disabled' ? profilingExecutor : null;
const profiles =
input.mode === 'profiling_disabled'
? emptyKloRelationshipProfileArtifact({
? emptyKtxRelationshipProfileArtifact({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
reason: 'relationship_benchmark_profiling_disabled',
})
: await profileKloRelationshipSchema({
: await profileKtxRelationshipSchema({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
schema: input.schema,
executor: profilingExecutor,
ctx: { runId: `relationship-benchmark:${input.fixtureId}:${input.mode}:profile` },
});
const broadRelationshipCandidates = generateKloRelationshipDiscoveryCandidates(input.schema, {
const broadRelationshipCandidates = generateKtxRelationshipDiscoveryCandidates(input.schema, {
profiles,
useEmbeddings: input.mode !== 'embeddings_disabled',
});
@ -700,7 +700,7 @@ export function currentKloRelationshipBenchmarkDetector(): KloRelationshipBenchm
: input.validationBudget === undefined
? 'all'
: Math.max(0, input.validationBudget - profiles.queryCount);
const validatedBroadCandidates = await validateKloRelationshipDiscoveryCandidates({
const validatedBroadCandidates = await validateKtxRelationshipDiscoveryCandidates({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
candidates: broadRelationshipCandidates,
@ -717,7 +717,7 @@ export function currentKloRelationshipBenchmarkDetector(): KloRelationshipBenchm
validationExecutor &&
input.mode !== 'profiling_disabled' &&
input.mode !== 'validation_disabled'
? await discoverKloCompositeRelationships({
? await discoverKtxCompositeRelationships({
connectionId: input.snapshot.connectionId,
driver: input.snapshot.driver,
schema: input.schema,
@ -727,7 +727,7 @@ export function currentKloRelationshipBenchmarkDetector(): KloRelationshipBenchm
})
: { primaryKeys: [], relationships: [], queryCount: 0, warnings: [] };
profilingExecutor?.close();
const graph = resolveKloRelationshipGraph({
const graph = resolveKtxRelationshipGraph({
schema: input.schema,
profiles,
candidates: validatedBroadCandidates,
@ -785,9 +785,9 @@ export function currentKloRelationshipBenchmarkDetector(): KloRelationshipBenchm
};
}
export async function loadKloRelationshipBenchmarkFixture(
export async function loadKtxRelationshipBenchmarkFixture(
fixtureDir: string,
): Promise<KloRelationshipBenchmarkFixture> {
): Promise<KtxRelationshipBenchmarkFixture> {
const [fixtureRaw, snapshotRaw, expectedRaw] = await Promise.all([
fixtureText(fixtureDir, 'fixture.yaml'),
fixtureText(fixtureDir, 'snapshot.json'),
@ -795,7 +795,7 @@ export async function loadKloRelationshipBenchmarkFixture(
]);
const fixture = fixtureConfigSchema.parse(YAML.parse(fixtureRaw));
const expected = expectedLinksSchema.parse(YAML.parse(expectedRaw));
const snapshot = JSON.parse(snapshotRaw) as KloSchemaSnapshot;
const snapshot = JSON.parse(snapshotRaw) as KtxSchemaSnapshot;
return {
...fixture,
@ -806,30 +806,30 @@ export async function loadKloRelationshipBenchmarkFixture(
};
}
export async function loadKloRelationshipBenchmarkFixtures(
export async function loadKtxRelationshipBenchmarkFixtures(
fixtureRoot: string,
): Promise<KloRelationshipBenchmarkFixture[]> {
): Promise<KtxRelationshipBenchmarkFixture[]> {
const entries = await readdir(fixtureRoot, { withFileTypes: true });
const fixtureDirs = entries
.filter((entry) => entry.isDirectory())
.map((entry) => join(fixtureRoot, entry.name))
.sort((left, right) => left.localeCompare(right));
return Promise.all(fixtureDirs.map((fixtureDir) => loadKloRelationshipBenchmarkFixture(fixtureDir)));
return Promise.all(fixtureDirs.map((fixtureDir) => loadKtxRelationshipBenchmarkFixture(fixtureDir)));
}
export async function runKloRelationshipBenchmarkCase(input: {
fixture: KloRelationshipBenchmarkFixture;
mode: KloRelationshipBenchmarkMode;
detector?: KloRelationshipBenchmarkDetector;
}): Promise<KloRelationshipBenchmarkCaseResult> {
const snapshot = maskKloRelationshipBenchmarkSnapshot(input.fixture.snapshot, input.mode);
export async function runKtxRelationshipBenchmarkCase(input: {
fixture: KtxRelationshipBenchmarkFixture;
mode: KtxRelationshipBenchmarkMode;
detector?: KtxRelationshipBenchmarkDetector;
}): Promise<KtxRelationshipBenchmarkCaseResult> {
const snapshot = maskKtxRelationshipBenchmarkSnapshot(input.fixture.snapshot, input.mode);
const embeddings =
input.mode === 'embeddings_disabled'
? new Map<string, number[]>()
: new Map(Object.entries(input.fixture.columnEmbeddings));
const schema = snapshotToKloEnrichedSchema(snapshot, embeddings);
const detected = await (input.detector ?? currentKloRelationshipBenchmarkDetector()).detect({
const schema = snapshotToKtxEnrichedSchema(snapshot, embeddings);
const detected = await (input.detector ?? currentKtxRelationshipBenchmarkDetector()).detect({
fixtureId: input.fixture.id,
mode: input.mode,
snapshot,
@ -846,15 +846,15 @@ export async function runKloRelationshipBenchmarkCase(input: {
});
}
export async function runKloRelationshipBenchmarkSuite(input: {
fixtures: KloRelationshipBenchmarkFixture[];
detector?: KloRelationshipBenchmarkDetector;
}): Promise<KloRelationshipBenchmarkSuiteResult> {
const cases: KloRelationshipBenchmarkCaseResult[] = [];
export async function runKtxRelationshipBenchmarkSuite(input: {
fixtures: KtxRelationshipBenchmarkFixture[];
detector?: KtxRelationshipBenchmarkDetector;
}): Promise<KtxRelationshipBenchmarkSuiteResult> {
const cases: KtxRelationshipBenchmarkCaseResult[] = [];
for (const fixture of input.fixtures) {
for (const mode of fixture.defaultModes) {
cases.push(
await runKloRelationshipBenchmarkCase({
await runKtxRelationshipBenchmarkCase({
fixture,
mode,
detector: input.detector,
@ -867,7 +867,7 @@ export async function runKloRelationshipBenchmarkSuite(input: {
const headlineCases = cases.filter((item) => {
const fixture = fixtureById.get(item.fixtureId);
return fixture
? isKloRelationshipBenchmarkTuningEligible({
? isKtxRelationshipBenchmarkTuningEligible({
fixture,
mode: item.mode,
validationBlocked: item.validationBlocked,

22
packages/context/src/scan/relationship-budget.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { applyKloRelationshipValidationBudget, defaultKloRelationshipValidationBudget } from './relationship-budget.js';
import { applyKtxRelationshipValidationBudget, defaultKtxRelationshipValidationBudget } from './relationship-budget.js';
interface Candidate {
id: string;
@ -8,16 +8,16 @@ interface Candidate {
describe('relationship validation budget', () => {
it('computes the default validation budget from table count', () => {
expect(defaultKloRelationshipValidationBudget(0)).toBe(0);
expect(defaultKloRelationshipValidationBudget(3)).toBe(6);
expect(defaultKloRelationshipValidationBudget(400)).toBe(800);
expect(defaultKloRelationshipValidationBudget(900)).toBe(1000);
expect(defaultKloRelationshipValidationBudget(-4)).toBe(0);
expect(defaultKloRelationshipValidationBudget(3.8)).toBe(6);
expect(defaultKtxRelationshipValidationBudget(0)).toBe(0);
expect(defaultKtxRelationshipValidationBudget(3)).toBe(6);
expect(defaultKtxRelationshipValidationBudget(400)).toBe(800);
expect(defaultKtxRelationshipValidationBudget(900)).toBe(1000);
expect(defaultKtxRelationshipValidationBudget(-4)).toBe(0);
expect(defaultKtxRelationshipValidationBudget(3.8)).toBe(6);
});
it('splits candidates by descending score with stable tie ordering', () => {
const result = applyKloRelationshipValidationBudget<Candidate>({
const result = applyKtxRelationshipValidationBudget<Candidate>({
candidates: [
{ id: 'first', confidence: 0.8 },
{ id: 'second', confidence: 0.9 },
@ -41,7 +41,7 @@ describe('relationship validation budget', () => {
confidence: 1 - index / 10,
}));
const result = applyKloRelationshipValidationBudget<Candidate>({
const result = applyKtxRelationshipValidationBudget<Candidate>({
candidates,
tableCount: 2,
score: (candidate) => candidate.confidence,
@ -53,7 +53,7 @@ describe('relationship validation budget', () => {
});
it('treats budget zero as disabling SQL validation', () => {
const result = applyKloRelationshipValidationBudget<Candidate>({
const result = applyKtxRelationshipValidationBudget<Candidate>({
candidates: [
{ id: 'first', confidence: 1 },
{ id: 'second', confidence: 0.5 },
@ -69,7 +69,7 @@ describe('relationship validation budget', () => {
});
it('treats budget all as validating every candidate', () => {
const result = applyKloRelationshipValidationBudget<Candidate>({
const result = applyKtxRelationshipValidationBudget<Candidate>({
candidates: [
{ id: 'first', confidence: 0.1 },
{ id: 'second', confidence: 0.9 },

24
packages/context/src/scan/relationship-budget.ts
View file

@ -1,32 +1,32 @@
export type KloRelationshipValidationBudget = number | 'all' | undefined;
export type KtxRelationshipValidationBudget = number | 'all' | undefined;
export interface KloRelationshipBudgetedCandidate<TCandidate> {
export interface KtxRelationshipBudgetedCandidate<TCandidate> {
candidate: TCandidate;
originalIndex: number;
score: number;
}
export interface KloRelationshipValidationBudgetResult<TCandidate> {
export interface KtxRelationshipValidationBudgetResult<TCandidate> {
effectiveBudget: number | 'all';
toValidate: KloRelationshipBudgetedCandidate<TCandidate>[];
deferred: KloRelationshipBudgetedCandidate<TCandidate>[];
toValidate: KtxRelationshipBudgetedCandidate<TCandidate>[];
deferred: KtxRelationshipBudgetedCandidate<TCandidate>[];
}
export interface ApplyKloRelationshipValidationBudgetInput<TCandidate> {
export interface ApplyKtxRelationshipValidationBudgetInput<TCandidate> {
candidates: readonly TCandidate[];
tableCount: number;
budget?: KloRelationshipValidationBudget;
budget?: KtxRelationshipValidationBudget;
score: (candidate: TCandidate) => number;
}
export function defaultKloRelationshipValidationBudget(tableCount: number): number {
export function defaultKtxRelationshipValidationBudget(tableCount: number): number {
const safeTableCount = Number.isFinite(tableCount) ? Math.max(0, Math.floor(tableCount)) : 0;
return Math.min(2 * safeTableCount, 1000);
}
export function applyKloRelationshipValidationBudget<TCandidate>(
input: ApplyKloRelationshipValidationBudgetInput<TCandidate>,
): KloRelationshipValidationBudgetResult<TCandidate> {
export function applyKtxRelationshipValidationBudget<TCandidate>(
input: ApplyKtxRelationshipValidationBudgetInput<TCandidate>,
): KtxRelationshipValidationBudgetResult<TCandidate> {
const ranked = input.candidates
.map((candidate, originalIndex) => ({
candidate,
@ -50,7 +50,7 @@ export function applyKloRelationshipValidationBudget<TCandidate>(
};
}
const effectiveBudget = input.budget ?? defaultKloRelationshipValidationBudget(input.tableCount);
const effectiveBudget = input.budget ?? defaultKtxRelationshipValidationBudget(input.tableCount);
const safeBudget = Math.max(0, Math.floor(effectiveBudget));
return {
effectiveBudget: safeBudget,

72
packages/context/src/scan/relationship-candidates.test.ts
View file

@ -1,19 +1,19 @@
import { describe, expect, it } from 'vitest';
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable } from './enrichment-types.js';
import { normalizeKloRelationshipName } from './relationship-name-similarity.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js';
import { normalizeKtxRelationshipName } from './relationship-name-similarity.js';
import {
generateKloRelationshipDiscoveryCandidates,
inferKloRelationshipTargetPks,
mergeKloRelationshipDiscoveryCandidates,
generateKtxRelationshipDiscoveryCandidates,
inferKtxRelationshipTargetPks,
mergeKtxRelationshipDiscoveryCandidates,
} from './relationship-candidates.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
function column(
tableId: string,
id: string,
name: string,
options: Partial<KloEnrichedColumn> = {},
): KloEnrichedColumn {
options: Partial<KtxEnrichedColumn> = {},
): KtxEnrichedColumn {
const tableRef = options.tableRef ?? { catalog: null, db: 'public', name: tableId };
return {
id,
@ -33,7 +33,7 @@ function column(
};
}
function table(id: string, name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
function table(id: string, name: string, columns: KtxEnrichedColumn[]): KtxEnrichedTable {
const ref = { catalog: null, db: 'public', name };
return {
id,
@ -44,7 +44,7 @@ function table(id: string, name: string, columns: KloEnrichedColumn[]): KloEnric
};
}
function schema(tables: KloEnrichedTable[]): KloEnrichedSchema {
function schema(tables: KtxEnrichedTable[]): KtxEnrichedSchema {
return {
connectionId: 'warehouse',
tables,
@ -52,7 +52,7 @@ function schema(tables: KloEnrichedTable[]): KloEnrichedSchema {
};
}
function planCodeProfiles(): KloRelationshipProfileArtifact {
function planCodeProfiles(): KtxRelationshipProfileArtifact {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -192,7 +192,7 @@ describe('relationship discovery candidates', () => {
column('invoices-id', 'account-id-col', 'account_id', { primaryKey: false }),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([accounts, invoices]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([accounts, invoices]));
expect(candidates).toHaveLength(1);
expect(candidates[0]).toMatchObject({
@ -232,7 +232,7 @@ describe('relationship discovery candidates', () => {
column('album-id', 'artist-id-fk-col', 'ArtistId', { primaryKey: false }),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([artists, albums]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([artists, albums]));
expect(
candidates.map(
@ -260,7 +260,7 @@ describe('relationship discovery candidates', () => {
column('invoices-id', 'account-id-col', 'account_id'),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([accounts, invoices]), {
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([accounts, invoices]), {
maxCandidateParentTables: 0,
});
@ -282,7 +282,7 @@ describe('relationship discovery candidates', () => {
]),
);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([albums, ...fillerTables, artists]), {
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([albums, ...fillerTables, artists]), {
maxCandidateParentTables: 1,
});
@ -304,7 +304,7 @@ describe('relationship discovery candidates', () => {
column('order-id', 'customer-id-fk-col', 'CustomerID', { primaryKey: false }),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([customers, orders]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([customers, orders]));
expect(
candidates.map(
@ -337,7 +337,7 @@ describe('relationship discovery candidates', () => {
column('subscriptions-id', 'customer-account-id-col', 'CustomerAccountID', { primaryKey: false }),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([customerAccounts, subscriptions]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([customerAccounts, subscriptions]));
expect(
candidates.map(
@ -383,7 +383,7 @@ describe('relationship discovery candidates', () => {
}),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([customerAccounts, subscriptions]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([customerAccounts, subscriptions]));
expect(
candidates.map(
@ -403,7 +403,7 @@ describe('relationship discovery candidates', () => {
}),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([customerAccounts]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([customerAccounts]));
expect(
candidates.map(
@ -471,9 +471,9 @@ describe('relationship discovery candidates', () => {
maxTextLength: 2,
},
},
} satisfies KloRelationshipProfileArtifact;
} satisfies KtxRelationshipProfileArtifact;
const candidates = generateKloRelationshipDiscoveryCandidates(schema([countries, accounts]), { profiles });
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([countries, accounts]), { profiles });
expect(candidates).toHaveLength(1);
expect(candidates[0]).toMatchObject({
@ -508,7 +508,7 @@ describe('relationship discovery candidates', () => {
}),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([accounts]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([accounts]));
expect(
candidates.map(
@ -524,7 +524,7 @@ describe('relationship discovery candidates', () => {
column('employees-id', 'employees-parent-id-col', 'parent_id', { primaryKey: false }),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([employees]));
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([employees]));
expect(
candidates.map(
@ -603,7 +603,7 @@ describe('relationship discovery candidates', () => {
}),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([plans, accountSegments, mapping]), {
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([plans, accountSegments, mapping]), {
profiles: planCodeProfiles(),
});
const candidateKeys = candidates.map(
@ -698,9 +698,9 @@ describe('relationship discovery candidates', () => {
maxTextLength: 1,
},
},
} satisfies KloRelationshipProfileArtifact;
} satisfies KtxRelationshipProfileArtifact;
const candidates = generateKloRelationshipDiscoveryCandidates(schema([users, plans, accounts]), { profiles });
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([users, plans, accounts]), { profiles });
const candidateKeys = candidates.map(
(candidate) =>
`${candidate.from.table.name}.${candidate.from.columns[0]}->${candidate.to.table.name}.${candidate.to.columns[0]}`,
@ -734,7 +734,7 @@ describe('relationship discovery candidates', () => {
}),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([customers, orders]), {
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([customers, orders]), {
embeddingSimilarityThreshold: 0.95,
});
@ -769,7 +769,7 @@ describe('relationship discovery candidates', () => {
column('events-id', 'account-id-col', 'account_id'),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([events, archivedAccounts, accounts]), {
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([events, archivedAccounts, accounts]), {
maxCandidatesPerColumn: 1,
});
@ -790,8 +790,8 @@ describe('relationship discovery candidates', () => {
column('events-id', 'user-id-col', 'user_id'),
]);
const candidates = generateKloRelationshipDiscoveryCandidates(schema([accounts, users, events]));
const inferredPks = inferKloRelationshipTargetPks(candidates);
const candidates = generateKtxRelationshipDiscoveryCandidates(schema([accounts, users, events]));
const inferredPks = inferKtxRelationshipTargetPks(candidates);
expect(inferredPks).toEqual([
{
@ -821,21 +821,21 @@ describe('relationship discovery candidates', () => {
column('invoices-id', 'account-id-col', 'account_id', { nativeType: 'INTEGER', normalizedType: 'integer' }),
]);
expect(generateKloRelationshipDiscoveryCandidates(schema([accounts, invoices]))).toEqual([]);
expect(generateKtxRelationshipDiscoveryCandidates(schema([accounts, invoices]))).toEqual([]);
});
it('normalizes layer prefixes, punctuation, plural forms, and non-plural trailing s words', () => {
expect(normalizeKloRelationshipName('mart__Sales_Accounts')).toMatchObject({
expect(normalizeKtxRelationshipName('mart__Sales_Accounts')).toMatchObject({
normalized: 'sales_accounts',
singular: 'sales_account',
tokens: ['sales', 'accounts'],
});
expect(normalizeKloRelationshipName('dim_users')).toMatchObject({
expect(normalizeKtxRelationshipName('dim_users')).toMatchObject({
normalized: 'users',
singular: 'user',
tokens: ['users'],
});
expect(normalizeKloRelationshipName('Address')).toMatchObject({
expect(normalizeKtxRelationshipName('Address')).toMatchObject({
normalized: 'address',
singular: 'address',
plural: 'addresses',
@ -846,7 +846,7 @@ describe('relationship discovery candidates', () => {
it('merges duplicate deterministic and LLM proposal candidates without losing LLM rationale', () => {
const accounts = table('accounts-id', 'accounts', [column('accounts-id', 'accounts-id-col', 'id')]);
const invoices = table('invoices-id', 'invoices', [column('invoices-id', 'account-id-col', 'account_id')]);
const [deterministic] = generateKloRelationshipDiscoveryCandidates(schema([accounts, invoices]));
const [deterministic] = generateKtxRelationshipDiscoveryCandidates(schema([accounts, invoices]));
if (!deterministic) {
throw new Error('Expected deterministic relationship candidate');
}
@ -862,7 +862,7 @@ describe('relationship discovery candidates', () => {
},
};
const merged = mergeKloRelationshipDiscoveryCandidates([deterministic, llmCandidate]);
const merged = mergeKtxRelationshipDiscoveryCandidates([deterministic, llmCandidate]);
expect(merged).toHaveLength(1);
expect(merged[0]).toMatchObject({

246
packages/context/src/scan/relationship-candidates.ts
View file

@ -1,26 +1,26 @@
import type {
KloEnrichedColumn,
KloEnrichedSchema,
KloEnrichedTable,
KloRelationshipEndpoint,
KloRelationshipType,
KtxEnrichedColumn,
KtxEnrichedSchema,
KtxEnrichedTable,
KtxRelationshipEndpoint,
KtxRelationshipType,
} from './enrichment-types.js';
import { localCandidateTables } from './relationship-locality.js';
import {
normalizeKloRelationshipName,
pluralizeKloRelationshipToken,
singularizeKloRelationshipToken,
normalizeKtxRelationshipName,
pluralizeKtxRelationshipToken,
singularizeKtxRelationshipToken,
} from './relationship-name-similarity.js';
export type { KloRelationshipNormalizedName } from './relationship-name-similarity.js';
export { normalizeKloRelationshipName } from './relationship-name-similarity.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
export type { KtxRelationshipNormalizedName } from './relationship-name-similarity.js';
export { normalizeKtxRelationshipName } from './relationship-name-similarity.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import {
scoreKloRelationshipCandidate,
type KloRelationshipScoreBreakdown,
type KloRelationshipSignalVector,
scoreKtxRelationshipCandidate,
type KtxRelationshipScoreBreakdown,
type KtxRelationshipSignalVector,
} from './relationship-scoring.js';
export type KloRelationshipDiscoveryCandidateSource =
export type KtxRelationshipDiscoveryCandidateSource =
| 'exact_column_match'
| 'normalized_table_match'
| 'parent_table_name_match'
@ -31,44 +31,44 @@ export type KloRelationshipDiscoveryCandidateSource =
| 'embedding_similarity'
| 'llm_proposal';
export type KloRelationshipDiscoveryCandidateStatus = 'review';
export type KtxRelationshipDiscoveryCandidateStatus = 'review';
export interface KloRelationshipDiscoveryCandidateEvidence {
export interface KtxRelationshipDiscoveryCandidateEvidence {
sourceColumnBase: string;
targetTableBase: string;
targetColumnBase: string;
targetKeyScore: number;
nameScore: number;
reasons: string[];
signalVector?: KloRelationshipSignalVector;
scoreBreakdown?: KloRelationshipScoreBreakdown;
signalVector?: KtxRelationshipSignalVector;
scoreBreakdown?: KtxRelationshipScoreBreakdown;
embeddingSimilarity?: number;
llmConfidence?: number;
llmRationale?: string;
}
export interface KloRelationshipDiscoveryCandidate {
export interface KtxRelationshipDiscoveryCandidate {
id: string;
from: KloRelationshipEndpoint;
to: KloRelationshipEndpoint;
relationshipType: KloRelationshipType;
from: KtxRelationshipEndpoint;
to: KtxRelationshipEndpoint;
relationshipType: KtxRelationshipType;
confidence: number;
source: KloRelationshipDiscoveryCandidateSource;
status: KloRelationshipDiscoveryCandidateStatus;
evidence: KloRelationshipDiscoveryCandidateEvidence;
source: KtxRelationshipDiscoveryCandidateSource;
status: KtxRelationshipDiscoveryCandidateStatus;
evidence: KtxRelationshipDiscoveryCandidateEvidence;
}
export interface KloRelationshipDiscoveryCandidateOptions {
export interface KtxRelationshipDiscoveryCandidateOptions {
maxCandidatesPerColumn?: number;
maxCandidateParentTables?: number;
maxEmbeddingCandidatesPerColumn?: number;
minConfidence?: number;
embeddingSimilarityThreshold?: number;
useEmbeddings?: boolean;
profiles?: KloRelationshipProfileArtifact;
profiles?: KtxRelationshipProfileArtifact;
}
export interface KloRelationshipInferredTargetPk {
export interface KtxRelationshipInferredTargetPk {
table: string;
columns: string[];
score: number;
@ -76,12 +76,12 @@ export interface KloRelationshipInferredTargetPk {
incomingCandidateCount: number;
}
interface KloRelationshipSourceColumnReference {
interface KtxRelationshipSourceColumnReference {
base: string;
reason: string;
}
interface KloRelationshipTargetKeyEvidence {
interface KtxRelationshipTargetKeyEvidence {
score: number;
reasons: string[];
}
@ -98,26 +98,26 @@ const REFERENCE_SUFFIXES: Array<{ suffix: string; reason: string }> = [
];
const RELATIONSHIP_KEY_TARGET_SUFFIXES = ['_id', '_key', '_code', '_uuid'] as const;
function isRelationshipKeyShapedTarget(column: KloEnrichedColumn): boolean {
const normalized = normalizeKloRelationshipName(column.name);
function isRelationshipKeyShapedTarget(column: KtxEnrichedColumn): boolean {
const normalized = normalizeKtxRelationshipName(column.name);
return (
normalized.tokens.length >= 2 &&
RELATIONSHIP_KEY_TARGET_SUFFIXES.some((suffix) => normalized.normalized.endsWith(suffix))
);
}
function columnSuffixMatchesTarget(input: { fromColumn: KloEnrichedColumn; toColumn: KloEnrichedColumn }): boolean {
const source = normalizeKloRelationshipName(input.fromColumn.name).normalized;
const target = normalizeKloRelationshipName(input.toColumn.name).normalized;
function columnSuffixMatchesTarget(input: { fromColumn: KtxEnrichedColumn; toColumn: KtxEnrichedColumn }): boolean {
const source = normalizeKtxRelationshipName(input.fromColumn.name).normalized;
const target = normalizeKtxRelationshipName(input.toColumn.name).normalized;
return source !== target && target.length > 0 && source.endsWith(`_${target}`);
}
function normalizeType(column: KloEnrichedColumn): string {
function normalizeType(column: KtxEnrichedColumn): string {
const rawType = (column.normalizedType || column.nativeType || '').toLowerCase().trim();
return rawType.includes('(') ? (rawType.split('(')[0] ?? '') : rawType;
}
function typesCompatible(left: KloEnrichedColumn, right: KloEnrichedColumn): boolean {
function typesCompatible(left: KtxEnrichedColumn, right: KtxEnrichedColumn): boolean {
const leftType = normalizeType(left);
const rightType = normalizeType(right);
if (leftType === rightType) {
@ -155,12 +155,12 @@ function cosineSimilarity(left: readonly number[] | null, right: readonly number
return dot / (Math.sqrt(leftMagnitude) * Math.sqrt(rightMagnitude));
}
function hasUsableEmbedding(column: KloEnrichedColumn): boolean {
function hasUsableEmbedding(column: KtxEnrichedColumn): boolean {
return Array.isArray(column.embedding) && column.embedding.length > 0;
}
function sourceColumnReference(column: KloEnrichedColumn): KloRelationshipSourceColumnReference | null {
const normalized = normalizeKloRelationshipName(column.name);
function sourceColumnReference(column: KtxEnrichedColumn): KtxRelationshipSourceColumnReference | null {
const normalized = normalizeKtxRelationshipName(column.name);
if (SELF_REFERENCE_NAMES.has(normalized.normalized)) {
return { base: normalized.normalized.replace(/_id$/u, ''), reason: 'foreign_key_suffix' };
}
@ -171,7 +171,7 @@ function sourceColumnReference(column: KloEnrichedColumn): KloRelationshipSource
}
const base = normalized.normalized.slice(0, -item.suffix.length);
if (base.length > 1) {
return { base: singularizeKloRelationshipToken(base), reason: item.reason };
return { base: singularizeKtxRelationshipToken(base), reason: item.reason };
}
}
@ -179,7 +179,7 @@ function sourceColumnReference(column: KloEnrichedColumn): KloRelationshipSource
}
function addNormalizedTableAlias(aliases: Set<string>, name: string): void {
const normalized = normalizeKloRelationshipName(name);
const normalized = normalizeKtxRelationshipName(name);
if (normalized.normalized.length > 0) {
aliases.add(normalized.normalized);
}
@ -191,34 +191,34 @@ function addNormalizedTableAlias(aliases: Set<string>, name: string): void {
}
}
function tableAliases(table: KloEnrichedTable): Set<string> {
const normalized = normalizeKloRelationshipName(table.ref.name);
function tableAliases(table: KtxEnrichedTable): Set<string> {
const normalized = normalizeKtxRelationshipName(table.ref.name);
const aliases = new Set([normalized.normalized, normalized.singular, normalized.plural]);
if (normalized.tokens.length > 1) {
const lastToken = normalized.tokens[normalized.tokens.length - 1];
if (lastToken) {
aliases.add(lastToken);
const singularLastToken = singularizeKloRelationshipToken(lastToken);
const singularLastToken = singularizeKtxRelationshipToken(lastToken);
aliases.add(singularLastToken);
aliases.add(pluralizeKloRelationshipToken(singularLastToken));
aliases.add(pluralizeKtxRelationshipToken(singularLastToken));
}
}
return aliases;
}
function finalTableNamePart(table: KloEnrichedTable): string {
function finalTableNamePart(table: KtxEnrichedTable): string {
const parts = table.ref.name.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
return parts[parts.length - 1] ?? table.ref.name;
}
function parentTableNameAliases(table: KloEnrichedTable): Set<string> {
function parentTableNameAliases(table: KtxEnrichedTable): Set<string> {
const aliases = tableAliases(table);
addNormalizedTableAlias(aliases, finalTableNamePart(table));
return aliases;
}
function targetKeyScore(table: KloEnrichedTable, column: KloEnrichedColumn): number {
const columnName = normalizeKloRelationshipName(column.name).normalized;
function targetKeyScore(table: KtxEnrichedTable, column: KtxEnrichedColumn): number {
const columnName = normalizeKtxRelationshipName(column.name).normalized;
const tableKeyBases = parentTableNameAliases(table);
if (column.primaryKey) {
return 1;
@ -239,7 +239,7 @@ function targetKeyScore(table: KloEnrichedTable, column: KloEnrichedColumn): num
}
function profileColumn(
profiles: KloRelationshipProfileArtifact | undefined,
profiles: KtxRelationshipProfileArtifact | undefined,
tableName: string,
columnName: string,
) {
@ -247,11 +247,11 @@ function profileColumn(
}
function profileSampleOverlap(input: {
profiles: KloRelationshipProfileArtifact | undefined;
fromTable: KloEnrichedTable;
fromColumn: KloEnrichedColumn;
toTable: KloEnrichedTable;
toColumn: KloEnrichedColumn;
profiles: KtxRelationshipProfileArtifact | undefined;
fromTable: KtxEnrichedTable;
fromColumn: KtxEnrichedColumn;
toTable: KtxEnrichedTable;
toColumn: KtxEnrichedColumn;
}): number {
const source = profileColumn(input.profiles, input.fromTable.ref.name, input.fromColumn.name);
const target = profileColumn(input.profiles, input.toTable.ref.name, input.toColumn.name);
@ -263,14 +263,14 @@ function profileSampleOverlap(input: {
return overlap / source.sampleValues.length;
}
function tableProfileRowCount(profiles: KloRelationshipProfileArtifact | undefined, tableName: string): number | null {
function tableProfileRowCount(profiles: KtxRelationshipProfileArtifact | undefined, tableName: string): number | null {
return profiles?.tables.find((table) => table.table.name === tableName)?.rowCount ?? null;
}
function structuralPriorScore(input: {
profiles: KloRelationshipProfileArtifact | undefined;
fromTable: KloEnrichedTable;
toTable: KloEnrichedTable;
profiles: KtxRelationshipProfileArtifact | undefined;
fromTable: KtxEnrichedTable;
toTable: KtxEnrichedTable;
}): number {
if (input.fromTable.id === input.toTable.id) {
return 0.72;
@ -290,16 +290,16 @@ function structuralPriorScore(input: {
}
function candidateSignalVector(input: {
profiles: KloRelationshipProfileArtifact | undefined;
fromTable: KloEnrichedTable;
fromColumn: KloEnrichedColumn;
toTable: KloEnrichedTable;
toColumn: KloEnrichedColumn;
profiles: KtxRelationshipProfileArtifact | undefined;
fromTable: KtxEnrichedTable;
fromColumn: KtxEnrichedColumn;
toTable: KtxEnrichedTable;
toColumn: KtxEnrichedColumn;
targetKeyScore: number;
nameScore: number;
valueOverlap: number;
embeddingSimilarity?: number;
}): KloRelationshipSignalVector {
}): KtxRelationshipSignalVector {
const sourceProfile = profileColumn(input.profiles, input.fromTable.ref.name, input.fromColumn.name);
const targetProfile = profileColumn(input.profiles, input.toTable.ref.name, input.toColumn.name);
const targetUniqueness = targetProfile?.uniquenessRatio ?? input.targetKeyScore;
@ -321,11 +321,11 @@ function candidateSignalVector(input: {
}
function candidateParentTables(input: {
tables: readonly KloEnrichedTable[];
fromTable: KloEnrichedTable;
fromColumn: KloEnrichedColumn;
options: KloRelationshipDiscoveryCandidateOptions;
}): KloEnrichedTable[] {
tables: readonly KtxEnrichedTable[];
fromTable: KtxEnrichedTable;
fromColumn: KtxEnrichedColumn;
options: KtxRelationshipDiscoveryCandidateOptions;
}): KtxEnrichedTable[] {
const maxParentTables = input.options.maxCandidateParentTables ?? 20;
if (maxParentTables <= 0) {
return [];
@ -338,7 +338,7 @@ function candidateParentTables(input: {
maxParentTables,
}).map((item) => item.table);
const normalizedColumn = normalizeKloRelationshipName(input.fromColumn.name).normalized;
const normalizedColumn = normalizeKtxRelationshipName(input.fromColumn.name).normalized;
if (!SELF_REFERENCE_NAMES.has(normalizedColumn) || ranked.some((table) => table.id === input.fromTable.id)) {
return ranked;
}
@ -350,10 +350,10 @@ function candidateParentTables(input: {
}
function targetKeyEvidence(
table: KloEnrichedTable,
column: KloEnrichedColumn,
profiles: KloRelationshipProfileArtifact | undefined,
): KloRelationshipTargetKeyEvidence {
table: KtxEnrichedTable,
column: KtxEnrichedColumn,
profiles: KtxRelationshipProfileArtifact | undefined,
): KtxRelationshipTargetKeyEvidence {
const deterministicScore = targetKeyScore(table, column);
if (deterministicScore > 0) {
return { score: deterministicScore, reasons: ['target_key_like'] };
@ -364,7 +364,7 @@ function targetKeyEvidence(
return { score: 0, reasons: [] };
}
const columnName = normalizeKloRelationshipName(column.name).normalized;
const columnName = normalizeKtxRelationshipName(column.name).normalized;
if (columnName === 'code' || columnName.endsWith('_code') || columnName === 'key' || columnName.endsWith('_key')) {
return { score: 0.86, reasons: ['profile_unique_target'] };
}
@ -372,7 +372,7 @@ function targetKeyEvidence(
return { score: 0.78, reasons: ['profile_unique_target'] };
}
function endpoint(table: KloEnrichedTable, column: KloEnrichedColumn): KloRelationshipEndpoint {
function endpoint(table: KtxEnrichedTable, column: KtxEnrichedColumn): KtxRelationshipEndpoint {
return {
tableId: table.id,
columnIds: [column.id],
@ -381,11 +381,11 @@ function endpoint(table: KloEnrichedTable, column: KloEnrichedColumn): KloRelati
};
}
function relationshipId(from: KloRelationshipEndpoint, to: KloRelationshipEndpoint): string {
function relationshipId(from: KtxRelationshipEndpoint, to: KtxRelationshipEndpoint): string {
return `${from.tableId}:(${from.columnIds.join(',')})->${to.tableId}:(${to.columnIds.join(',')})`;
}
function endpointsHaveSameOrderedColumns(left: KloRelationshipEndpoint, right: KloRelationshipEndpoint): boolean {
function endpointsHaveSameOrderedColumns(left: KtxRelationshipEndpoint, right: KtxRelationshipEndpoint): boolean {
if (left.columnIds.length !== right.columnIds.length || left.columns.length !== right.columns.length) {
return false;
}
@ -394,11 +394,11 @@ function endpointsHaveSameOrderedColumns(left: KloRelationshipEndpoint, right: K
);
}
function isDegenerateSameColumnSelfLink(candidate: Pick<KloRelationshipDiscoveryCandidate, 'from' | 'to'>): boolean {
function isDegenerateSameColumnSelfLink(candidate: Pick<KtxRelationshipDiscoveryCandidate, 'from' | 'to'>): boolean {
return candidate.from.tableId === candidate.to.tableId && endpointsHaveSameOrderedColumns(candidate.from, candidate.to);
}
function singleRelationshipColumn(endpointValue: KloRelationshipEndpoint): string {
function singleRelationshipColumn(endpointValue: KtxRelationshipEndpoint): string {
const column = endpointValue.columns[0];
if (!column) {
throw new Error(`Expected relationship endpoint ${endpointValue.table.name} to contain one column`);
@ -406,7 +406,7 @@ function singleRelationshipColumn(endpointValue: KloRelationshipEndpoint): strin
return column;
}
function candidateSortKey(candidate: KloRelationshipDiscoveryCandidate): string {
function candidateSortKey(candidate: KtxRelationshipDiscoveryCandidate): string {
return `${candidate.from.table.name}.${singleRelationshipColumn(candidate.from)}->${candidate.to.table.name}.${singleRelationshipColumn(candidate.to)}`;
}
@ -415,9 +415,9 @@ function uniqueReasons(values: readonly string[]): string[] {
}
function mergeCandidateEvidence(
left: KloRelationshipDiscoveryCandidate,
right: KloRelationshipDiscoveryCandidate,
): KloRelationshipDiscoveryCandidate {
left: KtxRelationshipDiscoveryCandidate,
right: KtxRelationshipDiscoveryCandidate,
): KtxRelationshipDiscoveryCandidate {
const preferred = right.confidence > left.confidence && left.source === 'llm_proposal' ? right : left;
const supplement = preferred === left ? right : left;
return {
@ -432,7 +432,7 @@ function mergeCandidateEvidence(
};
}
function sourceForEvidence(reasons: string[]): KloRelationshipDiscoveryCandidateSource {
function sourceForEvidence(reasons: string[]): KtxRelationshipDiscoveryCandidateSource {
if (reasons.includes('self_reference')) {
return 'self_reference';
}
@ -461,19 +461,19 @@ function sourceForEvidence(reasons: string[]): KloRelationshipDiscoveryCandidate
}
function createCandidate(input: {
fromTable: KloEnrichedTable;
fromColumn: KloEnrichedColumn;
toTable: KloEnrichedTable;
toColumn: KloEnrichedColumn;
fromTable: KtxEnrichedTable;
fromColumn: KtxEnrichedColumn;
toTable: KtxEnrichedTable;
toColumn: KtxEnrichedColumn;
sourceBase: string;
targetBase: string;
targetKeyScore: number;
nameScore: number;
reasons: string[];
profiles: KloRelationshipProfileArtifact | undefined;
profiles: KtxRelationshipProfileArtifact | undefined;
valueOverlap: number;
embeddingSimilarity?: number;
}): KloRelationshipDiscoveryCandidate {
}): KtxRelationshipDiscoveryCandidate {
const from = endpoint(input.fromTable, input.fromColumn);
const to = endpoint(input.toTable, input.toColumn);
const signalVector = candidateSignalVector({
@ -487,7 +487,7 @@ function createCandidate(input: {
valueOverlap: input.valueOverlap,
embeddingSimilarity: input.embeddingSimilarity,
});
const scoreBreakdown = scoreKloRelationshipCandidate(signalVector);
const scoreBreakdown = scoreKtxRelationshipCandidate(signalVector);
return {
id: relationshipId(from, to),
@ -500,7 +500,7 @@ function createCandidate(input: {
evidence: {
sourceColumnBase: input.sourceBase,
targetTableBase: input.targetBase,
targetColumnBase: normalizeKloRelationshipName(input.toColumn.name).normalized,
targetColumnBase: normalizeKtxRelationshipName(input.toColumn.name).normalized,
targetKeyScore: input.targetKeyScore,
nameScore: input.nameScore,
reasons: input.reasons,
@ -513,10 +513,10 @@ function createCandidate(input: {
};
}
function generateKloEmbeddingRelationshipCandidates(
schema: KloEnrichedSchema,
options: KloRelationshipDiscoveryCandidateOptions,
): KloRelationshipDiscoveryCandidate[] {
function generateKtxEmbeddingRelationshipCandidates(
schema: KtxEnrichedSchema,
options: KtxRelationshipDiscoveryCandidateOptions,
): KtxRelationshipDiscoveryCandidate[] {
if (options.useEmbeddings === false) {
return [];
}
@ -524,7 +524,7 @@ function generateKloEmbeddingRelationshipCandidates(
const threshold = options.embeddingSimilarityThreshold ?? 0.92;
const maxCandidatesPerColumn = options.maxEmbeddingCandidatesPerColumn ?? options.maxCandidatesPerColumn ?? 25;
const tables = schema.tables.filter((table) => table.enabled);
const candidates: KloRelationshipDiscoveryCandidate[] = [];
const candidates: KtxRelationshipDiscoveryCandidate[] = [];
for (const fromTable of tables) {
for (const fromColumn of fromTable.columns) {
@ -532,7 +532,7 @@ function generateKloEmbeddingRelationshipCandidates(
continue;
}
const columnCandidates: KloRelationshipDiscoveryCandidate[] = [];
const columnCandidates: KtxRelationshipDiscoveryCandidate[] = [];
for (const toTable of candidateParentTables({ tables, fromTable, fromColumn, options })) {
if (fromTable.id === toTable.id) {
continue;
@ -553,8 +553,8 @@ function generateKloEmbeddingRelationshipCandidates(
continue;
}
const sourceBase = normalizeKloRelationshipName(fromColumn.name).normalized;
const targetBase = normalizeKloRelationshipName(toTable.ref.name).singular;
const sourceBase = normalizeKtxRelationshipName(fromColumn.name).normalized;
const targetBase = normalizeKtxRelationshipName(toTable.ref.name).singular;
const reasons = ['embedding_similarity', ...keyEvidence.reasons];
const candidate = createCandidate({
fromTable,
@ -592,14 +592,14 @@ function generateKloEmbeddingRelationshipCandidates(
return candidates;
}
export function generateKloRelationshipDiscoveryCandidates(
schema: KloEnrichedSchema,
options: KloRelationshipDiscoveryCandidateOptions = {},
): KloRelationshipDiscoveryCandidate[] {
export function generateKtxRelationshipDiscoveryCandidates(
schema: KtxEnrichedSchema,
options: KtxRelationshipDiscoveryCandidateOptions = {},
): KtxRelationshipDiscoveryCandidate[] {
const maxCandidatesPerColumn = options.maxCandidatesPerColumn ?? 25;
const minConfidence = options.minConfidence ?? 0.72;
const tables = schema.tables.filter((table) => table.enabled);
const candidates: KloRelationshipDiscoveryCandidate[] = [];
const candidates: KtxRelationshipDiscoveryCandidate[] = [];
for (const fromTable of tables) {
for (const fromColumn of fromTable.columns) {
@ -612,15 +612,15 @@ export function generateKloRelationshipDiscoveryCandidates(
}
const sourceBase = sourceReference.base;
const columnCandidates: KloRelationshipDiscoveryCandidate[] = [];
const columnCandidates: KtxRelationshipDiscoveryCandidate[] = [];
for (const toTable of candidateParentTables({ tables, fromTable, fromColumn, options })) {
const strictAliases = tableAliases(toTable);
const parentAliases = parentTableNameAliases(toTable);
const targetBase = normalizeKloRelationshipName(toTable.ref.name).singular;
const targetBase = normalizeKtxRelationshipName(toTable.ref.name).singular;
const sameTable = fromTable.id === toTable.id;
const nameMatchesTarget = strictAliases.has(sourceBase);
const parentTableNameMatcher = !sameTable && !nameMatchesTarget && parentAliases.has(sourceBase);
const selfReference = sameTable && SELF_REFERENCE_NAMES.has(normalizeKloRelationshipName(fromColumn.name).normalized);
const selfReference = sameTable && SELF_REFERENCE_NAMES.has(normalizeKtxRelationshipName(fromColumn.name).normalized);
const strictTableMatcher = (!sameTable && nameMatchesTarget) || selfReference;
for (const toColumn of toTable.columns) {
@ -665,7 +665,7 @@ export function generateKloRelationshipDiscoveryCandidates(
} else if (selfReference) {
reasons.push('self_reference');
nameScore = 0.82;
} else if (!suffixMatcher && normalizeKloRelationshipName(toTable.ref.name).singular === sourceBase) {
} else if (!suffixMatcher && normalizeKtxRelationshipName(toTable.ref.name).singular === sourceBase) {
reasons.push('normalized_table_name');
nameScore = 0.92;
} else if (!suffixMatcher && strictAliases.has(sourceBase)) {
@ -675,7 +675,7 @@ export function generateKloRelationshipDiscoveryCandidates(
if (
!suffixMatcher &&
!parentTableNameMatcher &&
normalizeKloRelationshipName(fromColumn.name).normalized === normalizeKloRelationshipName(toColumn.name).normalized
normalizeKtxRelationshipName(fromColumn.name).normalized === normalizeKtxRelationshipName(toColumn.name).normalized
) {
reasons.push('exact_column_name');
nameScore = Math.max(nameScore, 0.9);
@ -707,9 +707,9 @@ export function generateKloRelationshipDiscoveryCandidates(
}
}
candidates.push(...generateKloEmbeddingRelationshipCandidates(schema, options));
candidates.push(...generateKtxEmbeddingRelationshipCandidates(schema, options));
const byId = new Map<string, KloRelationshipDiscoveryCandidate>();
const byId = new Map<string, KtxRelationshipDiscoveryCandidate>();
for (const candidate of candidates) {
const existing = byId.get(candidate.id);
if (!existing || candidate.confidence > existing.confidence) {
@ -721,10 +721,10 @@ export function generateKloRelationshipDiscoveryCandidates(
);
}
export function mergeKloRelationshipDiscoveryCandidates(
candidates: readonly KloRelationshipDiscoveryCandidate[],
): KloRelationshipDiscoveryCandidate[] {
const byId = new Map<string, KloRelationshipDiscoveryCandidate>();
export function mergeKtxRelationshipDiscoveryCandidates(
candidates: readonly KtxRelationshipDiscoveryCandidate[],
): KtxRelationshipDiscoveryCandidate[] {
const byId = new Map<string, KtxRelationshipDiscoveryCandidate>();
for (const candidate of candidates) {
const existing = byId.get(candidate.id);
byId.set(candidate.id, existing ? mergeCandidateEvidence(existing, candidate) : candidate);
@ -732,9 +732,9 @@ export function mergeKloRelationshipDiscoveryCandidates(
return Array.from(byId.values()).sort((left, right) => candidateSortKey(left).localeCompare(candidateSortKey(right)));
}
export function inferKloRelationshipTargetPks(
candidates: readonly KloRelationshipDiscoveryCandidate[],
): KloRelationshipInferredTargetPk[] {
export function inferKtxRelationshipTargetPks(
candidates: readonly KtxRelationshipDiscoveryCandidate[],
): KtxRelationshipInferredTargetPk[] {
const incoming = new Map<string, { table: string; column: string; scores: number[] }>();
for (const candidate of candidates) {
const toColumn = singleRelationshipColumn(candidate.to);

24
packages/context/src/scan/relationship-composite-candidates.test.ts
View file

@ -1,20 +1,20 @@
import Database from 'better-sqlite3';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { snapshotToKloEnrichedSchema } from './local-enrichment.js';
import { loadKloRelationshipBenchmarkFixture, maskKloRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js';
import { discoverKloCompositeRelationships } from './relationship-composite-candidates.js';
import { profileKloRelationshipSchema, type KloRelationshipReadOnlyExecutor } from './relationship-profiling.js';
import type { KloQueryResult, KloReadOnlyQueryInput, KloScanContext } from './types.js';
import { snapshotToKtxEnrichedSchema } from './local-enrichment.js';
import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js';
import { discoverKtxCompositeRelationships } from './relationship-composite-candidates.js';
import { profileKtxRelationshipSchema, type KtxRelationshipReadOnlyExecutor } from './relationship-profiling.js';
import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from './types.js';
class TestSqliteExecutor implements KloRelationshipReadOnlyExecutor {
class TestSqliteExecutor implements KtxRelationshipReadOnlyExecutor {
private readonly db: Database.Database;
constructor(dataPath: string) {
this.db = new Database(dataPath, { readonly: true, fileMustExist: true });
}
async executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
async executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
return {
@ -33,13 +33,13 @@ class TestSqliteExecutor implements KloRelationshipReadOnlyExecutor {
describe('composite relationship discovery detector', () => {
it('infers composite primary keys and validates composite foreign keys from row evidence', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(
const fixture = await loadKtxRelationshipBenchmarkFixture(
join(fixtureRoot.pathname, 'composite_keys_no_declared_constraints'),
);
const snapshot = maskKloRelationshipBenchmarkSnapshot(fixture.snapshot, 'declared_pks_and_declared_fks_removed');
const schema = snapshotToKloEnrichedSchema(snapshot, new Map());
const snapshot = maskKtxRelationshipBenchmarkSnapshot(fixture.snapshot, 'declared_pks_and_declared_fks_removed');
const schema = snapshotToKtxEnrichedSchema(snapshot, new Map());
const executor = new TestSqliteExecutor(fixture.dataPath ?? '');
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: snapshot.connectionId,
driver: snapshot.driver,
schema,
@ -47,7 +47,7 @@ describe('composite relationship discovery detector', () => {
ctx: { runId: 'test:composite-profile' },
});
const result = await discoverKloCompositeRelationships({
const result = await discoverKtxCompositeRelationships({
connectionId: snapshot.connectionId,
driver: snapshot.driver,
schema,

160
packages/context/src/scan/relationship-composite-candidates.ts
View file

@ -1,29 +1,29 @@
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable, KloRelationshipType } from './enrichment-types.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable, KtxRelationshipType } from './enrichment-types.js';
import {
formatKloRelationshipTableRef,
quoteKloRelationshipIdentifier,
type KloRelationshipProfileArtifact,
type KloRelationshipReadOnlyExecutor,
formatKtxRelationshipTableRef,
quoteKtxRelationshipIdentifier,
type KtxRelationshipProfileArtifact,
type KtxRelationshipReadOnlyExecutor,
} from './relationship-profiling.js';
import type { KloConnectionDriver, KloQueryResult, KloScanContext, KloTableRef } from './types.js';
import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext, KtxTableRef } from './types.js';
export type KloCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected';
export type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected';
export interface KloCompositeRelationshipTupleEndpoint {
export interface KtxCompositeRelationshipTupleEndpoint {
tableId: string;
columnIds: string[];
table: KloTableRef;
table: KtxTableRef;
columns: string[];
}
export interface KloCompositePrimaryKeyCandidate {
export interface KtxCompositePrimaryKeyCandidate {
id: string;
tableId: string;
table: KloTableRef;
table: KtxTableRef;
columns: string[];
columnIds: string[];
score: number;
status: KloCompositeRelationshipStatus;
status: KtxCompositeRelationshipStatus;
evidence: {
rowCount: number;
distinctCount: number;
@ -33,7 +33,7 @@ export interface KloCompositePrimaryKeyCandidate {
};
}
export interface KloCompositeRelationshipValidationEvidence {
export interface KtxCompositeRelationshipValidationEvidence {
targetUniqueness: number;
sourceCoverage: number;
violationCount: number;
@ -44,24 +44,24 @@ export interface KloCompositeRelationshipValidationEvidence {
reasons: string[];
}
export interface KloCompositeRelationshipCandidate {
export interface KtxCompositeRelationshipCandidate {
id: string;
from: KloCompositeRelationshipTupleEndpoint;
to: KloCompositeRelationshipTupleEndpoint;
relationshipType: KloRelationshipType;
from: KtxCompositeRelationshipTupleEndpoint;
to: KtxCompositeRelationshipTupleEndpoint;
relationshipType: KtxRelationshipType;
confidence: number;
status: KloCompositeRelationshipStatus;
status: KtxCompositeRelationshipStatus;
source: 'composite_profile_match';
validation: KloCompositeRelationshipValidationEvidence;
validation: KtxCompositeRelationshipValidationEvidence;
}
export interface DiscoverKloCompositeRelationshipsInput {
export interface DiscoverKtxCompositeRelationshipsInput {
connectionId: string;
driver: KloConnectionDriver;
schema: KloEnrichedSchema;
profiles: KloRelationshipProfileArtifact;
executor: KloRelationshipReadOnlyExecutor | null;
ctx: KloScanContext;
driver: KtxConnectionDriver;
schema: KtxEnrichedSchema;
profiles: KtxRelationshipProfileArtifact;
executor: KtxRelationshipReadOnlyExecutor | null;
ctx: KtxScanContext;
maxCompositeWidth?: number;
maxColumnsPerTable?: number;
minPrimaryKeyUniqueness?: number;
@ -69,9 +69,9 @@ export interface DiscoverKloCompositeRelationshipsInput {
maxViolationRatio?: number;
}
export interface DiscoverKloCompositeRelationshipsResult {
primaryKeys: KloCompositePrimaryKeyCandidate[];
relationships: KloCompositeRelationshipCandidate[];
export interface DiscoverKtxCompositeRelationshipsResult {
primaryKeys: KtxCompositePrimaryKeyCandidate[];
relationships: KtxCompositeRelationshipCandidate[];
queryCount: number;
warnings: string[];
}
@ -83,11 +83,11 @@ const DEFAULT_MIN_PRIMARY_KEY_UNIQUENESS = 0.98;
const DEFAULT_MIN_SOURCE_COVERAGE = 0.9;
const DEFAULT_MAX_VIOLATION_RATIO = 0.01;
function enabledTables(schema: KloEnrichedSchema): KloEnrichedTable[] {
function enabledTables(schema: KtxEnrichedSchema): KtxEnrichedTable[] {
return schema.tables.filter((table) => table.enabled);
}
function tableRowCount(profiles: KloRelationshipProfileArtifact, tableName: string): number {
function tableRowCount(profiles: KtxRelationshipProfileArtifact, tableName: string): number {
return profiles.tables.find((item) => item.table.name === tableName)?.rowCount ?? 0;
}
@ -95,7 +95,7 @@ function profileKey(tableName: string, columnName: string): string {
return `${tableName}.${columnName}`;
}
function profileNullRate(profiles: KloRelationshipProfileArtifact, tableName: string, columnName: string): number {
function profileNullRate(profiles: KtxRelationshipProfileArtifact, tableName: string, columnName: string): number {
return profiles.columns[profileKey(tableName, columnName)]?.nullRate ?? 1;
}
@ -106,7 +106,7 @@ function normalizedColumnName(name: string): string {
.replace(/^_+|_+$/gu, '');
}
function columnNameScore(column: KloEnrichedColumn): number {
function columnNameScore(column: KtxEnrichedColumn): number {
const parts = normalizedColumnName(column.name).split('_').filter(Boolean);
if (parts.some((part) => KEY_NAME_PARTS.has(part))) {
return 1;
@ -122,7 +122,7 @@ function keyLikeTableNameParts(tableName: string): Set<string> {
return new Set(nameParts(tableName).filter((part) => KEY_NAME_PARTS.has(part)));
}
function tupleCoversTableNameKeyParts(tableName: string, columns: readonly KloEnrichedColumn[]): boolean {
function tupleCoversTableNameKeyParts(tableName: string, columns: readonly KtxEnrichedColumn[]): boolean {
const required = keyLikeTableNameParts(tableName);
if (required.size === 0) {
return true;
@ -132,10 +132,10 @@ function tupleCoversTableNameKeyParts(tableName: string, columns: readonly KloEn
}
function candidateKeyColumns(input: {
table: KloEnrichedTable;
profiles: KloRelationshipProfileArtifact;
table: KtxEnrichedTable;
profiles: KtxRelationshipProfileArtifact;
maxColumnsPerTable: number;
}): KloEnrichedColumn[] {
}): KtxEnrichedColumn[] {
return input.table.columns
.map((column, index) => ({ column, index }))
.filter(({ column }) => {
@ -154,8 +154,8 @@ function candidateKeyColumns(input: {
}
function hasStrongSingleColumnKey(input: {
table: KloEnrichedTable;
profiles: KloRelationshipProfileArtifact;
table: KtxEnrichedTable;
profiles: KtxRelationshipProfileArtifact;
minPrimaryKeyUniqueness: number;
}): boolean {
return input.table.columns.some((column) => {
@ -196,7 +196,7 @@ function relationshipKey(input: {
return `${tupleKey(input.fromTable, input.fromColumns)}->${tupleKey(input.toTable, input.toColumns)}`;
}
function tupleEndpoint(table: KloEnrichedTable, columns: readonly KloEnrichedColumn[]): KloCompositeRelationshipTupleEndpoint {
function tupleEndpoint(table: KtxEnrichedTable, columns: readonly KtxEnrichedColumn[]): KtxCompositeRelationshipTupleEndpoint {
return {
tableId: table.id,
columnIds: columns.map((column) => column.id),
@ -205,11 +205,11 @@ function tupleEndpoint(table: KloEnrichedTable, columns: readonly KloEnrichedCol
};
}
function row(result: KloQueryResult): unknown[] {
function row(result: KtxQueryResult): unknown[] {
return result.rows[0] ?? [];
}
function numberAt(result: KloQueryResult, header: string): number {
function numberAt(result: KtxQueryResult, header: string): number {
const index = result.headers.findIndex((candidate) => candidate.toLowerCase() === header.toLowerCase());
const value = row(result)[index];
if (typeof value === 'number') {
@ -224,28 +224,28 @@ function numberAt(result: KloQueryResult, header: string): number {
return 0;
}
function topSql(driver: KloConnectionDriver, limit: number): string {
function topSql(driver: KtxConnectionDriver, limit: number): string {
if (driver === 'sqlserver') {
return ` TOP (${Math.max(1, Math.floor(limit))})`;
}
return '';
}
function limitSql(driver: KloConnectionDriver, limit: number): string {
function limitSql(driver: KtxConnectionDriver, limit: number): string {
if (driver === 'sqlserver') {
return '';
}
return ` LIMIT ${Math.max(1, Math.floor(limit))}`;
}
function aliasedTupleSelect(driver: KloConnectionDriver, columns: readonly string[]): string {
function aliasedTupleSelect(driver: KtxConnectionDriver, columns: readonly string[]): string {
return columns
.map((column, index) => `${quoteKloRelationshipIdentifier(driver, column)} AS c${index}`)
.map((column, index) => `${quoteKtxRelationshipIdentifier(driver, column)} AS c${index}`)
.join(', ');
}
function nonNullPredicate(driver: KloConnectionDriver, columns: readonly string[]): string {
return columns.map((column) => `${quoteKloRelationshipIdentifier(driver, column)} IS NOT NULL`).join(' AND ');
function nonNullPredicate(driver: KtxConnectionDriver, columns: readonly string[]): string {
return columns.map((column) => `${quoteKtxRelationshipIdentifier(driver, column)} IS NOT NULL`).join(' AND ');
}
function tupleEquality(columns: number): string {
@ -255,11 +255,11 @@ function tupleEquality(columns: number): string {
}
function buildTupleDistinctSql(input: {
driver: KloConnectionDriver;
table: KloTableRef;
driver: KtxConnectionDriver;
table: KtxTableRef;
columns: readonly string[];
}): string {
const tableSql = formatKloRelationshipTableRef(input.driver, input.table);
const tableSql = formatKtxRelationshipTableRef(input.driver, input.table);
return [
'WITH tuple_values AS (',
`SELECT DISTINCT ${aliasedTupleSelect(input.driver, input.columns)} FROM ${tableSql}`,
@ -270,15 +270,15 @@ function buildTupleDistinctSql(input: {
}
function buildCompositeCoverageSql(input: {
driver: KloConnectionDriver;
childTable: KloTableRef;
driver: KtxConnectionDriver;
childTable: KtxTableRef;
childColumns: readonly string[];
parentTable: KloTableRef;
parentTable: KtxTableRef;
parentColumns: readonly string[];
maxDistinctSourceValues: number;
}): string {
const childTableSql = formatKloRelationshipTableRef(input.driver, input.childTable);
const parentTableSql = formatKloRelationshipTableRef(input.driver, input.parentTable);
const childTableSql = formatKtxRelationshipTableRef(input.driver, input.childTable);
const parentTableSql = formatKtxRelationshipTableRef(input.driver, input.parentTable);
const top = topSql(input.driver, input.maxDistinctSourceValues);
const limit = limitSql(input.driver, input.maxDistinctSourceValues);
return [
@ -305,7 +305,7 @@ function relationshipStatus(input: {
violationRatio: number;
minSourceCoverage: number;
maxViolationRatio: number;
}): KloCompositeRelationshipStatus {
}): KtxCompositeRelationshipStatus {
if (
input.targetUniqueness >= DEFAULT_MIN_PRIMARY_KEY_UNIQUENESS &&
input.sourceCoverage >= input.minSourceCoverage &&
@ -320,7 +320,7 @@ function relationshipStatus(input: {
}
function hasAcceptedSubset(
accepted: readonly KloCompositePrimaryKeyCandidate[],
accepted: readonly KtxCompositePrimaryKeyCandidate[],
tableName: string,
columns: readonly string[],
): boolean {
@ -335,15 +335,15 @@ function hasAcceptedSubset(
async function detectCompositePrimaryKeys(input: {
connectionId: string;
driver: KloConnectionDriver;
table: KloEnrichedTable;
profiles: KloRelationshipProfileArtifact;
executor: KloRelationshipReadOnlyExecutor;
ctx: KloScanContext;
driver: KtxConnectionDriver;
table: KtxEnrichedTable;
profiles: KtxRelationshipProfileArtifact;
executor: KtxRelationshipReadOnlyExecutor;
ctx: KtxScanContext;
maxCompositeWidth: number;
maxColumnsPerTable: number;
minPrimaryKeyUniqueness: number;
}): Promise<{ primaryKeys: KloCompositePrimaryKeyCandidate[]; queryCount: number }> {
}): Promise<{ primaryKeys: KtxCompositePrimaryKeyCandidate[]; queryCount: number }> {
const rowCount = tableRowCount(input.profiles, input.table.ref.name);
if (rowCount === 0) {
return { primaryKeys: [], queryCount: 0 };
@ -363,7 +363,7 @@ async function detectCompositePrimaryKeys(input: {
profiles: input.profiles,
maxColumnsPerTable: input.maxColumnsPerTable,
});
const primaryKeys: KloCompositePrimaryKeyCandidate[] = [];
const primaryKeys: KtxCompositePrimaryKeyCandidate[] = [];
let queryCount = 0;
for (let width = 2; width <= input.maxCompositeWidth; width += 1) {
@ -423,11 +423,11 @@ async function detectCompositePrimaryKeys(input: {
};
}
function columnsByName(table: KloEnrichedTable): Map<string, KloEnrichedColumn> {
function columnsByName(table: KtxEnrichedTable): Map<string, KtxEnrichedColumn> {
return new Map(table.columns.map((column) => [column.name, column]));
}
function compatibleTuple(sourceColumns: readonly KloEnrichedColumn[], targetColumns: readonly KloEnrichedColumn[]): boolean {
function compatibleTuple(sourceColumns: readonly KtxEnrichedColumn[], targetColumns: readonly KtxEnrichedColumn[]): boolean {
if (sourceColumns.length !== targetColumns.length) {
return false;
}
@ -439,17 +439,17 @@ function compatibleTuple(sourceColumns: readonly KloEnrichedColumn[], targetColu
async function validateCompositeRelationship(input: {
connectionId: string;
driver: KloConnectionDriver;
sourceTable: KloEnrichedTable;
sourceColumns: readonly KloEnrichedColumn[];
targetKey: KloCompositePrimaryKeyCandidate;
targetTable: KloEnrichedTable;
targetColumns: readonly KloEnrichedColumn[];
executor: KloRelationshipReadOnlyExecutor;
ctx: KloScanContext;
driver: KtxConnectionDriver;
sourceTable: KtxEnrichedTable;
sourceColumns: readonly KtxEnrichedColumn[];
targetKey: KtxCompositePrimaryKeyCandidate;
targetTable: KtxEnrichedTable;
targetColumns: readonly KtxEnrichedColumn[];
executor: KtxRelationshipReadOnlyExecutor;
ctx: KtxScanContext;
minSourceCoverage: number;
maxViolationRatio: number;
}): Promise<{ relationship: KloCompositeRelationshipCandidate; queryCount: number }> {
}): Promise<{ relationship: KtxCompositeRelationshipCandidate; queryCount: number }> {
const result = await input.executor.executeReadOnly(
{
connectionId: input.connectionId,
@ -525,9 +525,9 @@ async function validateCompositeRelationship(input: {
};
}
export async function discoverKloCompositeRelationships(
input: DiscoverKloCompositeRelationshipsInput,
): Promise<DiscoverKloCompositeRelationshipsResult> {
export async function discoverKtxCompositeRelationships(
input: DiscoverKtxCompositeRelationshipsInput,
): Promise<DiscoverKtxCompositeRelationshipsResult> {
if (!input.executor || !input.profiles.sqlAvailable) {
return {
primaryKeys: [],
@ -546,7 +546,7 @@ export async function discoverKloCompositeRelationships(
};
const tables = enabledTables(input.schema);
const tableByName = new Map(tables.map((table) => [table.ref.name, table]));
const primaryKeys: KloCompositePrimaryKeyCandidate[] = [];
const primaryKeys: KtxCompositePrimaryKeyCandidate[] = [];
let queryCount = 0;
for (const table of tables) {
@ -565,7 +565,7 @@ export async function discoverKloCompositeRelationships(
queryCount += result.queryCount;
}
const relationships: KloCompositeRelationshipCandidate[] = [];
const relationships: KtxCompositeRelationshipCandidate[] = [];
for (const targetKey of primaryKeys) {
const targetTable = tableByName.get(targetKey.table.name);
if (!targetTable) {

38
packages/context/src/scan/relationship-diagnostics.test.ts
View file

@ -1,13 +1,13 @@
import { describe, expect, it } from 'vitest';
import type { KloEnrichedRelationship, KloRelationshipEndpoint } from './enrichment-types.js';
import type { KloResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
import type { KtxEnrichedRelationship, KtxRelationshipEndpoint } from './enrichment-types.js';
import type { KtxResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
import {
buildKloRelationshipArtifacts,
buildKloRelationshipDiagnostics,
emptyKloRelationshipProfileArtifact,
buildKtxRelationshipArtifacts,
buildKtxRelationshipDiagnostics,
emptyKtxRelationshipProfileArtifact,
} from './relationship-diagnostics.js';
function endpoint(table: string, column: string): KloRelationshipEndpoint {
function endpoint(table: string, column: string): KtxRelationshipEndpoint {
return {
tableId: table,
columnIds: [`${table}.${column}`],
@ -23,7 +23,7 @@ function enrichedRelationship(input: {
toTable: string;
toColumn: string;
confidence?: number;
}): KloEnrichedRelationship {
}): KtxEnrichedRelationship {
return {
id: input.id,
source: 'inferred',
@ -43,7 +43,7 @@ function resolvedRelationship(input: {
pkScore?: number;
validationReasons?: string[];
graphReasons?: string[];
}): KloResolvedRelationshipDiscoveryCandidate {
}): KtxResolvedRelationshipDiscoveryCandidate {
return {
id: input.id,
from: endpoint('orders', 'customer_id'),
@ -99,7 +99,7 @@ function resolvedRelationship(input: {
describe('relationship diagnostics artifacts', () => {
it('groups graph-resolved relationships and preserves evidence reasons', () => {
const artifacts = buildKloRelationshipArtifacts({
const artifacts = buildKtxRelationshipArtifacts({
connectionId: 'warehouse',
resolvedRelationships: [
resolvedRelationship({ id: 'accepted-edge', status: 'accepted', source: 'llm_proposal' }),
@ -142,7 +142,7 @@ describe('relationship diagnostics artifacts', () => {
});
it('adapts legacy relationship updates into the richer artifact shape', () => {
const artifacts = buildKloRelationshipArtifacts({
const artifacts = buildKtxRelationshipArtifacts({
connectionId: 'warehouse',
relationshipUpdate: {
connectionId: 'warehouse',
@ -184,7 +184,7 @@ describe('relationship diagnostics artifacts', () => {
});
it('deduplicates resolved and formal relationship update artifacts by edge id', () => {
const artifacts = buildKloRelationshipArtifacts({
const artifacts = buildKtxRelationshipArtifacts({
connectionId: 'warehouse',
resolvedRelationships: [
{
@ -254,7 +254,7 @@ describe('relationship diagnostics artifacts', () => {
});
it('explains validation-unavailable review candidates', () => {
const artifacts = buildKloRelationshipArtifacts({
const artifacts = buildKtxRelationshipArtifacts({
connectionId: 'warehouse',
resolvedRelationships: [
resolvedRelationship({
@ -265,13 +265,13 @@ describe('relationship diagnostics artifacts', () => {
}),
],
});
const profile = emptyKloRelationshipProfileArtifact({
const profile = emptyKtxRelationshipProfileArtifact({
connectionId: 'warehouse',
driver: 'sqlite',
reason: 'read_only_sql_unavailable',
});
const diagnostics = buildKloRelationshipDiagnostics({
const diagnostics = buildKtxRelationshipDiagnostics({
connectionId: 'warehouse',
generatedAt: '2026-05-07T12:00:00.000Z',
artifacts,
@ -279,7 +279,7 @@ describe('relationship diagnostics artifacts', () => {
warnings: [
{
code: 'connector_capability_missing',
message: 'KLO scan connector cannot run standalone statistical relationship validation',
message: 'KTX scan connector cannot run standalone statistical relationship validation',
recoverable: true,
metadata: { capability: 'readOnlySql' },
},
@ -300,12 +300,12 @@ describe('relationship diagnostics artifacts', () => {
});
it('explains empty relationship output as a no-candidate outcome', () => {
const artifacts = buildKloRelationshipArtifacts({ connectionId: 'warehouse' });
const diagnostics = buildKloRelationshipDiagnostics({
const artifacts = buildKtxRelationshipArtifacts({ connectionId: 'warehouse' });
const diagnostics = buildKtxRelationshipDiagnostics({
connectionId: 'warehouse',
generatedAt: '2026-05-07T12:00:00.000Z',
artifacts,
profile: emptyKloRelationshipProfileArtifact({
profile: emptyKtxRelationshipProfileArtifact({
connectionId: 'warehouse',
driver: 'sqlite',
reason: 'relationship_profiling_not_run',
@ -318,7 +318,7 @@ describe('relationship diagnostics artifacts', () => {
});
it('records composite relationship endpoints in relationship artifacts', () => {
const artifacts = buildKloRelationshipArtifacts({
const artifacts = buildKtxRelationshipArtifacts({
connectionId: 'warehouse',
compositeRelationships: [
{

130
packages/context/src/scan/relationship-diagnostics.ts
View file

@ -1,18 +1,18 @@
import type {
KloEnrichedRelationship,
KloRelationshipEndpoint,
KloRelationshipType,
KloRelationshipUpdate,
KtxEnrichedRelationship,
KtxRelationshipEndpoint,
KtxRelationshipType,
KtxRelationshipUpdate,
} from './enrichment-types.js';
import type {
KloResolvedRelationshipDiscoveryCandidate,
KloResolvedRelationshipStatus,
KtxResolvedRelationshipDiscoveryCandidate,
KtxResolvedRelationshipStatus,
} from './relationship-graph-resolver.js';
import type { KloCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KloConnectionDriver, KloScanWarning } from './types.js';
import type { KtxCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxConnectionDriver, KtxScanWarning } from './types.js';
export interface KloRelationshipArtifactEndpoint {
export interface KtxRelationshipArtifactEndpoint {
tableId: string;
columnIds: string[];
table: {
@ -23,13 +23,13 @@ export interface KloRelationshipArtifactEndpoint {
columns: string[];
}
export interface KloRelationshipArtifactEdge {
export interface KtxRelationshipArtifactEdge {
id: string;
status: KloResolvedRelationshipStatus;
status: KtxResolvedRelationshipStatus;
source: string;
from: KloRelationshipArtifactEndpoint;
to: KloRelationshipArtifactEndpoint;
relationshipType: KloRelationshipType;
from: KtxRelationshipArtifactEndpoint;
to: KtxRelationshipArtifactEndpoint;
relationshipType: KtxRelationshipType;
confidence: number;
pkScore: number | null;
fkScore: number | null;
@ -40,88 +40,88 @@ export interface KloRelationshipArtifactEdge {
reasons: string[];
}
export interface KloRelationshipArtifact {
export interface KtxRelationshipArtifact {
connectionId: string;
accepted: KloRelationshipArtifactEdge[];
review: KloRelationshipArtifactEdge[];
rejected: KloRelationshipArtifactEdge[];
skipped: KloRelationshipUpdate['skipped'];
accepted: KtxRelationshipArtifactEdge[];
review: KtxRelationshipArtifactEdge[];
rejected: KtxRelationshipArtifactEdge[];
skipped: KtxRelationshipUpdate['skipped'];
}
export interface KloRelationshipDiagnosticsSummary {
export interface KtxRelationshipDiagnosticsSummary {
accepted: number;
review: number;
rejected: number;
skipped: number;
}
export interface KloRelationshipDiagnosticsValidation {
export interface KtxRelationshipDiagnosticsValidation {
available: boolean;
sqlAvailable: boolean;
queryCount: number;
}
export interface KloRelationshipDiagnosticsThresholds {
export interface KtxRelationshipDiagnosticsThresholds {
acceptThreshold: number;
reviewThreshold: number;
}
export interface KloRelationshipDiagnosticsPolicy {
export interface KtxRelationshipDiagnosticsPolicy {
validationRequiredForManifest: boolean;
maxCandidatesPerColumn: number;
profileSampleRows: number;
validationConcurrency: number;
}
export interface KloRelationshipDiagnosticsArtifact {
export interface KtxRelationshipDiagnosticsArtifact {
connectionId: string;
generatedAt: string;
summary: KloRelationshipDiagnosticsSummary;
summary: KtxRelationshipDiagnosticsSummary;
noAcceptedReason: string | null;
candidateCountsBySource: Record<string, number>;
validation: KloRelationshipDiagnosticsValidation;
thresholds: KloRelationshipDiagnosticsThresholds;
policy: KloRelationshipDiagnosticsPolicy;
warnings: KloScanWarning[];
validation: KtxRelationshipDiagnosticsValidation;
thresholds: KtxRelationshipDiagnosticsThresholds;
policy: KtxRelationshipDiagnosticsPolicy;
warnings: KtxScanWarning[];
profileWarnings: string[];
}
export interface BuildKloRelationshipArtifactsInput {
export interface BuildKtxRelationshipArtifactsInput {
connectionId: string;
relationshipUpdate?: KloRelationshipUpdate | null;
resolvedRelationships?: readonly KloResolvedRelationshipDiscoveryCandidate[];
compositeRelationships?: readonly KloCompositeRelationshipCandidate[];
relationshipUpdate?: KtxRelationshipUpdate | null;
resolvedRelationships?: readonly KtxResolvedRelationshipDiscoveryCandidate[];
compositeRelationships?: readonly KtxCompositeRelationshipCandidate[];
}
export interface BuildKloRelationshipDiagnosticsInput {
export interface BuildKtxRelationshipDiagnosticsInput {
connectionId: string;
artifacts: KloRelationshipArtifact;
profile: KloRelationshipProfileArtifact;
warnings?: readonly KloScanWarning[];
thresholds?: Partial<KloRelationshipDiagnosticsThresholds>;
policy?: Partial<KloRelationshipDiagnosticsPolicy>;
artifacts: KtxRelationshipArtifact;
profile: KtxRelationshipProfileArtifact;
warnings?: readonly KtxScanWarning[];
thresholds?: Partial<KtxRelationshipDiagnosticsThresholds>;
policy?: Partial<KtxRelationshipDiagnosticsPolicy>;
generatedAt?: string;
}
export interface EmptyKloRelationshipProfileArtifactInput {
export interface EmptyKtxRelationshipProfileArtifactInput {
connectionId: string;
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
reason: string;
}
const DEFAULT_THRESHOLDS: KloRelationshipDiagnosticsThresholds = {
const DEFAULT_THRESHOLDS: KtxRelationshipDiagnosticsThresholds = {
acceptThreshold: 0.85,
reviewThreshold: 0.55,
};
const DEFAULT_POLICY: KloRelationshipDiagnosticsPolicy = {
const DEFAULT_POLICY: KtxRelationshipDiagnosticsPolicy = {
validationRequiredForManifest: true,
maxCandidatesPerColumn: 25,
profileSampleRows: 10000,
validationConcurrency: 4,
};
function endpointArtifact(endpoint: KloRelationshipEndpoint): KloRelationshipArtifactEndpoint {
function endpointArtifact(endpoint: KtxRelationshipEndpoint): KtxRelationshipArtifactEndpoint {
return {
tableId: endpoint.tableId,
columnIds: endpoint.columnIds,
@ -139,9 +139,9 @@ function uniqueReasons(values: readonly string[]): string[] {
}
function relationshipUpdateEdge(
relationship: KloEnrichedRelationship,
relationship: KtxEnrichedRelationship,
status: 'accepted' | 'rejected',
): KloRelationshipArtifactEdge {
): KtxRelationshipArtifactEdge {
const acceptedReason = relationship.source === 'formal' ? 'formal_metadata_accepted' : 'accepted_relationship_update';
return {
id: relationship.id,
@ -161,7 +161,7 @@ function relationshipUpdateEdge(
};
}
function resolvedEdge(candidate: KloResolvedRelationshipDiscoveryCandidate): KloRelationshipArtifactEdge {
function resolvedEdge(candidate: KtxResolvedRelationshipDiscoveryCandidate): KtxRelationshipArtifactEdge {
return {
id: candidate.id,
status: candidate.status,
@ -184,7 +184,7 @@ function resolvedEdge(candidate: KloResolvedRelationshipDiscoveryCandidate): Klo
};
}
function compositeEndpointArtifact(endpoint: KloCompositeRelationshipCandidate['from']): KloRelationshipArtifactEndpoint {
function compositeEndpointArtifact(endpoint: KtxCompositeRelationshipCandidate['from']): KtxRelationshipArtifactEndpoint {
return {
tableId: endpoint.tableId,
columnIds: endpoint.columnIds,
@ -197,7 +197,7 @@ function compositeEndpointArtifact(endpoint: KloCompositeRelationshipCandidate['
};
}
function compositeEdge(candidate: KloCompositeRelationshipCandidate): KloRelationshipArtifactEdge {
function compositeEdge(candidate: KtxCompositeRelationshipCandidate): KtxRelationshipArtifactEdge {
return {
id: candidate.id,
status: candidate.status,
@ -216,7 +216,7 @@ function compositeEdge(candidate: KloCompositeRelationshipCandidate): KloRelatio
};
}
function emptyArtifacts(connectionId: string): KloRelationshipArtifact {
function emptyArtifacts(connectionId: string): KtxRelationshipArtifact {
return {
connectionId,
accepted: [],
@ -226,13 +226,13 @@ function emptyArtifacts(connectionId: string): KloRelationshipArtifact {
};
}
function pushUniqueEdge(edges: KloRelationshipArtifactEdge[], edge: KloRelationshipArtifactEdge): void {
function pushUniqueEdge(edges: KtxRelationshipArtifactEdge[], edge: KtxRelationshipArtifactEdge): void {
if (!edges.some((item) => item.id === edge.id)) {
edges.push(edge);
}
}
export function buildKloRelationshipArtifacts(input: BuildKloRelationshipArtifactsInput): KloRelationshipArtifact {
export function buildKtxRelationshipArtifacts(input: BuildKtxRelationshipArtifactsInput): KtxRelationshipArtifact {
const artifacts = emptyArtifacts(input.connectionId);
if (input.resolvedRelationships) {
@ -279,11 +279,11 @@ export function buildKloRelationshipArtifacts(input: BuildKloRelationshipArtifac
};
}
function allEdges(artifacts: KloRelationshipArtifact): KloRelationshipArtifactEdge[] {
function allEdges(artifacts: KtxRelationshipArtifact): KtxRelationshipArtifactEdge[] {
return [...artifacts.accepted, ...artifacts.review, ...artifacts.rejected];
}
function candidateCountsBySource(artifacts: KloRelationshipArtifact): Record<string, number> {
function candidateCountsBySource(artifacts: KtxRelationshipArtifact): Record<string, number> {
const counts: Record<string, number> = {};
for (const edge of allEdges(artifacts)) {
counts[edge.source] = (counts[edge.source] ?? 0) + 1;
@ -291,13 +291,13 @@ function candidateCountsBySource(artifacts: KloRelationshipArtifact): Record<str
return Object.fromEntries(Object.entries(counts).sort(([left], [right]) => left.localeCompare(right)));
}
function hasReason(artifacts: KloRelationshipArtifact, reason: string): boolean {
function hasReason(artifacts: KtxRelationshipArtifact, reason: string): boolean {
return allEdges(artifacts).some((edge) => edge.reasons.includes(reason));
}
function noAcceptedReason(input: {
artifacts: KloRelationshipArtifact;
profile: KloRelationshipProfileArtifact;
artifacts: KtxRelationshipArtifact;
profile: KtxRelationshipProfileArtifact;
}): string | null {
if (input.artifacts.accepted.length > 0) {
return null;
@ -319,9 +319,9 @@ function noAcceptedReason(input: {
return 'no candidate pairs passed type compatibility';
}
export function emptyKloRelationshipProfileArtifact(
input: EmptyKloRelationshipProfileArtifactInput,
): KloRelationshipProfileArtifact {
export function emptyKtxRelationshipProfileArtifact(
input: EmptyKtxRelationshipProfileArtifactInput,
): KtxRelationshipProfileArtifact {
return {
connectionId: input.connectionId,
driver: input.driver,
@ -333,12 +333,12 @@ export function emptyKloRelationshipProfileArtifact(
};
}
export function buildKloRelationshipDiagnostics(
input: BuildKloRelationshipDiagnosticsInput,
): KloRelationshipDiagnosticsArtifact {
export function buildKtxRelationshipDiagnostics(
input: BuildKtxRelationshipDiagnosticsInput,
): KtxRelationshipDiagnosticsArtifact {
const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds };
const policy = { ...DEFAULT_POLICY, ...input.policy };
const summary: KloRelationshipDiagnosticsSummary = {
const summary: KtxRelationshipDiagnosticsSummary = {
accepted: input.artifacts.accepted.length,
review: input.artifacts.review.length,
rejected: input.artifacts.rejected.length,

92
packages/context/src/scan/relationship-discovery.test.ts
View file

@ -1,21 +1,21 @@
import type { KloLlmProvider } from '@klo/llm';
import type { KtxLlmProvider } from '@ktx/llm';
import Database from 'better-sqlite3';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { buildDefaultKloProjectConfig } from '../project/config.js';
import { snapshotToKloEnrichedSchema } from './local-enrichment.js';
import { buildDefaultKtxProjectConfig } from '../project/config.js';
import { snapshotToKtxEnrichedSchema } from './local-enrichment.js';
import {
loadKloRelationshipBenchmarkFixture,
maskKloRelationshipBenchmarkSnapshot,
loadKtxRelationshipBenchmarkFixture,
maskKtxRelationshipBenchmarkSnapshot,
} from './relationship-benchmarks.js';
import { discoverKloRelationships } from './relationship-discovery.js';
import { createKloConnectorCapabilities } from './types.js';
import type { KloQueryResult, KloReadOnlyQueryInput, KloScanConnector, KloScanContext, KloSchemaSnapshot } from './types.js';
import { discoverKtxRelationships } from './relationship-discovery.js';
import { createKtxConnectorCapabilities } from './types.js';
import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanConnector, KtxScanContext, KtxSchemaSnapshot } from './types.js';
class InMemorySqliteExecutor {
readonly db = new Database(':memory:');
queryCount = 0;
executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
this.queryCount += 1;
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
@ -32,7 +32,7 @@ class InMemorySqliteExecutor {
}
}
function snapshot(): KloSchemaSnapshot {
function snapshot(): KtxSchemaSnapshot {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -102,7 +102,7 @@ function snapshot(): KloSchemaSnapshot {
};
}
function declaredForeignKeySnapshot(): KloSchemaSnapshot {
function declaredForeignKeySnapshot(): KtxSchemaSnapshot {
const source = snapshot();
return {
...source,
@ -131,7 +131,7 @@ function declaredForeignKeySnapshot(): KloSchemaSnapshot {
};
}
function naturalKeySnapshot(): KloSchemaSnapshot {
function naturalKeySnapshot(): KtxSchemaSnapshot {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -201,11 +201,11 @@ function naturalKeySnapshot(): KloSchemaSnapshot {
};
}
function connector(executor: InMemorySqliteExecutor | null): KloScanConnector {
function connector(executor: InMemorySqliteExecutor | null): KtxScanConnector {
return {
id: 'sqlite:test',
driver: 'sqlite',
capabilities: createKloConnectorCapabilities({
capabilities: createKtxConnectorCapabilities({
readOnlySql: executor !== null,
columnStats: executor !== null,
tableSampling: false,
@ -216,11 +216,11 @@ function connector(executor: InMemorySqliteExecutor | null): KloScanConnector {
};
}
function llmProvider(): KloLlmProvider {
function llmProvider(): KtxLlmProvider {
const model = { modelId: 'claude-sonnet-4-6', provider: 'anthropic' };
return {
getModel: vi.fn(() => model as ReturnType<KloLlmProvider['getModel']>),
getModelByName: vi.fn(() => model as ReturnType<KloLlmProvider['getModelByName']>),
getModel: vi.fn(() => model as ReturnType<KtxLlmProvider['getModel']>),
getModelByName: vi.fn(() => model as ReturnType<KtxLlmProvider['getModelByName']>),
cacheMarker: vi.fn(),
repairToolCallHandler: vi.fn(),
thinkingProviderOptions: vi.fn(() => ({})),
@ -236,17 +236,17 @@ function llmProvider(): KloLlmProvider {
cacheTools: true,
cacheHistory: true,
vertexFallbackTo5m: false,
}) as ReturnType<KloLlmProvider['promptCachingConfig']>,
}) as ReturnType<KtxLlmProvider['promptCachingConfig']>,
),
activeBackend: vi.fn(() => 'anthropic' as ReturnType<KloLlmProvider['activeBackend']>),
activeBackend: vi.fn(() => 'anthropic' as ReturnType<KtxLlmProvider['activeBackend']>),
};
}
function relationshipSettings() {
return buildDefaultKloProjectConfig('warehouse').scan.relationships;
return buildDefaultKtxProjectConfig('warehouse').scan.relationships;
}
function llmOnlyRelationshipSnapshot(): KloSchemaSnapshot {
function llmOnlyRelationshipSnapshot(): KtxSchemaSnapshot {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -324,11 +324,11 @@ describe('production relationship discovery', () => {
INSERT INTO orders (id, account_id) VALUES (10, 1), (11, 1), (12, 2);
`);
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: connector(executor),
schema: snapshotToKloEnrichedSchema(snapshot()),
schema: snapshotToKtxEnrichedSchema(snapshot()),
context: { runId: 'relationship-run-1' },
settings: relationshipSettings(),
});
@ -363,14 +363,14 @@ describe('production relationship discovery', () => {
`);
const schema = naturalKeySnapshot();
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: {
...connector(executor),
introspect: async () => schema,
},
schema: snapshotToKloEnrichedSchema(schema),
schema: snapshotToKtxEnrichedSchema(schema),
context: { runId: 'natural-key-relationship-run' },
settings: relationshipSettings(),
});
@ -403,7 +403,7 @@ describe('production relationship discovery', () => {
`);
const sourceSnapshot = llmOnlyRelationshipSnapshot();
const schema = snapshotToKloEnrichedSchema(
const schema = snapshotToKtxEnrichedSchema(
sourceSnapshot,
new Map([
['customers.id', [1, 0, 0]],
@ -413,7 +413,7 @@ describe('production relationship discovery', () => {
]),
);
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: {
@ -446,11 +446,11 @@ describe('production relationship discovery', () => {
});
it('keeps candidates review-only when read-only SQL is unavailable', async () => {
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: connector(null),
schema: snapshotToKloEnrichedSchema(snapshot()),
schema: snapshotToKtxEnrichedSchema(snapshot()),
context: { runId: 'relationship-run-no-sql' },
settings: relationshipSettings(),
});
@ -464,7 +464,7 @@ describe('production relationship discovery', () => {
});
expect(result.warnings).toContainEqual({
code: 'connector_capability_missing',
message: 'KLO scan connector cannot run read-only SQL relationship validation',
message: 'KTX scan connector cannot run read-only SQL relationship validation',
recoverable: true,
metadata: { capability: 'readOnlySql' },
});
@ -472,11 +472,11 @@ describe('production relationship discovery', () => {
it('accepts formal metadata relationships when read-only SQL is unavailable', async () => {
const sourceSnapshot = declaredForeignKeySnapshot();
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: connector(null),
schema: snapshotToKloEnrichedSchema(sourceSnapshot),
schema: snapshotToKtxEnrichedSchema(sourceSnapshot),
context: { runId: 'formal-metadata-no-sql' },
settings: relationshipSettings(),
});
@ -521,11 +521,11 @@ describe('production relationship discovery', () => {
},
}));
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: connector(executor),
schema: snapshotToKloEnrichedSchema(llmOnlyRelationshipSnapshot()),
schema: snapshotToKtxEnrichedSchema(llmOnlyRelationshipSnapshot()),
context: { runId: 'llm-relationship-orchestrator' },
settings: relationshipSettings(),
llmProvider: llmProvider(),
@ -557,16 +557,16 @@ describe('production relationship discovery', () => {
`);
const settings = {
...buildDefaultKloProjectConfig('warehouse').scan.relationships,
...buildDefaultKtxProjectConfig('warehouse').scan.relationships,
acceptThreshold: 0.99,
reviewThreshold: 0.55,
};
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: connector(executor),
schema: snapshotToKloEnrichedSchema(snapshot()),
schema: snapshotToKtxEnrichedSchema(snapshot()),
context: { runId: 'configured-thresholds' },
settings,
});
@ -623,17 +623,17 @@ describe('production relationship discovery', () => {
],
});
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: 'warehouse',
driver: 'sqlite',
connector: {
...connector(executor),
introspect: async () => richSnapshot,
},
schema: snapshotToKloEnrichedSchema(richSnapshot),
schema: snapshotToKtxEnrichedSchema(richSnapshot),
context: { runId: 'candidate-cap' },
settings: {
...buildDefaultKloProjectConfig('warehouse').scan.relationships,
...buildDefaultKtxProjectConfig('warehouse').scan.relationships,
maxCandidatesPerColumn: 1,
},
});
@ -652,13 +652,13 @@ describe('production relationship discovery', () => {
'../../test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints',
import.meta.url,
);
const fixture = await loadKloRelationshipBenchmarkFixture(fixtureRoot.pathname);
const maskedSnapshot = maskKloRelationshipBenchmarkSnapshot(fixture.snapshot, 'declared_pks_and_declared_fks_removed');
const fixture = await loadKtxRelationshipBenchmarkFixture(fixtureRoot.pathname);
const maskedSnapshot = maskKtxRelationshipBenchmarkSnapshot(fixture.snapshot, 'declared_pks_and_declared_fks_removed');
const database = new Database(fixture.dataPath ?? '', { readonly: true, fileMustExist: true });
const testConnector: KloScanConnector = {
const testConnector: KtxScanConnector = {
id: 'sqlite:composite',
driver: 'sqlite',
capabilities: createKloConnectorCapabilities({
capabilities: createKtxConnectorCapabilities({
readOnlySql: true,
columnStats: true,
tableSampling: false,
@ -677,11 +677,11 @@ describe('production relationship discovery', () => {
},
};
const result = await discoverKloRelationships({
const result = await discoverKtxRelationships({
connectionId: maskedSnapshot.connectionId,
driver: maskedSnapshot.driver,
connector: testConnector,
schema: snapshotToKloEnrichedSchema(maskedSnapshot, new Map()),
schema: snapshotToKtxEnrichedSchema(maskedSnapshot, new Map()),
context: { runId: 'test:production-composite' },
settings: relationshipSettings(),
});

152
packages/context/src/scan/relationship-discovery.ts
View file

@ -1,63 +1,63 @@
import type { KloLlmProvider } from '@klo/llm';
import type { KloScanRelationshipConfig } from '../project/config.js';
import type { KloEnrichedRelationship, KloEnrichedSchema, KloRelationshipUpdate } from './enrichment-types.js';
import type { KtxLlmProvider } from '@ktx/llm';
import type { KtxScanRelationshipConfig } from '../project/config.js';
import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js';
import {
generateKloRelationshipDiscoveryCandidates,
type KloRelationshipDiscoveryCandidate,
mergeKloRelationshipDiscoveryCandidates,
generateKtxRelationshipDiscoveryCandidates,
type KtxRelationshipDiscoveryCandidate,
mergeKtxRelationshipDiscoveryCandidates,
} from './relationship-candidates.js';
import {
discoverKloCompositeRelationships,
type KloCompositeRelationshipCandidate,
discoverKtxCompositeRelationships,
type KtxCompositeRelationshipCandidate,
} from './relationship-composite-candidates.js';
import { collectKloFormalMetadataRelationships } from './relationship-formal-metadata.js';
import { collectKtxFormalMetadataRelationships } from './relationship-formal-metadata.js';
import {
type KloResolvedRelationshipDiscoveryCandidate,
resolveKloRelationshipGraph,
type KtxResolvedRelationshipDiscoveryCandidate,
resolveKtxRelationshipGraph,
} from './relationship-graph-resolver.js';
import {
type KloRelationshipLlmProposalGenerateText,
proposeKloRelationshipCandidatesWithLlm,
type KtxRelationshipLlmProposalGenerateText,
proposeKtxRelationshipCandidatesWithLlm,
} from './relationship-llm-proposal.js';
import {
createKloRelationshipProfileCache,
type KloRelationshipProfileArtifact,
type KloRelationshipReadOnlyExecutor,
profileKloRelationshipSchema,
createKtxRelationshipProfileCache,
type KtxRelationshipProfileArtifact,
type KtxRelationshipReadOnlyExecutor,
profileKtxRelationshipSchema,
} from './relationship-profiling.js';
import { validateKloRelationshipDiscoveryCandidates } from './relationship-validation.js';
import { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js';
import type {
KloConnectionDriver,
KloScanConnector,
KloScanContext,
KloScanEnrichmentSummary,
KloScanRelationshipSummary,
KloScanWarning,
KtxConnectionDriver,
KtxScanConnector,
KtxScanContext,
KtxScanEnrichmentSummary,
KtxScanRelationshipSummary,
KtxScanWarning,
} from './types.js';
export interface DiscoverKloRelationshipsInput {
export interface DiscoverKtxRelationshipsInput {
connectionId: string;
driver: KloConnectionDriver;
connector: KloScanConnector;
schema: KloEnrichedSchema;
context: KloScanContext;
settings: KloScanRelationshipConfig;
llmProvider?: KloLlmProvider | null;
generateText?: KloRelationshipLlmProposalGenerateText;
driver: KtxConnectionDriver;
connector: KtxScanConnector;
schema: KtxEnrichedSchema;
context: KtxScanContext;
settings: KtxScanRelationshipConfig;
llmProvider?: KtxLlmProvider | null;
generateText?: KtxRelationshipLlmProposalGenerateText;
}
export interface DiscoverKloRelationshipsResult {
relationshipUpdate: KloRelationshipUpdate;
relationships: KloScanRelationshipSummary;
profile: KloRelationshipProfileArtifact;
resolvedRelationships: KloResolvedRelationshipDiscoveryCandidate[];
compositeRelationships: KloCompositeRelationshipCandidate[];
statisticalValidation: KloScanEnrichmentSummary['statisticalValidation'];
llmRelationshipValidation: KloScanEnrichmentSummary['llmRelationshipValidation'];
warnings: KloScanWarning[];
export interface DiscoverKtxRelationshipsResult {
relationshipUpdate: KtxRelationshipUpdate;
relationships: KtxScanRelationshipSummary;
profile: KtxRelationshipProfileArtifact;
resolvedRelationships: KtxResolvedRelationshipDiscoveryCandidate[];
compositeRelationships: KtxCompositeRelationshipCandidate[];
statisticalValidation: KtxScanEnrichmentSummary['statisticalValidation'];
llmRelationshipValidation: KtxScanEnrichmentSummary['llmRelationshipValidation'];
warnings: KtxScanWarning[];
}
function relationshipFromResolved(candidate: KloResolvedRelationshipDiscoveryCandidate): KloEnrichedRelationship {
function relationshipFromResolved(candidate: KtxResolvedRelationshipDiscoveryCandidate): KtxEnrichedRelationship {
return {
id: candidate.id,
source: 'inferred',
@ -69,7 +69,7 @@ function relationshipFromResolved(candidate: KloResolvedRelationshipDiscoveryCan
};
}
function relationshipFromComposite(candidate: KloCompositeRelationshipCandidate): KloEnrichedRelationship {
function relationshipFromComposite(candidate: KtxCompositeRelationshipCandidate): KtxEnrichedRelationship {
return {
id: candidate.id,
source: 'inferred',
@ -91,22 +91,22 @@ function relationshipFromComposite(candidate: KloCompositeRelationshipCandidate)
};
}
function relationshipId(input: Pick<KloEnrichedRelationship, 'from' | 'to'>): string {
function relationshipId(input: Pick<KtxEnrichedRelationship, 'from' | 'to'>): string {
return `${input.from.tableId}:(${input.from.columnIds.join(',')})->${input.to.tableId}:(${input.to.columnIds.join(',')})`;
}
function nonFormalAcceptedRelationships(input: {
formalIds: ReadonlySet<string>;
resolvedRelationships: readonly KloResolvedRelationshipDiscoveryCandidate[];
}): KloEnrichedRelationship[] {
resolvedRelationships: readonly KtxResolvedRelationshipDiscoveryCandidate[];
}): KtxEnrichedRelationship[] {
return input.resolvedRelationships
.filter((candidate) => candidate.status === 'accepted' && !input.formalIds.has(candidate.id))
.map(relationshipFromResolved);
}
function relationshipSummary(
resolvedRelationships: readonly KloResolvedRelationshipDiscoveryCandidate[],
): KloScanRelationshipSummary {
resolvedRelationships: readonly KtxResolvedRelationshipDiscoveryCandidate[],
): KtxScanRelationshipSummary {
return {
accepted: resolvedRelationships.filter((candidate) => candidate.status === 'accepted').length,
review: resolvedRelationships.filter((candidate) => candidate.status === 'review').length,
@ -115,7 +115,7 @@ function relationshipSummary(
};
}
function compositeSummary(relationships: readonly KloCompositeRelationshipCandidate[]): KloScanRelationshipSummary {
function compositeSummary(relationships: readonly KtxCompositeRelationshipCandidate[]): KtxScanRelationshipSummary {
return {
accepted: relationships.filter((candidate) => candidate.status === 'accepted').length,
review: relationships.filter((candidate) => candidate.status === 'review').length,
@ -126,18 +126,18 @@ function compositeSummary(relationships: readonly KloCompositeRelationshipCandid
async function detectCompositeRelationships(input: {
connectionId: string;
driver: DiscoverKloRelationshipsInput['driver'];
schema: KloEnrichedSchema;
profile: KloRelationshipProfileArtifact;
executor: KloRelationshipReadOnlyExecutor | null;
context: DiscoverKloRelationshipsInput['context'];
warnings: KloScanWarning[];
}): Promise<KloCompositeRelationshipCandidate[]> {
driver: DiscoverKtxRelationshipsInput['driver'];
schema: KtxEnrichedSchema;
profile: KtxRelationshipProfileArtifact;
executor: KtxRelationshipReadOnlyExecutor | null;
context: DiscoverKtxRelationshipsInput['context'];
warnings: KtxScanWarning[];
}): Promise<KtxCompositeRelationshipCandidate[]> {
if (!input.executor || !input.profile.sqlAvailable) {
return [];
}
try {
const compositeDetection = await discoverKloCompositeRelationships({
const compositeDetection = await discoverKtxCompositeRelationships({
connectionId: input.connectionId,
driver: input.driver,
schema: input.schema,
@ -157,7 +157,7 @@ async function detectCompositeRelationships(input: {
} catch (error) {
input.warnings.push({
code: 'relationship_validation_failed',
message: `KLO composite relationship detection failed: ${error instanceof Error ? error.message : String(error)}`,
message: `KTX composite relationship detection failed: ${error instanceof Error ? error.message : String(error)}`,
recoverable: true,
metadata: { source: 'composite_relationship_detection' },
});
@ -168,8 +168,8 @@ async function detectCompositeRelationships(input: {
function combinedRelationshipSummary(input: {
formalAccepted: number;
formalSkipped: number;
resolvedRelationships: readonly KloResolvedRelationshipDiscoveryCandidate[];
}): KloScanRelationshipSummary {
resolvedRelationships: readonly KtxResolvedRelationshipDiscoveryCandidate[];
}): KtxScanRelationshipSummary {
const graph = relationshipSummary(input.resolvedRelationships);
return {
accepted: input.formalAccepted + graph.accepted,
@ -179,9 +179,9 @@ function combinedRelationshipSummary(input: {
};
}
function sqlExecutor(input: DiscoverKloRelationshipsInput): {
executor: KloRelationshipReadOnlyExecutor | null;
warnings: KloScanWarning[];
function sqlExecutor(input: DiscoverKtxRelationshipsInput): {
executor: KtxRelationshipReadOnlyExecutor | null;
warnings: KtxScanWarning[];
} {
if (!input.connector.capabilities.readOnlySql) {
return {
@ -189,7 +189,7 @@ function sqlExecutor(input: DiscoverKloRelationshipsInput): {
warnings: [
{
code: 'connector_capability_missing',
message: 'KLO scan connector cannot run read-only SQL relationship validation',
message: 'KTX scan connector cannot run read-only SQL relationship validation',
recoverable: true,
metadata: { capability: 'readOnlySql' },
},
@ -203,7 +203,7 @@ function sqlExecutor(input: DiscoverKloRelationshipsInput): {
warnings: [
{
code: 'relationship_validation_failed',
message: 'KLO scan connector advertises readOnlySql but does not expose executeReadOnly',
message: 'KTX scan connector advertises readOnlySql but does not expose executeReadOnly',
recoverable: true,
metadata: { capability: 'readOnlySql' },
},
@ -219,13 +219,13 @@ function sqlExecutor(input: DiscoverKloRelationshipsInput): {
};
}
export async function discoverKloRelationships(
input: DiscoverKloRelationshipsInput,
): Promise<DiscoverKloRelationshipsResult> {
export async function discoverKtxRelationships(
input: DiscoverKtxRelationshipsInput,
): Promise<DiscoverKtxRelationshipsResult> {
const { executor, warnings } = sqlExecutor(input);
const formalMetadata = collectKloFormalMetadataRelationships(input.schema);
const profileCache = createKloRelationshipProfileCache();
const profile = await profileKloRelationshipSchema({
const formalMetadata = collectKtxFormalMetadataRelationships(input.schema);
const profileCache = createKtxRelationshipProfileCache();
const profile = await profileKtxRelationshipSchema({
connectionId: input.connectionId,
driver: input.driver,
schema: input.schema,
@ -234,7 +234,7 @@ export async function discoverKloRelationships(
profileSampleRows: input.settings.profileSampleRows,
cache: profileCache,
});
const deterministicCandidates: KloRelationshipDiscoveryCandidate[] = generateKloRelationshipDiscoveryCandidates(
const deterministicCandidates: KtxRelationshipDiscoveryCandidate[] = generateKtxRelationshipDiscoveryCandidates(
input.schema,
{
maxCandidatesPerColumn: input.settings.maxCandidatesPerColumn,
@ -242,7 +242,7 @@ export async function discoverKloRelationships(
},
);
const llmProposalResult = input.settings.llmProposals
? await proposeKloRelationshipCandidatesWithLlm({
? await proposeKtxRelationshipCandidatesWithLlm({
connectionId: input.connectionId,
schema: input.schema,
profile,
@ -253,12 +253,12 @@ export async function discoverKloRelationships(
generateText: input.generateText,
})
: { candidates: [], warnings: [], llmCalls: 0, summary: 'skipped' as const };
const candidates = mergeKloRelationshipDiscoveryCandidates([
const candidates = mergeKtxRelationshipDiscoveryCandidates([
...deterministicCandidates,
...llmProposalResult.candidates,
]).filter((candidate) => !formalMetadata.acceptedIds.has(candidate.id));
warnings.push(...llmProposalResult.warnings);
const validated = await validateKloRelationshipDiscoveryCandidates({
const validated = await validateKtxRelationshipDiscoveryCandidates({
connectionId: input.connectionId,
driver: input.driver,
candidates,
@ -274,7 +274,7 @@ export async function discoverKloRelationships(
validationBudget: input.settings.validationBudget,
},
});
const graph = resolveKloRelationshipGraph({
const graph = resolveKtxRelationshipGraph({
schema: input.schema,
profiles: profile,
candidates: validated,

34
packages/context/src/scan/relationship-feedback-calibration.test.ts
View file

@ -1,19 +1,19 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import { describe, expect, it, vi } from 'vitest';
import {
buildKloRelationshipFeedbackCalibrationReport,
buildKtxRelationshipFeedbackCalibrationReport,
calibrateLocalRelationshipFeedbackLabels,
formatKloRelationshipFeedbackCalibrationMarkdown,
formatKtxRelationshipFeedbackCalibrationMarkdown,
} from './relationship-feedback-calibration.js';
import type {
ExportLocalRelationshipFeedbackLabelsResult,
KloRelationshipFeedbackLabel,
KtxRelationshipFeedbackLabel,
} from './relationship-feedback-export.js';
function label(
input: Partial<KloRelationshipFeedbackLabel> &
Pick<KloRelationshipFeedbackLabel, 'candidateId' | 'decision' | 'score'>,
): KloRelationshipFeedbackLabel {
input: Partial<KtxRelationshipFeedbackLabel> &
Pick<KtxRelationshipFeedbackLabel, 'candidateId' | 'decision' | 'score'>,
): KtxRelationshipFeedbackLabel {
return {
schemaVersion: 1,
previousStatus: 'review',
@ -38,7 +38,7 @@ function label(
};
}
function feedback(labels: KloRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult {
function feedback(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult {
return {
generatedAt: '2026-05-07T13:00:00.000Z',
filters: { connectionId: null, decision: 'all' },
@ -56,7 +56,7 @@ function feedback(labels: KloRelationshipFeedbackLabel[]): ExportLocalRelationsh
describe('relationship feedback calibration', () => {
it('builds score buckets and threshold-band summary from feedback labels', () => {
const report = buildKloRelationshipFeedbackCalibrationReport(
const report = buildKtxRelationshipFeedbackCalibrationReport(
feedback([
label({
candidateId: 'orders:orders.customer_id->customers:customers.id',
@ -124,7 +124,7 @@ describe('relationship feedback calibration', () => {
});
it('keeps unscored labels visible without treating them as threshold predictions', () => {
const report = buildKloRelationshipFeedbackCalibrationReport(
const report = buildKtxRelationshipFeedbackCalibrationReport(
feedback([
label({
candidateId: 'orders:orders.note_id->notes:notes.id',
@ -161,7 +161,7 @@ describe('relationship feedback calibration', () => {
});
it('formats a stable markdown summary for human CLI output', () => {
const report = buildKloRelationshipFeedbackCalibrationReport(
const report = buildKtxRelationshipFeedbackCalibrationReport(
feedback([
label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }),
label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }),
@ -172,18 +172,18 @@ describe('relationship feedback calibration', () => {
},
);
expect(formatKloRelationshipFeedbackCalibrationMarkdown(report)).toContain(
'KLO relationship feedback calibration',
expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain(
'KTX relationship feedback calibration',
);
expect(formatKloRelationshipFeedbackCalibrationMarkdown(report)).toContain('Total labels: 2');
expect(formatKloRelationshipFeedbackCalibrationMarkdown(report)).toContain('Accepted-band precision: 1.000');
expect(formatKloRelationshipFeedbackCalibrationMarkdown(report)).toContain(
expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain('Total labels: 2');
expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain('Accepted-band precision: 1.000');
expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain(
'0.75-1.00: total=1 accepted=1 rejected=0 acceptanceRate=1.000',
);
});
it('wraps the feedback exporter and preserves exporter warnings', async () => {
const project = { projectDir: '/tmp/klo-project' } as KloLocalProject;
const project = { projectDir: '/tmp/ktx-project' } as KtxLocalProject;
const exportLocalRelationshipFeedbackLabels = vi.fn(async () => ({
...feedback([
label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }),

64
packages/context/src/scan/relationship-feedback-calibration.ts
View file

@ -1,36 +1,36 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import {
exportLocalRelationshipFeedbackLabels,
type ExportLocalRelationshipFeedbackLabelsInput,
type ExportLocalRelationshipFeedbackLabelsResult,
type KloRelationshipFeedbackExportWarning,
type KloRelationshipFeedbackLabel,
type KtxRelationshipFeedbackExportWarning,
type KtxRelationshipFeedbackLabel,
} from './relationship-feedback-export.js';
import type { KloResolvedRelationshipStatus } from './relationship-graph-resolver.js';
import type { KloRelationshipReviewDecisionValue } from './relationship-review-decisions.js';
import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js';
import type { KtxRelationshipReviewDecisionValue } from './relationship-review-decisions.js';
const DEFAULT_ACCEPT_THRESHOLD = 0.85;
const DEFAULT_REVIEW_THRESHOLD = 0.55;
type CalibrationPredictedStatus = KloResolvedRelationshipStatus | 'unscored';
type CalibrationPredictedStatus = KtxResolvedRelationshipStatus | 'unscored';
interface Thresholds {
accept: number;
review: number;
}
export interface BuildKloRelationshipFeedbackCalibrationReportInput {
export interface BuildKtxRelationshipFeedbackCalibrationReportInput {
acceptThreshold?: number;
reviewThreshold?: number;
}
export interface CalibrateLocalRelationshipFeedbackLabelsInput
extends ExportLocalRelationshipFeedbackLabelsInput,
BuildKloRelationshipFeedbackCalibrationReportInput {
BuildKtxRelationshipFeedbackCalibrationReportInput {
exportLocalRelationshipFeedbackLabels?: typeof exportLocalRelationshipFeedbackLabels;
}
export interface KloRelationshipFeedbackCalibrationBucket {
export interface KtxRelationshipFeedbackCalibrationBucket {
label: string;
minInclusive: number;
maxInclusive: number;
@ -40,10 +40,10 @@ export interface KloRelationshipFeedbackCalibrationBucket {
acceptanceRate: number | null;
}
export interface KloRelationshipFeedbackCalibrationLabel {
export interface KtxRelationshipFeedbackCalibrationLabel {
candidateId: string;
decision: KloRelationshipReviewDecisionValue;
previousStatus: KloRelationshipFeedbackLabel['previousStatus'];
decision: KtxRelationshipReviewDecisionValue;
previousStatus: KtxRelationshipFeedbackLabel['previousStatus'];
predictedStatus: CalibrationPredictedStatus;
bucket: string;
score: number | null;
@ -59,7 +59,7 @@ export interface KloRelationshipFeedbackCalibrationLabel {
reasons: string[];
}
export interface KloRelationshipFeedbackCalibrationReport {
export interface KtxRelationshipFeedbackCalibrationReport {
generatedAt: string;
filters: ExportLocalRelationshipFeedbackLabelsResult['filters'];
thresholds: Thresholds;
@ -78,9 +78,9 @@ export interface KloRelationshipFeedbackCalibrationReport {
meanAcceptedScore: number | null;
meanRejectedScore: number | null;
};
buckets: KloRelationshipFeedbackCalibrationBucket[];
labels: KloRelationshipFeedbackCalibrationLabel[];
warnings: KloRelationshipFeedbackExportWarning[];
buckets: KtxRelationshipFeedbackCalibrationBucket[];
labels: KtxRelationshipFeedbackCalibrationLabel[];
warnings: KtxRelationshipFeedbackExportWarning[];
}
const BUCKETS = [
@ -90,7 +90,7 @@ const BUCKETS = [
{ label: '0.75-1.00', minInclusive: 0.75, maxInclusive: 1 },
] as const;
function thresholds(input: BuildKloRelationshipFeedbackCalibrationReportInput): Thresholds {
function thresholds(input: BuildKtxRelationshipFeedbackCalibrationReportInput): Thresholds {
return {
accept: input.acceptThreshold ?? DEFAULT_ACCEPT_THRESHOLD,
review: input.reviewThreshold ?? DEFAULT_REVIEW_THRESHOLD,
@ -133,9 +133,9 @@ function predictedStatus(score: number | null, currentThresholds: Thresholds): C
}
function calibrationLabel(
label: KloRelationshipFeedbackLabel,
label: KtxRelationshipFeedbackLabel,
currentThresholds: Thresholds,
): KloRelationshipFeedbackCalibrationLabel {
): KtxRelationshipFeedbackCalibrationLabel {
return {
candidateId: label.candidateId,
decision: label.decision,
@ -157,8 +157,8 @@ function calibrationLabel(
}
function summarize(
labels: readonly KloRelationshipFeedbackCalibrationLabel[],
): KloRelationshipFeedbackCalibrationReport['summary'] {
labels: readonly KtxRelationshipFeedbackCalibrationLabel[],
): KtxRelationshipFeedbackCalibrationReport['summary'] {
const scored = labels.filter((label) => label.score !== null);
const predictedAccepted = scored.filter((label) => label.predictedStatus === 'accepted');
const predictedReview = scored.filter((label) => label.predictedStatus === 'review');
@ -193,8 +193,8 @@ function summarize(
}
function buildBuckets(
labels: readonly KloRelationshipFeedbackCalibrationLabel[],
): KloRelationshipFeedbackCalibrationBucket[] {
labels: readonly KtxRelationshipFeedbackCalibrationLabel[],
): KtxRelationshipFeedbackCalibrationBucket[] {
return BUCKETS.map((bucket) => {
const bucketLabels = labels.filter((label) => label.bucket === bucket.label);
const accepted = bucketLabels.filter((label) => label.decision === 'accepted').length;
@ -218,10 +218,10 @@ function buildBuckets(
});
}
export function buildKloRelationshipFeedbackCalibrationReport(
export function buildKtxRelationshipFeedbackCalibrationReport(
feedback: ExportLocalRelationshipFeedbackLabelsResult,
input: BuildKloRelationshipFeedbackCalibrationReportInput = {},
): KloRelationshipFeedbackCalibrationReport {
input: BuildKtxRelationshipFeedbackCalibrationReportInput = {},
): KtxRelationshipFeedbackCalibrationReport {
const currentThresholds = thresholds(input);
const labels = feedback.labels
.map((label) => calibrationLabel(label, currentThresholds))
@ -244,26 +244,26 @@ export function buildKloRelationshipFeedbackCalibrationReport(
}
export async function calibrateLocalRelationshipFeedbackLabels(
project: KloLocalProject,
project: KtxLocalProject,
input: CalibrateLocalRelationshipFeedbackLabelsInput = {},
): Promise<KloRelationshipFeedbackCalibrationReport> {
): Promise<KtxRelationshipFeedbackCalibrationReport> {
const exporter = input.exportLocalRelationshipFeedbackLabels ?? exportLocalRelationshipFeedbackLabels;
const feedback = await exporter(project, {
connectionId: input.connectionId,
decision: input.decision,
});
return buildKloRelationshipFeedbackCalibrationReport(feedback, input);
return buildKtxRelationshipFeedbackCalibrationReport(feedback, input);
}
function formatMetric(value: number | null): string {
return value === null ? 'n/a' : value.toFixed(3);
}
export function formatKloRelationshipFeedbackCalibrationMarkdown(
report: KloRelationshipFeedbackCalibrationReport,
export function formatKtxRelationshipFeedbackCalibrationMarkdown(
report: KtxRelationshipFeedbackCalibrationReport,
): string {
const lines = [
'KLO relationship feedback calibration',
'KTX relationship feedback calibration',
`Generated: ${report.generatedAt}`,
`Filter connection: ${report.filters.connectionId ?? 'all'}`,
`Filter decision: ${report.filters.decision}`,

20
packages/context/src/scan/relationship-feedback-export.test.ts
View file

@ -1,12 +1,12 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import { describe, expect, it, vi } from 'vitest';
import {
exportLocalRelationshipFeedbackLabels,
formatKloRelationshipFeedbackLabelsJsonl,
formatKtxRelationshipFeedbackLabelsJsonl,
} from './relationship-feedback-export.js';
import type { KloRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js';
import type { KtxRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js';
function projectWithFiles(files: Record<string, unknown>): KloLocalProject {
function projectWithFiles(files: Record<string, unknown>): KtxLocalProject {
const contentByPath = new Map(
Object.entries(files).map(([path, value]) => [
path,
@ -14,7 +14,7 @@ function projectWithFiles(files: Record<string, unknown>): KloLocalProject {
]),
);
return {
projectDir: '/tmp/klo-project',
projectDir: '/tmp/ktx-project',
fileStore: {
async listFiles(path: string) {
return {
@ -33,15 +33,15 @@ function projectWithFiles(files: Record<string, unknown>): KloLocalProject {
getFileHistory: vi.fn(),
forWorktree: vi.fn(),
},
} as unknown as KloLocalProject;
} as unknown as KtxLocalProject;
}
function decisionsArtifact(input: {
connectionId: string;
runId: string;
syncId: string;
decisions: KloRelationshipReviewDecisionArtifact['decisions'];
}): KloRelationshipReviewDecisionArtifact {
decisions: KtxRelationshipReviewDecisionArtifact['decisions'];
}): KtxRelationshipReviewDecisionArtifact {
return {
connectionId: input.connectionId,
runId: input.runId,
@ -121,7 +121,7 @@ const acceptedInvoiceAccount = {
runId: 'scan-run-b',
syncId: 'sync-b',
decidedAt: '2026-05-07T12:10:00.000Z',
reviewer: 'klo',
reviewer: 'ktx',
note: null,
from: {
tableId: 'invoices',
@ -232,7 +232,7 @@ describe('relationship feedback export', () => {
now: () => new Date('2026-05-07T13:00:00.000Z'),
});
const lines = formatKloRelationshipFeedbackLabelsJsonl(result).trim().split('\n').map((line) => JSON.parse(line));
const lines = formatKtxRelationshipFeedbackLabelsJsonl(result).trim().split('\n').map((line) => JSON.parse(line));
expect(lines).toHaveLength(1);
expect(lines[0]).toMatchObject({

54
packages/context/src/scan/relationship-feedback-export.ts
View file

@ -1,33 +1,33 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import type {
KloRelationshipReviewDecisionArtifact,
KloRelationshipReviewDecisionEntry,
KloRelationshipReviewDecisionValue,
KtxRelationshipReviewDecisionArtifact,
KtxRelationshipReviewDecisionEntry,
KtxRelationshipReviewDecisionValue,
} from './relationship-review-decisions.js';
const DECISION_ARTIFACT_SUFFIX = '/enrichment/relationship-review-decisions.json';
const FEEDBACK_SCHEMA_VERSION = 1;
export type KloRelationshipFeedbackDecisionFilter = KloRelationshipReviewDecisionValue | 'all';
export type KtxRelationshipFeedbackDecisionFilter = KtxRelationshipReviewDecisionValue | 'all';
export interface ExportLocalRelationshipFeedbackLabelsInput {
connectionId?: string | null;
decision?: KloRelationshipFeedbackDecisionFilter;
decision?: KtxRelationshipFeedbackDecisionFilter;
now?: () => Date;
}
export interface KloRelationshipFeedbackLabel {
export interface KtxRelationshipFeedbackLabel {
schemaVersion: 1;
candidateId: string;
decision: KloRelationshipReviewDecisionValue;
previousStatus: KloRelationshipReviewDecisionEntry['previousStatus'];
decision: KtxRelationshipReviewDecisionValue;
previousStatus: KtxRelationshipReviewDecisionEntry['previousStatus'];
connectionId: string;
runId: string;
syncId: string;
decidedAt: string;
reviewer: string;
note: string | null;
relationshipType: KloRelationshipReviewDecisionEntry['relationshipType'];
relationshipType: KtxRelationshipReviewDecisionEntry['relationshipType'];
source: string;
score: number | null;
confidence: number;
@ -41,7 +41,7 @@ export interface KloRelationshipFeedbackLabel {
artifactPath: string;
}
export interface KloRelationshipFeedbackExportWarning {
export interface KtxRelationshipFeedbackExportWarning {
path: string;
message: string;
}
@ -50,7 +50,7 @@ export interface ExportLocalRelationshipFeedbackLabelsResult {
generatedAt: string;
filters: {
connectionId: string | null;
decision: KloRelationshipFeedbackDecisionFilter;
decision: KtxRelationshipFeedbackDecisionFilter;
};
summary: {
total: number;
@ -59,16 +59,16 @@ export interface ExportLocalRelationshipFeedbackLabelsResult {
connections: number;
runs: number;
};
labels: KloRelationshipFeedbackLabel[];
warnings: KloRelationshipFeedbackExportWarning[];
labels: KtxRelationshipFeedbackLabel[];
warnings: KtxRelationshipFeedbackExportWarning[];
}
function qualifiedTableName(entry: KloRelationshipReviewDecisionEntry, side: 'from' | 'to'): string {
function qualifiedTableName(entry: KtxRelationshipReviewDecisionEntry, side: 'from' | 'to'): string {
const table = entry[side].table;
return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.');
}
function labelFromDecision(entry: KloRelationshipReviewDecisionEntry, artifactPath: string): KloRelationshipFeedbackLabel {
function labelFromDecision(entry: KtxRelationshipReviewDecisionEntry, artifactPath: string): KtxRelationshipFeedbackLabel {
return {
schemaVersion: FEEDBACK_SCHEMA_VERSION,
candidateId: entry.candidateId,
@ -95,7 +95,7 @@ function labelFromDecision(entry: KloRelationshipReviewDecisionEntry, artifactPa
};
}
function sortLabels(labels: KloRelationshipFeedbackLabel[]): KloRelationshipFeedbackLabel[] {
function sortLabels(labels: KtxRelationshipFeedbackLabel[]): KtxRelationshipFeedbackLabel[] {
return [...labels].sort((left, right) => {
return (
left.connectionId.localeCompare(right.connectionId) ||
@ -107,8 +107,8 @@ function sortLabels(labels: KloRelationshipFeedbackLabel[]): KloRelationshipFeed
}
function passesFilters(
label: KloRelationshipFeedbackLabel,
filters: { connectionId: string | null; decision: KloRelationshipFeedbackDecisionFilter },
label: KtxRelationshipFeedbackLabel,
filters: { connectionId: string | null; decision: KtxRelationshipFeedbackDecisionFilter },
): boolean {
if (filters.connectionId && label.connectionId !== filters.connectionId) {
return false;
@ -121,16 +121,16 @@ function messageFromUnknownError(error: unknown): string {
}
async function readDecisionLabels(
project: KloLocalProject,
project: KtxLocalProject,
artifactPath: string,
): Promise<KloRelationshipFeedbackLabel[]> {
): Promise<KtxRelationshipFeedbackLabel[]> {
const raw = await project.fileStore.readFile(artifactPath);
const parsed = JSON.parse(raw.content) as KloRelationshipReviewDecisionArtifact;
const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact;
const decisions = Array.isArray(parsed.decisions) ? parsed.decisions : [];
return decisions.map((entry) => labelFromDecision(entry, artifactPath));
}
function summarize(labels: KloRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult['summary'] {
function summarize(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult['summary'] {
return {
total: labels.length,
accepted: labels.filter((label) => label.decision === 'accepted').length,
@ -141,7 +141,7 @@ function summarize(labels: KloRelationshipFeedbackLabel[]): ExportLocalRelations
}
export async function exportLocalRelationshipFeedbackLabels(
project: KloLocalProject,
project: KtxLocalProject,
input: ExportLocalRelationshipFeedbackLabelsInput = {},
): Promise<ExportLocalRelationshipFeedbackLabelsResult> {
const filters = {
@ -150,8 +150,8 @@ export async function exportLocalRelationshipFeedbackLabels(
};
const listed = await project.fileStore.listFiles('raw-sources');
const artifactPaths = listed.files.filter((path) => path.endsWith(DECISION_ARTIFACT_SUFFIX)).sort();
const labels: KloRelationshipFeedbackLabel[] = [];
const warnings: KloRelationshipFeedbackExportWarning[] = [];
const labels: KtxRelationshipFeedbackLabel[] = [];
const warnings: KtxRelationshipFeedbackExportWarning[] = [];
for (const artifactPath of artifactPaths) {
try {
@ -171,7 +171,7 @@ export async function exportLocalRelationshipFeedbackLabels(
};
}
export function formatKloRelationshipFeedbackLabelsJsonl(result: ExportLocalRelationshipFeedbackLabelsResult): string {
export function formatKtxRelationshipFeedbackLabelsJsonl(result: ExportLocalRelationshipFeedbackLabelsResult): string {
if (result.labels.length === 0) {
return '';
}

12
packages/context/src/scan/relationship-formal-metadata.test.ts
View file

@ -1,8 +1,8 @@
import { describe, expect, it } from 'vitest';
import type { KloEnrichedRelationship, KloEnrichedSchema } from './enrichment-types.js';
import { collectKloFormalMetadataRelationships } from './relationship-formal-metadata.js';
import type { KtxEnrichedRelationship, KtxEnrichedSchema } from './enrichment-types.js';
import { collectKtxFormalMetadataRelationships } from './relationship-formal-metadata.js';
function schema(relationships: KloEnrichedRelationship[]): KloEnrichedSchema {
function schema(relationships: KtxEnrichedRelationship[]): KtxEnrichedSchema {
return {
connectionId: 'warehouse',
tables: [
@ -59,7 +59,7 @@ function schema(relationships: KloEnrichedRelationship[]): KloEnrichedSchema {
};
}
function formalRelationship(overrides: Partial<KloEnrichedRelationship> = {}): KloEnrichedRelationship {
function formalRelationship(overrides: Partial<KtxEnrichedRelationship> = {}): KtxEnrichedRelationship {
return {
id: 'orders:orders.account_id->accounts:accounts.id',
source: 'formal',
@ -84,7 +84,7 @@ function formalRelationship(overrides: Partial<KloEnrichedRelationship> = {}): K
describe('formal metadata relationship collection', () => {
it('accepts valid formal relationships with ground-truth confidence', () => {
const result = collectKloFormalMetadataRelationships(schema([formalRelationship()]));
const result = collectKtxFormalMetadataRelationships(schema([formalRelationship()]));
expect(result.accepted).toEqual([
expect.objectContaining({
@ -99,7 +99,7 @@ describe('formal metadata relationship collection', () => {
});
it('skips duplicate and invalid formal relationships with reasons', () => {
const result = collectKloFormalMetadataRelationships(
const result = collectKtxFormalMetadataRelationships(
schema([
formalRelationship(),
formalRelationship(),

20
packages/context/src/scan/relationship-formal-metadata.ts
View file

@ -1,12 +1,12 @@
import type { KloEnrichedRelationship, KloEnrichedSchema, KloSkippedRelationship } from './enrichment-types.js';
import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxSkippedRelationship } from './enrichment-types.js';
export interface KloFormalMetadataRelationshipCollection {
accepted: KloEnrichedRelationship[];
skipped: KloSkippedRelationship[];
export interface KtxFormalMetadataRelationshipCollection {
accepted: KtxEnrichedRelationship[];
skipped: KtxSkippedRelationship[];
acceptedIds: Set<string>;
}
function relationshipEndpointExists(schema: KloEnrichedSchema, relationship: KloEnrichedRelationship): boolean {
function relationshipEndpointExists(schema: KtxEnrichedSchema, relationship: KtxEnrichedRelationship): boolean {
const fromTable = schema.tables.find((table) => table.id === relationship.from.tableId && table.enabled);
const toTable = schema.tables.find((table) => table.id === relationship.to.tableId && table.enabled);
const fromColumn = fromTable?.columns.some(
@ -18,11 +18,11 @@ function relationshipEndpointExists(schema: KloEnrichedSchema, relationship: Klo
return Boolean(fromTable && toTable && fromColumn && toColumn);
}
export function collectKloFormalMetadataRelationships(
schema: KloEnrichedSchema,
): KloFormalMetadataRelationshipCollection {
const accepted: KloEnrichedRelationship[] = [];
const skipped: KloSkippedRelationship[] = [];
export function collectKtxFormalMetadataRelationships(
schema: KtxEnrichedSchema,
): KtxFormalMetadataRelationshipCollection {
const accepted: KtxEnrichedRelationship[] = [];
const skipped: KtxSkippedRelationship[] = [];
const acceptedIds = new Set<string>();
for (const relationship of schema.relationships) {

46
packages/context/src/scan/relationship-graph-resolver.test.ts
View file

@ -1,15 +1,15 @@
import { describe, expect, it } from 'vitest';
import type {
KloEnrichedColumn,
KloEnrichedSchema,
KloEnrichedTable,
KloRelationshipEndpoint,
KtxEnrichedColumn,
KtxEnrichedSchema,
KtxEnrichedTable,
KtxRelationshipEndpoint,
} from './enrichment-types.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KloValidatedRelationshipDiscoveryCandidate } from './relationship-validation.js';
import { resolveKloRelationshipGraph } from './relationship-graph-resolver.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxValidatedRelationshipDiscoveryCandidate } from './relationship-validation.js';
import { resolveKtxRelationshipGraph } from './relationship-graph-resolver.js';
function column(tableId: string, name: string, overrides: Partial<KloEnrichedColumn> = {}): KloEnrichedColumn {
function column(tableId: string, name: string, overrides: Partial<KtxEnrichedColumn> = {}): KtxEnrichedColumn {
const tableRef = overrides.tableRef ?? { catalog: null, db: null, name: tableId };
return {
id: `${tableId}.${name}`,
@ -30,7 +30,7 @@ function column(tableId: string, name: string, overrides: Partial<KloEnrichedCol
};
}
function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
function table(name: string, columns: KtxEnrichedColumn[]): KtxEnrichedTable {
const ref = { catalog: null, db: null, name };
return {
id: name,
@ -41,7 +41,7 @@ function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
};
}
function schema(overrides: { accountsPrimaryKey?: boolean } = {}): KloEnrichedSchema {
function schema(overrides: { accountsPrimaryKey?: boolean } = {}): KtxEnrichedSchema {
return {
connectionId: 'warehouse',
tables: [
@ -59,7 +59,7 @@ function schema(overrides: { accountsPrimaryKey?: boolean } = {}): KloEnrichedSc
};
}
function endpoint(tableName: string, columnName: string): KloRelationshipEndpoint {
function endpoint(tableName: string, columnName: string): KtxRelationshipEndpoint {
return {
tableId: tableName,
columnIds: [`${tableName}.${columnName}`],
@ -68,7 +68,7 @@ function endpoint(tableName: string, columnName: string): KloRelationshipEndpoin
};
}
function profiles(): KloRelationshipProfileArtifact {
function profiles(): KtxRelationshipProfileArtifact {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -128,8 +128,8 @@ function profiles(): KloRelationshipProfileArtifact {
}
function validatedCandidate(
overrides: Partial<KloValidatedRelationshipDiscoveryCandidate> = {},
): KloValidatedRelationshipDiscoveryCandidate {
overrides: Partial<KtxValidatedRelationshipDiscoveryCandidate> = {},
): KtxValidatedRelationshipDiscoveryCandidate {
const from = overrides.from ?? endpoint('users', 'account_id');
const to = overrides.to ?? endpoint('accounts', 'id');
return {
@ -170,7 +170,7 @@ function validatedCandidate(
describe('relationship graph resolver', () => {
it('promotes validated relationship discovery references to accepted relationships and inferred PKs', () => {
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: schema(),
profiles: profiles(),
candidates: [validatedCandidate()],
@ -206,7 +206,7 @@ describe('relationship graph resolver', () => {
});
it('keeps validation-unavailable candidates in review even when name evidence is strong', () => {
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: schema(),
profiles: { ...profiles(), sqlAvailable: false, columns: {}, warnings: ['read_only_sql_unavailable'] },
candidates: [
@ -257,7 +257,7 @@ describe('relationship graph resolver', () => {
},
});
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: schema(),
profiles: profiles(),
candidates: [loser, winner],
@ -275,7 +275,7 @@ describe('relationship graph resolver', () => {
});
it('preserves declared primary keys as accepted even without incoming candidates', () => {
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: schema({ accountsPrimaryKey: true }),
profiles: profiles(),
candidates: [],
@ -316,7 +316,7 @@ describe('relationship graph resolver', () => {
}),
]);
const baseProfiles = profiles();
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: { ...baseSchema, tables: [...baseSchema.tables, invoices] },
profiles: {
...baseProfiles,
@ -458,7 +458,7 @@ describe('relationship graph resolver', () => {
],
},
],
} satisfies KloEnrichedSchema;
} satisfies KtxEnrichedSchema;
const profiles = {
connectionId: 'warehouse',
driver: 'sqlite' as const,
@ -483,7 +483,7 @@ describe('relationship graph resolver', () => {
},
},
};
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema,
profiles,
candidates: [
@ -579,7 +579,7 @@ describe('relationship graph resolver', () => {
maxTextLength: 3,
};
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: baseSchema,
profiles: baseProfiles,
candidates: [],
@ -629,7 +629,7 @@ describe('relationship graph resolver', () => {
maxTextLength: 3,
};
const result = resolveKloRelationshipGraph({
const result = resolveKtxRelationshipGraph({
schema: baseSchema,
profiles: baseProfiles,
candidates: [],

142
packages/context/src/scan/relationship-graph-resolver.ts
View file

@ -1,24 +1,24 @@
import type {
KloEnrichedColumn,
KloEnrichedSchema,
KloEnrichedTable,
KloRelationshipEndpoint,
KtxEnrichedColumn,
KtxEnrichedSchema,
KtxEnrichedTable,
KtxRelationshipEndpoint,
} from './enrichment-types.js';
import { normalizeKloRelationshipName } from './relationship-candidates.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import { scoreKloRelationshipCandidate } from './relationship-scoring.js';
import type { KloValidatedRelationshipDiscoveryCandidate } from './relationship-validation.js';
import { normalizeKtxRelationshipName } from './relationship-candidates.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import { scoreKtxRelationshipCandidate } from './relationship-scoring.js';
import type { KtxValidatedRelationshipDiscoveryCandidate } from './relationship-validation.js';
export type KloResolvedRelationshipStatus = 'accepted' | 'review' | 'rejected';
export type KtxResolvedRelationshipStatus = 'accepted' | 'review' | 'rejected';
export interface KloRelationshipGraphResolverSettings {
export interface KtxRelationshipGraphResolverSettings {
acceptThreshold: number;
reviewThreshold: number;
minTargetPkScoreForAcceptance: number;
validationRequiredForManifest: boolean;
}
export interface KloResolvedRelationshipPkEvidence {
export interface KtxResolvedRelationshipPkEvidence {
declaredPrimaryKey: boolean;
targetUniqueness: number;
incomingAcceptedCount: number;
@ -26,43 +26,43 @@ export interface KloResolvedRelationshipPkEvidence {
reasons: string[];
}
export interface KloResolvedRelationshipPk {
export interface KtxResolvedRelationshipPk {
table: string;
columns: string[];
pkScore: number;
status: KloResolvedRelationshipStatus;
status: KtxResolvedRelationshipStatus;
incomingCandidateCount: number;
evidence: KloResolvedRelationshipPkEvidence;
evidence: KtxResolvedRelationshipPkEvidence;
}
export interface KloResolvedRelationshipGraphEvidence {
export interface KtxResolvedRelationshipGraphEvidence {
targetPkScore: number;
incomingCandidateCount: number;
conflictRank: number;
reasons: string[];
}
export interface KloResolvedRelationshipDiscoveryCandidate
extends Omit<KloValidatedRelationshipDiscoveryCandidate, 'status'> {
status: KloResolvedRelationshipStatus;
export interface KtxResolvedRelationshipDiscoveryCandidate
extends Omit<KtxValidatedRelationshipDiscoveryCandidate, 'status'> {
status: KtxResolvedRelationshipStatus;
pkScore: number;
fkScore: number;
graph: KloResolvedRelationshipGraphEvidence;
graph: KtxResolvedRelationshipGraphEvidence;
}
export interface KloRelationshipGraphResolutionResult {
pks: KloResolvedRelationshipPk[];
relationships: KloResolvedRelationshipDiscoveryCandidate[];
export interface KtxRelationshipGraphResolutionResult {
pks: KtxResolvedRelationshipPk[];
relationships: KtxResolvedRelationshipDiscoveryCandidate[];
}
export interface ResolveKloRelationshipGraphInput {
schema: KloEnrichedSchema;
profiles: KloRelationshipProfileArtifact;
candidates: readonly KloValidatedRelationshipDiscoveryCandidate[];
settings?: Partial<KloRelationshipGraphResolverSettings>;
export interface ResolveKtxRelationshipGraphInput {
schema: KtxEnrichedSchema;
profiles: KtxRelationshipProfileArtifact;
candidates: readonly KtxValidatedRelationshipDiscoveryCandidate[];
settings?: Partial<KtxRelationshipGraphResolverSettings>;
}
const DEFAULT_SETTINGS: KloRelationshipGraphResolverSettings = {
const DEFAULT_SETTINGS: KtxRelationshipGraphResolverSettings = {
acceptThreshold: 0.85,
reviewThreshold: 0.55,
minTargetPkScoreForAcceptance: 0.78,
@ -72,8 +72,8 @@ const DEFAULT_SETTINGS: KloRelationshipGraphResolverSettings = {
const PROFILE_ONLY_PK_MEASURE_NAME_TOKENS = new Set(['amount', 'count', 'price', 'quantity', 'subtotal', 'total']);
function mergeSettings(
settings: Partial<KloRelationshipGraphResolverSettings> | undefined,
): KloRelationshipGraphResolverSettings {
settings: Partial<KtxRelationshipGraphResolverSettings> | undefined,
): KtxRelationshipGraphResolverSettings {
return { ...DEFAULT_SETTINGS, ...settings };
}
@ -81,15 +81,15 @@ function roundScore(value: number): number {
return Number(Math.max(0, Math.min(1, value)).toFixed(3));
}
function endpointKey(endpoint: KloRelationshipEndpoint): string {
function endpointKey(endpoint: KtxRelationshipEndpoint): string {
return `${endpoint.table.name}.${singleRelationshipColumn(endpoint)}`;
}
function sourceKey(endpoint: KloRelationshipEndpoint): string {
function sourceKey(endpoint: KtxRelationshipEndpoint): string {
return `${endpoint.tableId}:${endpoint.columnIds.join(',')}`;
}
function singleRelationshipColumn(endpoint: KloRelationshipEndpoint): string {
function singleRelationshipColumn(endpoint: KtxRelationshipEndpoint): string {
const column = endpoint.columns[0];
if (!column) {
throw new Error(`Expected relationship endpoint ${endpoint.table.name} to contain one column`);
@ -97,19 +97,19 @@ function singleRelationshipColumn(endpoint: KloRelationshipEndpoint): string {
return column;
}
function pkKey(pk: Pick<KloResolvedRelationshipPk, 'table' | 'columns'>): string {
function pkKey(pk: Pick<KtxResolvedRelationshipPk, 'table' | 'columns'>): string {
return `${pk.table}.(${pk.columns.join(',')})`;
}
function candidateSortKey(candidate: Pick<KloValidatedRelationshipDiscoveryCandidate, 'from' | 'to'>): string {
function candidateSortKey(candidate: Pick<KtxValidatedRelationshipDiscoveryCandidate, 'from' | 'to'>): string {
return `${candidate.from.table.name}.${singleRelationshipColumn(candidate.from)}->${candidate.to.table.name}.${singleRelationshipColumn(candidate.to)}`;
}
function statusForScore(
score: number,
settings: KloRelationshipGraphResolverSettings,
settings: KtxRelationshipGraphResolverSettings,
acceptedAllowed: boolean,
): KloResolvedRelationshipStatus {
): KtxResolvedRelationshipStatus {
if (acceptedAllowed && score >= settings.acceptThreshold) {
return 'accepted';
}
@ -119,19 +119,19 @@ function statusForScore(
return 'rejected';
}
function candidateHasValidationPassed(candidate: KloValidatedRelationshipDiscoveryCandidate): boolean {
function candidateHasValidationPassed(candidate: KtxValidatedRelationshipDiscoveryCandidate): boolean {
return candidate.validation.reasons.includes('validation_passed');
}
function candidateIsValidationUnavailable(candidate: KloValidatedRelationshipDiscoveryCandidate): boolean {
function candidateIsValidationUnavailable(candidate: KtxValidatedRelationshipDiscoveryCandidate): boolean {
return (
candidate.validation.reasons.includes('validation_unavailable') ||
candidate.validation.reasons.includes('profile_unavailable')
);
}
function declaredPrimaryKeys(schema: KloEnrichedSchema): KloResolvedRelationshipPk[] {
const pks: KloResolvedRelationshipPk[] = [];
function declaredPrimaryKeys(schema: KtxEnrichedSchema): KtxResolvedRelationshipPk[] {
const pks: KtxResolvedRelationshipPk[] = [];
for (const table of schema.tables.filter((candidate) => candidate.enabled)) {
for (const column of table.columns.filter((candidate) => candidate.primaryKey)) {
pks.push({
@ -153,27 +153,27 @@ function declaredPrimaryKeys(schema: KloEnrichedSchema): KloResolvedRelationship
return pks;
}
function schemaTargetColumns(schema: KloEnrichedSchema): Array<{ table: KloEnrichedTable; column: KloEnrichedColumn }> {
function schemaTargetColumns(schema: KtxEnrichedSchema): Array<{ table: KtxEnrichedTable; column: KtxEnrichedColumn }> {
return schema.tables
.filter((table) => table.enabled)
.flatMap((table) => table.columns.map((column) => ({ table, column })));
}
function profileUniqueness(profiles: KloRelationshipProfileArtifact, tableName: string, columnName: string): number {
function profileUniqueness(profiles: KtxRelationshipProfileArtifact, tableName: string, columnName: string): number {
return profiles.columns[`${tableName}.${columnName}`]?.uniquenessRatio ?? 0;
}
function profileNullRate(profiles: KloRelationshipProfileArtifact, tableName: string, columnName: string): number {
function profileNullRate(profiles: KtxRelationshipProfileArtifact, tableName: string, columnName: string): number {
return profiles.columns[`${tableName}.${columnName}`]?.nullRate ?? 1;
}
function profileColumnExists(profiles: KloRelationshipProfileArtifact, tableName: string, columnName: string): boolean {
function profileColumnExists(profiles: KtxRelationshipProfileArtifact, tableName: string, columnName: string): boolean {
return Boolean(profiles.columns[`${tableName}.${columnName}`]);
}
function profileOnlyPkNameScore(tableName: string, columnName: string): number {
const table = normalizeKloRelationshipName(tableName).singular;
const column = normalizeKloRelationshipName(columnName).normalized;
const table = normalizeKtxRelationshipName(tableName).singular;
const column = normalizeKtxRelationshipName(columnName).normalized;
if (column === 'id') {
return 1;
}
@ -190,12 +190,12 @@ function profileOnlyPkNameScore(tableName: string, columnName: string): number {
}
function profileOnlyPkTypeCompatibility(columnName: string): number {
const tokens = normalizeKloRelationshipName(columnName).normalized.split('_').filter(Boolean);
const tokens = normalizeKtxRelationshipName(columnName).normalized.split('_').filter(Boolean);
return tokens.some((token) => PROFILE_ONLY_PK_MEASURE_NAME_TOKENS.has(token)) ? 0 : 1;
}
function profileOnlyPkEvidence(input: {
profiles: KloRelationshipProfileArtifact;
profiles: KtxRelationshipProfileArtifact;
tableName: string;
columnName: string;
}): { nameScore: number; nullRate: number; uniqueness: number; pkScore: number; weakName: boolean } | null {
@ -209,7 +209,7 @@ function profileOnlyPkEvidence(input: {
return null;
}
const typeCompatibility = profileOnlyPkTypeCompatibility(input.columnName);
const scoreBreakdown = scoreKloRelationshipCandidate(
const scoreBreakdown = scoreKtxRelationshipCandidate(
{
nameSimilarity: nameScore,
typeCompatibility,
@ -240,12 +240,12 @@ function profileOnlyPkEvidence(input: {
function resolveTargetPk(input: {
table: string;
column: string;
declared: KloResolvedRelationshipPk | undefined;
profiles: KloRelationshipProfileArtifact;
incoming: readonly KloValidatedRelationshipDiscoveryCandidate[];
settings: KloRelationshipGraphResolverSettings;
declared: KtxResolvedRelationshipPk | undefined;
profiles: KtxRelationshipProfileArtifact;
incoming: readonly KtxValidatedRelationshipDiscoveryCandidate[];
settings: KtxRelationshipGraphResolverSettings;
profileOnly?: { nameScore: number; nullRate: number; uniqueness: number; pkScore: number; weakName: boolean } | null;
}): KloResolvedRelationshipPk {
}): KtxResolvedRelationshipPk {
if (input.declared) {
return input.declared;
}
@ -322,10 +322,10 @@ function resolveTargetPk(input: {
}
function baseRelationshipResolution(input: {
candidate: KloValidatedRelationshipDiscoveryCandidate;
pk: KloResolvedRelationshipPk;
settings: KloRelationshipGraphResolverSettings;
}): KloResolvedRelationshipDiscoveryCandidate {
candidate: KtxValidatedRelationshipDiscoveryCandidate;
pk: KtxResolvedRelationshipPk;
settings: KtxRelationshipGraphResolverSettings;
}): KtxResolvedRelationshipDiscoveryCandidate {
const reasons: string[] = [];
if (input.candidate.status === 'rejected') {
reasons.push('candidate_validation_rejected');
@ -349,7 +349,7 @@ function baseRelationshipResolution(input: {
0.14 * input.candidate.confidence +
0.08 * validationPassBonus,
);
let status: KloResolvedRelationshipStatus;
let status: KtxResolvedRelationshipStatus;
if (input.candidate.status === 'rejected') {
status = 'rejected';
@ -387,8 +387,8 @@ function baseRelationshipResolution(input: {
}
function relationshipRank(
left: KloResolvedRelationshipDiscoveryCandidate,
right: KloResolvedRelationshipDiscoveryCandidate,
left: KtxResolvedRelationshipDiscoveryCandidate,
right: KtxResolvedRelationshipDiscoveryCandidate,
): number {
return (
right.fkScore - left.fkScore ||
@ -399,15 +399,15 @@ function relationshipRank(
}
function applySourceConflicts(
relationships: readonly KloResolvedRelationshipDiscoveryCandidate[],
): KloResolvedRelationshipDiscoveryCandidate[] {
const bySource = new Map<string, KloResolvedRelationshipDiscoveryCandidate[]>();
relationships: readonly KtxResolvedRelationshipDiscoveryCandidate[],
): KtxResolvedRelationshipDiscoveryCandidate[] {
const bySource = new Map<string, KtxResolvedRelationshipDiscoveryCandidate[]>();
for (const relationship of relationships) {
const key = sourceKey(relationship.from);
bySource.set(key, [...(bySource.get(key) ?? []), relationship]);
}
const resolved: KloResolvedRelationshipDiscoveryCandidate[] = [];
const resolved: KtxResolvedRelationshipDiscoveryCandidate[] = [];
for (const group of bySource.values()) {
const ranked = [...group].sort(relationshipRank);
let acceptedSeen = false;
@ -441,20 +441,20 @@ function applySourceConflicts(
return resolved.sort(relationshipRank);
}
export function resolveKloRelationshipGraph(
input: ResolveKloRelationshipGraphInput,
): KloRelationshipGraphResolutionResult {
export function resolveKtxRelationshipGraph(
input: ResolveKtxRelationshipGraphInput,
): KtxRelationshipGraphResolutionResult {
const settings = mergeSettings(input.settings);
const declared = declaredPrimaryKeys(input.schema);
const declaredByKey = new Map(declared.map((pk) => [pkKey(pk), pk]));
const incomingByTarget = new Map<string, KloValidatedRelationshipDiscoveryCandidate[]>();
const incomingByTarget = new Map<string, KtxValidatedRelationshipDiscoveryCandidate[]>();
for (const candidate of input.candidates) {
const key = endpointKey(candidate.to);
incomingByTarget.set(key, [...(incomingByTarget.get(key) ?? []), candidate]);
}
const pkCandidates = new Map<string, KloResolvedRelationshipPk>();
const pkCandidates = new Map<string, KtxResolvedRelationshipPk>();
for (const item of schemaTargetColumns(input.schema)) {
const key = `${item.table.ref.name}.(${item.column.name})`;
const incoming = incomingByTarget.get(`${item.table.ref.name}.${item.column.name}`) ?? [];

36
packages/context/src/scan/relationship-llm-proposal.test.ts
View file

@ -1,14 +1,14 @@
import type { KloLlmProvider } from '@klo/llm';
import type { KtxLlmProvider } from '@ktx/llm';
import { describe, expect, it, vi } from 'vitest';
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable } from './enrichment-types.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import { proposeKloRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
function llmProvider(provider = 'anthropic'): KloLlmProvider {
function llmProvider(provider = 'anthropic'): KtxLlmProvider {
const model = { modelId: 'claude-sonnet-4-6', provider };
return {
getModel: vi.fn(() => model as ReturnType<KloLlmProvider['getModel']>),
getModelByName: vi.fn(() => model as ReturnType<KloLlmProvider['getModelByName']>),
getModel: vi.fn(() => model as ReturnType<KtxLlmProvider['getModel']>),
getModelByName: vi.fn(() => model as ReturnType<KtxLlmProvider['getModelByName']>),
cacheMarker: vi.fn(),
repairToolCallHandler: vi.fn(),
thinkingProviderOptions: vi.fn(() => ({})),
@ -24,13 +24,13 @@ function llmProvider(provider = 'anthropic'): KloLlmProvider {
cacheTools: true,
cacheHistory: true,
vertexFallbackTo5m: false,
}) as ReturnType<KloLlmProvider['promptCachingConfig']>,
}) as ReturnType<KtxLlmProvider['promptCachingConfig']>,
),
activeBackend: vi.fn(() => provider as ReturnType<KloLlmProvider['activeBackend']>),
activeBackend: vi.fn(() => provider as ReturnType<KtxLlmProvider['activeBackend']>),
};
}
function column(tableId: string, name: string, overrides: Partial<KloEnrichedColumn> = {}): KloEnrichedColumn {
function column(tableId: string, name: string, overrides: Partial<KtxEnrichedColumn> = {}): KtxEnrichedColumn {
const tableRef = overrides.tableRef ?? { catalog: null, db: null, name: tableId };
return {
id: `${tableId}.${name}`,
@ -51,7 +51,7 @@ function column(tableId: string, name: string, overrides: Partial<KloEnrichedCol
};
}
function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
function table(name: string, columns: KtxEnrichedColumn[]): KtxEnrichedTable {
const ref = { catalog: null, db: null, name };
return {
id: name,
@ -62,7 +62,7 @@ function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
};
}
function schema(): KloEnrichedSchema {
function schema(): KtxEnrichedSchema {
return {
connectionId: 'warehouse',
relationships: [],
@ -79,7 +79,7 @@ function schema(): KloEnrichedSchema {
};
}
function profile(): KloRelationshipProfileArtifact {
function profile(): KtxRelationshipProfileArtifact {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -141,7 +141,7 @@ describe('relationship LLM proposals', () => {
},
}));
const result = await proposeKloRelationshipCandidatesWithLlm({
const result = await proposeKtxRelationshipCandidatesWithLlm({
connectionId: 'warehouse',
schema: schema(),
profile: profile(),
@ -179,7 +179,7 @@ describe('relationship LLM proposals', () => {
it('skips deterministic providers without calling generateText', async () => {
const generateText = vi.fn();
const result = await proposeKloRelationshipCandidatesWithLlm({
const result = await proposeKtxRelationshipCandidatesWithLlm({
connectionId: 'warehouse',
schema: schema(),
profile: profile(),
@ -193,7 +193,7 @@ describe('relationship LLM proposals', () => {
});
it('returns recoverable warnings for invalid references and generation failures', async () => {
const invalidReference = await proposeKloRelationshipCandidatesWithLlm({
const invalidReference = await proposeKtxRelationshipCandidatesWithLlm({
connectionId: 'warehouse',
schema: schema(),
profile: profile(),
@ -221,7 +221,7 @@ describe('relationship LLM proposals', () => {
recoverable: true,
});
const failed = await proposeKloRelationshipCandidatesWithLlm({
const failed = await proposeKtxRelationshipCandidatesWithLlm({
connectionId: 'warehouse',
schema: schema(),
profile: profile(),
@ -233,7 +233,7 @@ describe('relationship LLM proposals', () => {
expect(failed).toMatchObject({ candidates: [], llmCalls: 1, summary: 'failed' });
expect(failed.warnings[0]).toMatchObject({
code: 'relationship_llm_proposal_failed',
message: 'KLO relationship LLM proposal failed: model unavailable',
message: 'KTX relationship LLM proposal failed: model unavailable',
recoverable: true,
});
});

112
packages/context/src/scan/relationship-llm-proposal.ts
View file

@ -1,14 +1,14 @@
import type { KloLlmProvider } from '@klo/llm';
import type { KtxLlmProvider } from '@ktx/llm';
import type { generateText } from 'ai';
import { z } from 'zod';
import { generateKloObject } from '../llm/index.js';
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable } from './enrichment-types.js';
import { generateKtxObject } from '../llm/index.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js';
import {
normalizeKloRelationshipName,
type KloRelationshipDiscoveryCandidate,
normalizeKtxRelationshipName,
type KtxRelationshipDiscoveryCandidate,
} from './relationship-candidates.js';
import type { KloRelationshipColumnProfile, KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KloScanEnrichmentSummary, KloScanWarning, KloTableRef } from './types.js';
import type { KtxRelationshipColumnProfile, KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxScanEnrichmentSummary, KtxScanWarning, KtxTableRef } from './types.js';
const relationshipLlmProposalSchema = z.object({
pkCandidates: z.array(
@ -31,36 +31,36 @@ const relationshipLlmProposalSchema = z.object({
),
});
type KloRelationshipLlmProposalOutput = z.infer<typeof relationshipLlmProposalSchema>;
type KtxRelationshipLlmProposalOutput = z.infer<typeof relationshipLlmProposalSchema>;
type GenerateTextInput = Parameters<typeof generateText>[0];
export type KloRelationshipLlmProposalGenerateText = (
export type KtxRelationshipLlmProposalGenerateText = (
input: GenerateTextInput,
) => Promise<{ text?: string; output?: unknown }>;
export interface KloRelationshipLlmProposalSettings {
export interface KtxRelationshipLlmProposalSettings {
maxTablesPerBatch: number;
maxColumnsPerTable: number;
maxSampleValuesPerColumn: number;
minConfidence: number;
}
export interface ProposeKloRelationshipCandidatesWithLlmInput {
export interface ProposeKtxRelationshipCandidatesWithLlmInput {
connectionId: string;
schema: KloEnrichedSchema;
profile: KloRelationshipProfileArtifact;
llmProvider: KloLlmProvider | null;
settings?: Partial<KloRelationshipLlmProposalSettings>;
generateText?: KloRelationshipLlmProposalGenerateText;
schema: KtxEnrichedSchema;
profile: KtxRelationshipProfileArtifact;
llmProvider: KtxLlmProvider | null;
settings?: Partial<KtxRelationshipLlmProposalSettings>;
generateText?: KtxRelationshipLlmProposalGenerateText;
}
export interface KloRelationshipLlmProposalResult {
candidates: KloRelationshipDiscoveryCandidate[];
warnings: KloScanWarning[];
export interface KtxRelationshipLlmProposalResult {
candidates: KtxRelationshipDiscoveryCandidate[];
warnings: KtxScanWarning[];
llmCalls: number;
summary: KloScanEnrichmentSummary['llmRelationshipValidation'];
summary: KtxScanEnrichmentSummary['llmRelationshipValidation'];
}
const DEFAULT_SETTINGS: KloRelationshipLlmProposalSettings = {
const DEFAULT_SETTINGS: KtxRelationshipLlmProposalSettings = {
maxTablesPerBatch: 40,
maxColumnsPerTable: 80,
maxSampleValuesPerColumn: 5,
@ -68,8 +68,8 @@ const DEFAULT_SETTINGS: KloRelationshipLlmProposalSettings = {
};
function mergeSettings(
settings: Partial<KloRelationshipLlmProposalSettings> | undefined,
): KloRelationshipLlmProposalSettings {
settings: Partial<KtxRelationshipLlmProposalSettings> | undefined,
): KtxRelationshipLlmProposalSettings {
return { ...DEFAULT_SETTINGS, ...settings };
}
@ -77,41 +77,41 @@ function clampConfidence(value: number): number {
return Number(Math.max(0, Math.min(1, value)).toFixed(3));
}
function modelIsDeterministic(llmProvider: KloLlmProvider): boolean {
function modelIsDeterministic(llmProvider: KtxLlmProvider): boolean {
const model = llmProvider.getModel('candidateExtraction');
return (model as { provider?: string }).provider === 'deterministic';
}
function findTable(schema: KloEnrichedSchema, name: string): KloEnrichedTable | null {
function findTable(schema: KtxEnrichedSchema, name: string): KtxEnrichedTable | null {
const normalized = name.toLowerCase();
return schema.tables.find((table) => table.ref.name.toLowerCase() === normalized) ?? null;
}
function findColumn(table: KloEnrichedTable, name: string): KloEnrichedColumn | null {
function findColumn(table: KtxEnrichedTable, name: string): KtxEnrichedColumn | null {
const normalized = name.toLowerCase();
return table.columns.find((column) => column.name.toLowerCase() === normalized) ?? null;
}
function profileKey(table: KloTableRef, column: KloEnrichedColumn): string {
function profileKey(table: KtxTableRef, column: KtxEnrichedColumn): string {
return `${table.name}.${column.name}`;
}
function profileForColumn(
profile: KloRelationshipProfileArtifact,
table: KloEnrichedTable,
column: KloEnrichedColumn,
): KloRelationshipColumnProfile | null {
profile: KtxRelationshipProfileArtifact,
table: KtxEnrichedTable,
column: KtxEnrichedColumn,
): KtxRelationshipColumnProfile | null {
return profile.columns[profileKey(table.ref, column)] ?? null;
}
function rowCountForTable(profile: KloRelationshipProfileArtifact, table: KloEnrichedTable): number | null {
function rowCountForTable(profile: KtxRelationshipProfileArtifact, table: KtxEnrichedTable): number | null {
return profile.tables.find((item) => item.table.name.toLowerCase() === table.ref.name.toLowerCase())?.rowCount ?? null;
}
function buildEvidencePacket(
schema: KloEnrichedSchema,
profile: KloRelationshipProfileArtifact,
settings: KloRelationshipLlmProposalSettings,
schema: KtxEnrichedSchema,
profile: KtxRelationshipProfileArtifact,
settings: KtxRelationshipLlmProposalSettings,
): Record<string, unknown> {
return {
connectionId: schema.connectionId,
@ -153,7 +153,7 @@ function pkProposalKey(table: string, column: string): string {
return `${table.toLowerCase()}.${column.toLowerCase()}`;
}
function endpoint(table: KloEnrichedTable, column: KloEnrichedColumn) {
function endpoint(table: KtxEnrichedTable, column: KtxEnrichedColumn) {
return {
tableId: table.id,
columnIds: [column.id],
@ -162,11 +162,11 @@ function endpoint(table: KloEnrichedTable, column: KloEnrichedColumn) {
};
}
function relationshipId(fromTable: KloEnrichedTable, fromColumn: KloEnrichedColumn, toTable: KloEnrichedTable, toColumn: KloEnrichedColumn): string {
function relationshipId(fromTable: KtxEnrichedTable, fromColumn: KtxEnrichedColumn, toTable: KtxEnrichedTable, toColumn: KtxEnrichedColumn): string {
return `${fromTable.id}:(${fromColumn.id})->${toTable.id}:(${toColumn.id})`;
}
function invalidReferenceWarning(message: string, metadata: Record<string, unknown>): KloScanWarning {
function invalidReferenceWarning(message: string, metadata: Record<string, unknown>): KtxScanWarning {
return {
code: 'relationship_llm_invalid_reference',
message,
@ -176,13 +176,13 @@ function invalidReferenceWarning(message: string, metadata: Record<string, unkno
}
function mapValidProposals(
schema: KloEnrichedSchema,
output: KloRelationshipLlmProposalOutput,
settings: KloRelationshipLlmProposalSettings,
): { candidates: KloRelationshipDiscoveryCandidate[]; warnings: KloScanWarning[] } {
const warnings: KloScanWarning[] = [];
schema: KtxEnrichedSchema,
output: KtxRelationshipLlmProposalOutput,
settings: KtxRelationshipLlmProposalSettings,
): { candidates: KtxRelationshipDiscoveryCandidate[]; warnings: KtxScanWarning[] } {
const warnings: KtxScanWarning[] = [];
const pkProposals = new Set(output.pkCandidates.map((item) => pkProposalKey(item.table, item.column)));
const candidates: KloRelationshipDiscoveryCandidate[] = [];
const candidates: KtxRelationshipDiscoveryCandidate[] = [];
for (const item of output.fkCandidates) {
if (item.confidence < settings.minConfidence) {
@ -194,7 +194,7 @@ function mapValidProposals(
const toColumn = toTable ? findColumn(toTable, item.toColumn) : null;
if (!fromTable || !toTable || !fromColumn || !toColumn) {
warnings.push(
invalidReferenceWarning('KLO relationship LLM proposal referenced a table or column that is not in the schema.', {
invalidReferenceWarning('KTX relationship LLM proposal referenced a table or column that is not in the schema.', {
proposal: item,
}),
);
@ -211,9 +211,9 @@ function mapValidProposals(
relationshipType: 'many_to_one',
confidence: clampConfidence(item.confidence),
evidence: {
sourceColumnBase: normalizeKloRelationshipName(fromColumn.name).singular,
targetTableBase: normalizeKloRelationshipName(toTable.ref.name).singular,
targetColumnBase: normalizeKloRelationshipName(toColumn.name).singular,
sourceColumnBase: normalizeKtxRelationshipName(fromColumn.name).singular,
targetTableBase: normalizeKtxRelationshipName(toTable.ref.name).singular,
targetColumnBase: normalizeKtxRelationshipName(toColumn.name).singular,
targetKeyScore: pkProposalExists ? 0.88 : 0.68,
nameScore: 0.45,
reasons: pkProposalExists ? ['llm_proposal', 'llm_pk_proposal'] : ['llm_proposal'],
@ -226,18 +226,18 @@ function mapValidProposals(
return { candidates, warnings };
}
function generationFailureWarning(error: unknown): KloScanWarning {
function generationFailureWarning(error: unknown): KtxScanWarning {
const message = error instanceof Error ? error.message : String(error);
return {
code: 'relationship_llm_proposal_failed',
message: `KLO relationship LLM proposal failed: ${message}`,
message: `KTX relationship LLM proposal failed: ${message}`,
recoverable: true,
};
}
export async function proposeKloRelationshipCandidatesWithLlm(
input: ProposeKloRelationshipCandidatesWithLlmInput,
): Promise<KloRelationshipLlmProposalResult> {
export async function proposeKtxRelationshipCandidatesWithLlm(
input: ProposeKtxRelationshipCandidatesWithLlmInput,
): Promise<KtxRelationshipLlmProposalResult> {
if (!input.llmProvider || modelIsDeterministic(input.llmProvider)) {
return { candidates: [], warnings: [], llmCalls: 0, summary: 'skipped' };
}
@ -245,15 +245,15 @@ export async function proposeKloRelationshipCandidatesWithLlm(
const settings = mergeSettings(input.settings);
const evidence = buildEvidencePacket(input.schema, input.profile, settings);
const prompt = [
'You are helping KLO review possible SQL relationships before validation.',
'You are helping KTX review possible SQL relationships before validation.',
'Use only the compact schema evidence. Propose likely primary keys and foreign keys for later SQL validation.',
'Return structured output only; never assume a join is accepted.',
JSON.stringify(evidence),
].join('\n\n');
try {
const generated = await generateKloObject<
KloRelationshipLlmProposalOutput,
const generated = await generateKtxObject<
KtxRelationshipLlmProposalOutput,
typeof relationshipLlmProposalSchema
>({
llmProvider: input.llmProvider,

8
packages/context/src/scan/relationship-locality.test.ts
View file

@ -1,13 +1,13 @@
import { describe, expect, it } from 'vitest';
import type { KloEnrichedColumn, KloEnrichedTable } from './enrichment-types.js';
import type { KtxEnrichedColumn, KtxEnrichedTable } from './enrichment-types.js';
import { localCandidateTables } from './relationship-locality.js';
function column(
tableId: string,
id: string,
name: string,
options: Partial<KloEnrichedColumn> = {},
): KloEnrichedColumn {
options: Partial<KtxEnrichedColumn> = {},
): KtxEnrichedColumn {
const tableRef = options.tableRef ?? { catalog: null, db: 'public', name: tableId };
return {
id,
@ -27,7 +27,7 @@ function column(
};
}
function table(id: string, name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
function table(id: string, name: string, columns: KtxEnrichedColumn[]): KtxEnrichedTable {
const ref = { catalog: null, db: 'public', name };
return {
id,

44
packages/context/src/scan/relationship-locality.ts
View file

@ -1,18 +1,18 @@
import type { KloEnrichedColumn, KloEnrichedTable } from './enrichment-types.js';
import { normalizeKloRelationshipName, tokenizeKloRelationshipName } from './relationship-name-similarity.js';
import type { KtxEnrichedColumn, KtxEnrichedTable } from './enrichment-types.js';
import { normalizeKtxRelationshipName, tokenizeKtxRelationshipName } from './relationship-name-similarity.js';
export interface KloRelationshipLocalityCandidateTable {
table: KloEnrichedTable;
export interface KtxRelationshipLocalityCandidateTable {
table: KtxEnrichedTable;
score: number;
tokenScore: number;
embeddingScore: number;
reasons: string[];
}
export interface LocalKloRelationshipCandidateTablesInput {
childTable: KloEnrichedTable;
childColumn: KloEnrichedColumn;
parentTables: readonly KloEnrichedTable[];
export interface LocalKtxRelationshipCandidateTablesInput {
childTable: KtxEnrichedTable;
childColumn: KtxEnrichedColumn;
parentTables: readonly KtxEnrichedTable[];
maxParentTables?: number;
}
@ -24,17 +24,17 @@ function roundedScore(value: number): number {
}
function normalizedTokenVariants(name: string): string[] {
const normalized = normalizeKloRelationshipName(name);
const normalized = normalizeKtxRelationshipName(name);
return Array.from(
new Set([
...normalized.tokens,
...tokenizeKloRelationshipName(normalized.singular),
...tokenizeKloRelationshipName(normalized.plural),
...tokenizeKtxRelationshipName(normalized.singular),
...tokenizeKtxRelationshipName(normalized.plural),
]),
).filter(Boolean);
}
function childColumnLocalityTokens(column: KloEnrichedColumn): string[] {
function childColumnLocalityTokens(column: KtxEnrichedColumn): string[] {
const tokens = normalizedTokenVariants(column.name);
const withoutSuffix = tokens.filter((token) => !RELATIONSHIP_SUFFIX_TOKENS.has(token));
return withoutSuffix.length > 0 ? withoutSuffix : tokens;
@ -78,7 +78,7 @@ function cosineSimilarity(left: readonly number[] | null, right: readonly number
return dot / (Math.sqrt(leftMagnitude) * Math.sqrt(rightMagnitude));
}
function parentEmbeddingScore(childColumn: KloEnrichedColumn, parentTable: KloEnrichedTable): number {
function parentEmbeddingScore(childColumn: KtxEnrichedColumn, parentTable: KtxEnrichedTable): number {
if (!Array.isArray(childColumn.embedding) || childColumn.embedding.length === 0) {
return 0;
}
@ -91,9 +91,9 @@ function parentEmbeddingScore(childColumn: KloEnrichedColumn, parentTable: KloEn
}
function tableTokenScore(input: {
childTable: KloEnrichedTable;
childColumn: KloEnrichedColumn;
parentTable: KloEnrichedTable;
childTable: KtxEnrichedTable;
childColumn: KtxEnrichedColumn;
parentTable: KtxEnrichedTable;
}): number {
const childTableTokens = normalizedTokenVariants(input.childTable.ref.name);
const childColumnTokens = childColumnLocalityTokens(input.childColumn);
@ -107,10 +107,10 @@ function tableTokenScore(input: {
}
function localityScore(input: {
childTable: KloEnrichedTable;
childColumn: KloEnrichedColumn;
parentTable: KloEnrichedTable;
}): Omit<KloRelationshipLocalityCandidateTable, 'table'> {
childTable: KtxEnrichedTable;
childColumn: KtxEnrichedColumn;
parentTable: KtxEnrichedTable;
}): Omit<KtxRelationshipLocalityCandidateTable, 'table'> {
const tokenScore = roundedScore(tableTokenScore(input));
const embeddingScore = roundedScore(parentEmbeddingScore(input.childColumn, input.parentTable));
const score =
@ -136,8 +136,8 @@ function localityScore(input: {
}
export function localCandidateTables(
input: LocalKloRelationshipCandidateTablesInput,
): KloRelationshipLocalityCandidateTable[] {
input: LocalKtxRelationshipCandidateTablesInput,
): KtxRelationshipLocalityCandidateTable[] {
const limit = input.maxParentTables ?? DEFAULT_MAX_PARENT_TABLES;
if (!Number.isFinite(limit) || limit <= 0) {
return [];

40
packages/context/src/scan/relationship-name-similarity.test.ts
View file

@ -1,73 +1,73 @@
import { describe, expect, it } from 'vitest';
import {
normalizeKloRelationshipName,
pluralizeKloRelationshipToken,
singularizeKloRelationshipToken,
normalizeKtxRelationshipName,
pluralizeKtxRelationshipToken,
singularizeKtxRelationshipToken,
tokenSimilarity,
tokenizeKloRelationshipName,
tokenizeKtxRelationshipName,
} from './relationship-name-similarity.js';
describe('relationship name similarity', () => {
it('tokenizes common warehouse naming styles', () => {
expect(normalizeKloRelationshipName('AlbumId')).toMatchObject({
expect(normalizeKtxRelationshipName('AlbumId')).toMatchObject({
normalized: 'album_id',
singular: 'album_id',
plural: 'album_ids',
tokens: ['album', 'id'],
});
expect(normalizeKloRelationshipName('artistID')).toMatchObject({
expect(normalizeKtxRelationshipName('artistID')).toMatchObject({
normalized: 'artist_id',
tokens: ['artist', 'id'],
});
expect(normalizeKloRelationshipName('SalesLT.CustomerID')).toMatchObject({
expect(normalizeKtxRelationshipName('SalesLT.CustomerID')).toMatchObject({
normalized: 'sales_lt_customer_id',
singular: 'sales_lt_customer_id',
tokens: ['sales', 'lt', 'customer', 'id'],
});
expect(normalizeKloRelationshipName('SCREAMING_CUSTOMER_UUID')).toMatchObject({
expect(normalizeKtxRelationshipName('SCREAMING_CUSTOMER_UUID')).toMatchObject({
normalized: 'screaming_customer_uuid',
tokens: ['screaming', 'customer', 'uuid'],
});
expect(normalizeKloRelationshipName('billing-account-key')).toMatchObject({
expect(normalizeKtxRelationshipName('billing-account-key')).toMatchObject({
normalized: 'billing_account_key',
tokens: ['billing', 'account', 'key'],
});
});
it('removes only leading warehouse layer prefixes', () => {
expect(normalizeKloRelationshipName('mart__Sales_Accounts')).toMatchObject({
expect(normalizeKtxRelationshipName('mart__Sales_Accounts')).toMatchObject({
normalized: 'sales_accounts',
singular: 'sales_account',
plural: 'sales_accounts',
tokens: ['sales', 'accounts'],
});
expect(normalizeKloRelationshipName('dim_users')).toMatchObject({
expect(normalizeKtxRelationshipName('dim_users')).toMatchObject({
normalized: 'users',
singular: 'user',
plural: 'users',
tokens: ['users'],
});
expect(normalizeKloRelationshipName('customer_dim_id')).toMatchObject({
expect(normalizeKtxRelationshipName('customer_dim_id')).toMatchObject({
normalized: 'customer_dim_id',
tokens: ['customer', 'dim', 'id'],
});
});
it('folds accents and preserves non-suffix trailing s words', () => {
expect(normalizeKloRelationshipName('KundénID')).toMatchObject({
expect(normalizeKtxRelationshipName('KundénID')).toMatchObject({
normalized: 'kunden_id',
tokens: ['kunden', 'id'],
});
expect(singularizeKloRelationshipToken('address')).toBe('address');
expect(singularizeKloRelationshipToken('addresses')).toBe('address');
expect(singularizeKloRelationshipToken('status')).toBe('status');
expect(pluralizeKloRelationshipToken('address')).toBe('addresses');
expect(pluralizeKloRelationshipToken('company')).toBe('companies');
expect(singularizeKtxRelationshipToken('address')).toBe('address');
expect(singularizeKtxRelationshipToken('addresses')).toBe('address');
expect(singularizeKtxRelationshipToken('status')).toBe('status');
expect(pluralizeKtxRelationshipToken('address')).toBe('addresses');
expect(pluralizeKtxRelationshipToken('company')).toBe('companies');
});
it('returns deterministic tokens for direct tokenization calls', () => {
expect(tokenizeKloRelationshipName('HTTPResponseCode')).toEqual(['http', 'response', 'code']);
expect(tokenizeKloRelationshipName('customer2AddressID')).toEqual(['customer', '2', 'address', 'id']);
expect(tokenizeKtxRelationshipName('HTTPResponseCode')).toEqual(['http', 'response', 'code']);
expect(tokenizeKtxRelationshipName('customer2AddressID')).toEqual(['customer', '2', 'address', 'id']);
});
it('scores token overlap and ordered suffix similarity', () => {

26
packages/context/src/scan/relationship-name-similarity.ts
View file

@ -1,4 +1,4 @@
export interface KloRelationshipNormalizedName {
export interface KtxRelationshipNormalizedName {
raw: string;
normalized: string;
singular: string;
@ -6,7 +6,7 @@ export interface KloRelationshipNormalizedName {
tokens: string[];
}
export type KloRelationshipTokenInput = string | readonly string[] | KloRelationshipNormalizedName;
export type KtxRelationshipTokenInput = string | readonly string[] | KtxRelationshipNormalizedName;
const WAREHOUSE_LAYER_PREFIXES = new Set(['stg', 'stage', 'staging', 'dim', 'fct', 'fact', 'int', 'mart']);
@ -27,7 +27,7 @@ function foldAccents(value: string): string {
.replace(/œ/giu, 'oe');
}
export function singularizeKloRelationshipToken(value: string): string {
export function singularizeKtxRelationshipToken(value: string): string {
if (value.length <= 2) {
return value;
}
@ -46,7 +46,7 @@ export function singularizeKloRelationshipToken(value: string): string {
return value;
}
export function pluralizeKloRelationshipToken(value: string): string {
export function pluralizeKtxRelationshipToken(value: string): string {
if (value.endsWith('y')) {
return `${value.slice(0, -1)}ies`;
}
@ -63,7 +63,7 @@ function singularizeTokens(tokens: readonly string[]): string[] {
const result = [...tokens];
const last = result[result.length - 1];
if (last) {
result[result.length - 1] = singularizeKloRelationshipToken(last);
result[result.length - 1] = singularizeKtxRelationshipToken(last);
}
return result;
}
@ -75,12 +75,12 @@ function pluralizeTokens(tokens: readonly string[]): string[] {
const result = [...tokens];
const last = result[result.length - 1];
if (last) {
result[result.length - 1] = pluralizeKloRelationshipToken(last);
result[result.length - 1] = pluralizeKtxRelationshipToken(last);
}
return result;
}
export function tokenizeKloRelationshipName(name: string): string[] {
export function tokenizeKtxRelationshipName(name: string): string[] {
const boundarySeparated = splitCaseBoundaries(foldAccents(name.trim()));
const tokens = boundarySeparated
.toLowerCase()
@ -92,8 +92,8 @@ export function tokenizeKloRelationshipName(name: string): string[] {
return tokens.filter((token, index) => index > 0 || !WAREHOUSE_LAYER_PREFIXES.has(token));
}
export function normalizeKloRelationshipName(name: string): KloRelationshipNormalizedName {
const tokens = tokenizeKloRelationshipName(name);
export function normalizeKtxRelationshipName(name: string): KtxRelationshipNormalizedName {
const tokens = tokenizeKtxRelationshipName(name);
const singularTokens = singularizeTokens(tokens);
const pluralTokens = pluralizeTokens(singularTokens);
@ -106,14 +106,14 @@ export function normalizeKloRelationshipName(name: string): KloRelationshipNorma
};
}
function tokensFromInput(input: KloRelationshipTokenInput): string[] {
function tokensFromInput(input: KtxRelationshipTokenInput): string[] {
if (typeof input === 'string') {
return tokenizeKloRelationshipName(input);
return tokenizeKtxRelationshipName(input);
}
if ('tokens' in input) {
return input.tokens;
}
return input.map((token) => normalizeKloRelationshipName(token).normalized).filter(Boolean);
return input.map((token) => normalizeKtxRelationshipName(token).normalized).filter(Boolean);
}
function longestCommonSuffixLength(left: readonly string[], right: readonly string[]): number {
@ -132,7 +132,7 @@ function roundedScore(value: number): number {
return Number(Math.max(0, Math.min(1, value)).toFixed(3));
}
export function tokenSimilarity(leftInput: KloRelationshipTokenInput, rightInput: KloRelationshipTokenInput): number {
export function tokenSimilarity(leftInput: KtxRelationshipTokenInput, rightInput: KtxRelationshipTokenInput): number {
const left = tokensFromInput(leftInput);
const right = tokensFromInput(rightInput);
if (left.length === 0 || right.length === 0) {

60
packages/context/src/scan/relationship-profiling.test.ts
View file

@ -2,22 +2,22 @@ import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import Database from 'better-sqlite3';
import { afterEach, describe, expect, it } from 'vitest';
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable } from './enrichment-types.js';
import { snapshotToKloEnrichedSchema } from './local-enrichment.js';
import { loadKloRelationshipBenchmarkFixture, maskKloRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js';
import { snapshotToKtxEnrichedSchema } from './local-enrichment.js';
import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js';
import {
createKloRelationshipProfileCache,
formatKloRelationshipTableRef,
profileKloRelationshipSchema,
quoteKloRelationshipIdentifier,
createKtxRelationshipProfileCache,
formatKtxRelationshipTableRef,
profileKtxRelationshipSchema,
quoteKtxRelationshipIdentifier,
} from './relationship-profiling.js';
import type { KloQueryResult, KloReadOnlyQueryInput, KloScanContext } from './types.js';
import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from './types.js';
class InMemorySqliteExecutor {
readonly db = new Database(':memory:');
queryCount = 0;
executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
this.queryCount += 1;
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
@ -42,7 +42,7 @@ class FileSqliteExecutor {
this.db = new Database(dataPath, { readonly: true, fileMustExist: true });
}
executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
this.queryCount += 1;
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
@ -59,7 +59,7 @@ class FileSqliteExecutor {
}
}
function column(tableId: string, name: string, overrides: Partial<KloEnrichedColumn> = {}): KloEnrichedColumn {
function column(tableId: string, name: string, overrides: Partial<KtxEnrichedColumn> = {}): KtxEnrichedColumn {
const tableRef = overrides.tableRef ?? { catalog: null, db: null, name: tableId };
return {
id: `${tableId}.${name}`,
@ -80,7 +80,7 @@ function column(tableId: string, name: string, overrides: Partial<KloEnrichedCol
};
}
function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
function table(name: string, columns: KtxEnrichedColumn[]): KtxEnrichedTable {
const ref = { catalog: null, db: null, name };
return {
id: name,
@ -91,7 +91,7 @@ function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
};
}
function schema(tables: KloEnrichedTable[]): KloEnrichedSchema {
function schema(tables: KtxEnrichedTable[]): KtxEnrichedSchema {
return { connectionId: 'warehouse', tables, relationships: [] };
}
@ -113,11 +113,11 @@ describe('relationship profiling', () => {
});
it('quotes identifiers and formats table refs for supported local SQL drivers', () => {
expect(quoteKloRelationshipIdentifier('sqlite', 'odd"name')).toBe('"odd""name"');
expect(quoteKloRelationshipIdentifier('mysql', 'odd`name')).toBe('`odd``name`');
expect(quoteKloRelationshipIdentifier('sqlserver', 'odd]name')).toBe('[odd]]name]');
expect(formatKloRelationshipTableRef('sqlite', { catalog: null, db: null, name: 'accounts' })).toBe('"accounts"');
expect(formatKloRelationshipTableRef('postgres', { catalog: null, db: 'analytics', name: 'accounts' })).toBe(
expect(quoteKtxRelationshipIdentifier('sqlite', 'odd"name')).toBe('"odd""name"');
expect(quoteKtxRelationshipIdentifier('mysql', 'odd`name')).toBe('`odd``name`');
expect(quoteKtxRelationshipIdentifier('sqlserver', 'odd]name')).toBe('[odd]]name]');
expect(formatKtxRelationshipTableRef('sqlite', { catalog: null, db: null, name: 'accounts' })).toBe('"accounts"');
expect(formatKtxRelationshipTableRef('postgres', { catalog: null, db: 'analytics', name: 'accounts' })).toBe(
'"analytics"."accounts"',
);
});
@ -133,7 +133,7 @@ describe('relationship profiling', () => {
(4, 'C-3', 2);
`);
const result = await profileKloRelationshipSchema({
const result = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: schema([
@ -195,7 +195,7 @@ describe('relationship profiling', () => {
(12, 2);
`);
const result = await profileKloRelationshipSchema({
const result = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: schema([
@ -238,7 +238,7 @@ describe('relationship profiling', () => {
INSERT INTO accounts VALUES (1, 'a1'), (2, 'a2'), (3, 'a3'), (4, 'a4');
`);
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: schema([
@ -287,9 +287,9 @@ describe('relationship profiling', () => {
}),
]),
]);
const cache = createKloRelationshipProfileCache();
const cache = createKtxRelationshipProfileCache();
const first = await profileKloRelationshipSchema({
const first = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: relationshipSchema,
@ -297,7 +297,7 @@ describe('relationship profiling', () => {
ctx: { runId: 'profile-cache-run' },
cache,
});
const second = await profileKloRelationshipSchema({
const second = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: relationshipSchema,
@ -305,13 +305,13 @@ describe('relationship profiling', () => {
ctx: { runId: 'profile-cache-run' },
cache,
});
const third = await profileKloRelationshipSchema({
const third = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: relationshipSchema,
executor,
ctx: { runId: 'profile-cache-fresh-run' },
cache: createKloRelationshipProfileCache(),
cache: createKtxRelationshipProfileCache(),
});
expect(first.queryCount).toBe(1);
@ -324,20 +324,20 @@ describe('relationship profiling', () => {
it('profiles the checked-in scale stress fixture with one query per table', async () => {
const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
const fixture = await loadKloRelationshipBenchmarkFixture(join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'));
const fixture = await loadKtxRelationshipBenchmarkFixture(join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'));
if (!fixture.dataPath) {
throw new Error('scale_stress_no_declared_constraints is missing data.sqlite');
}
const maskedSnapshot = maskKloRelationshipBenchmarkSnapshot(
const maskedSnapshot = maskKtxRelationshipBenchmarkSnapshot(
fixture.snapshot,
'declared_pks_and_declared_fks_removed',
);
const scaleExecutor = new FileSqliteExecutor(fixture.dataPath);
try {
const result = await profileKloRelationshipSchema({
const result = await profileKtxRelationshipSchema({
connectionId: fixture.snapshot.connectionId,
driver: fixture.snapshot.driver,
schema: snapshotToKloEnrichedSchema(maskedSnapshot, new Map()),
schema: snapshotToKtxEnrichedSchema(maskedSnapshot, new Map()),
executor: scaleExecutor,
ctx: { runId: 'scale-stress-profile-query-count' },
profileSampleRows: 3,

134
packages/context/src/scan/relationship-profiling.ts
View file

@ -1,18 +1,18 @@
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable } from './enrichment-types.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js';
import type {
KloConnectionDriver,
KloQueryResult,
KloReadOnlyQueryInput,
KloScanContext,
KloTableRef,
KtxConnectionDriver,
KtxQueryResult,
KtxReadOnlyQueryInput,
KtxScanContext,
KtxTableRef,
} from './types.js';
export interface KloRelationshipReadOnlyExecutor {
executeReadOnly(input: KloReadOnlyQueryInput, ctx: KloScanContext): Promise<KloQueryResult>;
export interface KtxRelationshipReadOnlyExecutor {
executeReadOnly(input: KtxReadOnlyQueryInput, ctx: KtxScanContext): Promise<KtxQueryResult>;
}
export interface KloRelationshipColumnProfile {
table: KloTableRef;
export interface KtxRelationshipColumnProfile {
table: KtxTableRef;
column: string;
nativeType: string;
normalizedType: string;
@ -26,43 +26,43 @@ export interface KloRelationshipColumnProfile {
maxTextLength: number | null;
}
export interface KloRelationshipTableProfile {
table: KloTableRef;
export interface KtxRelationshipTableProfile {
table: KtxTableRef;
rowCount: number;
}
export interface KloRelationshipProfileArtifact {
export interface KtxRelationshipProfileArtifact {
connectionId: string;
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
sqlAvailable: boolean;
queryCount: number;
tables: KloRelationshipTableProfile[];
columns: Record<string, KloRelationshipColumnProfile>;
tables: KtxRelationshipTableProfile[];
columns: Record<string, KtxRelationshipColumnProfile>;
warnings: string[];
}
interface KloRelationshipCachedTableProfile {
table: KloRelationshipTableProfile;
columns: Record<string, KloRelationshipColumnProfile>;
interface KtxRelationshipCachedTableProfile {
table: KtxRelationshipTableProfile;
columns: Record<string, KtxRelationshipColumnProfile>;
warnings: string[];
}
export interface KloRelationshipProfileCache {
readonly tableProfiles: Map<string, KloRelationshipCachedTableProfile>;
export interface KtxRelationshipProfileCache {
readonly tableProfiles: Map<string, KtxRelationshipCachedTableProfile>;
}
export interface ProfileKloRelationshipSchemaInput {
export interface ProfileKtxRelationshipSchemaInput {
connectionId: string;
driver: KloConnectionDriver;
schema: KloEnrichedSchema;
executor: KloRelationshipReadOnlyExecutor | null;
ctx: KloScanContext;
driver: KtxConnectionDriver;
schema: KtxEnrichedSchema;
executor: KtxRelationshipReadOnlyExecutor | null;
ctx: KtxScanContext;
sampleValuesPerColumn?: number;
profileSampleRows?: number;
cache?: KloRelationshipProfileCache;
cache?: KtxRelationshipProfileCache;
}
export function createKloRelationshipProfileCache(): KloRelationshipProfileCache {
export function createKtxRelationshipProfileCache(): KtxRelationshipProfileCache {
return { tableProfiles: new Map() };
}
@ -70,7 +70,7 @@ const SAMPLE_VALUE_DELIMITER = '\u001f';
type QuoteStyle = 'double' | 'backtick' | 'bracket';
function quoteStyle(driver: KloConnectionDriver): QuoteStyle {
function quoteStyle(driver: KtxConnectionDriver): QuoteStyle {
if (driver === 'mysql' || driver === 'clickhouse' || driver === 'posthog') {
return 'backtick';
}
@ -80,7 +80,7 @@ function quoteStyle(driver: KloConnectionDriver): QuoteStyle {
return 'double';
}
export function quoteKloRelationshipIdentifier(driver: KloConnectionDriver, identifier: string): string {
export function quoteKtxRelationshipIdentifier(driver: KtxConnectionDriver, identifier: string): string {
switch (quoteStyle(driver)) {
case 'backtick':
return `\`${identifier.replace(/`/g, '``')}\``;
@ -91,15 +91,15 @@ export function quoteKloRelationshipIdentifier(driver: KloConnectionDriver, iden
}
}
export function formatKloRelationshipTableRef(driver: KloConnectionDriver, table: KloTableRef): string {
export function formatKtxRelationshipTableRef(driver: KtxConnectionDriver, table: KtxTableRef): string {
const parts =
driver === 'sqlite' || driver === 'posthog'
? [table.name]
: [table.catalog, table.db, table.name].filter((value): value is string => Boolean(value));
return parts.map((part) => quoteKloRelationshipIdentifier(driver, part)).join('.');
return parts.map((part) => quoteKtxRelationshipIdentifier(driver, part)).join('.');
}
function textLengthExpression(driver: KloConnectionDriver, columnSql: string): string {
function textLengthExpression(driver: KtxConnectionDriver, columnSql: string): string {
if (driver === 'mysql') {
return `CHAR_LENGTH(CAST(${columnSql} AS CHAR))`;
}
@ -115,21 +115,21 @@ function textLengthExpression(driver: KloConnectionDriver, columnSql: string): s
return `LENGTH(CAST(${columnSql} AS TEXT))`;
}
function limitSql(driver: KloConnectionDriver, limit: number): string {
function limitSql(driver: KtxConnectionDriver, limit: number): string {
if (driver === 'sqlserver') {
return '';
}
return ` LIMIT ${Math.max(1, Math.floor(limit))}`;
}
function topSql(driver: KloConnectionDriver, limit: number): string {
function topSql(driver: KtxConnectionDriver, limit: number): string {
if (driver === 'sqlserver') {
return ` TOP (${Math.max(1, Math.floor(limit))})`;
}
return '';
}
function sampledTableSql(driver: KloConnectionDriver, tableSql: string, limit: number): string {
function sampledTableSql(driver: KtxConnectionDriver, tableSql: string, limit: number): string {
const safeLimit = Math.max(1, Math.floor(limit));
if (driver === 'sqlserver') {
return `(SELECT TOP (${safeLimit}) * FROM ${tableSql}) AS relationship_profile_sample`;
@ -137,15 +137,15 @@ function sampledTableSql(driver: KloConnectionDriver, tableSql: string, limit: n
return `(SELECT * FROM ${tableSql}${limitSql(driver, safeLimit)}) AS relationship_profile_sample`;
}
function firstRow(result: KloQueryResult): unknown[] {
function firstRow(result: KtxQueryResult): unknown[] {
return result.rows[0] ?? [];
}
function headerIndex(result: KloQueryResult, header: string): number {
function headerIndex(result: KtxQueryResult, header: string): number {
return result.headers.findIndex((candidate) => candidate.toLowerCase() === header.toLowerCase());
}
function valueAt(result: KloQueryResult, row: unknown[], header: string): unknown {
function valueAt(result: KtxQueryResult, row: unknown[], header: string): unknown {
return row[headerIndex(result, header)];
}
@ -178,19 +178,19 @@ function nullableNumberFromValue(value: unknown): number | null {
return null;
}
function numberAt(result: KloQueryResult, header: string): number {
function numberAt(result: KtxQueryResult, header: string): number {
return numberFromValue(valueAt(result, firstRow(result), header));
}
function columnKey(table: KloEnrichedTable, column: KloEnrichedColumn): string {
function columnKey(table: KtxEnrichedTable, column: KtxEnrichedColumn): string {
return `${table.ref.name}.${column.name}`;
}
function tableProfileCacheKey(input: {
connectionId: string;
driver: KloConnectionDriver;
ctx: KloScanContext;
table: KloTableRef;
driver: KtxConnectionDriver;
ctx: KtxScanContext;
table: KtxTableRef;
sampleValuesPerColumn: number;
profileSampleRows: number;
}): string {
@ -210,7 +210,7 @@ function sqlStringLiteral(value: string): string {
return `'${value.replace(/'/g, "''")}'`;
}
function sampleAggregateSql(driver: KloConnectionDriver, innerSql: string): string {
function sampleAggregateSql(driver: KtxConnectionDriver, innerSql: string): string {
if (driver === 'postgres') {
return `(SELECT STRING_AGG(CAST(value AS TEXT), CHR(31)) FROM (${innerSql}) AS relationship_profile_values)`;
}
@ -230,7 +230,7 @@ function sampleAggregateSql(driver: KloConnectionDriver, innerSql: string): stri
}
function sampleValuesSql(input: {
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
tableSql: string;
columnSql: string;
limit: number;
@ -246,13 +246,13 @@ function sampleValuesSql(input: {
}
function columnProfileSelectSql(input: {
connectionDriver: KloConnectionDriver;
connectionDriver: KtxConnectionDriver;
tableSql: string;
profileTableSql: string;
column: KloEnrichedColumn;
column: KtxEnrichedColumn;
sampleValuesPerColumn: number;
}): string {
const columnSql = quoteKloRelationshipIdentifier(input.connectionDriver, input.column.name);
const columnSql = quoteKtxRelationshipIdentifier(input.connectionDriver, input.column.name);
const textLengthSql = textLengthExpression(input.connectionDriver, columnSql);
const samplesSql = sampleAggregateSql(
input.connectionDriver,
@ -290,12 +290,12 @@ function splitSampleValues(value: unknown): string[] {
async function queryCount(input: {
connectionId: string;
driver: KloConnectionDriver;
table: KloTableRef;
executor: KloRelationshipReadOnlyExecutor;
ctx: KloScanContext;
driver: KtxConnectionDriver;
table: KtxTableRef;
executor: KtxRelationshipReadOnlyExecutor;
ctx: KtxScanContext;
}): Promise<{ rowCount: number; queryCount: number }> {
const tableSql = formatKloRelationshipTableRef(input.driver, input.table);
const tableSql = formatKtxRelationshipTableRef(input.driver, input.table);
const result = await input.executor.executeReadOnly(
{ connectionId: input.connectionId, sql: `SELECT COUNT(*) AS row_count FROM ${tableSql}`, maxRows: 1 },
input.ctx,
@ -305,15 +305,15 @@ async function queryCount(input: {
async function queryTableProfile(input: {
connectionId: string;
driver: KloConnectionDriver;
table: KloEnrichedTable;
executor: KloRelationshipReadOnlyExecutor;
ctx: KloScanContext;
driver: KtxConnectionDriver;
table: KtxEnrichedTable;
executor: KtxRelationshipReadOnlyExecutor;
ctx: KtxScanContext;
sampleValuesPerColumn: number;
profileSampleRows: number;
}): Promise<{
table: KloRelationshipTableProfile;
columns: Record<string, KloRelationshipColumnProfile>;
table: KtxRelationshipTableProfile;
columns: Record<string, KtxRelationshipColumnProfile>;
queryCount: number;
}> {
if (input.table.columns.length === 0) {
@ -331,7 +331,7 @@ async function queryTableProfile(input: {
};
}
const tableSql = formatKloRelationshipTableRef(input.driver, input.table.ref);
const tableSql = formatKtxRelationshipTableRef(input.driver, input.table.ref);
const profileTableSql = sampledTableSql(input.driver, tableSql, input.profileSampleRows);
const sql = input.table.columns
.map((column) =>
@ -349,7 +349,7 @@ async function queryTableProfile(input: {
input.ctx,
);
const columnsByName = new Map(input.table.columns.map((column) => [column.name, column]));
const profiles: Record<string, KloRelationshipColumnProfile> = {};
const profiles: Record<string, KtxRelationshipColumnProfile> = {};
let tableRowCount = 0;
for (const row of result.rows) {
@ -385,9 +385,9 @@ async function queryTableProfile(input: {
};
}
export async function profileKloRelationshipSchema(
input: ProfileKloRelationshipSchemaInput,
): Promise<KloRelationshipProfileArtifact> {
export async function profileKtxRelationshipSchema(
input: ProfileKtxRelationshipSchemaInput,
): Promise<KtxRelationshipProfileArtifact> {
if (!input.executor) {
return {
connectionId: input.connectionId,
@ -401,8 +401,8 @@ export async function profileKloRelationshipSchema(
}
let queryTotal = 0;
const tables: KloRelationshipTableProfile[] = [];
const columns: Record<string, KloRelationshipColumnProfile> = {};
const tables: KtxRelationshipTableProfile[] = [];
const columns: Record<string, KtxRelationshipColumnProfile> = {};
const warnings: string[] = [];
for (const table of input.schema.tables.filter((candidate) => candidate.enabled)) {

22
packages/context/src/scan/relationship-review-apply.test.ts
View file

@ -1,16 +1,16 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { KloLocalProject } from '../project/index.js';
import { initKloProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import { initKtxProject } from '../project/index.js';
import { describe, expect, it, vi } from 'vitest';
import { applyLocalScanRelationshipReviewDecisions } from './relationship-review-apply.js';
import type { KloRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js';
import type { KtxRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js';
import type { ReadLocalScanRelationshipArtifactsResult } from './relationship-artifacts.js';
import type { WriteLocalScanManifestShardsResult } from './local-enrichment-artifacts.js';
import type { KloSchemaSnapshot } from './types.js';
import type { KtxSchemaSnapshot } from './types.js';
const acceptedDecisionArtifact: KloRelationshipReviewDecisionArtifact = {
const acceptedDecisionArtifact: KtxRelationshipReviewDecisionArtifact = {
connectionId: 'warehouse',
runId: 'scan-run-a',
syncId: 'sync-a',
@ -146,7 +146,7 @@ const artifacts: ReadLocalScanRelationshipArtifactsResult = {
},
};
const snapshot: KloSchemaSnapshot = {
const snapshot: KtxSchemaSnapshot = {
connectionId: 'warehouse',
driver: 'postgres',
extractedAt: '2026-05-07T12:00:00.000Z',
@ -198,17 +198,17 @@ const snapshot: KloSchemaSnapshot = {
async function projectWithDecisions(
decisions = acceptedDecisionArtifact,
): Promise<{ project: KloLocalProject; tempDir: string }> {
const tempDir = await mkdtemp(join(tmpdir(), 'klo-relationship-review-apply-'));
const project = await initKloProject({
): Promise<{ project: KtxLocalProject; tempDir: string }> {
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-apply-'));
const project = await initKtxProject({
projectDir: join(tempDir, 'project'),
projectName: 'warehouse',
});
await project.fileStore.writeFile(
'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json',
`${JSON.stringify(decisions)}\n`,
'klo',
'klo@example.com',
'ktx',
'ktx@example.com',
'Seed relationship review decisions',
);
return { project, tempDir };

36
packages/context/src/scan/relationship-review-apply.ts
View file

@ -1,4 +1,4 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import {
readLocalScanRelationshipArtifacts,
type ReadLocalScanRelationshipArtifactsResult,
@ -12,10 +12,10 @@ import {
type WriteLocalScanManifestShardsInput,
type WriteLocalScanManifestShardsResult,
} from './local-enrichment-artifacts.js';
import type { KloEnrichedRelationship, KloRelationshipUpdate } from './enrichment-types.js';
import type { KtxEnrichedRelationship, KtxRelationshipUpdate } from './enrichment-types.js';
import type {
KloRelationshipReviewDecisionArtifact,
KloRelationshipReviewDecisionEntry,
KtxRelationshipReviewDecisionArtifact,
KtxRelationshipReviewDecisionEntry,
} from './relationship-review-decisions.js';
const DECISIONS_FILE = 'relationship-review-decisions.json';
@ -39,7 +39,7 @@ export interface AppliedRelationshipReviewDecision {
decidedAt: string;
reviewer: string;
note: string | null;
relationship: KloEnrichedRelationship;
relationship: KtxEnrichedRelationship;
}
export interface ApplyLocalScanRelationshipReviewDecisionsResult {
@ -50,7 +50,7 @@ export interface ApplyLocalScanRelationshipReviewDecisionsResult {
decisionsPath: string;
selectedDecisions: number;
appliedRelationships: number;
relationships: KloEnrichedRelationship[];
relationships: KtxEnrichedRelationship[];
manifestShards: string[];
manifestShardsWritten: number;
}
@ -60,17 +60,17 @@ function decisionsPathFromRelationshipsPath(relationshipsPath: string): string {
}
async function readDecisionArtifact(
project: KloLocalProject,
project: KtxLocalProject,
path: string,
runId: string,
): Promise<KloRelationshipReviewDecisionArtifact> {
): Promise<KtxRelationshipReviewDecisionArtifact> {
let raw: { content: string };
try {
raw = await project.fileStore.readFile(path);
} catch {
throw new Error(`Relationship review decisions were not found for scan run "${runId}"`);
}
const parsed = JSON.parse(raw.content) as KloRelationshipReviewDecisionArtifact;
const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact;
return {
connectionId: parsed.connectionId,
runId: parsed.runId,
@ -91,16 +91,16 @@ function assertSelection(input: ApplyLocalScanRelationshipReviewDecisionsInput):
}
function selectAcceptedDecisions(
artifact: KloRelationshipReviewDecisionArtifact,
artifact: KtxRelationshipReviewDecisionArtifact,
input: ApplyLocalScanRelationshipReviewDecisionsInput,
): KloRelationshipReviewDecisionEntry[] {
): KtxRelationshipReviewDecisionEntry[] {
assertSelection(input);
if (input.applyAllAccepted === true) {
return artifact.decisions.filter((decision) => decision.decision === 'accepted');
}
const decisionsById = new Map(artifact.decisions.map((decision) => [decision.candidateId, decision]));
const selected: KloRelationshipReviewDecisionEntry[] = [];
const selected: KtxRelationshipReviewDecisionEntry[] = [];
for (const candidateId of input.candidateIds ?? []) {
const decision = decisionsById.get(candidateId);
if (!decision) {
@ -114,16 +114,16 @@ function selectAcceptedDecisions(
return selected;
}
function tableId(table: KloRelationshipReviewDecisionEntry['from']['table']): string {
function tableId(table: KtxRelationshipReviewDecisionEntry['from']['table']): string {
return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.');
}
function columnIds(table: KloRelationshipReviewDecisionEntry['from']['table'], columns: readonly string[]): string[] {
function columnIds(table: KtxRelationshipReviewDecisionEntry['from']['table'], columns: readonly string[]): string[] {
const prefix = tableId(table);
return columns.map((column) => `${prefix}.${column}`);
}
function relationshipFromDecision(decision: KloRelationshipReviewDecisionEntry): KloEnrichedRelationship {
function relationshipFromDecision(decision: KtxRelationshipReviewDecisionEntry): KtxEnrichedRelationship {
return {
id: decision.candidateId,
source: 'manual',
@ -147,8 +147,8 @@ function relationshipFromDecision(decision: KloRelationshipReviewDecisionEntry):
function relationshipUpdate(
connectionId: string,
relationships: readonly KloEnrichedRelationship[],
): KloRelationshipUpdate {
relationships: readonly KtxEnrichedRelationship[],
): KtxRelationshipUpdate {
return {
connectionId,
accepted: [...relationships],
@ -166,7 +166,7 @@ function assertApplyableArtifacts(artifacts: ReadLocalScanRelationshipArtifactsR
}
export async function applyLocalScanRelationshipReviewDecisions(
project: KloLocalProject,
project: KtxLocalProject,
input: ApplyLocalScanRelationshipReviewDecisionsInput,
): Promise<ApplyLocalScanRelationshipReviewDecisionsResult> {
const readArtifacts = input.readLocalScanRelationshipArtifacts ?? readLocalScanRelationshipArtifacts;

38
packages/context/src/scan/relationship-review-decisions.test.ts
View file

@ -2,12 +2,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { runLocalStageOnlyIngest, type SourceAdapter } from '../ingest/index.js';
import { initKloProject, loadKloProject } from '../project/index.js';
import { initKtxProject, loadKtxProject } from '../project/index.js';
import { describe, expect, it } from 'vitest';
import { writeLocalScanRelationshipReviewDecision } from './relationship-review-decisions.js';
import type { KloRelationshipArtifact, KloRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KloScanReport } from './types.js';
import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import type { KtxScanReport } from './types.js';
const RUN_ID = 'scan-run-review';
const SYNC_ID = '2026-05-07-100000-scan-run-review';
@ -19,9 +19,9 @@ async function writeProjectFile(projectDir: string, relativePath: string, conten
}
async function createProject(projectDir: string): Promise<void> {
await initKloProject({ projectDir, projectName: 'warehouse' });
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeFile(
join(projectDir, 'klo.yaml'),
join(projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
@ -73,7 +73,7 @@ function liveDatabaseAdapter(): SourceAdapter {
async function createLiveDatabaseRun(projectDir: string): Promise<void> {
await createProject(projectDir);
const project = await loadKloProject({ projectDir });
const project = await loadKtxProject({ projectDir });
await runLocalStageOnlyIngest({
project,
adapters: [liveDatabaseAdapter()],
@ -84,7 +84,7 @@ async function createLiveDatabaseRun(projectDir: string): Promise<void> {
});
}
function reviewRelationships(): KloRelationshipArtifact {
function reviewRelationships(): KtxRelationshipArtifact {
return {
connectionId: 'warehouse',
accepted: [],
@ -121,7 +121,7 @@ function reviewRelationships(): KloRelationshipArtifact {
};
}
function diagnostics(): KloRelationshipDiagnosticsArtifact {
function diagnostics(): KtxRelationshipDiagnosticsArtifact {
return {
connectionId: 'warehouse',
generatedAt: '2026-05-07T10:00:00.000Z',
@ -141,7 +141,7 @@ function diagnostics(): KloRelationshipDiagnosticsArtifact {
};
}
function profile(): KloRelationshipProfileArtifact {
function profile(): KtxRelationshipProfileArtifact {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -153,7 +153,7 @@ function profile(): KloRelationshipProfileArtifact {
};
}
function report(): KloScanReport {
function report(): KtxScanReport {
return {
connectionId: 'warehouse',
driver: 'sqlite',
@ -236,11 +236,11 @@ async function writeScanArtifacts(projectDir: string): Promise<void> {
describe('relationship review decisions', () => {
it('writes an accepted decision beside the scan relationship artifacts', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-review-decisions-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-decisions-'));
try {
await createLiveDatabaseRun(projectDir);
await writeScanArtifacts(projectDir);
const project = await loadKloProject({ projectDir });
const project = await loadKtxProject({ projectDir });
const result = await writeLocalScanRelationshipReviewDecision(project, {
runId: 'scan-run-review',
@ -280,11 +280,11 @@ describe('relationship review decisions', () => {
});
it('replaces the existing decision for the same candidate id', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-review-replace-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-replace-'));
try {
await createLiveDatabaseRun(projectDir);
await writeScanArtifacts(projectDir);
const project = await loadKloProject({ projectDir });
const project = await loadKtxProject({ projectDir });
await writeLocalScanRelationshipReviewDecision(project, {
runId: 'scan-run-review',
@ -319,10 +319,10 @@ describe('relationship review decisions', () => {
});
it('returns null when the scan run does not exist', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-review-missing-run-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-missing-run-'));
try {
await createProject(projectDir);
const project = await loadKloProject({ projectDir });
const project = await loadKtxProject({ projectDir });
await expect(
writeLocalScanRelationshipReviewDecision(project, {
@ -340,11 +340,11 @@ describe('relationship review decisions', () => {
});
it('rejects unknown candidate ids for an existing scan run', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'klo-relationship-review-missing-candidate-'));
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-missing-candidate-'));
try {
await createLiveDatabaseRun(projectDir);
await writeScanArtifacts(projectDir);
const project = await loadKloProject({ projectDir });
const project = await loadKtxProject({ projectDir });
await expect(
writeLocalScanRelationshipReviewDecision(project, {

64
packages/context/src/scan/relationship-review-decisions.ts
View file

@ -1,40 +1,40 @@
import type { KloLocalProject } from '../project/index.js';
import type { KloRelationshipType } from './enrichment-types.js';
import type { KtxLocalProject } from '../project/index.js';
import type { KtxRelationshipType } from './enrichment-types.js';
import { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js';
import type {
KloRelationshipArtifactEdge,
KloRelationshipArtifactEndpoint,
KtxRelationshipArtifactEdge,
KtxRelationshipArtifactEndpoint,
} from './relationship-diagnostics.js';
import type { KloResolvedRelationshipStatus } from './relationship-graph-resolver.js';
import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js';
const LOCAL_AUTHOR = 'klo';
const LOCAL_AUTHOR_EMAIL = 'klo@example.com';
const LOCAL_AUTHOR = 'ktx';
const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
const DECISIONS_FILE = 'relationship-review-decisions.json';
export type KloRelationshipReviewDecisionValue = 'accepted' | 'rejected';
export type KtxRelationshipReviewDecisionValue = 'accepted' | 'rejected';
export interface WriteLocalScanRelationshipReviewDecisionInput {
runId: string;
candidateId: string;
decision: KloRelationshipReviewDecisionValue;
decision: KtxRelationshipReviewDecisionValue;
reviewer: string;
note: string | null;
decidedAt?: string;
}
export interface KloRelationshipReviewDecisionEntry {
export interface KtxRelationshipReviewDecisionEntry {
candidateId: string;
decision: KloRelationshipReviewDecisionValue;
previousStatus: KloResolvedRelationshipStatus;
decision: KtxRelationshipReviewDecisionValue;
previousStatus: KtxResolvedRelationshipStatus;
connectionId: string;
runId: string;
syncId: string;
decidedAt: string;
reviewer: string;
note: string | null;
from: KloRelationshipArtifactEndpoint;
to: KloRelationshipArtifactEndpoint;
relationshipType: KloRelationshipType;
from: KtxRelationshipArtifactEndpoint;
to: KtxRelationshipArtifactEndpoint;
relationshipType: KtxRelationshipType;
source: string;
score: number | null;
confidence: number;
@ -43,25 +43,25 @@ export interface KloRelationshipReviewDecisionEntry {
reasons: string[];
}
export interface KloRelationshipReviewDecisionArtifact {
export interface KtxRelationshipReviewDecisionArtifact {
connectionId: string;
runId: string;
syncId: string;
generatedAt: string;
decisions: KloRelationshipReviewDecisionEntry[];
decisions: KtxRelationshipReviewDecisionEntry[];
}
export interface WriteLocalScanRelationshipReviewDecisionResult {
path: string;
decision: KloRelationshipReviewDecisionEntry;
artifact: KloRelationshipReviewDecisionArtifact;
decision: KtxRelationshipReviewDecisionEntry;
artifact: KtxRelationshipReviewDecisionArtifact;
}
function reviewDecisionPath(relationshipsPath: string): string {
return relationshipsPath.replace(/relationships\.json$/u, DECISIONS_FILE);
}
function allCandidateEdges(result: Awaited<ReturnType<typeof readLocalScanRelationshipArtifacts>>): KloRelationshipArtifactEdge[] {
function allCandidateEdges(result: Awaited<ReturnType<typeof readLocalScanRelationshipArtifacts>>): KtxRelationshipArtifactEdge[] {
if (!result) {
return [];
}
@ -69,13 +69,13 @@ function allCandidateEdges(result: Awaited<ReturnType<typeof readLocalScanRelati
}
async function readExistingDecisions(
project: KloLocalProject,
project: KtxLocalProject,
path: string,
fallback: Omit<KloRelationshipReviewDecisionArtifact, 'decisions'>,
): Promise<KloRelationshipReviewDecisionArtifact> {
fallback: Omit<KtxRelationshipReviewDecisionArtifact, 'decisions'>,
): Promise<KtxRelationshipReviewDecisionArtifact> {
try {
const raw = await project.fileStore.readFile(path);
const parsed = JSON.parse(raw.content) as KloRelationshipReviewDecisionArtifact;
const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact;
return {
connectionId: parsed.connectionId,
runId: parsed.runId,
@ -89,15 +89,15 @@ async function readExistingDecisions(
}
function decisionEntry(input: {
candidate: KloRelationshipArtifactEdge;
candidate: KtxRelationshipArtifactEdge;
connectionId: string;
runId: string;
syncId: string;
decision: KloRelationshipReviewDecisionValue;
decision: KtxRelationshipReviewDecisionValue;
reviewer: string;
note: string | null;
decidedAt: string;
}): KloRelationshipReviewDecisionEntry {
}): KtxRelationshipReviewDecisionEntry {
return {
candidateId: input.candidate.id,
decision: input.decision,
@ -121,16 +121,16 @@ function decisionEntry(input: {
}
function upsertDecision(
existing: readonly KloRelationshipReviewDecisionEntry[],
next: KloRelationshipReviewDecisionEntry,
): KloRelationshipReviewDecisionEntry[] {
existing: readonly KtxRelationshipReviewDecisionEntry[],
next: KtxRelationshipReviewDecisionEntry,
): KtxRelationshipReviewDecisionEntry[] {
return [...existing.filter((item) => item.candidateId !== next.candidateId), next].sort((left, right) =>
left.candidateId.localeCompare(right.candidateId),
);
}
export async function writeLocalScanRelationshipReviewDecision(
project: KloLocalProject,
project: KtxLocalProject,
input: WriteLocalScanRelationshipReviewDecisionInput,
): Promise<WriteLocalScanRelationshipReviewDecisionResult | null> {
const artifacts = await readLocalScanRelationshipArtifacts(project, input.runId);
@ -162,7 +162,7 @@ export async function writeLocalScanRelationshipReviewDecision(
note: input.note,
decidedAt,
});
const artifact: KloRelationshipReviewDecisionArtifact = {
const artifact: KtxRelationshipReviewDecisionArtifact = {
connectionId: artifacts.connectionId,
runId: artifacts.runId,
syncId: artifacts.syncId,

20
packages/context/src/scan/relationship-scoring.test.ts
View file

@ -1,13 +1,13 @@
import { describe, expect, it } from 'vitest';
import {
calibrateWeightsFromSyntheticFixtures,
defaultKloRelationshipScoreWeights,
normalizeKloRelationshipScoreWeights,
scoreKloRelationshipCandidate,
type KloRelationshipSignalVector,
defaultKtxRelationshipScoreWeights,
normalizeKtxRelationshipScoreWeights,
scoreKtxRelationshipCandidate,
type KtxRelationshipSignalVector,
} from './relationship-scoring.js';
function signals(overrides: Partial<KloRelationshipSignalVector> = {}): KloRelationshipSignalVector {
function signals(overrides: Partial<KtxRelationshipSignalVector> = {}): KtxRelationshipSignalVector {
return {
nameSimilarity: 0.5,
typeCompatibility: 1,
@ -22,7 +22,7 @@ function signals(overrides: Partial<KloRelationshipSignalVector> = {}): KloRelat
describe('relationship scoring', () => {
it('scores stronger evidence higher without hard-gating on names', () => {
const weakNameStrongProfile = scoreKloRelationshipCandidate(
const weakNameStrongProfile = scoreKtxRelationshipCandidate(
signals({
nameSimilarity: 0.05,
typeCompatibility: 1,
@ -32,7 +32,7 @@ describe('relationship scoring', () => {
structuralPrior: 0.7,
}),
);
const strongNameWeakProfile = scoreKloRelationshipCandidate(
const strongNameWeakProfile = scoreKtxRelationshipCandidate(
signals({
nameSimilarity: 0.95,
typeCompatibility: 1,
@ -49,7 +49,7 @@ describe('relationship scoring', () => {
});
it('normalizes partial and invalid weights into a usable vector', () => {
const weights = normalizeKloRelationshipScoreWeights({
const weights = normalizeKtxRelationshipScoreWeights({
nameSimilarity: 3,
typeCompatibility: -1,
valueOverlap: Number.POSITIVE_INFINITY,
@ -64,8 +64,8 @@ describe('relationship scoring', () => {
});
it('returns deterministic defaults as a defensive copy', () => {
const first = defaultKloRelationshipScoreWeights();
const second = defaultKloRelationshipScoreWeights();
const first = defaultKtxRelationshipScoreWeights();
const second = defaultKtxRelationshipScoreWeights();
expect(first).toEqual(second);
expect(first).not.toBe(second);

76
packages/context/src/scan/relationship-scoring.ts
View file

@ -1,4 +1,4 @@
export const KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS = [
export const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [
'nameSimilarity',
'typeCompatibility',
'valueOverlap',
@ -8,11 +8,11 @@ export const KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS = [
'structuralPrior',
] as const;
export type KloRelationshipScoreSignal = (typeof KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS)[number];
export type KtxRelationshipScoreSignal = (typeof KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS)[number];
export type KloRelationshipFixtureOrigin = 'synthetic' | 'public' | 'customer';
export type KtxRelationshipFixtureOrigin = 'synthetic' | 'public' | 'customer';
export interface KloRelationshipSignalVector {
export interface KtxRelationshipSignalVector {
nameSimilarity: number;
typeCompatibility: number;
valueOverlap: number;
@ -22,23 +22,23 @@ export interface KloRelationshipSignalVector {
structuralPrior: number;
}
export type KloRelationshipScoreWeights = Record<KloRelationshipScoreSignal, number>;
export type KtxRelationshipScoreWeights = Record<KtxRelationshipScoreSignal, number>;
export interface KloRelationshipScoreBreakdown {
export interface KtxRelationshipScoreBreakdown {
score: number;
signals: KloRelationshipSignalVector;
weights: KloRelationshipScoreWeights;
contributions: KloRelationshipScoreWeights;
signals: KtxRelationshipSignalVector;
weights: KtxRelationshipScoreWeights;
contributions: KtxRelationshipScoreWeights;
}
export interface KloRelationshipScoringCalibrationObservation {
export interface KtxRelationshipScoringCalibrationObservation {
fixtureId: string;
origin: KloRelationshipFixtureOrigin;
origin: KtxRelationshipFixtureOrigin;
expectedRelationship: boolean;
signals: KloRelationshipSignalVector;
signals: KtxRelationshipSignalVector;
}
const DEFAULT_WEIGHTS: KloRelationshipScoreWeights = {
const DEFAULT_WEIGHTS: KtxRelationshipScoreWeights = {
nameSimilarity: 0.24,
typeCompatibility: 0.1,
valueOverlap: 0.22,
@ -59,7 +59,7 @@ function roundScore(value: number): number {
return Number(clampScore(value).toFixed(3));
}
function sanitizeSignalVector(signals: KloRelationshipSignalVector): KloRelationshipSignalVector {
function sanitizeSignalVector(signals: KtxRelationshipSignalVector): KtxRelationshipSignalVector {
return {
nameSimilarity: roundScore(signals.nameSimilarity),
typeCompatibility: roundScore(signals.typeCompatibility),
@ -71,38 +71,38 @@ function sanitizeSignalVector(signals: KloRelationshipSignalVector): KloRelation
};
}
export function defaultKloRelationshipScoreWeights(): KloRelationshipScoreWeights {
export function defaultKtxRelationshipScoreWeights(): KtxRelationshipScoreWeights {
return { ...DEFAULT_WEIGHTS };
}
export function normalizeKloRelationshipScoreWeights(
weights: Partial<KloRelationshipScoreWeights> = DEFAULT_WEIGHTS,
): KloRelationshipScoreWeights {
const rawEntries = KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS.map((key) => {
export function normalizeKtxRelationshipScoreWeights(
weights: Partial<KtxRelationshipScoreWeights> = DEFAULT_WEIGHTS,
): KtxRelationshipScoreWeights {
const rawEntries = KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS.map((key) => {
const value = weights[key] ?? 0;
return [key, Number.isFinite(value) ? Math.max(0, value) : 0] as const;
});
const total = rawEntries.reduce((sum, [, value]) => sum + value, 0);
if (total <= 0) {
return defaultKloRelationshipScoreWeights();
return defaultKtxRelationshipScoreWeights();
}
return Object.fromEntries(rawEntries.map(([key, value]) => [key, value / total])) as KloRelationshipScoreWeights;
return Object.fromEntries(rawEntries.map(([key, value]) => [key, value / total])) as KtxRelationshipScoreWeights;
}
export function scoreKloRelationshipCandidate(
signals: KloRelationshipSignalVector,
weights: Partial<KloRelationshipScoreWeights> = DEFAULT_WEIGHTS,
): KloRelationshipScoreBreakdown {
export function scoreKtxRelationshipCandidate(
signals: KtxRelationshipSignalVector,
weights: Partial<KtxRelationshipScoreWeights> = DEFAULT_WEIGHTS,
): KtxRelationshipScoreBreakdown {
const sanitizedSignals = sanitizeSignalVector(signals);
const normalizedWeights = normalizeKloRelationshipScoreWeights(weights);
const normalizedWeights = normalizeKtxRelationshipScoreWeights(weights);
const contributions = Object.fromEntries(
KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS.map((key) => [
KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS.map((key) => [
key,
Number((sanitizedSignals[key] * normalizedWeights[key]).toFixed(6)),
]),
) as KloRelationshipScoreWeights;
const rawWeightedScore = KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS.reduce((sum, key) => sum + contributions[key], 0);
) as KtxRelationshipScoreWeights;
const rawWeightedScore = KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS.reduce((sum, key) => sum + contributions[key], 0);
const scoredConfidence = sanitizedSignals.typeCompatibility <= 0 ? 0 : 0.56 + rawWeightedScore * 0.65;
return {
@ -114,8 +114,8 @@ export function scoreKloRelationshipCandidate(
}
function averageSignal(
observations: readonly KloRelationshipScoringCalibrationObservation[],
key: KloRelationshipScoreSignal,
observations: readonly KtxRelationshipScoringCalibrationObservation[],
key: KtxRelationshipScoreSignal,
): number {
if (observations.length === 0) {
return 0;
@ -124,8 +124,8 @@ function averageSignal(
}
export function calibrateWeightsFromSyntheticFixtures(
observations: readonly KloRelationshipScoringCalibrationObservation[],
): KloRelationshipScoreWeights {
observations: readonly KtxRelationshipScoringCalibrationObservation[],
): KtxRelationshipScoreWeights {
const nonSynthetic = observations.find((observation) => observation.origin !== 'synthetic');
if (nonSynthetic) {
throw new Error(
@ -133,23 +133,23 @@ export function calibrateWeightsFromSyntheticFixtures(
);
}
if (observations.length === 0) {
return defaultKloRelationshipScoreWeights();
return defaultKtxRelationshipScoreWeights();
}
const positives = observations.filter((observation) => observation.expectedRelationship);
const negatives = observations.filter((observation) => !observation.expectedRelationship);
if (positives.length === 0 || negatives.length === 0) {
return defaultKloRelationshipScoreWeights();
return defaultKtxRelationshipScoreWeights();
}
const calibrated = Object.fromEntries(
KLO_RELATIONSHIP_SCORE_SIGNAL_KEYS.map((key) => {
KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS.map((key) => {
const positiveAverage = averageSignal(positives, key);
const negativeAverage = averageSignal(negatives, key);
const separation = Math.max(0, positiveAverage - negativeAverage);
return [key, separation + DEFAULT_WEIGHTS[key] * 0.25];
}),
) as KloRelationshipScoreWeights;
) as KtxRelationshipScoreWeights;
return normalizeKloRelationshipScoreWeights(calibrated);
return normalizeKtxRelationshipScoreWeights(calibrated);
}

32
packages/context/src/scan/relationship-threshold-advice.test.ts
View file

@ -1,18 +1,18 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import { describe, expect, it, vi } from 'vitest';
import {
adviseLocalRelationshipFeedbackThresholds,
buildKloRelationshipThresholdAdviceReport,
formatKloRelationshipThresholdAdviceMarkdown,
buildKtxRelationshipThresholdAdviceReport,
formatKtxRelationshipThresholdAdviceMarkdown,
} from './relationship-threshold-advice.js';
import type {
ExportLocalRelationshipFeedbackLabelsResult,
KloRelationshipFeedbackLabel,
KtxRelationshipFeedbackLabel,
} from './relationship-feedback-export.js';
function label(
input: Partial<KloRelationshipFeedbackLabel> & Pick<KloRelationshipFeedbackLabel, 'candidateId' | 'decision' | 'score'>,
): KloRelationshipFeedbackLabel {
input: Partial<KtxRelationshipFeedbackLabel> & Pick<KtxRelationshipFeedbackLabel, 'candidateId' | 'decision' | 'score'>,
): KtxRelationshipFeedbackLabel {
return {
schemaVersion: 1,
previousStatus: 'review',
@ -37,7 +37,7 @@ function label(
};
}
function feedback(labels: KloRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult {
function feedback(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult {
return {
generatedAt: '2026-05-07T13:00:00.000Z',
filters: { connectionId: null, decision: 'all' },
@ -55,7 +55,7 @@ function feedback(labels: KloRelationshipFeedbackLabel[]): ExportLocalRelationsh
describe('relationship threshold advice', () => {
it('selects the highest-quality threshold candidate when enough labels exist', () => {
const report = buildKloRelationshipThresholdAdviceReport(
const report = buildKtxRelationshipThresholdAdviceReport(
feedback([
label({
candidateId: 'orders:orders.customer_id->customers:customers.id',
@ -125,7 +125,7 @@ describe('relationship threshold advice', () => {
});
it('reports insufficient labels without hiding evaluated candidates', () => {
const report = buildKloRelationshipThresholdAdviceReport(
const report = buildKtxRelationshipThresholdAdviceReport(
feedback([
label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }),
label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }),
@ -157,7 +157,7 @@ describe('relationship threshold advice', () => {
});
it('reports no eligible thresholds when label counts pass but quality gates fail', () => {
const report = buildKloRelationshipThresholdAdviceReport(
const report = buildKtxRelationshipThresholdAdviceReport(
feedback([
label({ candidateId: 'a', decision: 'accepted', score: 0.92 }),
label({ candidateId: 'b', decision: 'accepted', score: 0.58 }),
@ -186,7 +186,7 @@ describe('relationship threshold advice', () => {
});
it('wraps the feedback exporter and preserves warnings', async () => {
const project = { projectDir: '/tmp/klo-project' } as KloLocalProject;
const project = { projectDir: '/tmp/ktx-project' } as KtxLocalProject;
const exportLocalRelationshipFeedbackLabels = vi.fn(async () => ({
...feedback([]),
warnings: [
@ -216,7 +216,7 @@ describe('relationship threshold advice', () => {
});
it('formats a stable human-readable report', () => {
const report = buildKloRelationshipThresholdAdviceReport(
const report = buildKtxRelationshipThresholdAdviceReport(
feedback([
label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }),
label({ candidateId: 'orders:orders.account_id->accounts:accounts.id', decision: 'accepted', score: 0.61 }),
@ -233,9 +233,9 @@ describe('relationship threshold advice', () => {
},
);
expect(formatKloRelationshipThresholdAdviceMarkdown(report)).toContain('KLO relationship threshold advice');
expect(formatKloRelationshipThresholdAdviceMarkdown(report)).toContain('Status: ready');
expect(formatKloRelationshipThresholdAdviceMarkdown(report)).toContain('Recommended: accept=0.90 review=0.55');
expect(formatKloRelationshipThresholdAdviceMarkdown(report)).toContain('acceptedPrecision=1.000');
expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('KTX relationship threshold advice');
expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('Status: ready');
expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('Recommended: accept=0.90 review=0.55');
expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('acceptedPrecision=1.000');
});
});

62
packages/context/src/scan/relationship-threshold-advice.ts
View file

@ -1,20 +1,20 @@
import type { KloLocalProject } from '../project/index.js';
import type { KtxLocalProject } from '../project/index.js';
import {
exportLocalRelationshipFeedbackLabels,
type ExportLocalRelationshipFeedbackLabelsInput,
type ExportLocalRelationshipFeedbackLabelsResult,
type KloRelationshipFeedbackExportWarning,
type KloRelationshipFeedbackLabel,
type KtxRelationshipFeedbackExportWarning,
type KtxRelationshipFeedbackLabel,
} from './relationship-feedback-export.js';
import type { KloResolvedRelationshipStatus } from './relationship-graph-resolver.js';
import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js';
const DEFAULT_ACCEPT_THRESHOLDS = [0.95, 0.9, 0.85, 0.8, 0.75] as const;
const DEFAULT_REVIEW_THRESHOLDS = [0.65, 0.6, 0.55, 0.5, 0.45] as const;
type AdvicePredictedStatus = KloResolvedRelationshipStatus;
export type KloRelationshipThresholdAdviceStatus = 'ready' | 'insufficient_labels' | 'no_eligible_thresholds';
type AdvicePredictedStatus = KtxResolvedRelationshipStatus;
export type KtxRelationshipThresholdAdviceStatus = 'ready' | 'insufficient_labels' | 'no_eligible_thresholds';
export interface BuildKloRelationshipThresholdAdviceReportInput {
export interface BuildKtxRelationshipThresholdAdviceReportInput {
acceptThresholds?: readonly number[];
reviewThresholds?: readonly number[];
minTotalLabels?: number;
@ -27,11 +27,11 @@ export interface BuildKloRelationshipThresholdAdviceReportInput {
export interface AdviseLocalRelationshipFeedbackThresholdsInput
extends Omit<ExportLocalRelationshipFeedbackLabelsInput, 'decision'>,
BuildKloRelationshipThresholdAdviceReportInput {
BuildKtxRelationshipThresholdAdviceReportInput {
exportLocalRelationshipFeedbackLabels?: typeof exportLocalRelationshipFeedbackLabels;
}
export interface KloRelationshipThresholdAdviceCandidate {
export interface KtxRelationshipThresholdAdviceCandidate {
acceptThreshold: number;
reviewThreshold: number;
eligible: boolean;
@ -47,10 +47,10 @@ export interface KloRelationshipThresholdAdviceCandidate {
falseRejectedAcceptedLabels: number;
}
export interface KloRelationshipThresholdAdviceReport {
export interface KtxRelationshipThresholdAdviceReport {
generatedAt: string;
filters: ExportLocalRelationshipFeedbackLabelsResult['filters'];
status: KloRelationshipThresholdAdviceStatus;
status: KtxRelationshipThresholdAdviceStatus;
gates: {
minTotalLabels: number;
minAcceptedLabels: number;
@ -68,10 +68,10 @@ export interface KloRelationshipThresholdAdviceReport {
evaluatedCandidates: number;
eligibleCandidates: number;
};
recommended: KloRelationshipThresholdAdviceCandidate | null;
candidates: KloRelationshipThresholdAdviceCandidate[];
recommended: KtxRelationshipThresholdAdviceCandidate | null;
candidates: KtxRelationshipThresholdAdviceCandidate[];
reasons: string[];
warnings: KloRelationshipFeedbackExportWarning[];
warnings: KtxRelationshipFeedbackExportWarning[];
}
interface ResolvedAdviceInput {
@ -85,7 +85,7 @@ interface ResolvedAdviceInput {
minRejectedBandPrecision: number;
}
function resolveInput(input: BuildKloRelationshipThresholdAdviceReportInput): ResolvedAdviceInput {
function resolveInput(input: BuildKtxRelationshipThresholdAdviceReportInput): ResolvedAdviceInput {
return {
acceptThresholds: [...(input.acceptThresholds ?? DEFAULT_ACCEPT_THRESHOLDS)].sort((left, right) => right - left),
reviewThresholds: [...(input.reviewThresholds ?? DEFAULT_REVIEW_THRESHOLDS)].sort((left, right) => right - left),
@ -121,12 +121,12 @@ function isMetricAtLeast(value: number | null, minimum: number): boolean {
}
function thresholdCandidate(
labels: readonly KloRelationshipFeedbackLabel[],
labels: readonly KtxRelationshipFeedbackLabel[],
acceptThreshold: number,
reviewThreshold: number,
gates: ResolvedAdviceInput,
): KloRelationshipThresholdAdviceCandidate {
const scored = labels.filter((label): label is KloRelationshipFeedbackLabel & { score: number } => label.score !== null);
): KtxRelationshipThresholdAdviceCandidate {
const scored = labels.filter((label): label is KtxRelationshipFeedbackLabel & { score: number } => label.score !== null);
const acceptedLabels = scored.filter((label) => label.decision === 'accepted');
const rejectedLabels = scored.filter((label) => label.decision === 'rejected');
const predictions = scored.map((label) => ({
@ -182,8 +182,8 @@ function metricRank(value: number | null): number {
}
function sortCandidates(
candidates: readonly KloRelationshipThresholdAdviceCandidate[],
): KloRelationshipThresholdAdviceCandidate[] {
candidates: readonly KtxRelationshipThresholdAdviceCandidate[],
): KtxRelationshipThresholdAdviceCandidate[] {
return [...candidates].sort(
(left, right) =>
Number(right.eligible) - Number(left.eligible) ||
@ -195,7 +195,7 @@ function sortCandidates(
);
}
function labelGateReasons(labels: readonly KloRelationshipFeedbackLabel[], gates: ResolvedAdviceInput): string[] {
function labelGateReasons(labels: readonly KtxRelationshipFeedbackLabel[], gates: ResolvedAdviceInput): string[] {
const scored = labels.filter((label) => label.score !== null);
const accepted = scored.filter((label) => label.decision === 'accepted');
const rejected = scored.filter((label) => label.decision === 'rejected');
@ -212,10 +212,10 @@ function labelGateReasons(labels: readonly KloRelationshipFeedbackLabel[], gates
return reasons;
}
export function buildKloRelationshipThresholdAdviceReport(
export function buildKtxRelationshipThresholdAdviceReport(
feedback: ExportLocalRelationshipFeedbackLabelsResult,
input: BuildKloRelationshipThresholdAdviceReportInput = {},
): KloRelationshipThresholdAdviceReport {
input: BuildKtxRelationshipThresholdAdviceReportInput = {},
): KtxRelationshipThresholdAdviceReport {
const gates = resolveInput(input);
const scored = feedback.labels.filter((label) => label.score !== null);
const acceptedLabels = scored.filter((label) => label.decision === 'accepted');
@ -231,7 +231,7 @@ export function buildKloRelationshipThresholdAdviceReport(
);
const labelReasons = labelGateReasons(feedback.labels, gates);
const eligibleCandidates = candidates.filter((candidate) => candidate.eligible);
const status: KloRelationshipThresholdAdviceStatus =
const status: KtxRelationshipThresholdAdviceStatus =
labelReasons.length > 0 ? 'insufficient_labels' : eligibleCandidates.length > 0 ? 'ready' : 'no_eligible_thresholds';
const reasons =
status === 'insufficient_labels'
@ -269,22 +269,22 @@ export function buildKloRelationshipThresholdAdviceReport(
}
export async function adviseLocalRelationshipFeedbackThresholds(
project: KloLocalProject,
project: KtxLocalProject,
input: AdviseLocalRelationshipFeedbackThresholdsInput = {},
): Promise<KloRelationshipThresholdAdviceReport> {
): Promise<KtxRelationshipThresholdAdviceReport> {
const exporter = input.exportLocalRelationshipFeedbackLabels ?? exportLocalRelationshipFeedbackLabels;
const feedback = await exporter(project, {
connectionId: input.connectionId,
decision: 'all',
});
return buildKloRelationshipThresholdAdviceReport(feedback, input);
return buildKtxRelationshipThresholdAdviceReport(feedback, input);
}
function formatMetric(value: number | null): string {
return value === null ? 'n/a' : value.toFixed(3);
}
function candidateLine(candidate: KloRelationshipThresholdAdviceCandidate): string {
function candidateLine(candidate: KtxRelationshipThresholdAdviceCandidate): string {
return [
`accept=${candidate.acceptThreshold.toFixed(2)}`,
`review=${candidate.reviewThreshold.toFixed(2)}`,
@ -299,9 +299,9 @@ function candidateLine(candidate: KloRelationshipThresholdAdviceCandidate): stri
].join(' ');
}
export function formatKloRelationshipThresholdAdviceMarkdown(report: KloRelationshipThresholdAdviceReport): string {
export function formatKtxRelationshipThresholdAdviceMarkdown(report: KtxRelationshipThresholdAdviceReport): string {
const lines = [
'KLO relationship threshold advice',
'KTX relationship threshold advice',
`Generated: ${report.generatedAt}`,
`Filter connection: ${report.filters.connectionId ?? 'all'}`,
`Status: ${report.status}`,

62
packages/context/src/scan/relationship-validation.test.ts
View file

@ -1,17 +1,17 @@
import Database from 'better-sqlite3';
import { afterEach, describe, expect, it } from 'vitest';
import type { KloEnrichedColumn, KloEnrichedSchema, KloEnrichedTable } from './enrichment-types.js';
import { generateKloRelationshipDiscoveryCandidates } from './relationship-candidates.js';
import type { KloRelationshipProfileArtifact } from './relationship-profiling.js';
import { profileKloRelationshipSchema } from './relationship-profiling.js';
import { validateKloRelationshipDiscoveryCandidates } from './relationship-validation.js';
import type { KloQueryResult, KloReadOnlyQueryInput, KloScanContext } from './types.js';
import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js';
import { generateKtxRelationshipDiscoveryCandidates } from './relationship-candidates.js';
import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
import { profileKtxRelationshipSchema } from './relationship-profiling.js';
import { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js';
import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from './types.js';
class InMemorySqliteExecutor {
readonly db = new Database(':memory:');
queryCount = 0;
executeReadOnly(input: KloReadOnlyQueryInput, _ctx: KloScanContext): Promise<KloQueryResult> {
executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
this.queryCount += 1;
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
const headers = Object.keys(rows[0] ?? {});
@ -28,7 +28,7 @@ class InMemorySqliteExecutor {
}
}
function column(tableId: string, name: string, overrides: Partial<KloEnrichedColumn> = {}): KloEnrichedColumn {
function column(tableId: string, name: string, overrides: Partial<KtxEnrichedColumn> = {}): KtxEnrichedColumn {
const tableRef = overrides.tableRef ?? { catalog: null, db: null, name: tableId };
return {
id: `${tableId}.${name}`,
@ -49,7 +49,7 @@ function column(tableId: string, name: string, overrides: Partial<KloEnrichedCol
};
}
function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
function table(name: string, columns: KtxEnrichedColumn[]): KtxEnrichedTable {
const ref = { catalog: null, db: null, name };
return {
id: name,
@ -60,7 +60,7 @@ function table(name: string, columns: KloEnrichedColumn[]): KloEnrichedTable {
};
}
function schema(tables?: KloEnrichedTable[]): KloEnrichedSchema {
function schema(tables?: KtxEnrichedTable[]): KtxEnrichedSchema {
return {
connectionId: 'warehouse',
tables: tables ?? [
@ -97,18 +97,18 @@ describe('relationship validation', () => {
INSERT INTO invoices (id, account_id) VALUES (20, 1), (21, 2), (22, 999);
`);
const testSchema = schema();
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: testSchema,
executor,
ctx: { runId: 'validate-test' },
});
const candidates = generateKloRelationshipDiscoveryCandidates(testSchema).filter(
const candidates = generateKtxRelationshipDiscoveryCandidates(testSchema).filter(
(candidate) => candidate.from.table.name === 'users',
);
const validated = await validateKloRelationshipDiscoveryCandidates({
const validated = await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates,
@ -145,18 +145,18 @@ describe('relationship validation', () => {
INSERT INTO invoices (id, account_id) VALUES (20, 1), (21, 999), (22, 1000);
`);
const testSchema = schema();
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: testSchema,
executor,
ctx: { runId: 'validate-test' },
});
const candidates = generateKloRelationshipDiscoveryCandidates(testSchema).filter(
const candidates = generateKtxRelationshipDiscoveryCandidates(testSchema).filter(
(candidate) => candidate.from.table.name === 'invoices',
);
const validated = await validateKloRelationshipDiscoveryCandidates({
const validated = await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates,
@ -194,7 +194,7 @@ describe('relationship validation', () => {
INSERT INTO invoices (id, account_id) VALUES (20, 1), (21, 2), (22, 3);
`);
const testSchema = schema();
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: testSchema,
@ -202,12 +202,12 @@ describe('relationship validation', () => {
ctx: { runId: 'validate-budget-profile' },
});
executor.queryCount = 0;
const candidates = generateKloRelationshipDiscoveryCandidates(testSchema).map((candidate) => ({
const candidates = generateKtxRelationshipDiscoveryCandidates(testSchema).map((candidate) => ({
...candidate,
confidence: candidate.from.table.name === 'users' ? 0.99 : 0.5,
}));
const validated = await validateKloRelationshipDiscoveryCandidates({
const validated = await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates,
@ -249,7 +249,7 @@ describe('relationship validation', () => {
]),
table('users', [column('users', 'id', { nullable: false }), column('users', 'account_id', { nullable: false })]),
]);
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: testSchema,
@ -257,9 +257,9 @@ describe('relationship validation', () => {
ctx: { runId: 'validate-zero-budget-profile' },
});
executor.queryCount = 0;
const candidates = generateKloRelationshipDiscoveryCandidates(testSchema);
const candidates = generateKtxRelationshipDiscoveryCandidates(testSchema);
const validated = await validateKloRelationshipDiscoveryCandidates({
const validated = await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates,
@ -296,14 +296,14 @@ describe('relationship validation', () => {
table('customers', [column('customers', 'id', { nullable: false })]),
table('orders', [column('orders', 'buyer_ref')]),
]);
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: testSchema,
executor,
ctx: { runId: 'llm-rejected-validation' },
});
const [candidate] = generateKloRelationshipDiscoveryCandidates(
const [candidate] = generateKtxRelationshipDiscoveryCandidates(
schema([
table('customers', [column('customers', 'id', { nullable: false })]),
table('orders', [column('orders', 'customer_id')]),
@ -325,7 +325,7 @@ describe('relationship validation', () => {
},
};
const [validated] = await validateKloRelationshipDiscoveryCandidates({
const [validated] = await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates: [llmCandidate],
@ -354,7 +354,7 @@ describe('relationship validation', () => {
let active = 0;
let maxActive = 0;
const throttled = {
executeReadOnly: async (input: KloReadOnlyQueryInput, ctx: KloScanContext) => {
executeReadOnly: async (input: KtxReadOnlyQueryInput, ctx: KtxScanContext) => {
active += 1;
maxActive = Math.max(maxActive, active);
await new Promise((resolve) => setTimeout(resolve, input.sql.includes('WITH child_values') ? 10 : 0));
@ -369,16 +369,16 @@ describe('relationship validation', () => {
table('orders', [column('orders', 'id', { nullable: false }), column('orders', 'account_id')]),
table('invoices', [column('invoices', 'id', { nullable: false }), column('invoices', 'account_id')]),
]);
const profiles = await profileKloRelationshipSchema({
const profiles = await profileKtxRelationshipSchema({
connectionId: 'warehouse',
driver: 'sqlite',
schema: testSchema,
executor,
ctx: { runId: 'validation-concurrency-profile' },
});
const candidates = generateKloRelationshipDiscoveryCandidates(testSchema);
const candidates = generateKtxRelationshipDiscoveryCandidates(testSchema);
await validateKloRelationshipDiscoveryCandidates({
await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates,
@ -457,7 +457,7 @@ describe('relationship validation', () => {
maxTextLength: 10,
},
},
} satisfies KloRelationshipProfileArtifact;
} satisfies KtxRelationshipProfileArtifact;
const executor = {
async executeReadOnly() {
return {
@ -469,7 +469,7 @@ describe('relationship validation', () => {
},
};
const [validated] = await validateKloRelationshipDiscoveryCandidates({
const [validated] = await validateKtxRelationshipDiscoveryCandidates({
connectionId: 'warehouse',
driver: 'sqlite',
candidates: [candidate],

98
packages/context/src/scan/relationship-validation.ts
View file

@ -1,17 +1,17 @@
import type { KloRelationshipEndpoint } from './enrichment-types.js';
import { applyKloRelationshipValidationBudget, type KloRelationshipValidationBudget } from './relationship-budget.js';
import type { KloRelationshipDiscoveryCandidate } from './relationship-candidates.js';
import type { KtxRelationshipEndpoint } from './enrichment-types.js';
import { applyKtxRelationshipValidationBudget, type KtxRelationshipValidationBudget } from './relationship-budget.js';
import type { KtxRelationshipDiscoveryCandidate } from './relationship-candidates.js';
import {
formatKloRelationshipTableRef,
type KloRelationshipProfileArtifact,
type KloRelationshipReadOnlyExecutor,
quoteKloRelationshipIdentifier,
formatKtxRelationshipTableRef,
type KtxRelationshipProfileArtifact,
type KtxRelationshipReadOnlyExecutor,
quoteKtxRelationshipIdentifier,
} from './relationship-profiling.js';
import type { KloConnectionDriver, KloQueryResult, KloScanContext } from './types.js';
import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext } from './types.js';
export type KloValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected';
export type KtxValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected';
export interface KloRelationshipValidationSettings {
export interface KtxRelationshipValidationSettings {
acceptThreshold: number;
reviewThreshold: number;
minTargetUniqueness: number;
@ -19,10 +19,10 @@ export interface KloRelationshipValidationSettings {
maxViolationRatio: number;
maxDistinctSourceValues: number;
concurrency: number;
validationBudget?: KloRelationshipValidationBudget;
validationBudget?: KtxRelationshipValidationBudget;
}
export interface KloRelationshipValidationEvidence {
export interface KtxRelationshipValidationEvidence {
targetUniqueness: number;
sourceCoverage: number;
violationCount: number;
@ -36,25 +36,25 @@ export interface KloRelationshipValidationEvidence {
reasons: string[];
}
export interface KloValidatedRelationshipDiscoveryCandidate
extends Omit<KloRelationshipDiscoveryCandidate, 'status'> {
status: KloValidatedRelationshipStatus;
export interface KtxValidatedRelationshipDiscoveryCandidate
extends Omit<KtxRelationshipDiscoveryCandidate, 'status'> {
status: KtxValidatedRelationshipStatus;
score: number;
validation: KloRelationshipValidationEvidence;
validation: KtxRelationshipValidationEvidence;
}
export interface ValidateKloRelationshipDiscoveryCandidatesInput {
export interface ValidateKtxRelationshipDiscoveryCandidatesInput {
connectionId: string;
driver: KloConnectionDriver;
candidates: readonly KloRelationshipDiscoveryCandidate[];
profiles: KloRelationshipProfileArtifact;
executor: KloRelationshipReadOnlyExecutor | null;
ctx: KloScanContext;
driver: KtxConnectionDriver;
candidates: readonly KtxRelationshipDiscoveryCandidate[];
profiles: KtxRelationshipProfileArtifact;
executor: KtxRelationshipReadOnlyExecutor | null;
ctx: KtxScanContext;
tableCount?: number;
settings?: Partial<KloRelationshipValidationSettings>;
settings?: Partial<KtxRelationshipValidationSettings>;
}
const DEFAULT_SETTINGS: KloRelationshipValidationSettings = {
const DEFAULT_SETTINGS: KtxRelationshipValidationSettings = {
acceptThreshold: 0.85,
reviewThreshold: 0.55,
minTargetUniqueness: 0.9,
@ -65,8 +65,8 @@ const DEFAULT_SETTINGS: KloRelationshipValidationSettings = {
};
function mergeSettings(
settings: Partial<KloRelationshipValidationSettings> | undefined,
): KloRelationshipValidationSettings {
settings: Partial<KtxRelationshipValidationSettings> | undefined,
): KtxRelationshipValidationSettings {
return { ...DEFAULT_SETTINGS, ...settings };
}
@ -74,7 +74,7 @@ function profileKey(table: string, column: string): string {
return `${table}.${column}`;
}
function singleRelationshipColumn(endpointValue: KloRelationshipEndpoint): string {
function singleRelationshipColumn(endpointValue: KtxRelationshipEndpoint): string {
const column = endpointValue.columns[0];
if (!column) {
throw new Error(`Expected relationship endpoint ${endpointValue.table.name} to contain one column`);
@ -82,15 +82,15 @@ function singleRelationshipColumn(endpointValue: KloRelationshipEndpoint): strin
return column;
}
function headerIndex(result: KloQueryResult, header: string): number {
function headerIndex(result: KtxQueryResult, header: string): number {
return result.headers.findIndex((candidate) => candidate.toLowerCase() === header.toLowerCase());
}
function firstRow(result: KloQueryResult): unknown[] {
function firstRow(result: KtxQueryResult): unknown[] {
return result.rows[0] ?? [];
}
function numberAt(result: KloQueryResult, header: string): number {
function numberAt(result: KtxQueryResult, header: string): number {
const value = firstRow(result)[headerIndex(result, header)];
if (typeof value === 'number') {
return value;
@ -104,14 +104,14 @@ function numberAt(result: KloQueryResult, header: string): number {
return 0;
}
function limitSql(driver: KloConnectionDriver, limit: number): string {
function limitSql(driver: KtxConnectionDriver, limit: number): string {
if (driver === 'sqlserver') {
return '';
}
return ` LIMIT ${Math.max(1, Math.floor(limit))}`;
}
function topSql(driver: KloConnectionDriver, limit: number): string {
function topSql(driver: KtxConnectionDriver, limit: number): string {
if (driver === 'sqlserver') {
return ` TOP (${Math.max(1, Math.floor(limit))})`;
}
@ -119,17 +119,17 @@ function topSql(driver: KloConnectionDriver, limit: number): string {
}
function buildCoverageSql(input: {
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
childTable: string;
childColumn: string;
parentTable: string;
parentColumn: string;
maxDistinctSourceValues: number;
}): string {
const childTable = formatKloRelationshipTableRef(input.driver, { catalog: null, db: null, name: input.childTable });
const parentTable = formatKloRelationshipTableRef(input.driver, { catalog: null, db: null, name: input.parentTable });
const childColumn = quoteKloRelationshipIdentifier(input.driver, input.childColumn);
const parentColumn = quoteKloRelationshipIdentifier(input.driver, input.parentColumn);
const childTable = formatKtxRelationshipTableRef(input.driver, { catalog: null, db: null, name: input.childTable });
const parentTable = formatKtxRelationshipTableRef(input.driver, { catalog: null, db: null, name: input.parentTable });
const childColumn = quoteKtxRelationshipIdentifier(input.driver, input.childColumn);
const parentColumn = quoteKtxRelationshipIdentifier(input.driver, input.parentColumn);
const limit = limitSql(input.driver, input.maxDistinctSourceValues);
const top = topSql(input.driver, input.maxDistinctSourceValues);
@ -170,8 +170,8 @@ function score(input: {
function statusFor(input: {
score: number;
reasons: readonly string[];
settings: KloRelationshipValidationSettings;
}): KloValidatedRelationshipStatus {
settings: KtxRelationshipValidationSettings;
}): KtxValidatedRelationshipStatus {
if (
input.reasons.includes('low_target_uniqueness') ||
input.reasons.includes('low_source_coverage') ||
@ -215,10 +215,10 @@ async function mapWithConcurrency<TInput, TOutput>(
}
function reviewWithoutValidation(
candidate: KloRelationshipDiscoveryCandidate,
profiles: KloRelationshipProfileArtifact,
candidate: KtxRelationshipDiscoveryCandidate,
profiles: KtxRelationshipProfileArtifact,
reason: 'validation_unavailable' | 'profile_unavailable' | 'validation_unattempted',
): KloValidatedRelationshipDiscoveryCandidate {
): KtxValidatedRelationshipDiscoveryCandidate {
const sourceColumn = singleRelationshipColumn(candidate.from);
const targetColumn = singleRelationshipColumn(candidate.to);
const sourceProfile = profiles.columns[profileKey(candidate.from.table.name, sourceColumn)];
@ -244,9 +244,9 @@ function reviewWithoutValidation(
};
}
export async function validateKloRelationshipDiscoveryCandidates(
input: ValidateKloRelationshipDiscoveryCandidatesInput,
): Promise<KloValidatedRelationshipDiscoveryCandidate[]> {
export async function validateKtxRelationshipDiscoveryCandidates(
input: ValidateKtxRelationshipDiscoveryCandidatesInput,
): Promise<KtxValidatedRelationshipDiscoveryCandidate[]> {
const settings = mergeSettings(input.settings);
if (!input.executor || !input.profiles.sqlAvailable) {
return input.candidates.map((candidate) =>
@ -257,8 +257,8 @@ export async function validateKloRelationshipDiscoveryCandidates(
const executor = input.executor;
async function validateCandidate(
candidate: KloRelationshipDiscoveryCandidate,
): Promise<KloValidatedRelationshipDiscoveryCandidate> {
candidate: KtxRelationshipDiscoveryCandidate,
): Promise<KtxValidatedRelationshipDiscoveryCandidate> {
const sourceColumn = singleRelationshipColumn(candidate.from);
const targetColumn = singleRelationshipColumn(candidate.to);
const sourceProfile = input.profiles.columns[profileKey(candidate.from.table.name, sourceColumn)];
@ -334,7 +334,7 @@ export async function validateKloRelationshipDiscoveryCandidates(
};
}
const budgeted = applyKloRelationshipValidationBudget({
const budgeted = applyKtxRelationshipValidationBudget({
candidates: input.candidates,
tableCount: input.tableCount ?? 0,
budget: settings.validationBudget ?? (input.tableCount === undefined ? 'all' : undefined),
@ -345,7 +345,7 @@ export async function validateKloRelationshipDiscoveryCandidates(
settings.concurrency,
validateCandidate,
);
const byOriginalIndex = new Map<number, KloValidatedRelationshipDiscoveryCandidate>();
const byOriginalIndex = new Map<number, KtxValidatedRelationshipDiscoveryCandidate>();
for (let index = 0; index < budgeted.toValidate.length; index += 1) {
const originalIndex = budgeted.toValidate[index]?.originalIndex;
const candidate = validated[index];

30
packages/context/src/scan/sqlite-local-enrichment-state-store.ts
View file

@ -2,13 +2,13 @@ import { mkdirSync } from 'node:fs';
import { dirname } from 'node:path';
import Database from 'better-sqlite3';
import type {
KloScanEnrichmentCompletedStage,
KloScanEnrichmentFailedStage,
KloScanEnrichmentStageLookup,
KloScanEnrichmentStageRecord,
KloScanEnrichmentStateStore,
KtxScanEnrichmentCompletedStage,
KtxScanEnrichmentFailedStage,
KtxScanEnrichmentStageLookup,
KtxScanEnrichmentStageRecord,
KtxScanEnrichmentStateStore,
} from './enrichment-state.js';
import type { KloScanEnrichmentStage, KloScanMode } from './types.js';
import type { KtxScanEnrichmentStage, KtxScanMode } from './types.js';
export interface SqliteLocalScanEnrichmentStateStoreOptions {
dbPath: string;
@ -18,8 +18,8 @@ interface StageRow {
run_id: string;
connection_id: string;
sync_id: string;
mode: KloScanMode;
stage: KloScanEnrichmentStage;
mode: KtxScanMode;
stage: KtxScanEnrichmentStage;
input_hash: string;
status: 'completed' | 'failed';
output_json: string | null;
@ -27,7 +27,7 @@ interface StageRow {
updated_at: string;
}
function parseStageRow<TOutput = unknown>(row: StageRow): KloScanEnrichmentStageRecord<TOutput> {
function parseStageRow<TOutput = unknown>(row: StageRow): KtxScanEnrichmentStageRecord<TOutput> {
if (row.status === 'completed') {
return {
runId: row.run_id,
@ -61,7 +61,7 @@ function isSafeRunId(runId: string): boolean {
return /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(runId);
}
export class SqliteLocalScanEnrichmentStateStore implements KloScanEnrichmentStateStore {
export class SqliteLocalScanEnrichmentStateStore implements KtxScanEnrichmentStateStore {
private readonly db: Database.Database;
constructor(options: SqliteLocalScanEnrichmentStateStoreOptions) {
@ -89,8 +89,8 @@ export class SqliteLocalScanEnrichmentStateStore implements KloScanEnrichmentSta
}
async findCompletedStage<TOutput = unknown>(
input: KloScanEnrichmentStageLookup,
): Promise<KloScanEnrichmentCompletedStage<TOutput> | null> {
input: KtxScanEnrichmentStageLookup,
): Promise<KtxScanEnrichmentCompletedStage<TOutput> | null> {
if (!isSafeRunId(input.runId)) {
return null;
}
@ -115,7 +115,7 @@ export class SqliteLocalScanEnrichmentStateStore implements KloScanEnrichmentSta
}
async saveCompletedStage<TOutput = unknown>(
input: Omit<KloScanEnrichmentCompletedStage<TOutput>, 'status' | 'errorMessage'>,
input: Omit<KtxScanEnrichmentCompletedStage<TOutput>, 'status' | 'errorMessage'>,
): Promise<void> {
this.db
.prepare(
@ -167,7 +167,7 @@ export class SqliteLocalScanEnrichmentStateStore implements KloScanEnrichmentSta
});
}
async saveFailedStage(input: Omit<KloScanEnrichmentFailedStage, 'status' | 'output'>): Promise<void> {
async saveFailedStage(input: Omit<KtxScanEnrichmentFailedStage, 'status' | 'output'>): Promise<void> {
this.db
.prepare(
`
@ -218,7 +218,7 @@ export class SqliteLocalScanEnrichmentStateStore implements KloScanEnrichmentSta
});
}
async listRunStages(runId: string): Promise<KloScanEnrichmentStageRecord[]> {
async listRunStages(runId: string): Promise<KtxScanEnrichmentStageRecord[]> {
if (!isSafeRunId(runId)) {
return [];
}

20
packages/context/src/scan/type-normalization.test.ts
View file

@ -1,22 +1,22 @@
import { describe, expect, it } from 'vitest';
import { inferKloDimensionType, kloColumnTypeMappingFromNative, normalizeKloNativeType } from './type-normalization.js';
import { inferKtxDimensionType, ktxColumnTypeMappingFromNative, normalizeKtxNativeType } from './type-normalization.js';
describe('KLO scan type normalization', () => {
describe('KTX scan type normalization', () => {
it('normalizes native database type strings', () => {
expect(normalizeKloNativeType(' NUMERIC(12, 2) ')).toBe('numeric');
expect(normalizeKloNativeType('TIMESTAMP WITH TIME ZONE')).toBe('timestamp with time zone');
expect(normalizeKloNativeType('')).toBe('unknown');
expect(normalizeKtxNativeType(' NUMERIC(12, 2) ')).toBe('numeric');
expect(normalizeKtxNativeType('TIMESTAMP WITH TIME ZONE')).toBe('timestamp with time zone');
expect(normalizeKtxNativeType('')).toBe('unknown');
});
it('infers dimension types from native types', () => {
expect(inferKloDimensionType('BOOLEAN')).toBe('boolean');
expect(inferKloDimensionType('timestamp with time zone')).toBe('time');
expect(inferKloDimensionType('decimal(10,2)')).toBe('number');
expect(inferKloDimensionType('varchar(255)')).toBe('string');
expect(inferKtxDimensionType('BOOLEAN')).toBe('boolean');
expect(inferKtxDimensionType('timestamp with time zone')).toBe('time');
expect(inferKtxDimensionType('decimal(10,2)')).toBe('number');
expect(inferKtxDimensionType('varchar(255)')).toBe('string');
});
it('builds a complete column type mapping', () => {
expect(kloColumnTypeMappingFromNative('BIGINT')).toEqual({
expect(ktxColumnTypeMappingFromNative('BIGINT')).toEqual({
normalizedType: 'bigint',
dimensionType: 'number',
});

18
packages/context/src/scan/type-normalization.ts
View file

@ -1,17 +1,17 @@
import type { KloSchemaDimensionType } from './types.js';
import type { KtxSchemaDimensionType } from './types.js';
export interface KloColumnTypeMapping {
export interface KtxColumnTypeMapping {
normalizedType: string;
dimensionType: KloSchemaDimensionType;
dimensionType: KtxSchemaDimensionType;
}
export function normalizeKloNativeType(nativeType: string): string {
export function normalizeKtxNativeType(nativeType: string): string {
const normalized = nativeType.toLowerCase().replace(/\([^)]*\)/g, '').replace(/\s+/g, ' ').trim();
return normalized.length > 0 ? normalized : 'unknown';
}
export function inferKloDimensionType(nativeType: string): KloSchemaDimensionType {
const normalized = normalizeKloNativeType(nativeType);
export function inferKtxDimensionType(nativeType: string): KtxSchemaDimensionType {
const normalized = normalizeKtxNativeType(nativeType);
if (/\b(bool|boolean)\b/.test(normalized)) {
return 'boolean';
}
@ -24,9 +24,9 @@ export function inferKloDimensionType(nativeType: string): KloSchemaDimensionTyp
return 'string';
}
export function kloColumnTypeMappingFromNative(nativeType: string): KloColumnTypeMapping {
export function ktxColumnTypeMappingFromNative(nativeType: string): KtxColumnTypeMapping {
return {
normalizedType: normalizeKloNativeType(nativeType),
dimensionType: inferKloDimensionType(nativeType),
normalizedType: normalizeKtxNativeType(nativeType),
dimensionType: inferKtxDimensionType(nativeType),
};
}

68
packages/context/src/scan/types.test.ts
View file

@ -1,25 +1,25 @@
import { describe, expect, it } from 'vitest';
import {
createKloConnectorCapabilities,
type KloEventPropertyDiscovery,
type KloEventPropertyDiscoveryInput,
type KloEventPropertyValuesInput,
type KloEventPropertyValuesResult,
type KloEventStreamDiscoveryPort,
type KloEventTypeDiscovery,
type KloEventTypeDiscoveryInput,
type KloNetworkEndpoint,
type KloNetworkTunnelPort,
type KloQueryResult,
type KloScanConnector,
type KloScanContext,
type KloScanInput,
type KloSchemaSnapshot,
createKtxConnectorCapabilities,
type KtxEventPropertyDiscovery,
type KtxEventPropertyDiscoveryInput,
type KtxEventPropertyValuesInput,
type KtxEventPropertyValuesResult,
type KtxEventStreamDiscoveryPort,
type KtxEventTypeDiscovery,
type KtxEventTypeDiscoveryInput,
type KtxNetworkEndpoint,
type KtxNetworkTunnelPort,
type KtxQueryResult,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaSnapshot,
} from './types.js';
describe('KLO scan contract types', () => {
describe('KTX scan contract types', () => {
it('defaults to structural-only connector capabilities', () => {
expect(createKloConnectorCapabilities()).toEqual({
expect(createKtxConnectorCapabilities()).toEqual({
structuralIntrospection: true,
tableSampling: false,
columnSampling: false,
@ -34,7 +34,7 @@ describe('KLO scan contract types', () => {
it('keeps structural introspection mandatory when optional capabilities are enabled', () => {
expect(
createKloConnectorCapabilities({
createKtxConnectorCapabilities({
tableSampling: true,
readOnlySql: true,
eventStreamDiscovery: true,
@ -54,7 +54,7 @@ describe('KLO scan contract types', () => {
});
it('describes the connector surface without requiring enrichment methods', async () => {
const snapshot: KloSchemaSnapshot = {
const snapshot: KtxSchemaSnapshot = {
connectionId: 'warehouse',
driver: 'postgres',
extractedAt: '2026-04-29T00:00:00.000Z',
@ -84,11 +84,11 @@ describe('KLO scan contract types', () => {
],
};
const connector: KloScanConnector = {
const connector: KtxScanConnector = {
id: 'test-postgres',
driver: 'postgres',
capabilities: createKloConnectorCapabilities({ estimatedRowCounts: true }),
async introspect(input: KloScanInput, ctx: KloScanContext) {
capabilities: createKtxConnectorCapabilities({ estimatedRowCounts: true }),
async introspect(input: KtxScanInput, ctx: KtxScanContext) {
expect(input.connectionId).toBe('warehouse');
expect(ctx.runId).toBe('scan-run-1');
return snapshot;
@ -109,11 +109,11 @@ describe('KLO scan contract types', () => {
});
it('models optional event-stream discovery as a connector capability and port', async () => {
const eventTypes: KloEventTypeDiscovery[] = [{ value: '$pageview', count: 42 }];
const propertyKeys: KloEventPropertyDiscovery[] = [{ key: '$browser', count: 31 }];
const propertyValues: KloEventPropertyValuesResult = { values: ['Chrome', 'Safari'], cardinality: 2 };
const discovery: KloEventStreamDiscoveryPort = {
async listEventTypes(input: KloEventTypeDiscoveryInput) {
const eventTypes: KtxEventTypeDiscovery[] = [{ value: '$pageview', count: 42 }];
const propertyKeys: KtxEventPropertyDiscovery[] = [{ key: '$browser', count: 31 }];
const propertyValues: KtxEventPropertyValuesResult = { values: ['Chrome', 'Safari'], cardinality: 2 };
const discovery: KtxEventStreamDiscoveryPort = {
async listEventTypes(input: KtxEventTypeDiscoveryInput) {
expect(input).toEqual({
connectionId: 'product',
table: { catalog: '157881', db: null, name: 'events' },
@ -124,7 +124,7 @@ describe('KLO scan contract types', () => {
});
return eventTypes;
},
async listPropertyKeys(input: KloEventPropertyDiscoveryInput) {
async listPropertyKeys(input: KtxEventPropertyDiscoveryInput) {
expect(input).toEqual({
connectionId: 'product',
table: { catalog: '157881', db: null, name: 'events' },
@ -135,7 +135,7 @@ describe('KLO scan contract types', () => {
});
return propertyKeys;
},
async listPropertyValues(input: KloEventPropertyValuesInput) {
async listPropertyValues(input: KtxEventPropertyValuesInput) {
expect(input).toEqual({
connectionId: 'product',
table: { catalog: '157881', db: null, name: 'events' },
@ -149,10 +149,10 @@ describe('KLO scan contract types', () => {
},
};
const connector: KloScanConnector = {
const connector: KtxScanConnector = {
id: 'posthog:product',
driver: 'posthog',
capabilities: createKloConnectorCapabilities({ eventStreamDiscovery: true }),
capabilities: createKtxConnectorCapabilities({ eventStreamDiscovery: true }),
eventStreamDiscovery: discovery,
async introspect() {
return {
@ -209,7 +209,7 @@ describe('KLO scan contract types', () => {
});
it('keeps read-only query results separate from schema snapshots', () => {
const result: KloQueryResult = {
const result: KtxQueryResult = {
headers: ['id', 'amount'],
headerTypes: ['integer', 'numeric'],
rows: [[1, 10.5]],
@ -227,12 +227,12 @@ describe('KLO scan contract types', () => {
});
it('models host-provided network tunnel endpoint resolution without app imports', async () => {
const endpoint: KloNetworkEndpoint = {
const endpoint: KtxNetworkEndpoint = {
host: '127.0.0.1',
port: 15432,
close: async () => undefined,
};
const tunnelPort: KloNetworkTunnelPort<{ networkProxy?: { type: 'ssh_tunnel' } }> = {
const tunnelPort: KtxNetworkTunnelPort<{ networkProxy?: { type: 'ssh_tunnel' } }> = {
async resolveEndpoint(input) {
expect(input).toEqual({
connectionId: 'warehouse',

218
packages/context/src/scan/types.ts
View file

@ -1,4 +1,4 @@
export type KloConnectionDriver =
export type KtxConnectionDriver =
| 'sqlite'
| 'postgres'
| 'postgresql'
@ -9,11 +9,11 @@ export type KloConnectionDriver =
| 'mysql'
| 'clickhouse';
export type KloScanMode = 'structural' | 'relationships' | 'enriched';
export type KtxScanMode = 'structural' | 'relationships' | 'enriched';
export type KloScanTrigger = 'cli' | 'mcp' | 'schema_scan' | 'scheduled' | 'manual';
export type KtxScanTrigger = 'cli' | 'mcp' | 'schema_scan' | 'scheduled' | 'manual';
export interface KloConnectorCapabilities {
export interface KtxConnectorCapabilities {
structuralIntrospection: true;
tableSampling: boolean;
columnSampling: boolean;
@ -25,11 +25,11 @@ export interface KloConnectorCapabilities {
estimatedRowCounts: boolean;
}
export type KloOptionalConnectorCapabilities = Partial<Omit<KloConnectorCapabilities, 'structuralIntrospection'>>;
export type KtxOptionalConnectorCapabilities = Partial<Omit<KtxConnectorCapabilities, 'structuralIntrospection'>>;
export function createKloConnectorCapabilities(
capabilities: KloOptionalConnectorCapabilities = {},
): KloConnectorCapabilities {
export function createKtxConnectorCapabilities(
capabilities: KtxOptionalConnectorCapabilities = {},
): KtxConnectorCapabilities {
return {
structuralIntrospection: true,
tableSampling: capabilities.tableSampling ?? false,
@ -43,27 +43,27 @@ export function createKloConnectorCapabilities(
};
}
export interface KloSchemaScope {
export interface KtxSchemaScope {
catalogs?: string[];
schemas?: string[];
datasets?: string[];
}
export type KloSchemaTableKind = 'table' | 'view' | 'external' | 'event_stream';
export type KtxSchemaTableKind = 'table' | 'view' | 'external' | 'event_stream';
export type KloSchemaDimensionType = 'time' | 'string' | 'number' | 'boolean';
export type KtxSchemaDimensionType = 'time' | 'string' | 'number' | 'boolean';
export interface KloSchemaColumn {
export interface KtxSchemaColumn {
name: string;
nativeType: string;
normalizedType: string;
dimensionType: KloSchemaDimensionType;
dimensionType: KtxSchemaDimensionType;
nullable: boolean;
primaryKey: boolean;
comment: string | null;
}
export interface KloSchemaForeignKey {
export interface KtxSchemaForeignKey {
fromColumn: string;
toCatalog: string | null;
toDb: string | null;
@ -72,139 +72,139 @@ export interface KloSchemaForeignKey {
constraintName: string | null;
}
export interface KloSchemaTable {
export interface KtxSchemaTable {
catalog: string | null;
db: string | null;
name: string;
kind: KloSchemaTableKind;
kind: KtxSchemaTableKind;
comment: string | null;
estimatedRows: number | null;
columns: KloSchemaColumn[];
foreignKeys: KloSchemaForeignKey[];
columns: KtxSchemaColumn[];
foreignKeys: KtxSchemaForeignKey[];
}
export interface KloSchemaSnapshot {
export interface KtxSchemaSnapshot {
connectionId: string;
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
extractedAt: string;
scope: KloSchemaScope;
tables: KloSchemaTable[];
scope: KtxSchemaScope;
tables: KtxSchemaTable[];
metadata: Record<string, unknown>;
}
export interface KloCredentialEnvReference {
export interface KtxCredentialEnvReference {
kind: 'env';
name: string;
}
export interface KloCredentialFileReference {
export interface KtxCredentialFileReference {
kind: 'file';
path: string;
}
export interface KloResolvedCredentialEnvelope {
export interface KtxResolvedCredentialEnvelope {
kind: 'resolved';
source: 'standalone' | 'host';
values: Record<string, unknown>;
redacted?: boolean;
}
export type KloCredentialEnvelope =
| KloCredentialEnvReference
| KloCredentialFileReference
| KloResolvedCredentialEnvelope;
export type KtxCredentialEnvelope =
| KtxCredentialEnvReference
| KtxCredentialFileReference
| KtxResolvedCredentialEnvelope;
export interface KloNetworkEndpoint {
export interface KtxNetworkEndpoint {
host: string;
port: number;
close?: () => Promise<void>;
}
export interface KloNetworkTunnelRequest<TConnection = Record<string, unknown>> {
export interface KtxNetworkTunnelRequest<TConnection = Record<string, unknown>> {
connectionId: string;
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
host: string;
port: number;
connection: TConnection;
}
export interface KloNetworkTunnelPort<TConnection = Record<string, unknown>> {
resolveEndpoint(input: KloNetworkTunnelRequest<TConnection>): Promise<KloNetworkEndpoint | null>;
export interface KtxNetworkTunnelPort<TConnection = Record<string, unknown>> {
resolveEndpoint(input: KtxNetworkTunnelRequest<TConnection>): Promise<KtxNetworkEndpoint | null>;
}
export interface KloScanInput {
export interface KtxScanInput {
connectionId: string;
driver: KloConnectionDriver;
scope?: KloSchemaScope;
mode?: KloScanMode;
driver: KtxConnectionDriver;
scope?: KtxSchemaScope;
mode?: KtxScanMode;
dryRun?: boolean;
detectRelationships?: boolean;
credentials?: KloCredentialEnvelope;
credentials?: KtxCredentialEnvelope;
metadata?: Record<string, unknown>;
}
export interface KloProgressUpdateOptions {
export interface KtxProgressUpdateOptions {
transient?: boolean;
}
export interface KloProgressPort {
update(progress: number, message?: string, options?: KloProgressUpdateOptions): Promise<void>;
startPhase(weight: number): KloProgressPort;
export interface KtxProgressPort {
update(progress: number, message?: string, options?: KtxProgressUpdateOptions): Promise<void>;
startPhase(weight: number): KtxProgressPort;
}
export interface KloScanLoggerPort {
export interface KtxScanLoggerPort {
debug(message: string, metadata?: Record<string, unknown>): void;
info(message: string, metadata?: Record<string, unknown>): void;
warn(message: string, metadata?: Record<string, unknown>): void;
error(message: string, metadata?: Record<string, unknown>): void;
}
export interface KloScanContext {
export interface KtxScanContext {
runId: string;
signal?: AbortSignal;
progress?: KloProgressPort;
logger?: KloScanLoggerPort;
progress?: KtxProgressPort;
logger?: KtxScanLoggerPort;
}
export interface KloTableRef {
export interface KtxTableRef {
catalog: string | null;
db: string | null;
name: string;
}
export interface KloTableSampleInput {
export interface KtxTableSampleInput {
connectionId: string;
table: KloTableRef;
table: KtxTableRef;
columns?: string[];
limit: number;
}
export interface KloTableSampleResult {
export interface KtxTableSampleResult {
headers: string[];
rows: unknown[][];
totalRows: number;
}
export interface KloColumnSampleInput {
export interface KtxColumnSampleInput {
connectionId: string;
table: KloTableRef;
table: KtxTableRef;
column: string;
limit: number;
}
export interface KloColumnSampleResult {
export interface KtxColumnSampleResult {
values: unknown[];
nullCount: number | null;
distinctCount: number | null;
}
export interface KloColumnStatsInput {
export interface KtxColumnStatsInput {
connectionId: string;
table: KloTableRef;
table: KtxTableRef;
column: string;
}
export interface KloColumnStatsResult {
export interface KtxColumnStatsResult {
min: unknown;
max: unknown;
average: number | null;
@ -212,37 +212,37 @@ export interface KloColumnStatsResult {
distinctCount: number | null;
}
export interface KloEventTypeDiscoveryInput {
export interface KtxEventTypeDiscoveryInput {
connectionId: string;
table: KloTableRef;
table: KtxTableRef;
eventColumn: string;
limit: number;
minCount?: number;
lookbackDays?: number;
}
export interface KloEventTypeDiscovery {
export interface KtxEventTypeDiscovery {
value: string;
count: number;
}
export interface KloEventPropertyDiscoveryInput {
export interface KtxEventPropertyDiscoveryInput {
connectionId: string;
table: KloTableRef;
table: KtxTableRef;
jsonColumn: string;
sampleSize: number;
limit: number;
lookbackDays?: number;
}
export interface KloEventPropertyDiscovery {
export interface KtxEventPropertyDiscovery {
key: string;
count: number;
}
export interface KloEventPropertyValuesInput {
export interface KtxEventPropertyValuesInput {
connectionId: string;
table: KloTableRef;
table: KtxTableRef;
jsonColumn: string;
propertyKey: string;
limit: number;
@ -250,27 +250,27 @@ export interface KloEventPropertyValuesInput {
lookbackDays?: number;
}
export interface KloEventPropertyValuesResult {
export interface KtxEventPropertyValuesResult {
values: string[];
cardinality: number;
}
export interface KloEventStreamDiscoveryPort {
listEventTypes(input: KloEventTypeDiscoveryInput, ctx: KloScanContext): Promise<KloEventTypeDiscovery[]>;
listPropertyKeys(input: KloEventPropertyDiscoveryInput, ctx: KloScanContext): Promise<KloEventPropertyDiscovery[]>;
export interface KtxEventStreamDiscoveryPort {
listEventTypes(input: KtxEventTypeDiscoveryInput, ctx: KtxScanContext): Promise<KtxEventTypeDiscovery[]>;
listPropertyKeys(input: KtxEventPropertyDiscoveryInput, ctx: KtxScanContext): Promise<KtxEventPropertyDiscovery[]>;
listPropertyValues(
input: KloEventPropertyValuesInput,
ctx: KloScanContext,
): Promise<KloEventPropertyValuesResult | null>;
input: KtxEventPropertyValuesInput,
ctx: KtxScanContext,
): Promise<KtxEventPropertyValuesResult | null>;
}
export interface KloReadOnlyQueryInput {
export interface KtxReadOnlyQueryInput {
connectionId: string;
sql: string;
maxRows?: number;
}
export interface KloQueryResult {
export interface KtxQueryResult {
headers: string[];
headerTypes?: string[];
rows: unknown[][];
@ -278,26 +278,26 @@ export interface KloQueryResult {
rowCount: number | null;
}
export interface KloScanConnector {
export interface KtxScanConnector {
id: string;
driver: KloConnectionDriver;
capabilities: KloConnectorCapabilities;
eventStreamDiscovery?: KloEventStreamDiscoveryPort;
introspect(input: KloScanInput, ctx: KloScanContext): Promise<KloSchemaSnapshot>;
sampleColumn?(input: KloColumnSampleInput, ctx: KloScanContext): Promise<KloColumnSampleResult>;
sampleTable?(input: KloTableSampleInput, ctx: KloScanContext): Promise<KloTableSampleResult>;
columnStats?(input: KloColumnStatsInput, ctx: KloScanContext): Promise<KloColumnStatsResult | null>;
executeReadOnly?(input: KloReadOnlyQueryInput, ctx: KloScanContext): Promise<KloQueryResult>;
driver: KtxConnectionDriver;
capabilities: KtxConnectorCapabilities;
eventStreamDiscovery?: KtxEventStreamDiscoveryPort;
introspect(input: KtxScanInput, ctx: KtxScanContext): Promise<KtxSchemaSnapshot>;
sampleColumn?(input: KtxColumnSampleInput, ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
sampleTable?(input: KtxTableSampleInput, ctx: KtxScanContext): Promise<KtxTableSampleResult>;
columnStats?(input: KtxColumnStatsInput, ctx: KtxScanContext): Promise<KtxColumnStatsResult | null>;
executeReadOnly?(input: KtxReadOnlyQueryInput, ctx: KtxScanContext): Promise<KtxQueryResult>;
cleanup?(): Promise<void>;
}
export interface KloEmbeddingPort {
export interface KtxEmbeddingPort {
dimensions: number;
maxBatchSize: number;
embedBatch(texts: string[]): Promise<number[][]>;
}
export interface KloStructuralSyncStats {
export interface KtxStructuralSyncStats {
tablesCreated: number;
tablesUpdated: number;
tablesDeleted: number;
@ -306,7 +306,7 @@ export interface KloStructuralSyncStats {
columnsDeleted: number;
}
export interface KloScanDiffSummary {
export interface KtxScanDiffSummary {
tablesAdded: number;
tablesModified: number;
tablesDeleted: number;
@ -316,14 +316,14 @@ export interface KloScanDiffSummary {
columnsDeleted: number;
}
export interface KloScanArtifactPaths {
export interface KtxScanArtifactPaths {
rawSourcesDir: string | null;
reportPath: string | null;
manifestShards: string[];
enrichmentArtifacts: string[];
}
export type KloScanWarningCode =
export type KtxScanWarningCode =
| 'connector_capability_missing'
| 'sampling_failed'
| 'statistics_failed'
@ -336,8 +336,8 @@ export type KloScanWarningCode =
| 'credential_redacted'
| 'enrichment_failed';
export interface KloScanWarning {
code: KloScanWarningCode;
export interface KtxScanWarning {
code: KtxScanWarningCode;
message: string;
table?: string;
column?: string;
@ -345,7 +345,7 @@ export interface KloScanWarning {
metadata?: Record<string, unknown>;
}
export interface KloScanEnrichmentSummary {
export interface KtxScanEnrichmentSummary {
dataDictionary: 'skipped' | 'completed' | 'failed';
tableDescriptions: 'skipped' | 'completed' | 'failed';
columnDescriptions: 'skipped' | 'completed' | 'failed';
@ -355,37 +355,37 @@ export interface KloScanEnrichmentSummary {
statisticalValidation: 'skipped' | 'completed' | 'failed';
}
export interface KloScanRelationshipSummary {
export interface KtxScanRelationshipSummary {
accepted: number;
review: number;
rejected: number;
skipped: number;
}
export type KloScanEnrichmentStage = 'descriptions' | 'embeddings' | 'relationships';
export type KtxScanEnrichmentStage = 'descriptions' | 'embeddings' | 'relationships';
export interface KloScanEnrichmentStateSummary {
resumedStages: KloScanEnrichmentStage[];
completedStages: KloScanEnrichmentStage[];
failedStages: KloScanEnrichmentStage[];
export interface KtxScanEnrichmentStateSummary {
resumedStages: KtxScanEnrichmentStage[];
completedStages: KtxScanEnrichmentStage[];
failedStages: KtxScanEnrichmentStage[];
}
export interface KloScanReport {
export interface KtxScanReport {
connectionId: string;
driver: KloConnectionDriver;
driver: KtxConnectionDriver;
syncId: string;
runId: string;
trigger: KloScanTrigger;
mode: KloScanMode;
trigger: KtxScanTrigger;
mode: KtxScanMode;
dryRun: boolean;
artifactPaths: KloScanArtifactPaths;
diffSummary: KloScanDiffSummary;
artifactPaths: KtxScanArtifactPaths;
diffSummary: KtxScanDiffSummary;
manifestShardsWritten: number;
structuralSyncStats: KloStructuralSyncStats;
enrichment: KloScanEnrichmentSummary;
capabilityGaps: Array<keyof Omit<KloConnectorCapabilities, 'structuralIntrospection'>>;
warnings: KloScanWarning[];
relationships: KloScanRelationshipSummary;
enrichmentState: KloScanEnrichmentStateSummary;
structuralSyncStats: KtxStructuralSyncStats;
enrichment: KtxScanEnrichmentSummary;
capabilityGaps: Array<keyof Omit<KtxConnectorCapabilities, 'structuralIntrospection'>>;
warnings: KtxScanWarning[];
relationships: KtxScanRelationshipSummary;
enrichmentState: KtxScanEnrichmentStateSummary;
createdAt: string;
}