mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-04 10:52:13 +02:00
805 lines
26 KiB
TypeScript
805 lines
26 KiB
TypeScript
import Database from 'better-sqlite3';
|
|
import { describe, expect, it, vi } from 'vitest';
|
|
import { buildDefaultKtxProjectConfig } from '../project/config.js';
|
|
import type {
|
|
KtxScanEnrichmentCompletedStage,
|
|
KtxScanEnrichmentFailedStage,
|
|
KtxScanEnrichmentStageLookup,
|
|
KtxScanEnrichmentStateStore,
|
|
} from './enrichment-state.js';
|
|
import {
|
|
createDeterministicLocalScanEnrichmentProviders,
|
|
runLocalScanEnrichment,
|
|
snapshotToKtxEnrichedSchema,
|
|
} from './local-enrichment.js';
|
|
import { createLocalScanEnrichmentProvidersFromConfig } from './local-scan.js';
|
|
import {
|
|
createKtxConnectorCapabilities,
|
|
type KtxQueryResult,
|
|
type KtxReadOnlyQueryInput,
|
|
type KtxScanConnector,
|
|
type KtxScanContext,
|
|
type KtxSchemaSnapshot,
|
|
} from './types.js';
|
|
|
|
const snapshot: KtxSchemaSnapshot = {
|
|
connectionId: 'warehouse',
|
|
driver: 'postgres',
|
|
extractedAt: '2026-04-29T12:00:00.000Z',
|
|
scope: { schemas: ['public'] },
|
|
metadata: {},
|
|
tables: [
|
|
{
|
|
catalog: null,
|
|
db: 'public',
|
|
name: 'customers',
|
|
kind: 'table',
|
|
comment: 'Customer accounts',
|
|
estimatedRows: 2,
|
|
foreignKeys: [],
|
|
columns: [
|
|
{
|
|
name: 'id',
|
|
nativeType: 'integer',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number',
|
|
nullable: false,
|
|
primaryKey: true,
|
|
comment: 'Customer id',
|
|
},
|
|
],
|
|
},
|
|
{
|
|
catalog: null,
|
|
db: 'public',
|
|
name: 'orders',
|
|
kind: 'table',
|
|
comment: 'Customer orders',
|
|
estimatedRows: 3,
|
|
foreignKeys: [],
|
|
columns: [
|
|
{
|
|
name: 'id',
|
|
nativeType: 'integer',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number',
|
|
nullable: false,
|
|
primaryKey: true,
|
|
comment: 'Order id',
|
|
},
|
|
{
|
|
name: 'customer_id',
|
|
nativeType: 'integer',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number',
|
|
nullable: false,
|
|
primaryKey: false,
|
|
comment: 'Customer id',
|
|
},
|
|
],
|
|
},
|
|
],
|
|
};
|
|
|
|
function connector(): KtxScanConnector {
|
|
return {
|
|
id: 'test:warehouse',
|
|
driver: 'postgres',
|
|
capabilities: createKtxConnectorCapabilities({
|
|
tableSampling: true,
|
|
columnSampling: true,
|
|
readOnlySql: true,
|
|
columnStats: true,
|
|
}),
|
|
introspect: vi.fn(async () => snapshot),
|
|
sampleTable: vi.fn(async () => ({
|
|
headers: ['id', 'customer_id'],
|
|
rows: [[1, 10]],
|
|
totalRows: 1,
|
|
})),
|
|
sampleColumn: vi.fn(async () => ({
|
|
values: ['10', '11'],
|
|
nullCount: 0,
|
|
distinctCount: 2,
|
|
})),
|
|
};
|
|
}
|
|
|
|
class InMemorySqliteExecutor {
|
|
readonly db = new Database(':memory:');
|
|
|
|
executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult> {
|
|
const rows = this.db.prepare(input.sql).all() as Record<string, unknown>[];
|
|
const headers = Object.keys(rows[0] ?? {});
|
|
return Promise.resolve({
|
|
headers,
|
|
rows: rows.map((row) => headers.map((header) => row[header])),
|
|
totalRows: rows.length,
|
|
rowCount: rows.length,
|
|
});
|
|
}
|
|
|
|
close(): void {
|
|
this.db.close();
|
|
}
|
|
}
|
|
|
|
function noDeclaredRelationshipSnapshot(): KtxSchemaSnapshot {
|
|
return {
|
|
connectionId: 'warehouse',
|
|
driver: 'sqlite',
|
|
extractedAt: '2026-05-07T00:00:00.000Z',
|
|
scope: {},
|
|
metadata: {},
|
|
tables: [
|
|
{
|
|
catalog: null,
|
|
db: null,
|
|
name: 'accounts',
|
|
kind: 'table',
|
|
comment: null,
|
|
estimatedRows: 2,
|
|
foreignKeys: [],
|
|
columns: [
|
|
{
|
|
name: 'id',
|
|
nativeType: 'INTEGER',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number',
|
|
nullable: false,
|
|
primaryKey: false,
|
|
comment: null,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
catalog: null,
|
|
db: null,
|
|
name: 'orders',
|
|
kind: 'table',
|
|
comment: null,
|
|
estimatedRows: 3,
|
|
foreignKeys: [],
|
|
columns: [
|
|
{
|
|
name: 'id',
|
|
nativeType: 'INTEGER',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number',
|
|
nullable: false,
|
|
primaryKey: false,
|
|
comment: null,
|
|
},
|
|
{
|
|
name: 'account_id',
|
|
nativeType: 'INTEGER',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number',
|
|
nullable: false,
|
|
primaryKey: false,
|
|
comment: null,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
function memoryEnrichmentStateStore(): KtxScanEnrichmentStateStore {
|
|
const records = new Map<string, KtxScanEnrichmentCompletedStage | KtxScanEnrichmentFailedStage>();
|
|
const key = (input: Pick<KtxScanEnrichmentStageLookup, 'runId' | 'stage'>) => `${input.runId}:${input.stage}`;
|
|
return {
|
|
async findCompletedStage<TOutput>(input: KtxScanEnrichmentStageLookup) {
|
|
const record = records.get(key(input));
|
|
if (!record || record.status !== 'completed' || record.inputHash !== input.inputHash) {
|
|
return null;
|
|
}
|
|
return record as KtxScanEnrichmentCompletedStage<TOutput>;
|
|
},
|
|
async saveCompletedStage(input) {
|
|
records.set(key(input), {
|
|
...input,
|
|
status: 'completed',
|
|
errorMessage: null,
|
|
});
|
|
},
|
|
async saveFailedStage(input) {
|
|
records.set(key(input), {
|
|
...input,
|
|
status: 'failed',
|
|
output: null,
|
|
});
|
|
},
|
|
async listRunStages(runId) {
|
|
return [...records.values()].filter((record) => record.runId === runId);
|
|
},
|
|
};
|
|
}
|
|
|
|
describe('local scan enrichment', () => {
|
|
it('maps a scan snapshot into relationship detector schema', () => {
|
|
const schema = snapshotToKtxEnrichedSchema(snapshot);
|
|
|
|
expect(schema.connectionId).toBe('warehouse');
|
|
expect(schema.tables).toHaveLength(2);
|
|
expect(schema.tables[1]?.columns.map((column) => column.name)).toEqual(['id', 'customer_id']);
|
|
expect(schema.tables[1]?.columns[1]).toMatchObject({
|
|
id: 'public.orders.customer_id',
|
|
tableId: 'public.orders',
|
|
primaryKey: false,
|
|
sampleValues: null,
|
|
embedding: null,
|
|
});
|
|
});
|
|
|
|
it('maps snapshot foreign keys into formal schema relationships', () => {
|
|
const source = noDeclaredRelationshipSnapshot();
|
|
const snapshotWithForeignKey = {
|
|
...source,
|
|
tables: source.tables.map((table) =>
|
|
table.name === 'orders'
|
|
? {
|
|
...table,
|
|
foreignKeys: [
|
|
{
|
|
fromColumn: 'account_id',
|
|
toCatalog: null,
|
|
toDb: null,
|
|
toTable: 'accounts',
|
|
toColumn: 'id',
|
|
constraintName: 'orders_account_id_fkey',
|
|
},
|
|
],
|
|
}
|
|
: table.name === 'accounts'
|
|
? {
|
|
...table,
|
|
columns: table.columns.map((column) =>
|
|
column.name === 'id' ? { ...column, primaryKey: true } : column,
|
|
),
|
|
}
|
|
: table,
|
|
),
|
|
};
|
|
|
|
const schema = snapshotToKtxEnrichedSchema(snapshotWithForeignKey);
|
|
|
|
expect(schema.relationships).toEqual([
|
|
{
|
|
id: 'orders:(orders.account_id)->accounts:(accounts.id)',
|
|
source: 'formal',
|
|
from: {
|
|
tableId: 'orders',
|
|
columnIds: ['orders.account_id'],
|
|
table: { catalog: null, db: null, name: 'orders' },
|
|
columns: ['account_id'],
|
|
},
|
|
to: {
|
|
tableId: 'accounts',
|
|
columnIds: ['accounts.id'],
|
|
table: { catalog: null, db: null, name: 'accounts' },
|
|
columns: ['id'],
|
|
},
|
|
relationshipType: 'many_to_one',
|
|
confidence: 1,
|
|
isPrimaryKeyReference: true,
|
|
},
|
|
]);
|
|
});
|
|
|
|
it('runs deterministic relationship detection for relationship scans', async () => {
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'relationships',
|
|
detectRelationships: true,
|
|
connector: connector(),
|
|
context: { runId: 'scan-run-1' },
|
|
providers: null,
|
|
});
|
|
|
|
expect(result.summary).toMatchObject({
|
|
deterministicRelationships: 'completed',
|
|
llmRelationshipValidation: 'skipped',
|
|
embeddings: 'skipped',
|
|
});
|
|
expect(result.relationships).toEqual({ accepted: 0, review: 1, rejected: 0, skipped: 0 });
|
|
expect(result.summary.statisticalValidation).toBe('skipped');
|
|
expect(result.warnings).toContainEqual({
|
|
code: 'relationship_validation_failed',
|
|
message: 'KTX scan connector advertises readOnlySql but does not expose executeReadOnly',
|
|
recoverable: true,
|
|
metadata: { capability: 'readOnlySql' },
|
|
});
|
|
});
|
|
|
|
it('runs relationship discovery with connector SQL evidence', async () => {
|
|
const executor = new InMemorySqliteExecutor();
|
|
try {
|
|
executor.db.exec(`
|
|
CREATE TABLE accounts (id INTEGER NOT NULL);
|
|
CREATE TABLE orders (id INTEGER NOT NULL, account_id INTEGER NOT NULL);
|
|
INSERT INTO accounts (id) VALUES (1), (2);
|
|
INSERT INTO orders (id, account_id) VALUES (10, 1), (11, 1), (12, 2);
|
|
`);
|
|
const scanConnector = {
|
|
...connector(),
|
|
driver: 'sqlite' as const,
|
|
capabilities: createKtxConnectorCapabilities({ readOnlySql: true, columnStats: true }),
|
|
introspect: vi.fn(async () => noDeclaredRelationshipSnapshot()),
|
|
executeReadOnly: executor.executeReadOnly.bind(executor),
|
|
};
|
|
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'relationships',
|
|
detectRelationships: true,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-relationship-discovery' },
|
|
providers: null,
|
|
});
|
|
|
|
expect(result.relationships).toEqual({ accepted: 1, review: 0, rejected: 0, skipped: 0 });
|
|
expect(result.summary.statisticalValidation).toBe('completed');
|
|
expect(result.relationshipProfile).toMatchObject({ sqlAvailable: true });
|
|
expect(result.resolvedRelationships).toEqual([
|
|
expect.objectContaining({
|
|
status: 'accepted',
|
|
from: expect.objectContaining({ table: expect.objectContaining({ name: 'orders' }), columns: ['account_id'] }),
|
|
to: expect.objectContaining({ table: expect.objectContaining({ name: 'accounts' }), columns: ['id'] }),
|
|
}),
|
|
]);
|
|
expect(result.relationshipUpdate?.accepted).toHaveLength(1);
|
|
} finally {
|
|
executor.close();
|
|
}
|
|
});
|
|
|
|
it('honors scan relationship config when LLM proposals are disabled', async () => {
|
|
const providers = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 3 });
|
|
const getModel = vi.fn(() => ({ modelId: 'provider/language-model', provider: 'gateway' }));
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'relationships',
|
|
detectRelationships: true,
|
|
connector: connector(),
|
|
context: { runId: 'scan-run-llm-disabled' },
|
|
providers: {
|
|
...providers,
|
|
llm: {
|
|
...providers.llm,
|
|
getModel: getModel as never,
|
|
},
|
|
},
|
|
relationshipSettings: {
|
|
...buildDefaultKtxProjectConfig('warehouse').scan.relationships,
|
|
llmProposals: false,
|
|
maxLlmTablesPerBatch: 40,
|
|
},
|
|
});
|
|
|
|
expect(result.summary.llmRelationshipValidation).toBe('skipped');
|
|
expect(getModel).not.toHaveBeenCalledWith('candidateExtraction');
|
|
});
|
|
|
|
it('skips relationship detection when scan relationships are disabled', async () => {
|
|
const settings = {
|
|
...buildDefaultKtxProjectConfig('warehouse').scan.relationships,
|
|
enabled: false,
|
|
};
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
connector: connector(),
|
|
context: { runId: 'disabled-relationships' },
|
|
providers: createDeterministicLocalScanEnrichmentProviders(),
|
|
relationshipSettings: settings,
|
|
});
|
|
|
|
expect(result.summary.deterministicRelationships).toBe('skipped');
|
|
expect(result.summary.statisticalValidation).toBe('skipped');
|
|
expect(result.summary.llmRelationshipValidation).toBe('skipped');
|
|
expect(result.relationships).toEqual({ accepted: 0, review: 0, rejected: 0, skipped: 0 });
|
|
expect(result.relationshipUpdate).toBeNull();
|
|
expect(result.relationshipProfile).toBeNull();
|
|
expect(result.resolvedRelationships).toBeNull();
|
|
});
|
|
|
|
it('runs configured deterministic enrichment with descriptions and embeddings', async () => {
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: true,
|
|
connector: connector(),
|
|
context: { runId: 'scan-run-2' },
|
|
providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }),
|
|
});
|
|
|
|
expect(result.summary).toMatchObject({
|
|
dataDictionary: 'completed',
|
|
tableDescriptions: 'completed',
|
|
columnDescriptions: 'completed',
|
|
embeddings: 'completed',
|
|
deterministicRelationships: 'completed',
|
|
});
|
|
expect(result.embeddingUpdates).toHaveLength(3);
|
|
expect(result.embeddingUpdates[0]?.embedding).toHaveLength(6);
|
|
expect(result.snapshot).toEqual(snapshot);
|
|
expect(result.relationships).toEqual({ accepted: 0, review: 1, rejected: 0, skipped: 0 });
|
|
});
|
|
|
|
it('generates table descriptions with bounded table-level concurrency', async () => {
|
|
const concurrentSnapshot: KtxSchemaSnapshot = {
|
|
...snapshot,
|
|
tables: Array.from({ length: 8 }, (_, index) => ({
|
|
catalog: null,
|
|
db: 'public',
|
|
name: `table_${index + 1}`,
|
|
kind: 'table' as const,
|
|
comment: null,
|
|
estimatedRows: 2,
|
|
foreignKeys: [],
|
|
columns: [
|
|
{
|
|
name: 'id',
|
|
nativeType: 'integer',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number' as const,
|
|
nullable: false,
|
|
primaryKey: true,
|
|
comment: null,
|
|
},
|
|
],
|
|
})),
|
|
};
|
|
let activeColumnSamples = 0;
|
|
let maxActiveColumnSamples = 0;
|
|
const scanConnector = {
|
|
...connector(),
|
|
introspect: vi.fn(async () => concurrentSnapshot),
|
|
sampleColumn: vi.fn(async () => {
|
|
activeColumnSamples += 1;
|
|
maxActiveColumnSamples = Math.max(maxActiveColumnSamples, activeColumnSamples);
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
activeColumnSamples -= 1;
|
|
return {
|
|
values: ['1'],
|
|
nullCount: 0,
|
|
distinctCount: 1,
|
|
};
|
|
}),
|
|
sampleTable: vi.fn(async () => ({
|
|
headers: ['id'],
|
|
rows: [[1]],
|
|
totalRows: 1,
|
|
})),
|
|
};
|
|
const settings = {
|
|
...buildDefaultKtxProjectConfig('test').scan.relationships,
|
|
enabled: false,
|
|
};
|
|
|
|
await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-concurrent-descriptions' },
|
|
providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 3 }),
|
|
relationshipSettings: settings,
|
|
});
|
|
|
|
expect(maxActiveColumnSamples).toBe(6);
|
|
});
|
|
|
|
it('reports enrichment progress for countable stages', async () => {
|
|
const events: Array<{ progress: number; message?: string; transient?: boolean }> = [];
|
|
const progress = {
|
|
async update(progressValue: number, message?: string, options?: { transient?: boolean }) {
|
|
events.push({ progress: progressValue, message, transient: options?.transient });
|
|
},
|
|
startPhase() {
|
|
return progress;
|
|
},
|
|
};
|
|
|
|
await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: true,
|
|
connector: connector(),
|
|
context: { runId: 'scan-run-progress', progress },
|
|
providers: createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 }),
|
|
});
|
|
|
|
expect(events).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ message: 'Generating descriptions 1/2 tables', transient: true }),
|
|
expect.objectContaining({ message: 'Generating descriptions 2/2 tables', transient: true }),
|
|
expect.objectContaining({ message: 'Building embeddings 1/1 batches', transient: true }),
|
|
expect.objectContaining({ message: 'Detecting relationships' }),
|
|
]),
|
|
);
|
|
});
|
|
|
|
it('reports progress before enrichment connector introspection starts', async () => {
|
|
const events: Array<{ progress: number; message?: string; transient?: boolean }> = [];
|
|
const progress = {
|
|
async update(progressValue: number, message?: string, options?: { transient?: boolean }) {
|
|
events.push({ progress: progressValue, message, transient: options?.transient });
|
|
},
|
|
startPhase() {
|
|
return progress;
|
|
},
|
|
};
|
|
const scanConnector = {
|
|
...connector(),
|
|
introspect: vi.fn(async () => {
|
|
expect(events).toContainEqual(expect.objectContaining({ message: 'Loading enrichment schema snapshot' }));
|
|
return snapshot;
|
|
}),
|
|
};
|
|
|
|
await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'relationships',
|
|
detectRelationships: true,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-progress-before-introspection', progress },
|
|
providers: null,
|
|
});
|
|
|
|
expect(scanConnector.introspect).toHaveBeenCalled();
|
|
});
|
|
|
|
it('splits enrichment embedding requests by provider batch size', async () => {
|
|
const manyColumnSnapshot: KtxSchemaSnapshot = {
|
|
...snapshot,
|
|
tables: [
|
|
{
|
|
catalog: null,
|
|
db: 'public',
|
|
name: 'wide_orders',
|
|
kind: 'table',
|
|
comment: 'Wide order facts',
|
|
estimatedRows: 3,
|
|
foreignKeys: [],
|
|
columns: Array.from({ length: 5 }, (_, index) => ({
|
|
name: `metric_${index + 1}`,
|
|
nativeType: 'integer',
|
|
normalizedType: 'integer',
|
|
dimensionType: 'number' as const,
|
|
nullable: false,
|
|
primaryKey: false,
|
|
comment: `Metric ${index + 1}`,
|
|
})),
|
|
},
|
|
],
|
|
};
|
|
const scanConnector = {
|
|
...connector(),
|
|
introspect: vi.fn(async () => manyColumnSnapshot),
|
|
};
|
|
const deterministicProviders = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 3 });
|
|
const embedBatch = vi.fn(async (texts: string[]) => {
|
|
if (texts.length > 2) {
|
|
throw new Error(`Embedding batch size ${texts.length} exceeds maximum 2`);
|
|
}
|
|
return texts.map((_, index) => [index, index + 1, index + 2]);
|
|
});
|
|
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: false,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-batched-embeddings' },
|
|
providers: {
|
|
llm: deterministicProviders.llm,
|
|
embedding: {
|
|
dimensions: 3,
|
|
maxBatchSize: 2,
|
|
embedBatch,
|
|
},
|
|
},
|
|
});
|
|
|
|
expect(result.embeddingUpdates).toHaveLength(5);
|
|
expect(embedBatch.mock.calls.map(([texts]) => texts).map((texts) => texts.length)).toEqual([2, 2, 1]);
|
|
});
|
|
|
|
it('reuses completed description and embedding stages for the same run id and snapshot hash', async () => {
|
|
const stateStore = memoryEnrichmentStateStore();
|
|
const scanConnector = connector();
|
|
const providers = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 });
|
|
|
|
const first = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: true,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-resume-1' },
|
|
providers,
|
|
stateStore,
|
|
syncId: 'sync-resume-1',
|
|
providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 },
|
|
});
|
|
|
|
const getModel = vi.spyOn(providers.llm, 'getModel');
|
|
const embedBatch = vi.spyOn(providers.embedding, 'embedBatch');
|
|
const second = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: true,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-resume-1' },
|
|
providers,
|
|
stateStore,
|
|
syncId: 'sync-resume-1',
|
|
providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 },
|
|
});
|
|
|
|
expect(first.state.completedStages).toEqual(['descriptions', 'embeddings', 'relationships']);
|
|
expect(first.state.resumedStages).toEqual([]);
|
|
expect(second.state.resumedStages).toEqual(['descriptions', 'embeddings', 'relationships']);
|
|
expect(second.state.completedStages).toEqual(['descriptions', 'embeddings', 'relationships']);
|
|
expect(getModel).not.toHaveBeenCalled();
|
|
expect(embedBatch).not.toHaveBeenCalled();
|
|
expect(second.descriptionUpdates).toEqual(first.descriptionUpdates);
|
|
expect(second.embeddingUpdates).toEqual(first.embeddingUpdates);
|
|
expect(second.relationships).toEqual(first.relationships);
|
|
});
|
|
|
|
it('does not reuse completed stages when the snapshot changes', async () => {
|
|
const stateStore = memoryEnrichmentStateStore();
|
|
const providers = createDeterministicLocalScanEnrichmentProviders({ embeddingDimensions: 6 });
|
|
const scanConnector = connector();
|
|
|
|
await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: false,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-resume-hash' },
|
|
providers,
|
|
stateStore,
|
|
syncId: 'sync-resume-hash',
|
|
providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 },
|
|
});
|
|
|
|
const firstTable = snapshot.tables[0];
|
|
if (!firstTable) {
|
|
throw new Error('Expected test snapshot table');
|
|
}
|
|
const changedConnector = {
|
|
...connector(),
|
|
introspect: vi.fn(async () => ({
|
|
...snapshot,
|
|
tables: [{ ...firstTable, name: 'customers' }],
|
|
})),
|
|
};
|
|
const getModel = vi.spyOn(providers.llm, 'getModel');
|
|
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: false,
|
|
connector: changedConnector,
|
|
context: { runId: 'scan-run-resume-hash' },
|
|
providers,
|
|
stateStore,
|
|
syncId: 'sync-resume-hash',
|
|
providerIdentity: { provider: 'deterministic', embeddingDimensions: 6 },
|
|
});
|
|
|
|
expect(result.state.resumedStages).toEqual([]);
|
|
expect(result.state.completedStages).toEqual(['descriptions', 'embeddings', 'relationships']);
|
|
expect(getModel).toHaveBeenCalled();
|
|
});
|
|
|
|
it('runs providerless enriched scans as relationship-only discovery enrichment', async () => {
|
|
const executor = new InMemorySqliteExecutor();
|
|
try {
|
|
executor.db.exec(`
|
|
CREATE TABLE accounts (id INTEGER NOT NULL);
|
|
CREATE TABLE orders (id INTEGER NOT NULL, account_id INTEGER NOT NULL);
|
|
INSERT INTO accounts (id) VALUES (1), (2);
|
|
INSERT INTO orders (id, account_id) VALUES (10, 1), (11, 1), (12, 2);
|
|
`);
|
|
const scanConnector = {
|
|
...connector(),
|
|
driver: 'sqlite' as const,
|
|
capabilities: createKtxConnectorCapabilities({ readOnlySql: true, columnStats: true }),
|
|
introspect: vi.fn(async () => noDeclaredRelationshipSnapshot()),
|
|
executeReadOnly: executor.executeReadOnly.bind(executor),
|
|
};
|
|
|
|
const result = await runLocalScanEnrichment({
|
|
connectionId: 'warehouse',
|
|
mode: 'enriched',
|
|
detectRelationships: false,
|
|
connector: scanConnector,
|
|
context: { runId: 'scan-run-providerless-enriched' },
|
|
providers: null,
|
|
});
|
|
|
|
expect(result.summary).toEqual({
|
|
dataDictionary: 'skipped',
|
|
tableDescriptions: 'skipped',
|
|
columnDescriptions: 'skipped',
|
|
embeddings: 'skipped',
|
|
deterministicRelationships: 'completed',
|
|
llmRelationshipValidation: 'skipped',
|
|
statisticalValidation: 'completed',
|
|
});
|
|
expect(result.descriptionUpdates).toEqual([]);
|
|
expect(result.embeddingUpdates).toEqual([]);
|
|
expect(result.relationships).toEqual({ accepted: 1, review: 0, rejected: 0, skipped: 0 });
|
|
expect(result.relationshipUpdate?.accepted).toHaveLength(1);
|
|
expect(result.relationshipProfile).toMatchObject({ sqlAvailable: true });
|
|
expect(result.resolvedRelationships).toEqual([
|
|
expect.objectContaining({
|
|
status: 'accepted',
|
|
from: expect.objectContaining({ table: expect.objectContaining({ name: 'orders' }), columns: ['account_id'] }),
|
|
to: expect.objectContaining({ table: expect.objectContaining({ name: 'accounts' }), columns: ['id'] }),
|
|
}),
|
|
]);
|
|
expect(result.warnings).toContainEqual({
|
|
code: 'scan_enrichment_backend_not_configured',
|
|
message:
|
|
'Skipping description and embedding enrichment because scan.enrichment.mode is not configured; relationship discovery still ran.',
|
|
recoverable: true,
|
|
metadata: {
|
|
skippedStages: ['descriptions', 'embeddings'],
|
|
relationshipDetection: true,
|
|
},
|
|
});
|
|
} finally {
|
|
executor.close();
|
|
}
|
|
});
|
|
|
|
it('resolves gateway LLM providers and OpenAI embeddings from local scan config', () => {
|
|
const createKtxLlmProvider = vi.fn(() => ({
|
|
getModel: vi.fn().mockReturnValue({ modelId: 'provider/language-model', provider: 'gateway' }),
|
|
}));
|
|
const createKtxEmbeddingProvider = vi.fn(() => ({
|
|
dimensions: 1536,
|
|
maxBatchSize: 8,
|
|
embed: vi.fn(),
|
|
[['embed', 'Many'].join('')]: vi.fn(),
|
|
}));
|
|
|
|
const providers = createLocalScanEnrichmentProvidersFromConfig(
|
|
{
|
|
mode: 'llm',
|
|
embeddings: {
|
|
backend: 'openai',
|
|
model: 'provider/embedding-model',
|
|
dimensions: 1536,
|
|
batchSize: 8,
|
|
openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret
|
|
},
|
|
},
|
|
{
|
|
provider: {
|
|
backend: 'gateway',
|
|
gateway: {},
|
|
},
|
|
models: { default: 'provider/language-model' },
|
|
},
|
|
{
|
|
createKtxLlmProvider: createKtxLlmProvider as any,
|
|
createKtxEmbeddingProvider: createKtxEmbeddingProvider as any,
|
|
env: { OPENAI_API_KEY: 'openai-key' }, // pragma: allowlist secret
|
|
},
|
|
);
|
|
|
|
expect(providers?.embedding.dimensions).toBe(1536);
|
|
expect(providers?.embedding.maxBatchSize).toBe(8);
|
|
expect(createKtxLlmProvider).toHaveBeenCalledWith(
|
|
expect.objectContaining({ backend: 'gateway', modelSlots: { default: 'provider/language-model' } }),
|
|
);
|
|
expect(createKtxEmbeddingProvider).toHaveBeenCalledWith(
|
|
expect.objectContaining({ backend: 'openai', model: 'provider/embedding-model' }),
|
|
);
|
|
});
|
|
});
|