apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-07-07 11:02:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

test: split cli tests from source tree (#216)

Browse source 
* feat(cli): define full warehouse dialect contract

* test(cli): keep dialect edge tests focused

* fix(cli): stabilize dialect contract foundation

* refactor(connectors): own read-only query preparation

* refactor(connectors): resolve dialects through registry

* refactor(connectors): keep concrete dialect classes internal

* chore(workspace): enforce dialect import boundary

* refactor(cli): resolve relationship dialect at scan boundary

* refactor(cli): use dialect display parsing for entity details

* refactor(cli): use dialect display parsing for warehouse catalog

* refactor(cli): use dialect SQL in relationship workflows

* test(cli): verify solid dialect scan workflow closure

* test: split cli tests from source tree

* refactor(cli): standardize BigQuery scope listing

* feat(sqlite): implement connector scope listing

* test(connectors): cover required table listing

* feat(cli): add warehouse driver registry

* refactor(setup): route scope discovery through driver registry

* refactor(cli): route local query execution through driver registry

* refactor(historic-sql): route dialect support through driver registry

* refactor(cli): test warehouse connections through driver registry

* fix(cli): close driver registry type export gaps

* Improve setup daemon diagnostics

* refactor(setup): centralize rail-prefixed diagnostics + query-history fallback

Extract errorMessage, writePrefixedLines, and flushPrefixedBufferedCommandOutput
into clack.ts so the setup wizard, managed daemons, and embedding/agent steps
share one rail-formatted writer. setup-databases.ts also adds a
"disable query history and retry" option when the schema-context build fails
and query history is the likely culprit, surfaced via a new
failed-query-history-unavailable status.

* fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match

The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.

Align the picker boundary with the canonical 3-level KtxTableRef:

- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
  resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
  (resolveEnabledTables already accepts the 3-part shape) and
  schemasFromEnabledTables now goes through parseDottedTableEntry so it
  recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
  reuse.

Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).

* fix(cli): allow debug telemetry under opt-out env
This commit is contained in:
Andrey Avtomonov 2026-05-26 08:49:05 +02:00 • committed by GitHub
parent 924868841d
commit 56985b7e09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
548 changed files with 5048 additions and 2228 deletions
Download patch file Download diff file Expand all files Collapse all files

264
packages/cli/src/context/search/discover.test.ts
View file

@ -1,264 +0,0 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js';
import { writeLocalKnowledgePage } from '../wiki/local-knowledge.js';
import { createKtxDiscoverDataService } from './discover.js';
describe('createKtxDiscoverDataService', () => {
let tempDir: string;
let project: KtxLocalProject;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-discover-data-'));
project = await initKtxProject({ projectDir: join(tempDir, 'project') });
project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL' };
project.config.connections.billing = { driver: 'postgres', url: 'env:BILLING_DATABASE_URL' };
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
async function seedWiki(): Promise<void> {
await writeLocalKnowledgePage(project, {
key: 'orders-playbook',
scope: 'GLOBAL',
summary: 'Paid order operations',
content: 'Use paid orders and order_count to inspect monthly customer activity for Acme Corp.',
tags: ['orders'],
});
}
async function seedSl(): Promise<void> {
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
[
'name: orders',
'descriptions:',
' user: Paid order facts',
'table: public.orders',
'grain: [id]',
'columns:',
' - name: status',
' type: string',
' descriptions:',
' user: Payment status for the order',
' - name: ordered_at',
' type: time',
'measures:',
' - name: order_count',
' expr: count(*)',
' description: Number of paid orders',
'',
].join('\n'),
'ktx',
'ktx@example.com',
'seed sl source',
);
}
async function seedScan(input: {
connectionId?: string;
syncId: string;
tableName?: string;
comment?: string;
sampleValues?: string[];
}): Promise<void> {
const connectionId = input.connectionId ?? 'warehouse';
const root = `raw-sources/${connectionId}/live-database/${input.syncId}`;
const tableName = input.tableName ?? 'orders';
await project.fileStore.writeFile(
`${root}/connection.json`,
JSON.stringify(
{
connectionId,
driver: 'postgres',
extractedAt: `2026-05-14T09:00:00.000Z`,
scope: { schemas: ['public'] },
},
null,
2,
),
'ktx',
'ktx@example.com',
'seed scan connection',
);
await project.fileStore.writeFile(
`${root}/tables/public-${tableName}.json`,
JSON.stringify(
{
catalog: null,
db: 'public',
name: tableName,
kind: 'table',
comment: input.comment ?? 'Orders table from warehouse',
estimatedRows: 123,
descriptions: { db: input.comment ?? 'Orders table from warehouse' },
columns: [
{
name: 'id',
nativeType: 'integer',
normalizedType: 'integer',
dimensionType: 'number',
nullable: false,
primaryKey: true,
comment: 'Order id',
},
{
name: 'status',
nativeType: 'text',
normalizedType: 'text',
dimensionType: 'string',
nullable: false,
primaryKey: false,
comment: 'Order status',
sampleValues: input.sampleValues ?? ['paid', 'pending'],
},
],
foreignKeys: [],
},
null,
2,
),
'ktx',
'ktx@example.com',
'seed table',
);
await project.fileStore.writeFile(
`${root}/scan-report.json`,
JSON.stringify(
{
connectionId,
driver: 'postgres',
syncId: input.syncId,
runId: `scan-${input.syncId}`,
trigger: 'mcp',
mode: 'enriched',
dryRun: false,
artifactPaths: {
rawSourcesDir: root,
reportPath: `${root}/scan-report.json`,
manifestShards: [],
enrichmentArtifacts: [],
},
diffSummary: {
tablesAdded: 1,
tablesModified: 0,
tablesDeleted: 0,
tablesUnchanged: 0,
columnsAdded: 0,
columnsModified: 0,
columnsDeleted: 0,
},
manifestShardsWritten: 0,
structuralSyncStats: {
tablesCreated: 0,
tablesUpdated: 0,
tablesDeleted: 0,
columnsCreated: 0,
columnsUpdated: 0,
columnsDeleted: 0,
},
enrichment: {
dataDictionary: 'completed',
tableDescriptions: 'completed',
columnDescriptions: 'completed',
embeddings: 'skipped',
deterministicRelationships: 'skipped',
llmRelationshipValidation: 'skipped',
statisticalValidation: 'skipped',
},
capabilityGaps: [],
warnings: [],
relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 },
enrichmentState: { resumedStages: [], completedStages: [], failedStages: [] },
createdAt: '2026-05-14T09:00:00.000Z',
},
null,
2,
),
'ktx',
'ktx@example.com',
'seed scan report',
);
}
it('returns unified ranked refs across wiki, semantic-layer, and raw schema', async () => {
await seedWiki();
await seedSl();
await seedScan({ syncId: 'sync-1', sampleValues: ['paid', 'refunded'] });
const service = createKtxDiscoverDataService(project, { userId: 'local-user' });
const results = await service.search({ query: 'paid orders', connectionId: 'warehouse', limit: 10 });
expect(results.map((result) => result.kind)).toEqual(
expect.arrayContaining(['wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column']),
);
expect(results.every((result) => result.score >= 0 && result.score <= 1)).toBe(true);
expect(results.every((result) => result.snippet === null || result.snippet.length <= 200)).toBe(true);
expect(results).toContainEqual(
expect.objectContaining({
kind: 'table',
id: 'public.orders',
connectionId: 'warehouse',
tableRef: { catalog: null, db: 'public', name: 'orders' },
matchedOn: expect.stringMatching(/name|description|comment|display/),
}),
);
expect(results).toContainEqual(
expect.objectContaining({
kind: 'column',
id: 'public.orders.status',
connectionId: 'warehouse',
columnName: 'status',
matchedOn: expect.stringMatching(/name|comment|description|sample_value/),
}),
);
expect(results).toContainEqual(
expect.objectContaining({
kind: 'sl_measure',
id: 'orders.order_count',
connectionId: 'warehouse',
summary: 'Number of paid orders',
snippet: 'count(*)',
matchedOn: expect.stringMatching(/name|description|expr/),
}),
);
});
it('honors kind filters and connection scope', async () => {
await seedWiki();
await seedSl();
await seedScan({ syncId: 'sync-1', connectionId: 'warehouse', tableName: 'orders' });
await seedScan({ syncId: 'sync-2', connectionId: 'billing', tableName: 'invoices', comment: 'Billing invoices' });
const service = createKtxDiscoverDataService(project);
const results = await service.search({
query: 'orders',
connectionId: 'warehouse',
kinds: ['table', 'column'],
limit: 10,
});
expect(results.every((result) => result.kind === 'table' || result.kind === 'column')).toBe(true);
expect(results.every((result) => result.connectionId === 'warehouse')).toBe(true);
expect(results.some((result) => result.id.includes('invoices'))).toBe(false);
expect(results.some((result) => result.kind === 'wiki')).toBe(false);
});
it('re-reads the latest scan artifacts on each call', async () => {
await seedScan({ syncId: 'sync-1', tableName: 'orders', comment: 'Old orders table' });
const service = createKtxDiscoverDataService(project);
await expect(
service.search({ query: 'orders', connectionId: 'warehouse', kinds: ['table'], limit: 10 }),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'public.orders' })]));
await seedScan({ syncId: 'sync-2', tableName: 'invoices', comment: 'Invoice facts' });
const fresh = await service.search({ query: 'invoice', connectionId: 'warehouse', kinds: ['table'], limit: 10 });
expect(fresh).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'public.invoices' })]));
expect(fresh.some((result) => result.id === 'public.orders')).toBe(false);
});
});