apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-07-22 11:51:01 +02:00
Code Issues Projects Releases Packages Wiki Activity

test: split cli tests from source tree (#216)

Browse source 
* feat(cli): define full warehouse dialect contract

* test(cli): keep dialect edge tests focused

* fix(cli): stabilize dialect contract foundation

* refactor(connectors): own read-only query preparation

* refactor(connectors): resolve dialects through registry

* refactor(connectors): keep concrete dialect classes internal

* chore(workspace): enforce dialect import boundary

* refactor(cli): resolve relationship dialect at scan boundary

* refactor(cli): use dialect display parsing for entity details

* refactor(cli): use dialect display parsing for warehouse catalog

* refactor(cli): use dialect SQL in relationship workflows

* test(cli): verify solid dialect scan workflow closure

* test: split cli tests from source tree

* refactor(cli): standardize BigQuery scope listing

* feat(sqlite): implement connector scope listing

* test(connectors): cover required table listing

* feat(cli): add warehouse driver registry

* refactor(setup): route scope discovery through driver registry

* refactor(cli): route local query execution through driver registry

* refactor(historic-sql): route dialect support through driver registry

* refactor(cli): test warehouse connections through driver registry

* fix(cli): close driver registry type export gaps

* Improve setup daemon diagnostics

* refactor(setup): centralize rail-prefixed diagnostics + query-history fallback

Extract errorMessage, writePrefixedLines, and flushPrefixedBufferedCommandOutput
into clack.ts so the setup wizard, managed daemons, and embedding/agent steps
share one rail-formatted writer. setup-databases.ts also adds a
"disable query history and retry" option when the schema-context build fails
and query history is the likely culprit, surfaced via a new
failed-query-history-unavailable status.

* fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match

The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.

Align the picker boundary with the canonical 3-level KtxTableRef:

- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
  resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
  (resolveEnabledTables already accepts the 3-part shape) and
  schemasFromEnabledTables now goes through parseDottedTableEntry so it
  recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
  reuse.

Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).

* fix(cli): allow debug telemetry under opt-out env
This commit is contained in:
Andrey Avtomonov 2026-05-26 08:49:05 +02:00 • committed by GitHub
parent 924868841d
commit 56985b7e09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
548 changed files with 5048 additions and 2228 deletions
Download patch file Download diff file Expand all files Collapse all files

332
packages/cli/src/context/ingest/memory-flow/events.test.ts
View file

@ -1,332 +0,0 @@
import { describe, expect, it } from 'vitest';
import type { LocalIngestRunRecord } from '../local-stage-ingest.js';
import type { IngestReportSnapshot } from '../reports.js';
import { ingestReportToMemoryFlowReplay, localIngestRunToMemoryFlowReplay } from './events.js';
function localRecord(): LocalIngestRunRecord {
return {
runId: 'local-run-1',
jobId: 'local-run-1',
status: 'done',
adapter: 'metricflow',
connectionId: 'warehouse',
sourceDir: '/tmp/source',
syncId: 'sync-1',
startedAt: '2026-04-30T10:00:00.000Z',
completedAt: '2026-04-30T10:00:01.000Z',
progress: 1,
done: true,
previousRunId: null,
diffSummary: { added: 2, modified: 1, deleted: 1, unchanged: 4 },
diffPaths: {
added: ['models/orders.yml', 'models/revenue.yml'],
modified: ['models/customers.yml'],
deleted: ['models/old.yml'],
unchanged: ['models/a.yml', 'models/b.yml', 'models/c.yml', 'models/d.yml'],
},
workUnitCount: 2,
rawFileCount: 7,
workUnits: [
{
unitKey: 'orders',
rawFiles: ['models/orders.yml'],
peerFileIndex: ['models/customers.yml'],
dependencyPaths: ['models/base.yml'],
},
{
unitKey: 'revenue',
rawFiles: ['models/revenue.yml'],
peerFileIndex: [],
dependencyPaths: [],
},
],
evictionDeletedRawPaths: ['raw-sources/warehouse/metricflow/sync-1/models/old.yml'],
errors: [],
};
}
function reportSnapshot(): IngestReportSnapshot {
return {
id: 'report-1',
runId: 'run-1',
jobId: 'job-1',
connectionId: 'warehouse',
sourceKey: 'lookml',
createdAt: '2026-04-30T10:00:02.000Z',
body: {
syncId: 'sync-2',
diffSummary: { added: 1, modified: 1, deleted: 0, unchanged: 3 },
commitSha: 'abc123456789', // pragma: allowlist secret
failedWorkUnits: ['customers'],
reconciliationSkipped: false,
conflictsResolved: [
{
kind: 'near_duplicate',
artifactKey: 'warehouse.orders',
detail: 'kept candidate definition',
flaggedForHuman: false,
},
],
evictionsApplied: [],
unmappedFallbacks: [{ rawPath: 'cards/42.json', reason: 'no_connection_mapping', fallback: 'flagged' }],
evictionInputs: [],
unresolvedCards: [],
supersededBy: null,
overrideOf: null,
provenanceRows: [
{
rawPath: 'views/orders.view.lkml',
artifactKind: 'wiki',
artifactKey: 'wiki/global/orders.md',
actionType: 'wiki_written',
},
{
rawPath: 'views/orders.view.lkml',
artifactKind: 'sl',
artifactKey: 'warehouse.orders',
actionType: 'measure_added',
},
{
rawPath: 'views/customers.view.lkml',
artifactKind: null,
artifactKey: null,
actionType: 'skipped',
},
],
toolTranscripts: [
{
unitKey: 'orders',
path: '/tmp/ktx/run/wu-transcripts/job-1/orders.jsonl',
toolCallCount: 3,
errorCount: 0,
toolNames: ['read_raw_span', 'wiki_write', 'sl_write_source'],
},
{
unitKey: 'customers',
path: '/tmp/ktx/run/wu-transcripts/job-1/customers.jsonl',
toolCallCount: 2,
errorCount: 1,
toolNames: ['read_raw_span', 'sl_write_source'],
},
],
workUnits: [
{
unitKey: 'orders',
rawFiles: ['views/orders.view.lkml'],
status: 'success',
actions: [
{ target: 'wiki', type: 'created', key: 'wiki/global/orders.md', detail: 'order facts' },
{ target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'order measures' },
],
touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'warehouse.orders' }],
},
{
unitKey: 'customers',
rawFiles: ['views/customers.view.lkml'],
status: 'failed',
reason: 'semantic-layer validation failed',
actions: [{ target: 'sl', type: 'created', key: 'warehouse.customers', detail: 'invalid source' }],
touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'warehouse.customers' }],
},
],
},
};
}
describe('memory-flow event mapping', () => {
it('maps a local ingest run to source, snapshot, diff, chunk, and report events', () => {
const replay = localIngestRunToMemoryFlowReplay(localRecord());
expect(replay).toMatchObject({
runId: 'local-run-1',
connectionId: 'warehouse',
adapter: 'metricflow',
status: 'done',
sourceDir: '/tmp/source',
syncId: 'sync-1',
plannedWorkUnits: [
{ unitKey: 'orders', rawFiles: ['models/orders.yml'], peerFileCount: 1, dependencyCount: 1 },
{ unitKey: 'revenue', rawFiles: ['models/revenue.yml'], peerFileCount: 0, dependencyCount: 0 },
],
});
expect(replay.events).toEqual([
{ type: 'source_acquired', adapter: 'metricflow', trigger: 'manual_resync', fileCount: 7 },
{ type: 'scope_detected', fingerprint: null },
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 7 },
{ type: 'diff_computed', added: 2, modified: 1, deleted: 1, unchanged: 4 },
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 1 },
{ type: 'report_created', runId: 'local-run-1' },
]);
});
it('maps an ingest report snapshot to work-unit, candidate, gate, saved, provenance, and report events', () => {
const replay = ingestReportToMemoryFlowReplay(reportSnapshot(), { provenanceRowCount: 5 });
expect(replay).toMatchObject({
runId: 'run-1',
connectionId: 'warehouse',
adapter: 'lookml',
status: 'error',
sourceDir: null,
syncId: 'sync-2',
reportId: 'report-1',
plannedWorkUnits: [
{ unitKey: 'orders', rawFiles: ['views/orders.view.lkml'], peerFileCount: 0, dependencyCount: 0 },
{ unitKey: 'customers', rawFiles: ['views/customers.view.lkml'], peerFileCount: 0, dependencyCount: 0 },
],
});
expect(replay.events).toContainEqual({
type: 'candidate_action',
unitKey: 'orders',
target: 'wiki',
action: 'created',
key: 'wiki/global/orders.md',
});
expect(replay.events).toContainEqual({
type: 'work_unit_finished',
unitKey: 'customers',
status: 'failed',
reason: 'semantic-layer validation failed',
});
expect(replay.events).toContainEqual({ type: 'reconciliation_finished', conflictCount: 1, fallbackCount: 1 });
expect(replay.events).toContainEqual({ type: 'saved', commitSha: 'abc123456789', wikiCount: 1, slCount: 2 }); // pragma: allowlist secret
expect(replay.events).toContainEqual({ type: 'provenance_recorded', rowCount: 5 });
expect(replay.events).toContainEqual({ type: 'report_created', runId: 'run-1', reportPath: 'report-1' });
expect(replay.details.actions).toEqual([
{
unitKey: 'orders',
target: 'wiki',
action: 'created',
key: 'wiki/global/orders.md',
summary: 'order facts',
rawFiles: ['views/orders.view.lkml'],
status: 'success',
},
{
unitKey: 'orders',
target: 'sl',
action: 'updated',
key: 'warehouse.orders',
summary: 'order measures',
rawFiles: ['views/orders.view.lkml'],
status: 'success',
},
{
unitKey: 'customers',
target: 'sl',
action: 'created',
key: 'warehouse.customers',
summary: 'invalid source',
rawFiles: ['views/customers.view.lkml'],
status: 'failed',
},
]);
expect(replay.details.provenance).toEqual([
{
rawPath: 'views/orders.view.lkml',
artifactKind: 'wiki',
artifactKey: 'wiki/global/orders.md',
actionType: 'wiki_written',
},
{
rawPath: 'views/orders.view.lkml',
artifactKind: 'sl',
artifactKey: 'warehouse.orders',
actionType: 'measure_added',
},
{
rawPath: 'views/customers.view.lkml',
artifactKind: null,
artifactKey: null,
actionType: 'skipped',
},
]);
expect(replay.details.transcripts).toEqual([
{
unitKey: 'orders',
path: '/tmp/ktx/run/wu-transcripts/job-1/orders.jsonl',
toolCallCount: 3,
errorCount: 0,
toolNames: ['read_raw_span', 'wiki_write', 'sl_write_source'],
},
{
unitKey: 'customers',
path: '/tmp/ktx/run/wu-transcripts/job-1/customers.jsonl',
toolCallCount: 2,
errorCount: 1,
toolNames: ['read_raw_span', 'sl_write_source'],
},
]);
});
it('prefers captured memory-flow snapshots from report bodies', () => {
const report = reportSnapshot();
Object.assign(report.body, {
memoryFlow: {
metadata: {
schemaVersion: 1,
mode: 'full',
origin: 'captured',
timing: 'captured',
capturedAt: '2026-05-01T10:00:03.000Z',
sourceReportId: null,
sourceReportPath: null,
fallbackReason: null,
},
runId: 'run-1',
connectionId: 'warehouse',
adapter: 'lookml',
status: 'running',
sourceDir: null,
syncId: 'sync-2',
errors: [],
plannedWorkUnits: [
{ unitKey: 'orders', rawFiles: ['views/orders.view.lkml'], peerFileCount: 1, dependencyCount: 2 },
],
details: { actions: [], provenance: [], transcripts: [] },
events: [
{
type: 'source_acquired',
adapter: 'lookml',
trigger: 'manual_resync',
fileCount: 1,
emittedAt: '2026-05-01T10:00:00.000Z',
},
],
},
});
const replay = ingestReportToMemoryFlowReplay(report);
expect(replay.metadata).toEqual({
schemaVersion: 1,
mode: 'full',
origin: 'captured',
timing: 'captured',
capturedAt: '2026-05-01T10:00:03.000Z',
sourceReportId: 'report-1',
sourceReportPath: 'report-1',
fallbackReason: null,
});
expect(replay.status).toBe('error');
expect(replay.reportId).toBe('report-1');
expect(replay.reportPath).toBe('report-1');
expect(replay.events[0]).toMatchObject({ type: 'source_acquired', emittedAt: '2026-05-01T10:00:00.000Z' });
expect(replay.events).toContainEqual({ type: 'report_created', runId: 'run-1', reportPath: 'report-1' });
});
it('labels reconstructed report replays as synthetic when no captured snapshot exists', () => {
const replay = ingestReportToMemoryFlowReplay(reportSnapshot(), { provenanceRowCount: 5 });
expect(replay.metadata).toEqual({
schemaVersion: 1,
mode: 'full',
origin: 'synthetic-report',
timing: 'synthetic',
capturedAt: '2026-04-30T10:00:02.000Z',
sourceReportId: 'report-1',
sourceReportPath: 'report-1',
fallbackReason: 'report did not include captured memory-flow events',
});
});
});