mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-04 10:52:13 +02:00
acd20ac248
* feat(sigma): add Sigma Computing context-source adapter Closes #168 Adds a full ingest adapter for Sigma Computing so `ktx ingest` can pull data model specs and workbook summaries into the ktx context layer. The implementation follows the same fetch → chunk → project → LLM pattern used by the Looker, Metabase, and MetricFlow adapters. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(sigma): address PR review comments - Remove manifest from rawFiles; moves to peerFileIndex so fetchedAt changes don't mark all work units dirty every run - Fix workbookFilter.updatedSince eviction bug: fetch full universe first, apply filter client-side, evict only on archived/deleted - Remove measure projection entirely; project() writes measures: [] and the sigma_ingest skill surfaces Lookup/aggregation formulas as wiki prose - Remove joins projection (v1 limitation); project() writes joins: [] and Lookup relationships are described in wiki prose instead - Remove write-back dead code: createDataModel, updateDataModel, SigmaDataModelPushResult, mutate/post/put - Fix emitBatches notes pluralization bug ('2 data modelss' → '2 data models') - Add tokenInflight dedup on ensureToken to coalesce concurrent auth requests - Retry spec fetch when existing staged spec is null (transient failure cache) - Drop unused WorkbookFilter import from client-port.ts - Note in docs that joins are not projected from Sigma data models in this release Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * updates * fix(sigma): restore sigma in local adapter test + small cleanups The gdrive↔sigma merge dropped 'sigma' from the expected adapter source list in local-adapters.test.ts while keeping gdrive, so the slow TS suite failed even though the source registers both. Add 'sigma' back at its registration position (after metabase, before gdrive). Also: - Move the orphaned SigmaPullConfig docstring onto the schema it documents and drop the stale BullMQ reference (standalone ktx has no BullMQ; the config lives in the ingest job's bundleRef.config). - Drop an O(n^2) find() round-trip in fetch() when building the active data-model list; filter once and reuse for the eviction id set. --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Andrey Avtomonov <andreybavt@gmail.com> Co-authored-by: Luca Martial <48870843+luca-martial@users.noreply.github.com>
61 lines
2.4 KiB
TypeScript
61 lines
2.4 KiB
TypeScript
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
import { detectSigmaStagedDir } from '../../../../../src/context/ingest/adapters/sigma/detect.js';
|
|
|
|
async function touch(dir: string, relPath: string, body = '{}'): Promise<void> {
|
|
const abs = join(dir, relPath);
|
|
await mkdir(join(abs, '..'), { recursive: true });
|
|
await writeFile(abs, body, 'utf-8');
|
|
}
|
|
|
|
describe('detectSigmaStagedDir', () => {
|
|
let stagedDir: string;
|
|
|
|
beforeEach(async () => {
|
|
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-detect-'));
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await rm(stagedDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('returns true when manifest and at least one data-model file are present', async () => {
|
|
await touch(stagedDir, 'sigma-manifest.json');
|
|
await touch(stagedDir, 'data-models/dm-aaa111.json');
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(true);
|
|
});
|
|
|
|
it('returns true when manifest and at least one workbook file are present', async () => {
|
|
await touch(stagedDir, 'sigma-manifest.json');
|
|
await touch(stagedDir, 'workbooks/wb-xxx111.json');
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(true);
|
|
});
|
|
|
|
it('returns false when sigma-manifest.json is absent', async () => {
|
|
await touch(stagedDir, 'data-models/dm-aaa111.json');
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
|
});
|
|
|
|
it('returns false for a completely empty directory', async () => {
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
|
});
|
|
|
|
it('returns false when manifest is present but both entity dirs are empty', async () => {
|
|
await touch(stagedDir, 'sigma-manifest.json');
|
|
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
|
await mkdir(join(stagedDir, 'workbooks'), { recursive: true });
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
|
});
|
|
|
|
it('returns false when manifest is present but entity dirs are absent', async () => {
|
|
await touch(stagedDir, 'sigma-manifest.json');
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
|
});
|
|
|
|
it('returns false when only unrelated files are present', async () => {
|
|
await touch(stagedDir, 'data-models/dm-aaa111.json');
|
|
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
|
});
|
|
});
|