mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-01 08:59:39 +02:00
feat(sigma): add Sigma Computing context-source adapter (#316)
* feat(sigma): add Sigma Computing context-source adapter Closes #168 Adds a full ingest adapter for Sigma Computing so `ktx ingest` can pull data model specs and workbook summaries into the ktx context layer. The implementation follows the same fetch → chunk → project → LLM pattern used by the Looker, Metabase, and MetricFlow adapters. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(sigma): address PR review comments - Remove manifest from rawFiles; moves to peerFileIndex so fetchedAt changes don't mark all work units dirty every run - Fix workbookFilter.updatedSince eviction bug: fetch full universe first, apply filter client-side, evict only on archived/deleted - Remove measure projection entirely; project() writes measures: [] and the sigma_ingest skill surfaces Lookup/aggregation formulas as wiki prose - Remove joins projection (v1 limitation); project() writes joins: [] and Lookup relationships are described in wiki prose instead - Remove write-back dead code: createDataModel, updateDataModel, SigmaDataModelPushResult, mutate/post/put - Fix emitBatches notes pluralization bug ('2 data modelss' → '2 data models') - Add tokenInflight dedup on ensureToken to coalesce concurrent auth requests - Retry spec fetch when existing staged spec is null (transient failure cache) - Drop unused WorkbookFilter import from client-port.ts - Note in docs that joins are not projected from Sigma data models in this release Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * updates * fix(sigma): restore sigma in local adapter test + small cleanups The gdrive↔sigma merge dropped 'sigma' from the expected adapter source list in local-adapters.test.ts while keeping gdrive, so the slow TS suite failed even though the source registers both. Add 'sigma' back at its registration position (after metabase, before gdrive). Also: - Move the orphaned SigmaPullConfig docstring onto the schema it documents and drop the stale BullMQ reference (standalone ktx has no BullMQ; the config lives in the ingest job's bundleRef.config). - Drop an O(n^2) find() round-trip in fetch() when building the active data-model list; filter once and reuse for the eviction id set. --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Andrey Avtomonov <andreybavt@gmail.com> Co-authored-by: Luca Martial <48870843+luca-martial@users.noreply.github.com>
This commit is contained in:
Matt Senick (Sigma)
GitHub
parent
139ac08320
commit
acd20ac248
41 changed files with 3610 additions and 6 deletions
325
packages/cli/test/context/ingest/adapters/sigma/chunk.test.ts
Normal file
325
packages/cli/test/context/ingest/adapters/sigma/chunk.test.ts
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { chunkSigmaStagedDir } from '../../../../../src/context/ingest/adapters/sigma/chunk.js';
|
||||
|
||||
// Keep in sync with constants in chunk.ts
|
||||
const DATA_MODELS_PER_UNIT = 50;
|
||||
const WORKBOOKS_PER_UNIT = 2000;
|
||||
|
||||
const FIXTURES = resolve(import.meta.dirname, '../../../../fixtures/sigma');
|
||||
const SINGLE = join(FIXTURES, 'single-folder');
|
||||
const MULTI = join(FIXTURES, 'multi-folder');
|
||||
const EMPTY = join(FIXTURES, 'empty-manifest');
|
||||
|
||||
describe('chunkSigmaStagedDir — first run', () => {
|
||||
it('single-folder fixture emits two WUs (data-models and workbooks)', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
expect(result.workUnits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('data-models WU has correct unitKey and displayLabel', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models')!;
|
||||
expect(wu).toBeDefined();
|
||||
expect(wu.displayLabel).toBe('Sigma: data models');
|
||||
});
|
||||
|
||||
it('workbooks WU has correct unitKey and displayLabel', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks')!;
|
||||
expect(wu).toBeDefined();
|
||||
expect(wu.displayLabel).toBe('Sigma: workbooks');
|
||||
});
|
||||
|
||||
it('data-models WU rawFiles contains data model files but not the manifest', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models')!;
|
||||
expect(wu.rawFiles).toContain('data-models/dm-aaa111.json');
|
||||
expect(wu.rawFiles).toContain('data-models/dm-bbb222.json');
|
||||
expect(wu.rawFiles).not.toContain('sigma-manifest.json');
|
||||
expect(wu.rawFiles).not.toContain('workbooks/wb-xxx111.json');
|
||||
});
|
||||
|
||||
it('manifest is in peerFileIndex so the LLM can read it without affecting the hash', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const dmWu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models')!;
|
||||
const wbWu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks')!;
|
||||
expect(dmWu.peerFileIndex).toContain('sigma-manifest.json');
|
||||
expect(wbWu.peerFileIndex).toContain('sigma-manifest.json');
|
||||
});
|
||||
|
||||
it('workbooks WU rawFiles contains workbook files but not the manifest', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks')!;
|
||||
expect(wu.rawFiles).toContain('workbooks/wb-xxx111.json');
|
||||
expect(wu.rawFiles).not.toContain('sigma-manifest.json');
|
||||
expect(wu.rawFiles).not.toContain('data-models/dm-aaa111.json');
|
||||
});
|
||||
|
||||
it('data-models WU peerFileIndex contains workbook files', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models')!;
|
||||
expect(wu.peerFileIndex).toContain('workbooks/wb-xxx111.json');
|
||||
});
|
||||
|
||||
it('workbooks WU peerFileIndex contains data model files', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks')!;
|
||||
expect(wu.peerFileIndex).toContain('data-models/dm-aaa111.json');
|
||||
expect(wu.peerFileIndex).toContain('data-models/dm-bbb222.json');
|
||||
});
|
||||
|
||||
it('data-models WU notes describes model count', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models')!;
|
||||
expect(wu.notes).toBe('2 data models');
|
||||
});
|
||||
|
||||
it('workbooks WU notes describes workbook count', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks')!;
|
||||
expect(wu.notes).toBe('1 workbook');
|
||||
});
|
||||
|
||||
it('dependencyPaths is empty on first run for both WUs', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
for (const wu of result.workUnits) {
|
||||
expect(wu.dependencyPaths).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it('multi-folder fixture still emits two WUs (data-models and workbooks)', async () => {
|
||||
const result = await chunkSigmaStagedDir(MULTI);
|
||||
expect(result.workUnits).toHaveLength(2);
|
||||
expect(result.workUnits.map((w) => w.unitKey).sort()).toEqual(['sigma-data-models', 'sigma-workbooks']);
|
||||
});
|
||||
|
||||
it('multi-folder: data-models WU contains all data models regardless of folder', async () => {
|
||||
const result = await chunkSigmaStagedDir(MULTI);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models')!;
|
||||
expect(wu.rawFiles).toContain('data-models/dm-aaa111.json');
|
||||
expect(wu.rawFiles).toContain('data-models/dm-bbb222.json');
|
||||
expect(wu.rawFiles).toContain('data-models/dm-ccc333.json');
|
||||
});
|
||||
|
||||
it('multi-folder: workbooks WU contains all workbooks regardless of folder', async () => {
|
||||
const result = await chunkSigmaStagedDir(MULTI);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks')!;
|
||||
expect(wu.rawFiles).toContain('workbooks/wb-yyy222.json');
|
||||
expect(wu.rawFiles).toContain('workbooks/wb-zzz333.json');
|
||||
});
|
||||
|
||||
it('unitKey is slug-safe (no slashes or spaces)', async () => {
|
||||
const result = await chunkSigmaStagedDir(SINGLE);
|
||||
for (const wu of result.workUnits) {
|
||||
expect(wu.unitKey).toMatch(/^[a-zA-Z0-9_-]+$/);
|
||||
}
|
||||
});
|
||||
|
||||
it('empty-manifest fixture emits zero WUs', async () => {
|
||||
const result = await chunkSigmaStagedDir(EMPTY);
|
||||
expect(result.workUnits).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('missing manifest directory emits zero WUs without crashing', async () => {
|
||||
const result = await chunkSigmaStagedDir('/tmp/sigma-nonexistent-dir-ktx-test');
|
||||
expect(result.workUnits).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('is deterministic: two identical calls produce structurally equal output', async () => {
|
||||
const r1 = await chunkSigmaStagedDir(SINGLE);
|
||||
const r2 = await chunkSigmaStagedDir(SINGLE);
|
||||
expect(JSON.stringify(r1)).toBe(JSON.stringify(r2));
|
||||
});
|
||||
});
|
||||
|
||||
describe('chunkSigmaStagedDir — data model batching', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-dm-batch-'));
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
const manifest = JSON.stringify({
|
||||
fetchedAt: new Date().toISOString(),
|
||||
dataModelCount: DATA_MODELS_PER_UNIT + 1,
|
||||
workbookCount: 0,
|
||||
sigmaConnectionId: 'conn-1',
|
||||
});
|
||||
await writeFile(join(stagedDir, 'sigma-manifest.json'), manifest);
|
||||
for (let i = 0; i < DATA_MODELS_PER_UNIT + 1; i++) {
|
||||
const dm = JSON.stringify({
|
||||
sigmaId: `dm-${i}`,
|
||||
name: `Data Model ${i}`,
|
||||
path: 'Engineering',
|
||||
latestVersion: 1,
|
||||
updatedAt: '2026-01-01T00:00:00Z',
|
||||
isArchived: false,
|
||||
dataModelUrlId: `url-${i}`,
|
||||
spec: null,
|
||||
});
|
||||
await writeFile(join(stagedDir, 'data-models', `dm-${String(i).padStart(6, '0')}.json`), dm);
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('splits into two data model WUs when count exceeds DATA_MODELS_PER_UNIT', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const dmUnits = result.workUnits.filter((w) => w.unitKey.startsWith('sigma-data-models'));
|
||||
expect(dmUnits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('batched data model WUs get indexed unitKeys (sigma-data-models-0, sigma-data-models-1)', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const keys = result.workUnits.map((w) => w.unitKey).filter((k) => k.startsWith('sigma-data-models')).sort();
|
||||
expect(keys).toEqual(['sigma-data-models-0', 'sigma-data-models-1']);
|
||||
});
|
||||
|
||||
it('first batch has exactly DATA_MODELS_PER_UNIT files (manifest excluded from rawFiles)', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models-0')!;
|
||||
expect(wu.rawFiles).toHaveLength(DATA_MODELS_PER_UNIT);
|
||||
});
|
||||
|
||||
it('displayLabel includes batch position when split', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-data-models-0')!;
|
||||
expect(wu.displayLabel).toMatch(/\(1\/2\)/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('chunkSigmaStagedDir — workbook batching', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-batch-'));
|
||||
await mkdir(join(stagedDir, 'workbooks'), { recursive: true });
|
||||
const manifest = JSON.stringify({
|
||||
fetchedAt: new Date().toISOString(),
|
||||
dataModelCount: 0,
|
||||
workbookCount: WORKBOOKS_PER_UNIT + 1,
|
||||
sigmaConnectionId: 'conn-1',
|
||||
});
|
||||
await writeFile(join(stagedDir, 'sigma-manifest.json'), manifest);
|
||||
for (let i = 0; i < WORKBOOKS_PER_UNIT + 1; i++) {
|
||||
const wb = JSON.stringify({
|
||||
sigmaId: `wb-${i}`,
|
||||
name: `Workbook ${i}`,
|
||||
path: 'Finance',
|
||||
latestVersion: 1,
|
||||
updatedAt: '2026-01-01T00:00:00Z',
|
||||
isArchived: false,
|
||||
workbookUrlId: `url-${i}`,
|
||||
});
|
||||
await writeFile(join(stagedDir, 'workbooks', `wb-${String(i).padStart(6, '0')}.json`), wb);
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('splits into two workbook WUs when count exceeds WORKBOOKS_PER_UNIT', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const wbUnits = result.workUnits.filter((w) => w.unitKey.startsWith('sigma-workbooks'));
|
||||
expect(wbUnits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('batched WUs get indexed unitKeys (sigma-workbooks-0, sigma-workbooks-1)', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const keys = result.workUnits.map((w) => w.unitKey).filter((k) => k.startsWith('sigma-workbooks')).sort();
|
||||
expect(keys).toEqual(['sigma-workbooks-0', 'sigma-workbooks-1']);
|
||||
});
|
||||
|
||||
it('first batch has exactly WORKBOOKS_PER_UNIT files (manifest excluded from rawFiles)', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks-0')!;
|
||||
expect(wu.rawFiles).toHaveLength(WORKBOOKS_PER_UNIT);
|
||||
});
|
||||
|
||||
it('second batch has the remainder only', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks-1')!;
|
||||
expect(wu.rawFiles).toHaveLength(1); // 1 overflow workbook
|
||||
});
|
||||
|
||||
it('displayLabel includes batch position when split', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir);
|
||||
const wu = result.workUnits.find((w) => w.unitKey === 'sigma-workbooks-0')!;
|
||||
expect(wu.displayLabel).toMatch(/\(1\/2\)/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('chunkSigmaStagedDir — diffSet re-sync', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-chunk-diff-'));
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
const fs = await import('node:fs/promises');
|
||||
const manifestBody = await fs.readFile(join(SINGLE, 'sigma-manifest.json'), 'utf-8');
|
||||
await writeFile(join(stagedDir, 'sigma-manifest.json'), manifestBody);
|
||||
for (const file of ['dm-aaa111.json', 'dm-bbb222.json']) {
|
||||
const body = await fs.readFile(join(SINGLE, 'data-models', file), 'utf-8');
|
||||
await writeFile(join(stagedDir, 'data-models', file), body);
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('only the WU containing the modified file is kept', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir, {
|
||||
diffSet: {
|
||||
added: [],
|
||||
modified: ['data-models/dm-aaa111.json'],
|
||||
deleted: [],
|
||||
unchanged: ['data-models/dm-bbb222.json', 'sigma-manifest.json'],
|
||||
},
|
||||
});
|
||||
expect(result.workUnits).toHaveLength(1);
|
||||
expect(result.workUnits[0]!.rawFiles).toEqual(['data-models/dm-aaa111.json']);
|
||||
});
|
||||
|
||||
it('unchanged sibling data-model moves to dependencyPaths', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir, {
|
||||
diffSet: {
|
||||
added: [],
|
||||
modified: ['data-models/dm-aaa111.json'],
|
||||
deleted: [],
|
||||
unchanged: ['data-models/dm-bbb222.json', 'sigma-manifest.json'],
|
||||
},
|
||||
});
|
||||
expect(result.workUnits[0]!.dependencyPaths).toContain('data-models/dm-bbb222.json');
|
||||
});
|
||||
|
||||
it('all-unchanged diffSet produces zero WUs and no eviction', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir, {
|
||||
diffSet: {
|
||||
added: [],
|
||||
modified: [],
|
||||
deleted: [],
|
||||
unchanged: ['data-models/dm-aaa111.json', 'data-models/dm-bbb222.json', 'sigma-manifest.json'],
|
||||
},
|
||||
});
|
||||
expect(result.workUnits).toHaveLength(0);
|
||||
expect(result.eviction).toBeUndefined();
|
||||
});
|
||||
|
||||
it('deleted paths produce an eviction unit listing those paths', async () => {
|
||||
const result = await chunkSigmaStagedDir(stagedDir, {
|
||||
diffSet: {
|
||||
added: [],
|
||||
modified: [],
|
||||
deleted: ['data-models/dm-aaa111.json'],
|
||||
unchanged: ['data-models/dm-bbb222.json', 'sigma-manifest.json'],
|
||||
},
|
||||
});
|
||||
expect(result.eviction?.deletedRawPaths).toContain('data-models/dm-aaa111.json');
|
||||
});
|
||||
});
|
||||
309
packages/cli/test/context/ingest/adapters/sigma/client.test.ts
Normal file
309
packages/cli/test/context/ingest/adapters/sigma/client.test.ts
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { DefaultSigmaClient } from '../../../../../src/context/ingest/adapters/sigma/client.js';
|
||||
|
||||
const BASE = 'https://api.sigmacomputing.com';
|
||||
|
||||
const TOKEN_RESPONSE = {
|
||||
access_token: 'test-token',
|
||||
token_type: 'Bearer',
|
||||
expires_in: 3600,
|
||||
};
|
||||
|
||||
function makeResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
function makeClient(): DefaultSigmaClient {
|
||||
return new DefaultSigmaClient(
|
||||
{ apiUrl: BASE, clientId: 'cid', clientSecret: 'csec' }, // pragma: allowlist secret
|
||||
{ maxRetries: 1, baseDelayMs: 0, maxDelayMs: 0, timeoutMs: 5000 },
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
globalThis.fetch = vi.fn<typeof fetch>();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('DefaultSigmaClient.testConnection', () => {
|
||||
it('returns success:true when auth succeeds', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE));
|
||||
const client = makeClient();
|
||||
const result = await client.testConnection();
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('returns success:false with error message when auth fails', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(makeResponse({ error: 'unauthorized' }, 401));
|
||||
const client = makeClient();
|
||||
const result = await client.testConnection();
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toMatch(/401/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DefaultSigmaClient.listDataModels', () => {
|
||||
it('returns entries from a single page', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // auth
|
||||
.mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
entries: [
|
||||
{
|
||||
dataModelId: 'dm-1',
|
||||
dataModelUrlId: 'url-1',
|
||||
name: 'Revenue Model',
|
||||
path: 'Finance/Revenue',
|
||||
latestVersion: 1,
|
||||
ownerId: 'user-1',
|
||||
createdAt: '2026-01-01T00:00:00Z',
|
||||
updatedAt: '2026-01-01T00:00:00Z',
|
||||
isArchived: false,
|
||||
},
|
||||
],
|
||||
nextPage: null,
|
||||
}),
|
||||
);
|
||||
const client = makeClient();
|
||||
const models = await client.listDataModels();
|
||||
expect(models).toHaveLength(1);
|
||||
expect(models[0]!.name).toBe('Revenue Model');
|
||||
});
|
||||
|
||||
it('paginates across multiple pages', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
entries: [
|
||||
{
|
||||
dataModelId: 'dm-1',
|
||||
dataModelUrlId: 'url-1',
|
||||
name: 'Model A',
|
||||
path: 'Finance/A',
|
||||
latestVersion: 1,
|
||||
ownerId: 'u1',
|
||||
createdAt: '2026-01-01T00:00:00Z',
|
||||
updatedAt: '2026-01-01T00:00:00Z',
|
||||
},
|
||||
],
|
||||
nextPage: 'cursor-abc',
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
entries: [
|
||||
{
|
||||
dataModelId: 'dm-2',
|
||||
dataModelUrlId: 'url-2',
|
||||
name: 'Model B',
|
||||
path: 'Finance/B',
|
||||
latestVersion: 1,
|
||||
ownerId: 'u1',
|
||||
createdAt: '2026-01-02T00:00:00Z',
|
||||
updatedAt: '2026-01-02T00:00:00Z',
|
||||
},
|
||||
],
|
||||
nextPage: null,
|
||||
}),
|
||||
);
|
||||
const client = makeClient();
|
||||
const models = await client.listDataModels();
|
||||
expect(models).toHaveLength(2);
|
||||
expect(models.map((m) => m.name)).toEqual(['Model A', 'Model B']);
|
||||
});
|
||||
|
||||
it('second page request includes cursor in query string', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [{ dataModelId: 'dm-1', dataModelUrlId: 'url-1', name: 'A', path: 'F/A', latestVersion: 1, ownerId: 'u', createdAt: '', updatedAt: '' }], nextPage: 'cursor-xyz' }))
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null }));
|
||||
const client = makeClient();
|
||||
await client.listDataModels();
|
||||
const calls = vi.mocked(fetch).mock.calls;
|
||||
const pageCall = calls[calls.length - 1]!;
|
||||
expect(String(pageCall[0])).toContain('cursor-xyz');
|
||||
});
|
||||
});
|
||||
|
||||
function makeWorkbook(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
workbookId: 'wb-1',
|
||||
workbookUrlId: 'Sales-Dashboard-wb1',
|
||||
name: 'Sales Dashboard',
|
||||
url: 'https://app.sigmacomputing.com/workbooks/wb-1',
|
||||
path: 'Finance',
|
||||
latestVersion: 3,
|
||||
ownerId: 'user-1',
|
||||
createdAt: '2026-01-01T00:00:00Z',
|
||||
updatedAt: '2026-01-15T00:00:00Z',
|
||||
createdBy: 'user-1',
|
||||
updatedBy: 'user-1',
|
||||
isArchived: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('DefaultSigmaClient.listWorkbooks', () => {
|
||||
it('returns entries from a single page', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [makeWorkbook()], nextPage: null }));
|
||||
const client = makeClient();
|
||||
const workbooks = await client.listWorkbooks();
|
||||
expect(workbooks).toHaveLength(1);
|
||||
expect(workbooks[0]!.name).toBe('Sales Dashboard');
|
||||
});
|
||||
|
||||
it('passes excludeExplorations=true by default', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null }));
|
||||
const client = makeClient();
|
||||
await client.listWorkbooks();
|
||||
const url = String(vi.mocked(fetch).mock.calls[1]![0]);
|
||||
expect(url).toContain('excludeExplorations=true');
|
||||
});
|
||||
|
||||
it('omits excludeExplorations when includeExplorations=true', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null }));
|
||||
const client = makeClient();
|
||||
await client.listWorkbooks({ includeExplorations: true });
|
||||
const url = String(vi.mocked(fetch).mock.calls[1]![0]);
|
||||
expect(url).not.toContain('excludeExplorations');
|
||||
});
|
||||
|
||||
it('filters out archived workbooks by default', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
entries: [makeWorkbook({ isArchived: false }), makeWorkbook({ workbookId: 'wb-2', name: 'Old', isArchived: true })],
|
||||
nextPage: null,
|
||||
}),
|
||||
);
|
||||
const client = makeClient();
|
||||
const workbooks = await client.listWorkbooks();
|
||||
expect(workbooks).toHaveLength(1);
|
||||
expect(workbooks[0]!.name).toBe('Sales Dashboard');
|
||||
});
|
||||
|
||||
it('includes archived workbooks when includeArchived=true', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
entries: [makeWorkbook({ isArchived: false }), makeWorkbook({ workbookId: 'wb-2', name: 'Old', isArchived: true })],
|
||||
nextPage: null,
|
||||
}),
|
||||
);
|
||||
const client = makeClient();
|
||||
const workbooks = await client.listWorkbooks({ includeArchived: true });
|
||||
expect(workbooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('filters workbooks by updatedSince', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
entries: [
|
||||
makeWorkbook({ workbookId: 'wb-1', updatedAt: '2026-01-10T00:00:00Z' }),
|
||||
makeWorkbook({ workbookId: 'wb-2', updatedAt: '2026-01-20T00:00:00Z' }),
|
||||
],
|
||||
nextPage: null,
|
||||
}),
|
||||
);
|
||||
const client = makeClient();
|
||||
const workbooks = await client.listWorkbooks({ updatedSince: '2026-01-15T00:00:00Z' });
|
||||
expect(workbooks).toHaveLength(1);
|
||||
expect(workbooks[0]!.workbookId).toBe('wb-2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('DefaultSigmaClient.getDataModelSpec', () => {
|
||||
it('calls the correct URL with encoded id', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse({ schemaVersion: 1 }));
|
||||
const client = makeClient();
|
||||
const spec = await client.getDataModelSpec('dm/123');
|
||||
expect(spec).toEqual({ schemaVersion: 1 });
|
||||
const calls = vi.mocked(fetch).mock.calls;
|
||||
expect(String(calls[1]![0])).toContain('/v2/dataModels/dm%2F123/spec');
|
||||
});
|
||||
});
|
||||
|
||||
describe('DefaultSigmaClient — error handling', () => {
|
||||
it('retries on 500 and succeeds on retry', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // auth
|
||||
.mockResolvedValueOnce(makeResponse({ error: 'server error' }, 500)) // first attempt
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })); // retry
|
||||
const client = makeClient();
|
||||
const models = await client.listDataModels();
|
||||
expect(models).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('throws after exhausting retries on 500', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValue(makeResponse({ error: 'server error' }, 500));
|
||||
const client = makeClient();
|
||||
await expect(client.listDataModels()).rejects.toThrow(/500/);
|
||||
});
|
||||
|
||||
it('throws immediately on service_error 500 without retrying', async () => {
|
||||
const serviceError = { requestId: 'abc', message: 'dataSource subtype not supported in data model read', code: 'service_error' };
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse(serviceError, 500));
|
||||
const client = makeClient();
|
||||
await expect(client.getDataModelSpec('dm-1')).rejects.toThrow(/service_error/);
|
||||
// Only 2 calls: auth + one request. No retries.
|
||||
expect(vi.mocked(fetch)).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('throws immediately on 404 (non-retryable)', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
||||
.mockResolvedValueOnce(makeResponse({ error: 'not found' }, 404));
|
||||
const client = makeClient();
|
||||
await expect(client.getDataModelSpec('dm-999')).rejects.toThrow(/404/);
|
||||
});
|
||||
|
||||
it('re-authenticates and retries on 401', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // initial auth
|
||||
.mockResolvedValueOnce(makeResponse({ error: 'expired' }, 401)) // 401 on first request
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // re-auth
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })); // retried request
|
||||
const client = makeClient();
|
||||
const models = await client.listDataModels();
|
||||
expect(models).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DefaultSigmaClient.cleanup', () => {
|
||||
it('clears cached token so next call re-authenticates', async () => {
|
||||
vi.mocked(fetch)
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // first auth
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })) // first list
|
||||
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // second auth after cleanup
|
||||
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })); // second list
|
||||
const client = makeClient();
|
||||
await client.listDataModels();
|
||||
await client.cleanup();
|
||||
await client.listDataModels();
|
||||
// 4 calls total: 2 auths + 2 lists
|
||||
expect(vi.mocked(fetch)).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { detectSigmaStagedDir } from '../../../../../src/context/ingest/adapters/sigma/detect.js';
|
||||
|
||||
async function touch(dir: string, relPath: string, body = '{}'): Promise<void> {
|
||||
const abs = join(dir, relPath);
|
||||
await mkdir(join(abs, '..'), { recursive: true });
|
||||
await writeFile(abs, body, 'utf-8');
|
||||
}
|
||||
|
||||
describe('detectSigmaStagedDir', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-detect-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns true when manifest and at least one data-model file are present', async () => {
|
||||
await touch(stagedDir, 'sigma-manifest.json');
|
||||
await touch(stagedDir, 'data-models/dm-aaa111.json');
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when manifest and at least one workbook file are present', async () => {
|
||||
await touch(stagedDir, 'sigma-manifest.json');
|
||||
await touch(stagedDir, 'workbooks/wb-xxx111.json');
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when sigma-manifest.json is absent', async () => {
|
||||
await touch(stagedDir, 'data-models/dm-aaa111.json');
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for a completely empty directory', async () => {
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when manifest is present but both entity dirs are empty', async () => {
|
||||
await touch(stagedDir, 'sigma-manifest.json');
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
await mkdir(join(stagedDir, 'workbooks'), { recursive: true });
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when manifest is present but entity dirs are absent', async () => {
|
||||
await touch(stagedDir, 'sigma-manifest.json');
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when only unrelated files are present', async () => {
|
||||
await touch(stagedDir, 'data-models/dm-aaa111.json');
|
||||
expect(await detectSigmaStagedDir(stagedDir)).toBe(false);
|
||||
});
|
||||
});
|
||||
493
packages/cli/test/context/ingest/adapters/sigma/fetch.test.ts
Normal file
493
packages/cli/test/context/ingest/adapters/sigma/fetch.test.ts
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
import { mkdtemp, readFile, rm, writeFile, mkdir } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { SigmaClientFactory, SigmaRuntimeClient } from '../../../../../src/context/ingest/adapters/sigma/client-port.js';
|
||||
import { fetchSigmaBundle } from '../../../../../src/context/ingest/adapters/sigma/fetch.js';
|
||||
import type { SigmaPullConfig } from '../../../../../src/context/ingest/adapters/sigma/types.js';
|
||||
|
||||
const TEST_PULL_CONFIG = { sigmaConnectionId: 'sigma-prod' };
|
||||
|
||||
function makeSummary(id: string, name: string, path: string, isArchived = false) {
|
||||
return {
|
||||
dataModelId: id,
|
||||
dataModelUrlId: `${name.replace(/\s+/g, '-')}-${id}`,
|
||||
name,
|
||||
path,
|
||||
latestVersion: 1,
|
||||
ownerId: 'user-1',
|
||||
createdAt: '2026-01-01T00:00:00Z',
|
||||
updatedAt: '2026-01-15T00:00:00Z',
|
||||
isArchived,
|
||||
};
|
||||
}
|
||||
|
||||
function makeFactory(client: Partial<SigmaRuntimeClient>): SigmaClientFactory {
|
||||
const fullClient: SigmaRuntimeClient = {
|
||||
testConnection: vi.fn().mockResolvedValue({ success: true }),
|
||||
listDataModels: vi.fn().mockResolvedValue([]),
|
||||
listWorkbooks: vi.fn().mockResolvedValue([]),
|
||||
getDataModelSpec: vi.fn().mockResolvedValue(null),
|
||||
cleanup: vi.fn().mockResolvedValue(undefined),
|
||||
...client,
|
||||
};
|
||||
return {
|
||||
createClient: vi.fn().mockResolvedValue(fullClient),
|
||||
};
|
||||
}
|
||||
|
||||
describe('fetchSigmaBundle', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-fetch-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('creates sigma-manifest.json after a successful fetch', async () => {
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
makeSummary('dm-1', 'Revenue Model', 'Finance/Revenue'),
|
||||
]),
|
||||
getDataModelSpec: vi.fn().mockResolvedValue({ schemaVersion: 1 }),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.sigmaConnectionId).toBe('sigma-prod');
|
||||
expect(manifest.dataModelCount).toBe(1);
|
||||
expect(manifest.fetchedAt).toBeDefined();
|
||||
});
|
||||
|
||||
it('writes one data-model file per active model', async () => {
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
makeSummary('dm-1', 'Revenue Model', 'Finance/Revenue'),
|
||||
makeSummary('dm-2', 'ARR Model', 'Finance/ARR'),
|
||||
]),
|
||||
getDataModelSpec: vi.fn().mockResolvedValue({ schemaVersion: 1 }),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const dm1 = JSON.parse(await readFile(join(stagedDir, 'data-models', 'dm-1.json'), 'utf-8'));
|
||||
const dm2 = JSON.parse(await readFile(join(stagedDir, 'data-models', 'dm-2.json'), 'utf-8'));
|
||||
expect(dm1.name).toBe('Revenue Model');
|
||||
expect(dm2.name).toBe('ARR Model');
|
||||
});
|
||||
|
||||
it('skips archived models and does not write their files', async () => {
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
makeSummary('dm-1', 'Active Model', 'Finance/Active', false),
|
||||
makeSummary('dm-archived', 'Archived Model', 'Finance/Old', true),
|
||||
]),
|
||||
getDataModelSpec: vi.fn().mockResolvedValue({ schemaVersion: 1 }),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.dataModelCount).toBe(1);
|
||||
await expect(readFile(join(stagedDir, 'data-models', 'dm-archived.json'), 'utf-8')).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('logs a specific message for unsupported data source subtype (service_error)', async () => {
|
||||
const warnMessages: string[] = [];
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
makeSummary('dm-1', 'CSV Upload Model', 'Finance/CSV'),
|
||||
]),
|
||||
getDataModelSpec: vi.fn().mockRejectedValue(
|
||||
new Error('Sigma API error (500): {"code":"service_error","message":"dataSource subtype not supported in data model read"}'),
|
||||
),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
logger: { log: () => undefined, warn: (m) => warnMessages.push(m) },
|
||||
});
|
||||
expect(warnMessages[0]).toContain('data source type not supported');
|
||||
expect(warnMessages[0]).not.toContain('Sigma API error (500)');
|
||||
});
|
||||
|
||||
it('writes null spec when getDataModelSpec throws, and does not abort the whole fetch', async () => {
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
makeSummary('dm-1', 'Good Model', 'Finance/Good'),
|
||||
makeSummary('dm-2', 'Broken Model', 'Finance/Broken'),
|
||||
]),
|
||||
getDataModelSpec: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ schemaVersion: 1 })
|
||||
.mockRejectedValueOnce(new Error('Spec fetch failed')),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const dm2 = JSON.parse(await readFile(join(stagedDir, 'data-models', 'dm-2.json'), 'utf-8'));
|
||||
expect(dm2.spec).toBeNull();
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.dataModelCount).toBe(2);
|
||||
});
|
||||
|
||||
it('calls cleanup on the client even when an error is thrown', async () => {
|
||||
const cleanupMock = vi.fn().mockResolvedValue(undefined);
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockRejectedValue(new Error('Network failure')),
|
||||
cleanup: cleanupMock,
|
||||
});
|
||||
await expect(
|
||||
fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
}),
|
||||
).rejects.toThrow('Network failure');
|
||||
expect(cleanupMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('passes the resolved config to clientFactory.createClient', async () => {
|
||||
const createClientMock = vi.fn().mockResolvedValue({
|
||||
testConnection: vi.fn(),
|
||||
listDataModels: vi.fn().mockResolvedValue([]),
|
||||
listWorkbooks: vi.fn().mockResolvedValue([]),
|
||||
getDataModelSpec: vi.fn(),
|
||||
cleanup: vi.fn().mockResolvedValue(undefined),
|
||||
} satisfies SigmaRuntimeClient);
|
||||
const factory: SigmaClientFactory = { createClient: createClientMock };
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const calledConfig = createClientMock.mock.calls[0]![0] as SigmaPullConfig;
|
||||
expect(calledConfig.sigmaConnectionId).toBe('sigma-prod');
|
||||
});
|
||||
|
||||
it('writes sigma-projection-config.json with connectionMappings from pullConfig', async () => {
|
||||
const factory = makeFactory({});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: { sigmaConnectionId: 'sigma-prod', connectionMappings: { 'uuid-1': 'snowflake-prod' } },
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const config = JSON.parse(await readFile(join(stagedDir, 'sigma-projection-config.json'), 'utf-8'));
|
||||
expect(config.connectionMappings['uuid-1']).toBe('snowflake-prod');
|
||||
});
|
||||
|
||||
it('writes sigma-projection-config.json with empty mappings when none are provided', async () => {
|
||||
const factory = makeFactory({});
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
const config = JSON.parse(await readFile(join(stagedDir, 'sigma-projection-config.json'), 'utf-8'));
|
||||
expect(config.connectionMappings).toEqual({});
|
||||
});
|
||||
|
||||
it('writes workbookFilter defaults to projection config when not specified', async () => {
|
||||
const factory = makeFactory({});
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
const config = JSON.parse(await readFile(join(stagedDir, 'sigma-projection-config.json'), 'utf-8'));
|
||||
expect(config.workbookFilter.includeArchived).toBe(false);
|
||||
expect(config.workbookFilter.includeExplorations).toBe(false);
|
||||
expect(config.workbookFilter.updatedSince).toBeUndefined();
|
||||
});
|
||||
|
||||
it('writes explicit workbookFilter settings to projection config', async () => {
|
||||
const factory = makeFactory({});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: {
|
||||
sigmaConnectionId: 'sigma-prod',
|
||||
workbookFilter: { includeArchived: true, includeExplorations: false, updatedSince: '2026-01-01T00:00:00Z' },
|
||||
},
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const config = JSON.parse(await readFile(join(stagedDir, 'sigma-projection-config.json'), 'utf-8'));
|
||||
expect(config.workbookFilter.includeArchived).toBe(true);
|
||||
expect(config.workbookFilter.updatedSince).toBe('2026-01-01T00:00:00Z');
|
||||
});
|
||||
|
||||
it('throws on invalid pullConfig', async () => {
|
||||
const factory = makeFactory({});
|
||||
await expect(
|
||||
fetchSigmaBundle({
|
||||
pullConfig: { sigmaConnectionId: 'invalid id with spaces' },
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('handles zero active models gracefully', async () => {
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([]),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.dataModelCount).toBe(0);
|
||||
});
|
||||
|
||||
it('skips spec fetch for a model whose updatedAt matches the existing staged file', async () => {
|
||||
const summary = makeSummary('dm-1', 'Revenue Model', 'Finance/Revenue');
|
||||
// Pre-populate a staged file with the same updatedAt.
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
const existingStaged = {
|
||||
sigmaId: 'dm-1',
|
||||
name: 'Revenue Model',
|
||||
path: 'Finance/Revenue',
|
||||
latestVersion: 1,
|
||||
updatedAt: summary.updatedAt,
|
||||
isArchived: false,
|
||||
spec: { schemaVersion: 1, name: 'old' },
|
||||
};
|
||||
await writeFile(
|
||||
join(stagedDir, 'data-models', 'dm-1.json'),
|
||||
JSON.stringify(existingStaged),
|
||||
'utf-8',
|
||||
);
|
||||
const getSpecMock = vi.fn().mockResolvedValue({ schemaVersion: 1 });
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([summary]),
|
||||
getDataModelSpec: getSpecMock,
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
// Spec fetch must be skipped for the unchanged model.
|
||||
expect(getSpecMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retries spec fetch for a model whose updatedAt matches but staged spec is null (transient failure)', async () => {
|
||||
const summary = makeSummary('dm-1', 'Revenue Model', 'Finance/Revenue');
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
const existingStaged = {
|
||||
sigmaId: 'dm-1',
|
||||
name: 'Revenue Model',
|
||||
path: 'Finance/Revenue',
|
||||
latestVersion: 1,
|
||||
updatedAt: summary.updatedAt,
|
||||
isArchived: false,
|
||||
spec: null,
|
||||
};
|
||||
await writeFile(
|
||||
join(stagedDir, 'data-models', 'dm-1.json'),
|
||||
JSON.stringify(existingStaged),
|
||||
'utf-8',
|
||||
);
|
||||
const freshSpec = { schemaVersion: 1, name: 'Revenue Model' };
|
||||
const getSpecMock = vi.fn().mockResolvedValue(freshSpec);
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([summary]),
|
||||
getDataModelSpec: getSpecMock,
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
expect(getSpecMock).toHaveBeenCalledWith('dm-1');
|
||||
const written = JSON.parse(await readFile(join(stagedDir, 'data-models', 'dm-1.json'), 'utf-8'));
|
||||
expect(written.spec).toEqual(freshSpec);
|
||||
});
|
||||
|
||||
it('writes workbook count to manifest', async () => {
|
||||
const factory = makeFactory({
|
||||
listWorkbooks: vi.fn().mockResolvedValue([
|
||||
{ workbookId: 'wb-1', workbookUrlId: 'wb-url-1', name: 'Sales Dashboard', path: 'Finance/Dashboards', latestVersion: 1, ownerId: 'u1', createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-01-15T00:00:00Z', isArchived: false },
|
||||
{ workbookId: 'wb-2', workbookUrlId: 'wb-url-2', name: 'ARR Tracker', path: 'Finance/Dashboards', latestVersion: 2, ownerId: 'u1', createdAt: '2026-01-02T00:00:00Z', updatedAt: '2026-01-16T00:00:00Z', isArchived: false },
|
||||
]),
|
||||
});
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.workbookCount).toBe(2);
|
||||
});
|
||||
|
||||
it('writes one staged file per active workbook', async () => {
|
||||
const factory = makeFactory({
|
||||
listWorkbooks: vi.fn().mockResolvedValue([
|
||||
{ workbookId: 'wb-1', workbookUrlId: 'wb-url-1', name: 'Sales Dashboard', path: 'Finance/Dashboards', latestVersion: 1, ownerId: 'u1', createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-01-15T00:00:00Z', isArchived: false, description: 'Finance overview' },
|
||||
]),
|
||||
});
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
const wb = JSON.parse(await readFile(join(stagedDir, 'workbooks', 'wb-1.json'), 'utf-8'));
|
||||
expect(wb.name).toBe('Sales Dashboard');
|
||||
expect(wb.description).toBe('Finance overview');
|
||||
});
|
||||
|
||||
it('skips workbook re-staging when updatedAt is unchanged', async () => {
|
||||
await mkdir(join(stagedDir, 'workbooks'), { recursive: true });
|
||||
const existing = { sigmaId: 'wb-1', name: 'Sales Dashboard', path: 'Finance', latestVersion: 1, updatedAt: '2026-01-15T00:00:00Z', isArchived: false };
|
||||
await writeFile(join(stagedDir, 'workbooks', 'wb-1.json'), JSON.stringify(existing), 'utf-8');
|
||||
const listWorkbooksMock = vi.fn().mockResolvedValue([
|
||||
{ workbookId: 'wb-1', workbookUrlId: 'wb-url-1', name: 'Sales Dashboard', path: 'Finance', latestVersion: 1, ownerId: 'u1', createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-01-15T00:00:00Z', isArchived: false },
|
||||
]);
|
||||
const factory = makeFactory({ listWorkbooks: listWorkbooksMock });
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
// File should still contain the pre-existing content (not overwritten).
|
||||
const wb = JSON.parse(await readFile(join(stagedDir, 'workbooks', 'wb-1.json'), 'utf-8'));
|
||||
expect(wb.sigmaId).toBe('wb-1');
|
||||
});
|
||||
|
||||
it('removes the staged file when a model is no longer in the active list', async () => {
|
||||
// Pre-populate a staged file for dm-stale.
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
const staleStaged = {
|
||||
sigmaId: 'dm-stale',
|
||||
name: 'Stale Model',
|
||||
path: 'Old/Stale',
|
||||
latestVersion: 1,
|
||||
updatedAt: '2026-01-01T00:00:00Z',
|
||||
isArchived: false,
|
||||
spec: null,
|
||||
};
|
||||
await writeFile(
|
||||
join(stagedDir, 'data-models', 'dm-stale.json'),
|
||||
JSON.stringify(staleStaged),
|
||||
'utf-8',
|
||||
);
|
||||
// API now returns only dm-1 (dm-stale was archived or deleted).
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([makeSummary('dm-1', 'Active Model', 'Finance/Active')]),
|
||||
getDataModelSpec: vi.fn().mockResolvedValue({ schemaVersion: 1 }),
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: TEST_PULL_CONFIG,
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
await expect(
|
||||
readFile(join(stagedDir, 'data-models', 'dm-stale.json'), 'utf-8'),
|
||||
).rejects.toThrow();
|
||||
// The active model's file must still exist.
|
||||
await expect(
|
||||
readFile(join(stagedDir, 'data-models', 'dm-1.json'), 'utf-8'),
|
||||
).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it('filters spec fetches by dataModelFilter.updatedSince but preserves existing staged files for filtered-out models', async () => {
|
||||
// Pre-stage the old model from a previous full fetch.
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
const oldStaged = {
|
||||
sigmaId: 'dm-old', name: 'Old Model', path: 'Finance/Old',
|
||||
latestVersion: 1, updatedAt: '2026-06-20T00:00:00Z', isArchived: false, spec: { schemaVersion: 0 },
|
||||
};
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-old.json'), JSON.stringify(oldStaged), 'utf-8');
|
||||
const getSpecMock = vi.fn().mockResolvedValue({ schemaVersion: 1 });
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
{ ...makeSummary('dm-old', 'Old Model', 'Finance/Old'), updatedAt: '2026-06-20T00:00:00Z' },
|
||||
{ ...makeSummary('dm-new', 'New Model', 'Finance/New'), updatedAt: '2026-06-26T00:00:00Z' },
|
||||
]),
|
||||
getDataModelSpec: getSpecMock,
|
||||
});
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: { sigmaConnectionId: 'sigma-prod', dataModelFilter: { updatedSince: '2026-06-25T00:00:00Z' } },
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
// Only the new model's spec is fetched (old one is outside the filter window).
|
||||
expect(getSpecMock).toHaveBeenCalledTimes(1);
|
||||
// Manifest reflects only the filtered count.
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.dataModelCount).toBe(1);
|
||||
// New model is staged.
|
||||
await expect(readFile(join(stagedDir, 'data-models', 'dm-new.json'), 'utf-8')).resolves.toBeDefined();
|
||||
// Old model's staged file is PRESERVED — it is still active, just outside the filter window.
|
||||
await expect(readFile(join(stagedDir, 'data-models', 'dm-old.json'), 'utf-8')).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it('includes all active models when dataModelFilter is not set', async () => {
|
||||
const factory = makeFactory({
|
||||
listDataModels: vi.fn().mockResolvedValue([
|
||||
{ ...makeSummary('dm-old', 'Old Model', 'Finance/Old'), updatedAt: '2026-01-01T00:00:00Z' },
|
||||
{ ...makeSummary('dm-new', 'New Model', 'Finance/New'), updatedAt: '2026-06-26T00:00:00Z' },
|
||||
]),
|
||||
getDataModelSpec: vi.fn().mockResolvedValue({ schemaVersion: 1 }),
|
||||
});
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
const manifest = JSON.parse(await readFile(join(stagedDir, 'sigma-manifest.json'), 'utf-8'));
|
||||
expect(manifest.dataModelCount).toBe(2);
|
||||
});
|
||||
|
||||
it('removes the staged file when a workbook is no longer returned by the API', async () => {
|
||||
await mkdir(join(stagedDir, 'workbooks'), { recursive: true });
|
||||
const stale = {
|
||||
sigmaId: 'wb-stale',
|
||||
name: 'Old Dashboard',
|
||||
path: 'Finance/Old',
|
||||
latestVersion: 1,
|
||||
updatedAt: '2026-01-01T00:00:00Z',
|
||||
isArchived: false,
|
||||
};
|
||||
await writeFile(join(stagedDir, 'workbooks', 'wb-stale.json'), JSON.stringify(stale), 'utf-8');
|
||||
const factory = makeFactory({
|
||||
listWorkbooks: vi.fn().mockResolvedValue([
|
||||
{ workbookId: 'wb-active', workbookUrlId: 'wb-url-active', name: 'Active Dashboard', path: 'Finance/Active', latestVersion: 1, ownerId: 'u1', createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-01-16T00:00:00Z', isArchived: false },
|
||||
]),
|
||||
});
|
||||
await fetchSigmaBundle({ pullConfig: TEST_PULL_CONFIG, stagedDir, ctx: {} as never, clientFactory: factory });
|
||||
await expect(readFile(join(stagedDir, 'workbooks', 'wb-stale.json'), 'utf-8')).rejects.toThrow();
|
||||
await expect(readFile(join(stagedDir, 'workbooks', 'wb-active.json'), 'utf-8')).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it('workbookFilter.updatedSince filters fetch but preserves existing staged files for older workbooks', async () => {
|
||||
// Pre-stage an old workbook from a previous full fetch.
|
||||
await mkdir(join(stagedDir, 'workbooks'), { recursive: true });
|
||||
const oldStaged = {
|
||||
sigmaId: 'wb-old', name: 'Old Dashboard', path: 'Finance/Old',
|
||||
latestVersion: 1, updatedAt: '2026-06-20T00:00:00Z', isArchived: false, workbookUrlId: 'wb-url-old',
|
||||
};
|
||||
await writeFile(join(stagedDir, 'workbooks', 'wb-old.json'), JSON.stringify(oldStaged), 'utf-8');
|
||||
const listWorkbooksMock = vi.fn().mockResolvedValue([
|
||||
{ workbookId: 'wb-old', workbookUrlId: 'wb-url-old', name: 'Old Dashboard', path: 'Finance/Old', latestVersion: 1, ownerId: 'u1', createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-06-20T00:00:00Z', isArchived: false },
|
||||
{ workbookId: 'wb-new', workbookUrlId: 'wb-url-new', name: 'New Dashboard', path: 'Finance/New', latestVersion: 1, ownerId: 'u1', createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-06-26T00:00:00Z', isArchived: false },
|
||||
]);
|
||||
const factory = makeFactory({ listWorkbooks: listWorkbooksMock });
|
||||
await fetchSigmaBundle({
|
||||
pullConfig: { sigmaConnectionId: 'sigma-prod', workbookFilter: { updatedSince: '2026-06-25T00:00:00Z' } },
|
||||
stagedDir,
|
||||
ctx: {} as never,
|
||||
clientFactory: factory,
|
||||
});
|
||||
// Only the new workbook is staged on this run.
|
||||
await expect(readFile(join(stagedDir, 'workbooks', 'wb-new.json'), 'utf-8')).resolves.toBeDefined();
|
||||
// Old workbook's staged file is PRESERVED — it is still active, just outside the filter window.
|
||||
await expect(readFile(join(stagedDir, 'workbooks', 'wb-old.json'), 'utf-8')).resolves.toBeDefined();
|
||||
// listWorkbooks is called without updatedSince to get the full universe for eviction.
|
||||
expect(listWorkbooksMock).toHaveBeenCalledWith(expect.not.objectContaining({ updatedSince: expect.anything() }));
|
||||
});
|
||||
});
|
||||
301
packages/cli/test/context/ingest/adapters/sigma/project.test.ts
Normal file
301
packages/cli/test/context/ingest/adapters/sigma/project.test.ts
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { projectSigmaDataModels } from '../../../../../src/context/ingest/adapters/sigma/project.js';
|
||||
import type { DeterministicProjectionContext } from '../../../../../src/context/ingest/types.js';
|
||||
import type { SemanticLayerService } from '../../../../../src/context/sl/semantic-layer.service.js';
|
||||
import type { SemanticLayerSource } from '../../../../../src/context/sl/types.js';
|
||||
|
||||
function makeCtx(
|
||||
stagedDir: string,
|
||||
writeSource: (connectionId: string, source: SemanticLayerSource, ...rest: string[]) => Promise<{ warnings: string[] }>,
|
||||
): DeterministicProjectionContext {
|
||||
const svc = {
|
||||
writeSource,
|
||||
forWorktree: () => ({ writeSource }),
|
||||
} as unknown as SemanticLayerService;
|
||||
|
||||
return {
|
||||
connectionId: 'sigma-prod',
|
||||
sourceKey: 'sigma-prod',
|
||||
syncId: 'sync-1',
|
||||
jobId: 'job-1',
|
||||
runId: 'run-1',
|
||||
stagedDir,
|
||||
workdir: '',
|
||||
semanticLayerService: svc,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSpec(elements: unknown[]) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
name: 'Test Model',
|
||||
pages: [{ id: 'p1', name: 'Main', elements }],
|
||||
};
|
||||
}
|
||||
|
||||
function makeStagedModel(id: string, name: string, spec: unknown) {
|
||||
return JSON.stringify({
|
||||
sigmaId: id,
|
||||
name,
|
||||
path: 'Finance/Models',
|
||||
latestVersion: 1,
|
||||
updatedAt: '2026-01-15T00:00:00Z',
|
||||
isArchived: false,
|
||||
spec,
|
||||
});
|
||||
}
|
||||
|
||||
/** Write a projection config that maps the given sigma connection IDs to 'warehouse-main'. */
|
||||
async function writeProjectionConfig(stagedDir: string, sigmaConnectionIds: string[]): Promise<void> {
|
||||
const mappings = Object.fromEntries(sigmaConnectionIds.map((id) => [id, 'warehouse-main']));
|
||||
await writeFile(
|
||||
join(stagedDir, 'sigma-projection-config.json'),
|
||||
JSON.stringify({ connectionMappings: mappings }),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
describe('projectSigmaDataModels', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-project-'));
|
||||
await mkdir(join(stagedDir, 'data-models'), { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns empty result when data-models directory is missing', async () => {
|
||||
const emptyDir = await mkdtemp(join(tmpdir(), 'sigma-project-empty-'));
|
||||
try {
|
||||
const writeSource = vi.fn().mockResolvedValue({ warnings: [] });
|
||||
const result = await projectSigmaDataModels(makeCtx(emptyDir, writeSource), makeCtx(emptyDir, writeSource).semanticLayerService as never);
|
||||
expect(result.touchedSources).toHaveLength(0);
|
||||
expect(writeSource).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await rm(emptyDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('converts a warehouse-table element to a semantic-layer source', async () => {
|
||||
await writeProjectionConfig(stagedDir, ['sigma-conn-uuid']);
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'elem1',
|
||||
kind: 'table',
|
||||
name: 'Opportunities',
|
||||
source: { kind: 'warehouse-table', connectionId: 'sigma-conn-uuid', path: ['FIVETRAN', 'SALESFORCE', 'OPPORTUNITIES'] },
|
||||
columns: [
|
||||
{ id: 'c1', formula: '[OPPORTUNITIES/Amount]', name: 'Deal Amount' },
|
||||
{ id: 'c2', formula: 'Sum([OPPORTUNITIES/Amount])', name: 'Total Amount' },
|
||||
],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Revenue Model', spec));
|
||||
|
||||
const written: Array<{ connectionId: string; source: SemanticLayerSource }> = [];
|
||||
const writeSource = vi.fn().mockImplementation((connectionId: string, source: SemanticLayerSource) => {
|
||||
written.push({ connectionId, source });
|
||||
return Promise.resolve({ warnings: [] });
|
||||
});
|
||||
|
||||
const result = await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
|
||||
expect(writeSource).toHaveBeenCalledOnce();
|
||||
expect(written[0]!.connectionId).toBe('warehouse-main');
|
||||
const source = written[0]!.source;
|
||||
expect(source.table).toBe('FIVETRAN.SALESFORCE.OPPORTUNITIES');
|
||||
expect(source.columns.some((c) => c.name === 'deal_amount')).toBe(true);
|
||||
expect(source.columns.some((c) => c.name === 'total_amount')).toBe(false);
|
||||
expect(source.measures).toEqual([]);
|
||||
expect(result.touchedSources).toHaveLength(1);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('skips elements whose source kind is not warehouse-table', async () => {
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'elem1',
|
||||
kind: 'table',
|
||||
name: 'Derived',
|
||||
source: { kind: 'data-model', dataModelId: 'dm-other', elementId: 'e1' },
|
||||
columns: [{ id: 'c1', formula: '[Derived/Revenue]' }],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Derived Model', spec));
|
||||
|
||||
const writeSource = vi.fn().mockResolvedValue({ warnings: [] });
|
||||
const result = await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
|
||||
expect(writeSource).not.toHaveBeenCalled();
|
||||
expect(result.touchedSources).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('skips hidden elements', async () => {
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'elem1',
|
||||
kind: 'table',
|
||||
name: 'Hidden',
|
||||
hidden: true,
|
||||
source: { kind: 'warehouse-table', connectionId: 'c', path: ['DB', 'SCHEMA', 'TABLE'] },
|
||||
columns: [{ id: 'c1', formula: '[TABLE/Col]' }],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Hidden Model', spec));
|
||||
|
||||
const writeSource = vi.fn().mockResolvedValue({ warnings: [] });
|
||||
const result = await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
expect(writeSource).not.toHaveBeenCalled();
|
||||
expect(result.touchedSources).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('skips hidden columns', async () => {
|
||||
await writeProjectionConfig(stagedDir, ['c']);
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'elem1',
|
||||
kind: 'table',
|
||||
name: 'Revenue',
|
||||
source: { kind: 'warehouse-table', connectionId: 'c', path: ['DB', 'S', 'T'] },
|
||||
columns: [
|
||||
{ id: 'c1', formula: '[T/Visible]', name: 'Visible' },
|
||||
{ id: 'c2', formula: '[T/Hidden]', name: 'Hidden', hidden: true },
|
||||
],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Revenue', spec));
|
||||
|
||||
const written: SemanticLayerSource[] = [];
|
||||
const writeSource = vi.fn().mockImplementation((_: string, source: SemanticLayerSource) => {
|
||||
written.push(source);
|
||||
return Promise.resolve({ warnings: [] });
|
||||
});
|
||||
await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
const source = written[0]!;
|
||||
expect(source.columns.some((c) => c.name === 'visible')).toBe(true);
|
||||
expect(source.columns.some((c) => c.name === 'hidden')).toBe(false);
|
||||
});
|
||||
|
||||
it('silently skips aggregation formula columns and never emits measures', async () => {
|
||||
await writeProjectionConfig(stagedDir, ['c']);
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'e1',
|
||||
kind: 'table',
|
||||
name: 'Sales',
|
||||
source: { kind: 'warehouse-table', connectionId: 'c', path: ['DB', 'S', 'ORDERS'] },
|
||||
columns: [
|
||||
{ id: 'c1', formula: 'Sum([ORDERS/Revenue])', name: 'Total Revenue' },
|
||||
{ id: 'c2', formula: 'CountDistinct([ORDERS/CustomerId])', name: 'Unique Customers' },
|
||||
{ id: 'c3', formula: '[ORDERS/OrderDate]', name: 'Order Date' },
|
||||
],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Sales', spec));
|
||||
|
||||
const written: SemanticLayerSource[] = [];
|
||||
const writeSource = vi.fn().mockImplementation((_: string, source: SemanticLayerSource) => {
|
||||
written.push(source);
|
||||
return Promise.resolve({ warnings: [] });
|
||||
});
|
||||
await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
|
||||
const source = written[0]!;
|
||||
expect(source.measures).toEqual([]);
|
||||
expect(source.columns.map((c) => c.name)).toContain('order_date');
|
||||
expect(source.columns.map((c) => c.name)).not.toContain('total_revenue');
|
||||
expect(source.columns.map((c) => c.name)).not.toContain('unique_customers');
|
||||
});
|
||||
|
||||
it('skips models with null spec', async () => {
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'No Spec Model', null));
|
||||
|
||||
const writeSource = vi.fn().mockResolvedValue({ warnings: [] });
|
||||
const result = await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
expect(writeSource).not.toHaveBeenCalled();
|
||||
expect(result.touchedSources).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('routes to the mapped warehouse connection when connectionMappings is set', async () => {
|
||||
// Write a projection config that maps the Sigma internal connection UUID to a ktx warehouse.
|
||||
await writeFile(
|
||||
join(stagedDir, 'sigma-projection-config.json'),
|
||||
JSON.stringify({ connectionMappings: { 'sigma-internal-uuid': 'snowflake-prod' } }),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'e1',
|
||||
kind: 'table',
|
||||
name: 'Accounts',
|
||||
source: { kind: 'warehouse-table', connectionId: 'sigma-internal-uuid', path: ['PROD', 'SF', 'ACCOUNTS'] },
|
||||
columns: [{ id: 'c1', formula: '[ACCOUNTS/Name]', name: 'Account Name' }],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Accounts', spec));
|
||||
|
||||
const written: Array<{ connectionId: string }> = [];
|
||||
const writeSource = vi.fn().mockImplementation((connectionId: string) => {
|
||||
written.push({ connectionId });
|
||||
return Promise.resolve({ warnings: [] });
|
||||
});
|
||||
|
||||
await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
|
||||
expect(written[0]!.connectionId).toBe('snowflake-prod');
|
||||
});
|
||||
|
||||
it('skips SL source and emits a warning when no connectionMappings entry exists for the element', async () => {
|
||||
await writeFile(
|
||||
join(stagedDir, 'sigma-projection-config.json'),
|
||||
JSON.stringify({ connectionMappings: { 'other-uuid': 'snowflake-prod' } }),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'e1',
|
||||
kind: 'table',
|
||||
name: 'Orders',
|
||||
source: { kind: 'warehouse-table', connectionId: 'unmapped-uuid', path: ['DB', 'S', 'ORDERS'] },
|
||||
columns: [{ id: 'c1', formula: '[ORDERS/Id]', name: 'Order Id' }],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Orders', spec));
|
||||
|
||||
const writeSource = vi.fn().mockResolvedValue({ warnings: [] });
|
||||
const result = await projectSigmaDataModels(
|
||||
makeCtx(stagedDir, writeSource),
|
||||
makeCtx(stagedDir, writeSource).semanticLayerService as never,
|
||||
);
|
||||
|
||||
expect(writeSource).not.toHaveBeenCalled();
|
||||
expect(result.touchedSources).toHaveLength(0);
|
||||
expect(result.warnings.some((w) => w.includes('no connectionMappings entry'))).toBe(true);
|
||||
});
|
||||
|
||||
it('surfaces writeSource warnings in result', async () => {
|
||||
await writeProjectionConfig(stagedDir, ['c']);
|
||||
const spec = makeSpec([
|
||||
{
|
||||
id: 'e1',
|
||||
kind: 'table',
|
||||
name: 'Revenue',
|
||||
source: { kind: 'warehouse-table', connectionId: 'c', path: ['DB', 'S', 'T'] },
|
||||
columns: [{ id: 'c1', formula: '[T/Amount]', name: 'Amount' }],
|
||||
},
|
||||
]);
|
||||
await writeFile(join(stagedDir, 'data-models', 'dm-1.json'), makeStagedModel('dm-1', 'Revenue', spec));
|
||||
|
||||
const writeSource = vi.fn().mockResolvedValue({ warnings: ['schema: some warning'] });
|
||||
const result = await projectSigmaDataModels(makeCtx(stagedDir, writeSource), makeCtx(stagedDir, writeSource).semanticLayerService as never);
|
||||
expect(result.warnings).toContain('schema: some warning');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { SigmaSourceAdapter } from '../../../../../src/context/ingest/adapters/sigma/sigma.adapter.js';
|
||||
import type { SigmaClientFactory } from '../../../../../src/context/ingest/adapters/sigma/client-port.js';
|
||||
|
||||
function makeFactory(): SigmaClientFactory {
|
||||
return { createClient: vi.fn() };
|
||||
}
|
||||
|
||||
describe('SigmaSourceAdapter.listTargetConnectionIds', () => {
|
||||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stagedDir = await mkdtemp(join(tmpdir(), 'sigma-adapter-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(stagedDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeProjectionConfig(mappings: Record<string, string>) {
|
||||
await writeFile(
|
||||
join(stagedDir, 'sigma-projection-config.json'),
|
||||
JSON.stringify({ connectionMappings: mappings }),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
it('returns mapped warehouse connection IDs when mappings are present', async () => {
|
||||
await writeProjectionConfig({ 'uuid-a': 'snowflake-prod', 'uuid-b': 'snowflake-prod', 'uuid-c': 'bigquery-prod' });
|
||||
const adapter = new SigmaSourceAdapter({ clientFactory: makeFactory() });
|
||||
const ids = await adapter.listTargetConnectionIds(stagedDir);
|
||||
expect(ids).toEqual(['bigquery-prod', 'snowflake-prod']);
|
||||
});
|
||||
|
||||
it('returns empty array when connectionMappings is empty', async () => {
|
||||
await writeProjectionConfig({});
|
||||
const adapter = new SigmaSourceAdapter({ clientFactory: makeFactory() });
|
||||
const ids = await adapter.listTargetConnectionIds(stagedDir);
|
||||
expect(ids).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array when the projection config file is missing', async () => {
|
||||
const adapter = new SigmaSourceAdapter({ clientFactory: makeFactory() });
|
||||
const ids = await adapter.listTargetConnectionIds(stagedDir);
|
||||
expect(ids).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array when the projection config is malformed', async () => {
|
||||
await mkdir(stagedDir, { recursive: true });
|
||||
await writeFile(join(stagedDir, 'sigma-projection-config.json'), 'not json', 'utf-8');
|
||||
const adapter = new SigmaSourceAdapter({ clientFactory: makeFactory() });
|
||||
const ids = await adapter.listTargetConnectionIds(stagedDir);
|
||||
expect(ids).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array when both projection config and manifest are missing', async () => {
|
||||
const adapter = new SigmaSourceAdapter({ clientFactory: makeFactory() });
|
||||
const ids = await adapter.listTargetConnectionIds(stagedDir);
|
||||
expect(ids).toEqual([]);
|
||||
});
|
||||
});
|
||||
113
packages/cli/test/context/ingest/adapters/sigma/types.test.ts
Normal file
113
packages/cli/test/context/ingest/adapters/sigma/types.test.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
parseSigmaPullConfig,
|
||||
sigmaManifestSchema,
|
||||
stagedDataModelFileSchema,
|
||||
} from '../../../../../src/context/ingest/adapters/sigma/types.js';
|
||||
|
||||
describe('parseSigmaPullConfig', () => {
|
||||
it('accepts a simple alphanumeric connection ID', () => {
|
||||
const result = parseSigmaPullConfig({ sigmaConnectionId: 'sigma-prod' });
|
||||
expect(result.sigmaConnectionId).toBe('sigma-prod');
|
||||
});
|
||||
|
||||
it('accepts IDs with underscores', () => {
|
||||
const result = parseSigmaPullConfig({ sigmaConnectionId: 'sigma_prod_2' });
|
||||
expect(result.sigmaConnectionId).toBe('sigma_prod_2');
|
||||
});
|
||||
|
||||
it('rejects IDs starting with a special char', () => {
|
||||
expect(() => parseSigmaPullConfig({ sigmaConnectionId: '../prod' })).toThrow();
|
||||
});
|
||||
|
||||
it('rejects IDs with spaces', () => {
|
||||
expect(() => parseSigmaPullConfig({ sigmaConnectionId: 'sigma prod' })).toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing sigmaConnectionId', () => {
|
||||
expect(() => parseSigmaPullConfig({})).toThrow();
|
||||
});
|
||||
|
||||
it('rejects null', () => {
|
||||
expect(() => parseSigmaPullConfig(null)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('stagedDataModelFileSchema', () => {
|
||||
const minimal = {
|
||||
sigmaId: 'dm-aaa111',
|
||||
name: 'Revenue Model',
|
||||
path: 'My Documents/Finance/Revenue Model',
|
||||
latestVersion: 3,
|
||||
updatedAt: '2026-01-15T10:00:00Z',
|
||||
isArchived: false,
|
||||
spec: { schemaVersion: 1, pages: [] },
|
||||
};
|
||||
|
||||
it('parses a fully-populated file', () => {
|
||||
const result = stagedDataModelFileSchema.parse(minimal);
|
||||
expect(result.sigmaId).toBe('dm-aaa111');
|
||||
expect(result.name).toBe('Revenue Model');
|
||||
expect(result.isArchived).toBe(false);
|
||||
});
|
||||
|
||||
it('coerces absent isArchived to false', () => {
|
||||
const { isArchived: _, ...rest } = minimal;
|
||||
void _;
|
||||
const result = stagedDataModelFileSchema.parse(rest);
|
||||
expect(result.isArchived).toBe(false);
|
||||
});
|
||||
|
||||
it('accepts null spec', () => {
|
||||
const result = stagedDataModelFileSchema.parse({ ...minimal, spec: null });
|
||||
expect(result.spec).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects missing sigmaId', () => {
|
||||
const { sigmaId: _, ...rest } = minimal;
|
||||
void _;
|
||||
expect(() => stagedDataModelFileSchema.parse(rest)).toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing name', () => {
|
||||
const { name: _, ...rest } = minimal;
|
||||
void _;
|
||||
expect(() => stagedDataModelFileSchema.parse(rest)).toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing path', () => {
|
||||
const { path: _, ...rest } = minimal;
|
||||
void _;
|
||||
expect(() => stagedDataModelFileSchema.parse(rest)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('sigmaManifestSchema', () => {
|
||||
const valid = {
|
||||
sigmaConnectionId: 'sigma-prod',
|
||||
fetchedAt: '2026-01-15T10:00:00Z',
|
||||
dataModelCount: 2,
|
||||
};
|
||||
|
||||
it('parses a valid manifest', () => {
|
||||
const result = sigmaManifestSchema.parse(valid);
|
||||
expect(result.sigmaConnectionId).toBe('sigma-prod');
|
||||
expect(result.dataModelCount).toBe(2);
|
||||
});
|
||||
|
||||
it('rejects missing fetchedAt', () => {
|
||||
const { fetchedAt: _, ...rest } = valid;
|
||||
void _;
|
||||
expect(() => sigmaManifestSchema.parse(rest)).toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing dataModelCount', () => {
|
||||
const { dataModelCount: _, ...rest } = valid;
|
||||
void _;
|
||||
expect(() => sigmaManifestSchema.parse(rest)).toThrow();
|
||||
});
|
||||
|
||||
it('rejects a non-integer dataModelCount', () => {
|
||||
expect(() => sigmaManifestSchema.parse({ ...valid, dataModelCount: 2.5 })).toThrow();
|
||||
});
|
||||
});
|
||||
|
|
@ -73,6 +73,7 @@ describe('local ingest adapters', () => {
|
|||
'lookml',
|
||||
'dbt',
|
||||
'metabase',
|
||||
'sigma',
|
||||
'gdrive',
|
||||
'looker',
|
||||
'metricflow',
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const expectedAdapterSkillHeadings: Record<string, string> = {
|
|||
lookml_ingest: '# LookML to ktx Semantic Layer',
|
||||
metabase_ingest: '# Metabase to ktx Semantic Layer',
|
||||
metricflow_ingest: '# MetricFlow to ktx Semantic Layer',
|
||||
sigma_ingest: '# Sigma Ingest',
|
||||
};
|
||||
const verificationWriterSkills = [
|
||||
'gdrive_synthesize',
|
||||
|
|
@ -33,6 +34,7 @@ const verificationWriterSkills = [
|
|||
'looker_ingest',
|
||||
'metabase_ingest',
|
||||
'metricflow_ingest',
|
||||
'sigma_ingest',
|
||||
'live_database_ingest',
|
||||
'historic_sql_table_digest',
|
||||
'historic_sql_patterns',
|
||||
|
|
|
|||
5
packages/cli/test/fixtures/sigma/empty-manifest/sigma-manifest.json
vendored
Normal file
5
packages/cli/test/fixtures/sigma/empty-manifest/sigma-manifest.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"sigmaConnectionId": "sigma-prod",
|
||||
"fetchedAt": "2026-01-15T10:00:00Z",
|
||||
"dataModelCount": 0
|
||||
}
|
||||
9
packages/cli/test/fixtures/sigma/multi-folder/data-models/dm-aaa111.json
vendored
Normal file
9
packages/cli/test/fixtures/sigma/multi-folder/data-models/dm-aaa111.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"sigmaId": "dm-aaa111",
|
||||
"name": "Revenue Model",
|
||||
"path": "Finance/Revenue Model",
|
||||
"latestVersion": 3,
|
||||
"updatedAt": "2026-01-15T10:00:00Z",
|
||||
"isArchived": false,
|
||||
"spec": { "schemaVersion": 1, "pages": [] }
|
||||
}
|
||||
9
packages/cli/test/fixtures/sigma/multi-folder/data-models/dm-bbb222.json
vendored
Normal file
9
packages/cli/test/fixtures/sigma/multi-folder/data-models/dm-bbb222.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"sigmaId": "dm-bbb222",
|
||||
"name": "ARR Model",
|
||||
"path": "Finance/ARR Model",
|
||||
"latestVersion": 1,
|
||||
"updatedAt": "2026-01-10T08:00:00Z",
|
||||
"isArchived": false,
|
||||
"spec": { "schemaVersion": 1, "pages": [] }
|
||||
}
|
||||
9
packages/cli/test/fixtures/sigma/multi-folder/data-models/dm-ccc333.json
vendored
Normal file
9
packages/cli/test/fixtures/sigma/multi-folder/data-models/dm-ccc333.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"sigmaId": "dm-ccc333",
|
||||
"name": "Usage Model",
|
||||
"path": "Product/Usage Model",
|
||||
"latestVersion": 2,
|
||||
"updatedAt": "2026-01-12T09:00:00Z",
|
||||
"isArchived": false,
|
||||
"spec": { "schemaVersion": 1, "pages": [] }
|
||||
}
|
||||
6
packages/cli/test/fixtures/sigma/multi-folder/sigma-manifest.json
vendored
Normal file
6
packages/cli/test/fixtures/sigma/multi-folder/sigma-manifest.json
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"sigmaConnectionId": "sigma-prod",
|
||||
"fetchedAt": "2026-01-15T10:00:00Z",
|
||||
"dataModelCount": 3,
|
||||
"workbookCount": 2
|
||||
}
|
||||
9
packages/cli/test/fixtures/sigma/multi-folder/workbooks/wb-yyy222.json
vendored
Normal file
9
packages/cli/test/fixtures/sigma/multi-folder/workbooks/wb-yyy222.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"sigmaId": "wb-yyy222",
|
||||
"name": "Revenue Tracker",
|
||||
"path": "Finance/Revenue Tracker",
|
||||
"latestVersion": 2,
|
||||
"updatedAt": "2026-01-13T08:00:00Z",
|
||||
"isArchived": false,
|
||||
"workbookUrlId": "Revenue-Tracker-yyy222"
|
||||
}
|
||||
10
packages/cli/test/fixtures/sigma/multi-folder/workbooks/wb-zzz333.json
vendored
Normal file
10
packages/cli/test/fixtures/sigma/multi-folder/workbooks/wb-zzz333.json
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"sigmaId": "wb-zzz333",
|
||||
"name": "Usage Dashboard",
|
||||
"path": "Product/Usage Dashboard",
|
||||
"latestVersion": 8,
|
||||
"updatedAt": "2026-01-12T11:00:00Z",
|
||||
"isArchived": false,
|
||||
"workbookUrlId": "Usage-Dashboard-zzz333",
|
||||
"description": "Product usage metrics"
|
||||
}
|
||||
9
packages/cli/test/fixtures/sigma/single-folder/data-models/dm-aaa111.json
vendored
Normal file
9
packages/cli/test/fixtures/sigma/single-folder/data-models/dm-aaa111.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"sigmaId": "dm-aaa111",
|
||||
"name": "Revenue Model",
|
||||
"path": "Finance/Revenue Model",
|
||||
"latestVersion": 3,
|
||||
"updatedAt": "2026-01-15T10:00:00Z",
|
||||
"isArchived": false,
|
||||
"spec": { "schemaVersion": 1, "pages": [] }
|
||||
}
|
||||
9
packages/cli/test/fixtures/sigma/single-folder/data-models/dm-bbb222.json
vendored
Normal file
9
packages/cli/test/fixtures/sigma/single-folder/data-models/dm-bbb222.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"sigmaId": "dm-bbb222",
|
||||
"name": "ARR Model",
|
||||
"path": "Finance/ARR Model",
|
||||
"latestVersion": 1,
|
||||
"updatedAt": "2026-01-10T08:00:00Z",
|
||||
"isArchived": false,
|
||||
"spec": { "schemaVersion": 1, "pages": [] }
|
||||
}
|
||||
6
packages/cli/test/fixtures/sigma/single-folder/sigma-manifest.json
vendored
Normal file
6
packages/cli/test/fixtures/sigma/single-folder/sigma-manifest.json
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"sigmaConnectionId": "sigma-prod",
|
||||
"fetchedAt": "2026-01-15T10:00:00Z",
|
||||
"dataModelCount": 2,
|
||||
"workbookCount": 1
|
||||
}
|
||||
10
packages/cli/test/fixtures/sigma/single-folder/workbooks/wb-xxx111.json
vendored
Normal file
10
packages/cli/test/fixtures/sigma/single-folder/workbooks/wb-xxx111.json
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"sigmaId": "wb-xxx111",
|
||||
"name": "Finance Overview",
|
||||
"path": "Finance/Finance Overview",
|
||||
"latestVersion": 5,
|
||||
"updatedAt": "2026-01-14T09:00:00Z",
|
||||
"isArchived": false,
|
||||
"workbookUrlId": "Finance-Overview-xxx111",
|
||||
"description": "Top-level finance dashboard"
|
||||
}
|
||||
|
|
@ -1364,6 +1364,18 @@ describe('setup sources step', () => {
|
|||
deps: { validateNotion: vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })) },
|
||||
expectedLabel: 'Notion',
|
||||
},
|
||||
{
|
||||
source: 'sigma',
|
||||
connectionId: 'sigma-main',
|
||||
connection: {
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'my-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
},
|
||||
deps: { validateSigma: vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' })) },
|
||||
expectedLabel: 'Sigma Computing',
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
|
|
@ -2035,4 +2047,206 @@ describe('setup sources step', () => {
|
|||
path: 'staging',
|
||||
});
|
||||
});
|
||||
|
||||
it('writes Sigma config in non-interactive mode', async () => {
|
||||
await addPrimarySource();
|
||||
const validateSigma = vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' }));
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-prod',
|
||||
sourceUrl: 'https://api.sigmacomputing.com',
|
||||
sourceClientId: 'my-client-id',
|
||||
sourceClientSecretRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
makeIo().io,
|
||||
{ validateSigma },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-prod'] });
|
||||
|
||||
expect((await readConfig()).connections['sigma-prod']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'my-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
expect(validateSigma).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('defaults Sigma api_url when --source-url is omitted', async () => {
|
||||
await addPrimarySource();
|
||||
const validateSigma = vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' }));
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-main',
|
||||
sourceClientId: 'my-client-id',
|
||||
sourceClientSecretRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
makeIo().io,
|
||||
{ validateSigma },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-main'] });
|
||||
|
||||
expect((await readConfig()).connections['sigma-main']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects --source-auth-token-ref for Sigma and points at --source-client-secret-ref', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-main',
|
||||
sourceClientId: 'my-client-id',
|
||||
sourceAuthTokenRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(io.stderr()).toContain('--source-auth-token-ref does not apply to --source sigma; use --source-client-secret-ref.');
|
||||
});
|
||||
|
||||
it('runs interactive Sigma setup with API URL, client ID, and env credential', async () => {
|
||||
await addPrimarySource();
|
||||
const validateSigma = vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' }));
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['sigma']],
|
||||
select: ['env', 'done'],
|
||||
// connection name, API URL (default accepted), client ID
|
||||
text: ['sigma-main', 'https://api.sigmacomputing.com', 'my-client-id'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{ prompts: testPrompts, validateSigma },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: textInputPrompt(connectionNamePrompt('Sigma Computing')),
|
||||
initialValue: 'sigma-main',
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: textInputPrompt('Sigma API URL') }),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: textInputPrompt('Sigma client ID') }),
|
||||
);
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'How should ktx find your Sigma client secret?',
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use SIGMA_CLIENT_SECRET from the environment' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect((await readConfig()).connections['sigma-main']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'my-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
|
||||
it('edits an existing Sigma source with current URL and client ID as defaults', async () => {
|
||||
await addPrimarySource();
|
||||
await addConnection('sigma-main', {
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'old-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['sigma']],
|
||||
select: ['edit:sigma-main', 'keep', 'done'],
|
||||
// API URL and new client ID
|
||||
text: ['https://api.sigmacomputing.com', 'new-client-id'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
validateSigma: vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' })),
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: textInputPrompt('Sigma API URL'),
|
||||
initialValue: 'https://api.sigmacomputing.com',
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: textInputPrompt('Sigma client ID'),
|
||||
initialValue: 'old-client-id', // pre-filled from existing connection
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.select).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'How should ktx find your Sigma client secret?',
|
||||
options: expect.arrayContaining([{ value: 'keep', label: 'Keep existing credential' }]),
|
||||
}),
|
||||
);
|
||||
expect((await readConfig()).connections['sigma-main']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
client_id: 'new-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
|
||||
it('fails Sigma setup when validation rejects the credentials', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
const result = await runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-main',
|
||||
sourceClientId: 'bad-client-id',
|
||||
sourceClientSecretRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{ validateSigma: vi.fn(async () => ({ ok: false as const, message: 'Sigma auth failed (401): Unauthorized' })) },
|
||||
);
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(io.stderr()).toContain('Sigma auth failed (401): Unauthorized');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue