mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-04 10:52:13 +02:00
* feat(sigma): add Sigma Computing context-source adapter Closes #168 Adds a full ingest adapter for Sigma Computing so `ktx ingest` can pull data model specs and workbook summaries into the ktx context layer. The implementation follows the same fetch → chunk → project → LLM pattern used by the Looker, Metabase, and MetricFlow adapters. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(sigma): address PR review comments - Remove manifest from rawFiles; moves to peerFileIndex so fetchedAt changes don't mark all work units dirty every run - Fix workbookFilter.updatedSince eviction bug: fetch full universe first, apply filter client-side, evict only on archived/deleted - Remove measure projection entirely; project() writes measures: [] and the sigma_ingest skill surfaces Lookup/aggregation formulas as wiki prose - Remove joins projection (v1 limitation); project() writes joins: [] and Lookup relationships are described in wiki prose instead - Remove write-back dead code: createDataModel, updateDataModel, SigmaDataModelPushResult, mutate/post/put - Fix emitBatches notes pluralization bug ('2 data modelss' → '2 data models') - Add tokenInflight dedup on ensureToken to coalesce concurrent auth requests - Retry spec fetch when existing staged spec is null (transient failure cache) - Drop unused WorkbookFilter import from client-port.ts - Note in docs that joins are not projected from Sigma data models in this release Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * updates * fix(sigma): restore sigma in local adapter test + small cleanups The gdrive↔sigma merge dropped 'sigma' from the expected adapter source list in local-adapters.test.ts while keeping gdrive, so the slow TS suite failed even though the source registers both. Add 'sigma' back at its registration position (after metabase, before gdrive). Also: - Move the orphaned SigmaPullConfig docstring onto the schema it documents and drop the stale BullMQ reference (standalone ktx has no BullMQ; the config lives in the ingest job's bundleRef.config). - Drop an O(n^2) find() round-trip in fetch() when building the active data-model list; filter once and reuse for the eviction id set. --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Andrey Avtomonov <andreybavt@gmail.com> Co-authored-by: Luca Martial <48870843+luca-martial@users.noreply.github.com>
309 lines
11 KiB
TypeScript
309 lines
11 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { DefaultSigmaClient } from '../../../../../src/context/ingest/adapters/sigma/client.js';
|
|
|
|
const BASE = 'https://api.sigmacomputing.com';
|
|
|
|
const TOKEN_RESPONSE = {
|
|
access_token: 'test-token',
|
|
token_type: 'Bearer',
|
|
expires_in: 3600,
|
|
};
|
|
|
|
function makeResponse(body: unknown, status = 200): Response {
|
|
return new Response(JSON.stringify(body), {
|
|
status,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
});
|
|
}
|
|
|
|
function makeClient(): DefaultSigmaClient {
|
|
return new DefaultSigmaClient(
|
|
{ apiUrl: BASE, clientId: 'cid', clientSecret: 'csec' }, // pragma: allowlist secret
|
|
{ maxRetries: 1, baseDelayMs: 0, maxDelayMs: 0, timeoutMs: 5000 },
|
|
);
|
|
}
|
|
|
|
beforeEach(() => {
|
|
globalThis.fetch = vi.fn<typeof fetch>();
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
describe('DefaultSigmaClient.testConnection', () => {
|
|
it('returns success:true when auth succeeds', async () => {
|
|
vi.mocked(fetch).mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE));
|
|
const client = makeClient();
|
|
const result = await client.testConnection();
|
|
expect(result.success).toBe(true);
|
|
});
|
|
|
|
it('returns success:false with error message when auth fails', async () => {
|
|
vi.mocked(fetch).mockResolvedValueOnce(makeResponse({ error: 'unauthorized' }, 401));
|
|
const client = makeClient();
|
|
const result = await client.testConnection();
|
|
expect(result.success).toBe(false);
|
|
expect(result.error).toMatch(/401/);
|
|
});
|
|
});
|
|
|
|
describe('DefaultSigmaClient.listDataModels', () => {
|
|
it('returns entries from a single page', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // auth
|
|
.mockResolvedValueOnce(
|
|
makeResponse({
|
|
entries: [
|
|
{
|
|
dataModelId: 'dm-1',
|
|
dataModelUrlId: 'url-1',
|
|
name: 'Revenue Model',
|
|
path: 'Finance/Revenue',
|
|
latestVersion: 1,
|
|
ownerId: 'user-1',
|
|
createdAt: '2026-01-01T00:00:00Z',
|
|
updatedAt: '2026-01-01T00:00:00Z',
|
|
isArchived: false,
|
|
},
|
|
],
|
|
nextPage: null,
|
|
}),
|
|
);
|
|
const client = makeClient();
|
|
const models = await client.listDataModels();
|
|
expect(models).toHaveLength(1);
|
|
expect(models[0]!.name).toBe('Revenue Model');
|
|
});
|
|
|
|
it('paginates across multiple pages', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(
|
|
makeResponse({
|
|
entries: [
|
|
{
|
|
dataModelId: 'dm-1',
|
|
dataModelUrlId: 'url-1',
|
|
name: 'Model A',
|
|
path: 'Finance/A',
|
|
latestVersion: 1,
|
|
ownerId: 'u1',
|
|
createdAt: '2026-01-01T00:00:00Z',
|
|
updatedAt: '2026-01-01T00:00:00Z',
|
|
},
|
|
],
|
|
nextPage: 'cursor-abc',
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(
|
|
makeResponse({
|
|
entries: [
|
|
{
|
|
dataModelId: 'dm-2',
|
|
dataModelUrlId: 'url-2',
|
|
name: 'Model B',
|
|
path: 'Finance/B',
|
|
latestVersion: 1,
|
|
ownerId: 'u1',
|
|
createdAt: '2026-01-02T00:00:00Z',
|
|
updatedAt: '2026-01-02T00:00:00Z',
|
|
},
|
|
],
|
|
nextPage: null,
|
|
}),
|
|
);
|
|
const client = makeClient();
|
|
const models = await client.listDataModels();
|
|
expect(models).toHaveLength(2);
|
|
expect(models.map((m) => m.name)).toEqual(['Model A', 'Model B']);
|
|
});
|
|
|
|
it('second page request includes cursor in query string', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse({ entries: [{ dataModelId: 'dm-1', dataModelUrlId: 'url-1', name: 'A', path: 'F/A', latestVersion: 1, ownerId: 'u', createdAt: '', updatedAt: '' }], nextPage: 'cursor-xyz' }))
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null }));
|
|
const client = makeClient();
|
|
await client.listDataModels();
|
|
const calls = vi.mocked(fetch).mock.calls;
|
|
const pageCall = calls[calls.length - 1]!;
|
|
expect(String(pageCall[0])).toContain('cursor-xyz');
|
|
});
|
|
});
|
|
|
|
function makeWorkbook(overrides: Record<string, unknown> = {}) {
|
|
return {
|
|
workbookId: 'wb-1',
|
|
workbookUrlId: 'Sales-Dashboard-wb1',
|
|
name: 'Sales Dashboard',
|
|
url: 'https://app.sigmacomputing.com/workbooks/wb-1',
|
|
path: 'Finance',
|
|
latestVersion: 3,
|
|
ownerId: 'user-1',
|
|
createdAt: '2026-01-01T00:00:00Z',
|
|
updatedAt: '2026-01-15T00:00:00Z',
|
|
createdBy: 'user-1',
|
|
updatedBy: 'user-1',
|
|
isArchived: false,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
describe('DefaultSigmaClient.listWorkbooks', () => {
|
|
it('returns entries from a single page', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse({ entries: [makeWorkbook()], nextPage: null }));
|
|
const client = makeClient();
|
|
const workbooks = await client.listWorkbooks();
|
|
expect(workbooks).toHaveLength(1);
|
|
expect(workbooks[0]!.name).toBe('Sales Dashboard');
|
|
});
|
|
|
|
it('passes excludeExplorations=true by default', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null }));
|
|
const client = makeClient();
|
|
await client.listWorkbooks();
|
|
const url = String(vi.mocked(fetch).mock.calls[1]![0]);
|
|
expect(url).toContain('excludeExplorations=true');
|
|
});
|
|
|
|
it('omits excludeExplorations when includeExplorations=true', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null }));
|
|
const client = makeClient();
|
|
await client.listWorkbooks({ includeExplorations: true });
|
|
const url = String(vi.mocked(fetch).mock.calls[1]![0]);
|
|
expect(url).not.toContain('excludeExplorations');
|
|
});
|
|
|
|
it('filters out archived workbooks by default', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(
|
|
makeResponse({
|
|
entries: [makeWorkbook({ isArchived: false }), makeWorkbook({ workbookId: 'wb-2', name: 'Old', isArchived: true })],
|
|
nextPage: null,
|
|
}),
|
|
);
|
|
const client = makeClient();
|
|
const workbooks = await client.listWorkbooks();
|
|
expect(workbooks).toHaveLength(1);
|
|
expect(workbooks[0]!.name).toBe('Sales Dashboard');
|
|
});
|
|
|
|
it('includes archived workbooks when includeArchived=true', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(
|
|
makeResponse({
|
|
entries: [makeWorkbook({ isArchived: false }), makeWorkbook({ workbookId: 'wb-2', name: 'Old', isArchived: true })],
|
|
nextPage: null,
|
|
}),
|
|
);
|
|
const client = makeClient();
|
|
const workbooks = await client.listWorkbooks({ includeArchived: true });
|
|
expect(workbooks).toHaveLength(2);
|
|
});
|
|
|
|
it('filters workbooks by updatedSince', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(
|
|
makeResponse({
|
|
entries: [
|
|
makeWorkbook({ workbookId: 'wb-1', updatedAt: '2026-01-10T00:00:00Z' }),
|
|
makeWorkbook({ workbookId: 'wb-2', updatedAt: '2026-01-20T00:00:00Z' }),
|
|
],
|
|
nextPage: null,
|
|
}),
|
|
);
|
|
const client = makeClient();
|
|
const workbooks = await client.listWorkbooks({ updatedSince: '2026-01-15T00:00:00Z' });
|
|
expect(workbooks).toHaveLength(1);
|
|
expect(workbooks[0]!.workbookId).toBe('wb-2');
|
|
});
|
|
});
|
|
|
|
describe('DefaultSigmaClient.getDataModelSpec', () => {
|
|
it('calls the correct URL with encoded id', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse({ schemaVersion: 1 }));
|
|
const client = makeClient();
|
|
const spec = await client.getDataModelSpec('dm/123');
|
|
expect(spec).toEqual({ schemaVersion: 1 });
|
|
const calls = vi.mocked(fetch).mock.calls;
|
|
expect(String(calls[1]![0])).toContain('/v2/dataModels/dm%2F123/spec');
|
|
});
|
|
});
|
|
|
|
describe('DefaultSigmaClient — error handling', () => {
|
|
it('retries on 500 and succeeds on retry', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // auth
|
|
.mockResolvedValueOnce(makeResponse({ error: 'server error' }, 500)) // first attempt
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })); // retry
|
|
const client = makeClient();
|
|
const models = await client.listDataModels();
|
|
expect(models).toHaveLength(0);
|
|
});
|
|
|
|
it('throws after exhausting retries on 500', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValue(makeResponse({ error: 'server error' }, 500));
|
|
const client = makeClient();
|
|
await expect(client.listDataModels()).rejects.toThrow(/500/);
|
|
});
|
|
|
|
it('throws immediately on service_error 500 without retrying', async () => {
|
|
const serviceError = { requestId: 'abc', message: 'dataSource subtype not supported in data model read', code: 'service_error' };
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse(serviceError, 500));
|
|
const client = makeClient();
|
|
await expect(client.getDataModelSpec('dm-1')).rejects.toThrow(/service_error/);
|
|
// Only 2 calls: auth + one request. No retries.
|
|
expect(vi.mocked(fetch)).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('throws immediately on 404 (non-retryable)', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE))
|
|
.mockResolvedValueOnce(makeResponse({ error: 'not found' }, 404));
|
|
const client = makeClient();
|
|
await expect(client.getDataModelSpec('dm-999')).rejects.toThrow(/404/);
|
|
});
|
|
|
|
it('re-authenticates and retries on 401', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // initial auth
|
|
.mockResolvedValueOnce(makeResponse({ error: 'expired' }, 401)) // 401 on first request
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // re-auth
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })); // retried request
|
|
const client = makeClient();
|
|
const models = await client.listDataModels();
|
|
expect(models).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('DefaultSigmaClient.cleanup', () => {
|
|
it('clears cached token so next call re-authenticates', async () => {
|
|
vi.mocked(fetch)
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // first auth
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })) // first list
|
|
.mockResolvedValueOnce(makeResponse(TOKEN_RESPONSE)) // second auth after cleanup
|
|
.mockResolvedValueOnce(makeResponse({ entries: [], nextPage: null })); // second list
|
|
const client = makeClient();
|
|
await client.listDataModels();
|
|
await client.cleanup();
|
|
await client.listDataModels();
|
|
// 4 calls total: 2 auths + 2 lists
|
|
expect(vi.mocked(fetch)).toHaveBeenCalledTimes(4);
|
|
});
|
|
});
|