mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-13 11:22:11 +02:00
feat(sigma): add Sigma Computing context-source adapter (#316)
* feat(sigma): add Sigma Computing context-source adapter Closes #168 Adds a full ingest adapter for Sigma Computing so `ktx ingest` can pull data model specs and workbook summaries into the ktx context layer. The implementation follows the same fetch → chunk → project → LLM pattern used by the Looker, Metabase, and MetricFlow adapters. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(sigma): address PR review comments - Remove manifest from rawFiles; moves to peerFileIndex so fetchedAt changes don't mark all work units dirty every run - Fix workbookFilter.updatedSince eviction bug: fetch full universe first, apply filter client-side, evict only on archived/deleted - Remove measure projection entirely; project() writes measures: [] and the sigma_ingest skill surfaces Lookup/aggregation formulas as wiki prose - Remove joins projection (v1 limitation); project() writes joins: [] and Lookup relationships are described in wiki prose instead - Remove write-back dead code: createDataModel, updateDataModel, SigmaDataModelPushResult, mutate/post/put - Fix emitBatches notes pluralization bug ('2 data modelss' → '2 data models') - Add tokenInflight dedup on ensureToken to coalesce concurrent auth requests - Retry spec fetch when existing staged spec is null (transient failure cache) - Drop unused WorkbookFilter import from client-port.ts - Note in docs that joins are not projected from Sigma data models in this release Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * updates * fix(sigma): restore sigma in local adapter test + small cleanups The gdrive↔sigma merge dropped 'sigma' from the expected adapter source list in local-adapters.test.ts while keeping gdrive, so the slow TS suite failed even though the source registers both. Add 'sigma' back at its registration position (after metabase, before gdrive). Also: - Move the orphaned SigmaPullConfig docstring onto the schema it documents and drop the stale BullMQ reference (standalone ktx has no BullMQ; the config lives in the ingest job's bundleRef.config). - Drop an O(n^2) find() round-trip in fetch() when building the active data-model list; filter once and reuse for the eviction id set. --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Andrey Avtomonov <andreybavt@gmail.com> Co-authored-by: Luca Martial <48870843+luca-martial@users.noreply.github.com>
This commit is contained in:
Matt Senick (Sigma)
GitHub
parent
139ac08320
commit
acd20ac248
41 changed files with 3610 additions and 6 deletions
|
|
@ -1364,6 +1364,18 @@ describe('setup sources step', () => {
|
|||
deps: { validateNotion: vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })) },
|
||||
expectedLabel: 'Notion',
|
||||
},
|
||||
{
|
||||
source: 'sigma',
|
||||
connectionId: 'sigma-main',
|
||||
connection: {
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'my-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
},
|
||||
deps: { validateSigma: vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' })) },
|
||||
expectedLabel: 'Sigma Computing',
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
|
|
@ -2035,4 +2047,206 @@ describe('setup sources step', () => {
|
|||
path: 'staging',
|
||||
});
|
||||
});
|
||||
|
||||
it('writes Sigma config in non-interactive mode', async () => {
|
||||
await addPrimarySource();
|
||||
const validateSigma = vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' }));
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-prod',
|
||||
sourceUrl: 'https://api.sigmacomputing.com',
|
||||
sourceClientId: 'my-client-id',
|
||||
sourceClientSecretRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
makeIo().io,
|
||||
{ validateSigma },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-prod'] });
|
||||
|
||||
expect((await readConfig()).connections['sigma-prod']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'my-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
expect(validateSigma).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('defaults Sigma api_url when --source-url is omitted', async () => {
|
||||
await addPrimarySource();
|
||||
const validateSigma = vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' }));
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-main',
|
||||
sourceClientId: 'my-client-id',
|
||||
sourceClientSecretRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
makeIo().io,
|
||||
{ validateSigma },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-main'] });
|
||||
|
||||
expect((await readConfig()).connections['sigma-main']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects --source-auth-token-ref for Sigma and points at --source-client-secret-ref', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-main',
|
||||
sourceClientId: 'my-client-id',
|
||||
sourceAuthTokenRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{},
|
||||
),
|
||||
).resolves.toEqual({ status: 'failed', projectDir });
|
||||
|
||||
expect(io.stderr()).toContain('--source-auth-token-ref does not apply to --source sigma; use --source-client-secret-ref.');
|
||||
});
|
||||
|
||||
it('runs interactive Sigma setup with API URL, client ID, and env credential', async () => {
|
||||
await addPrimarySource();
|
||||
const validateSigma = vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' }));
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['sigma']],
|
||||
select: ['env', 'done'],
|
||||
// connection name, API URL (default accepted), client ID
|
||||
text: ['sigma-main', 'https://api.sigmacomputing.com', 'my-client-id'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{ prompts: testPrompts, validateSigma },
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: textInputPrompt(connectionNamePrompt('Sigma Computing')),
|
||||
initialValue: 'sigma-main',
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: textInputPrompt('Sigma API URL') }),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: textInputPrompt('Sigma client ID') }),
|
||||
);
|
||||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'How should ktx find your Sigma client secret?',
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use SIGMA_CLIENT_SECRET from the environment' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
expect((await readConfig()).connections['sigma-main']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'my-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
|
||||
it('edits an existing Sigma source with current URL and client ID as defaults', async () => {
|
||||
await addPrimarySource();
|
||||
await addConnection('sigma-main', {
|
||||
driver: 'sigma',
|
||||
api_url: 'https://api.sigmacomputing.com',
|
||||
client_id: 'old-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
const testPrompts = prompts({
|
||||
multiselect: [['sigma']],
|
||||
select: ['edit:sigma-main', 'keep', 'done'],
|
||||
// API URL and new client ID
|
||||
text: ['https://api.sigmacomputing.com', 'new-client-id'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupSourcesStep(
|
||||
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
|
||||
makeIo().io,
|
||||
{
|
||||
prompts: testPrompts,
|
||||
validateSigma: vi.fn(async () => ({ ok: true as const, detail: 'Sigma API connection verified' })),
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['sigma-main'] });
|
||||
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: textInputPrompt('Sigma API URL'),
|
||||
initialValue: 'https://api.sigmacomputing.com',
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.text).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: textInputPrompt('Sigma client ID'),
|
||||
initialValue: 'old-client-id', // pre-filled from existing connection
|
||||
}),
|
||||
);
|
||||
expect(testPrompts.select).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'How should ktx find your Sigma client secret?',
|
||||
options: expect.arrayContaining([{ value: 'keep', label: 'Keep existing credential' }]),
|
||||
}),
|
||||
);
|
||||
expect((await readConfig()).connections['sigma-main']).toMatchObject({
|
||||
driver: 'sigma',
|
||||
client_id: 'new-client-id',
|
||||
client_secret_ref: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
|
||||
it('fails Sigma setup when validation rejects the credentials', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
|
||||
const result = await runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'disabled',
|
||||
source: 'sigma',
|
||||
sourceConnectionId: 'sigma-main',
|
||||
sourceClientId: 'bad-client-id',
|
||||
sourceClientSecretRef: 'env:SIGMA_CLIENT_SECRET', // pragma: allowlist secret
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{ validateSigma: vi.fn(async () => ({ ok: false as const, message: 'Sigma auth failed (401): Unauthorized' })) },
|
||||
);
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(io.stderr()).toContain('Sigma auth failed (401): Unauthorized');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue