apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-22 08:38:08 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into connector-solid-review

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-25 17:27:40 +02:00
commit 4c303db2a4
23 changed files with 86 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/cli/src/context/ingest/local-ingest.ts
View file

@ -336,7 +336,7 @@ export async function runLocalMetabaseIngest(
options: RunLocalMetabaseIngestOptions,
): Promise<LocalMetabaseFanoutResult> {
if ((options as RunLocalMetabaseIngestOptions & { sourceDir?: string }).sourceDir) {
throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
throw new Error('source-dir uploads are not supported for the Metabase fanout adapter');
}
const metabaseConnectionId = safeSegment('metabase connection id', options.metabaseConnectionId);

4
packages/cli/src/ingest.ts
View file

@ -222,7 +222,7 @@ function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIng
},
{ wikiCount: 0, slCount: 0 },
);
io.stdout.write(`Metabase fan-out: ${result.status}\n`);
io.stdout.write(`Metabase fanout: ${result.status}\n`);
io.stdout.write(`Source: ${result.metabaseConnectionId}\n`);
io.stdout.write(`Children: ${result.children.length}\n`);
if (result.totals) {
@ -719,7 +719,7 @@ export async function runKtxIngest(
localIngestOptions.queryExecutor ??
(deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(ingestProject);
if (args.adapter === 'metabase' && args.sourceDir) {
throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
throw new Error('source-dir uploads are not supported for the Metabase fanout adapter');
}
if (args.adapter === 'metabase') {
const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest;

4
packages/cli/src/skills/sl/SKILL.md
View file

@ -124,7 +124,7 @@ Every standalone column requires `name` and `type`. Overlays have computed colum
### Grain
`grain: [col_a, col_b]` - the set of columns that uniquely identify one row. The query engine uses grain to prevent fan-out in joins. Overlays inherit grain from the manifest unless they override.
`grain: [col_a, col_b]` - the set of columns that uniquely identify one row. The query engine uses grain to prevent fanout in joins. Overlays inherit grain from the manifest unless they override.
### Joins
@ -177,7 +177,7 @@ The reverse edge (wiki pages that cite this source) is derived automatically fro
## Part 2 - Querying via `sl_query`
The `sl_query` tool generates correct SQL from a structured query. It handles joins, fan-out prevention, aggregation correctness, and filter classification automatically. Prefer it over writing raw SQL whenever the SL has the relevant sources.
The `sl_query` tool generates correct SQL from a structured query. It handles joins, fanout prevention, aggregation correctness, and filter classification automatically. Prefer it over writing raw SQL whenever the SL has the relevant sources.
### When to prefer sl_query over raw SQL

2
packages/cli/test/context/ingest/local-adapters.test.ts
View file

@ -76,7 +76,7 @@ describe('local ingest adapters', () => {
expect(looker?.fetch).toBeTypeOf('function');
});
it('returns the explicit Metabase fan-out boundary before runner construction', async () => {
it('returns the explicit Metabase fanout boundary before runner construction', async () => {
const metabase = createDefaultLocalIngestAdapters(project).find((adapter) => adapter.source === 'metabase');
await expect(localPullConfigForAdapter(project, metabase!, 'warehouse')).rejects.toThrow(

6
packages/cli/test/context/ingest/local-metabase-ingest.test.ts
View file

@ -148,7 +148,7 @@ describe('runLocalMetabaseIngest', () => {
).rejects.toThrow('no sync-enabled mappings with a target connection');
});
it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => {
it('seeds yaml-only Metabase mappings before the unhydrated fanout preflight', async () => {
project.config.connections['prod-metabase'].mappings = {
databaseMappings: { '1': 'warehouse_a' },
syncEnabled: { '1': true },
@ -172,7 +172,7 @@ describe('runLocalMetabaseIngest', () => {
]);
});
it('rejects source-dir uploads through the Metabase fan-out runner', async () => {
it('rejects source-dir uploads through the Metabase fanout runner', async () => {
await expect(
runLocalMetabaseIngest({
project,
@ -181,7 +181,7 @@ describe('runLocalMetabaseIngest', () => {
agentRunner: new TestAgentRunner(),
sourceDir: tempDir,
} as Parameters<typeof runLocalMetabaseIngest>[0] & { sourceDir: string }),
).rejects.toThrow('source-dir uploads are not supported for the Metabase fan-out adapter');
).rejects.toThrow('source-dir uploads are not supported for the Metabase fanout adapter');
});
it('reports partial failure when a child job fails', async () => {

2
packages/cli/test/ingest.test-utils.ts
View file

@ -533,7 +533,7 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
).resolves.toBe(0);
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Metabase fanout: all_succeeded');
expect(io.stdout()).toContain(`target=warehouse_a database=1 status=done job=${jobId}`);
const report = await getLocalIngestStatus(project, jobId);

32
packages/cli/test/ingest.test.ts
View file

@ -346,7 +346,7 @@ describe('runKtxIngest', () => {
);
});
it('routes metabase scheduled pulls to the fan-out runner and prints child summaries', async () => {
it('routes metabase scheduled pulls to the fanout runner and prints child summaries', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -397,13 +397,13 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Metabase fanout: all_succeeded');
expect(io.stdout()).toContain('warehouse_a');
expect(io.stdout()).toContain('metabase-child-1');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
});
it('returns a non-zero code when Metabase fan-out has failed children', async () => {
it('returns a non-zero code when Metabase fanout has failed children', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -467,13 +467,13 @@ describe('runKtxIngest', () => {
),
).resolves.toBe(1);
expect(io.stdout()).toContain('Metabase fan-out: partial_failure');
expect(io.stdout()).toContain('Metabase fanout: partial_failure');
expect(io.stdout()).toContain('Failed tasks: 1');
expect(io.stdout()).toContain('status=error');
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
});
it('prints Metabase fan-out progress before the final summary', async () => {
it('prints Metabase fanout progress before the final summary', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -548,11 +548,11 @@ describe('runKtxIngest', () => {
expect(io.stderr()).toContain('Targets: 1 mapped database');
expect(io.stderr()).toContain('- database=1 target=warehouse_a status=running job=metabase-child-1');
expect(io.stderr()).toContain('- database=1 target=warehouse_a status=done job=metabase-child-1');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Metabase fanout: all_succeeded');
expect(io.stdout()).not.toContain('status=running job=metabase-child-1');
});
it('writes metabase fan-out progress to stderr and final result to stdout', async () => {
it('writes metabase fanout progress to stderr and final result to stdout', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo({ isTTY: true });
@ -592,11 +592,11 @@ describe('runKtxIngest', () => {
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stderr()).toContain('status=running job=metabase-child-1');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Metabase fanout: all_succeeded');
expect(io.stdout()).not.toContain('status=running job=metabase-child-1');
});
it('emits structured progress for Metabase fan-out without writing progress to JSON output', async () => {
it('emits structured progress for Metabase fanout without writing progress to JSON output', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -655,7 +655,7 @@ describe('runKtxIngest', () => {
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
});
it('emits structured child ingest progress during Metabase fan-out', async () => {
it('emits structured child ingest progress during Metabase fanout', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -766,7 +766,7 @@ describe('runKtxIngest', () => {
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
});
it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => {
it('runs Metabase scheduled ingest through the public CLI command path with real fanout', async () => {
const projectDir = join(tempDir, 'metabase-cli-project');
await writeWarehouseConfig(projectDir);
await writeFile(
@ -838,7 +838,7 @@ describe('runKtxIngest', () => {
expect(io.stderr()).toContain('Metabase ingest: prod-metabase');
expect(io.stderr()).toContain('Targets: 2 mapped databases');
expect(io.stdout()).toContain('Metabase fan-out: all_succeeded');
expect(io.stdout()).toContain('Metabase fanout: all_succeeded');
expect(io.stdout()).toContain('Source: prod-metabase');
expect(io.stdout()).toContain('Children: 2');
expect(io.stdout()).toContain('target=warehouse_a database=1 status=done job=metabase-child-1');
@ -893,7 +893,7 @@ describe('runKtxIngest', () => {
});
});
it('prints metabase fan-out JSON results', async () => {
it('prints metabase fanout JSON results', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -967,7 +967,7 @@ describe('runKtxIngest', () => {
expect(io.stderr()).toBe('');
});
it('rejects source-dir uploads through the metabase fan-out route', async () => {
it('rejects source-dir uploads through the metabase fanout route', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
@ -985,13 +985,13 @@ describe('runKtxIngest', () => {
io.io,
{
runLocalMetabaseIngest: async () => {
throw new Error('fan-out should not be called');
throw new Error('fanout should not be called');
},
},
),
).resolves.toBe(1);
expect(io.stderr()).toContain('source-dir uploads are not supported for the Metabase fan-out adapter');
expect(io.stderr()).toContain('source-dir uploads are not supported for the Metabase fanout adapter');
expect(io.stderr()).not.toContain('ktx ingest requires llm.provider.backend');
expect(io.stdout()).toBe('');
});