fix: store Metabase mappings in ktx.yaml

This commit is contained in:
Andrey Avtomonov 2026-05-13 13:08:59 +02:00
parent b9e0a746af
commit 477002805d
23 changed files with 638 additions and 1147 deletions

View file

@ -1,8 +1,8 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { LocalMetabaseDiscoveryCache } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnectionMapping } from './connection-mapping.js';
@ -79,19 +79,24 @@ describe('runKtxConnectionMapping', () => {
it('sets, lists, disables, and clears local Metabase mappings', async () => {
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
const setCode = await runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
);
expect(setCode, io.stderr()).toBe(0);
let config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toMatchObject({
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
});
const listIo = makeIo();
await expect(
@ -113,6 +118,12 @@ describe('runKtxConnectionMapping', () => {
),
).resolves.toBe(0);
config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toMatchObject({
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': false },
});
await expect(
runKtxConnectionMapping(
{
@ -124,6 +135,9 @@ describe('runKtxConnectionMapping', () => {
makeIo().io,
),
).resolves.toBe(0);
config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toBeUndefined();
});
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
@ -194,9 +208,11 @@ describe('runKtxConnectionMapping', () => {
expect(io.stdout()).toContain('Discovery: 1 database');
expect(client.cleanup).toHaveBeenCalledTimes(1);
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toBeUndefined();
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(discoveryCache.listDiscoveredDatabases('prod-metabase')).resolves.toMatchObject([
{ id: 1, name: 'Analytics', engine: 'postgres' },
]);
});

View file

@ -4,8 +4,9 @@ import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
KtxYamlMetabaseSourceStateReader,
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
@ -16,10 +17,19 @@ import {
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type LocalMetabaseMappingListRow,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
} from '@ktx/context/ingest';
import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import {
type KtxLocalProject,
type KtxProjectConfig,
ktxLocalStateDbPath,
loadKtxProject,
parseMetabaseMappingBootstrap,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
@ -84,6 +94,89 @@ function parseId(value: string, label: string): number {
return parsed;
}
interface MetabaseMappingsBlock {
databaseMappings: Record<string, string | null>;
syncEnabled: Record<string, boolean>;
syncMode: MetabaseSyncMode;
selections: { collections: number[]; items: number[] };
defaultTagNames: string[];
}
function currentMetabaseMappings(project: KtxLocalProject, connectionId: string): MetabaseMappingsBlock {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const bootstrap = parseMetabaseMappingBootstrap(connectionId, connection);
return {
databaseMappings: { ...bootstrap.databaseMappings },
syncEnabled: { ...bootstrap.syncEnabled },
syncMode: bootstrap.syncMode,
selections: {
collections: [...bootstrap.selections.collections],
items: [...bootstrap.selections.items],
},
defaultTagNames: [...bootstrap.defaultTagNames],
};
}
function hasMetabaseMappings(block: MetabaseMappingsBlock): boolean {
return (
Object.keys(block.databaseMappings).length > 0 ||
Object.keys(block.syncEnabled).length > 0 ||
block.syncMode !== 'ALL' ||
block.selections.collections.length > 0 ||
block.selections.items.length > 0 ||
block.defaultTagNames.length > 0
);
}
function serializeMetabaseMappingsBlock(block: MetabaseMappingsBlock): Record<string, unknown> | undefined {
if (!hasMetabaseMappings(block)) {
return undefined;
}
return {
databaseMappings: block.databaseMappings,
syncEnabled: block.syncEnabled,
syncMode: block.syncMode,
selections: block.selections,
defaultTagNames: block.defaultTagNames,
};
}
async function writeMetabaseMappings(
project: KtxLocalProject,
connectionId: string,
block: MetabaseMappingsBlock,
message: string,
): Promise<void> {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const mappings = serializeMetabaseMappingsBlock(block);
const nextConnection = { ...connection };
if (mappings) {
nextConnection.mappings = mappings;
} else {
delete nextConnection.mappings;
}
const nextConfig: KtxProjectConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: nextConnection,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(stripKtxSetupCompletedSteps(nextConfig)),
'ktx',
'ktx@example.com',
message,
);
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
@ -149,9 +242,7 @@ function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
};
}
function renderMapping(
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
): string {
function renderMapping(row: LocalMetabaseMappingListRow): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
@ -255,92 +346,78 @@ export async function runKtxConnectionMapping(
}
assertMetabaseConnection(project, args.connectionId);
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
const metabaseStateReader = new KtxYamlMetabaseSourceStateReader(project, { discoveryCache });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
const rows = await metabaseStateReader.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'databaseMappings') {
throw new Error('Metabase mapping set requires databaseMappings <metabaseDatabaseId>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
await store.upsertDatabaseMapping({
connectionId: args.connectionId,
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
targetConnectionId: args.value,
syncEnabled: true,
source: 'cli',
});
const block = currentMetabaseMappings(project, args.connectionId);
const metabaseDatabaseId = String(parseId(args.key, 'metabaseDatabaseId'));
block.databaseMappings[metabaseDatabaseId] = args.value;
block.syncEnabled[metabaseDatabaseId] = true;
await writeMetabaseMappings(project, args.connectionId, block, `Set Metabase mapping ${args.connectionId}.${metabaseDatabaseId}`);
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'apply-bulk') {
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
const existingState = await store.getSourceState(args.connectionId);
const existingRows = await store.listDatabaseMappings(args.connectionId);
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
const block = currentMetabaseMappings(project, args.connectionId);
const databaseMappings = payload.databaseMappings ?? {};
for (const targetConnectionId of Object.values(databaseMappings)) {
if (targetConnectionId) {
assertTargetConnection(project, targetConnectionId);
}
}
const mappingIds = new Set([
...existingRows.map((row) => row.metabaseDatabaseId),
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
]);
await store.replaceSourceState({
connectionId: args.connectionId,
syncMode: payload.syncMode ?? existingState.syncMode,
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
selections:
payload.selections === undefined
? existingState.selections
: [
...(payload.selections.collections ?? []).map((id) => ({
selectionType: 'collection' as const,
metabaseObjectId: id,
})),
...(payload.selections.items ?? []).map((id) => ({
selectionType: 'item' as const,
metabaseObjectId: id,
})),
],
mappings: [...mappingIds]
.sort((a, b) => a - b)
.map((id) => {
const existing = existingById.get(id);
return {
metabaseDatabaseId: id,
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
metabaseEngine: existing?.metabaseEngine ?? null,
metabaseHost: existing?.metabaseHost ?? null,
metabaseDbName: existing?.metabaseDbName ?? null,
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
source: 'cli',
};
}),
});
for (const id of Object.keys(databaseMappings)) {
parseId(id, 'metabaseDatabaseId');
block.databaseMappings[id] = databaseMappings[id] ?? null;
}
for (const [id, enabled] of Object.entries(payload.syncEnabled ?? {})) {
parseId(id, 'metabaseDatabaseId');
block.syncEnabled[id] = enabled;
}
if (payload.syncMode !== undefined) {
block.syncMode = payload.syncMode;
}
if (payload.defaultTagNames !== undefined) {
block.defaultTagNames = payload.defaultTagNames;
}
if (payload.selections !== undefined) {
block.selections = {
collections: payload.selections.collections ?? [],
items: payload.selections.items ?? [],
};
}
await writeMetabaseMappings(project, args.connectionId, block, `Apply Metabase mappings ${args.connectionId}`);
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'set-sync-enabled') {
await store.setMappingSyncEnabled({
connectionId: args.connectionId,
metabaseDatabaseId: args.metabaseDatabaseId,
syncEnabled: args.enabled,
});
const block = currentMetabaseMappings(project, args.connectionId);
block.syncEnabled[String(args.metabaseDatabaseId)] = args.enabled;
await writeMetabaseMappings(
project,
args.connectionId,
block,
`Set Metabase sync ${args.connectionId}.${args.metabaseDatabaseId}`,
);
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
return 0;
}
if (args.command === 'sync-state-get') {
const state = await store.getSourceState(args.connectionId);
const state = await metabaseStateReader.getSourceState(args.connectionId);
const payload = {
syncMode: state.syncMode,
selections: state.selections,
@ -351,15 +428,11 @@ export async function runKtxConnectionMapping(
}
if (args.command === 'sync-state-set') {
await store.setSyncState({
connectionId: args.connectionId,
syncMode: args.syncMode,
defaultTagNames: args.tagNames,
selections: [
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
],
});
const block = currentMetabaseMappings(project, args.connectionId);
block.syncMode = args.syncMode;
block.defaultTagNames = args.tagNames;
block.selections = { collections: args.collectionIds, items: args.itemIds };
await writeMetabaseMappings(project, args.connectionId, block, `Set Metabase sync state ${args.connectionId}`);
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
return 0;
}
@ -368,15 +441,11 @@ export async function runKtxConnectionMapping(
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const block = currentMetabaseMappings(project, args.connectionId);
const existing = block.databaseMappings;
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
await discoveryCache.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
@ -388,7 +457,9 @@ export async function runKtxConnectionMapping(
}
if (args.command === 'validate') {
const rows = await store.listDatabaseMappings(args.connectionId);
const rows = (await metabaseStateReader.listDatabaseMappings(args.connectionId)).filter(
(row) => row.source === 'ktx.yaml',
);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
@ -412,7 +483,18 @@ export async function runKtxConnectionMapping(
}
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
const block = currentMetabaseMappings(project, args.connectionId);
if (metabaseDatabaseId === undefined) {
block.databaseMappings = {};
block.syncEnabled = {};
block.syncMode = 'ALL';
block.selections = { collections: [], items: [] };
block.defaultTagNames = [];
} else {
delete block.databaseMappings[String(metabaseDatabaseId)];
delete block.syncEnabled[String(metabaseDatabaseId)];
}
await writeMetabaseMappings(project, args.connectionId, block, `Clear Metabase mappings ${args.connectionId}`);
io.stdout.write(
metabaseDatabaseId
? `Cleared databaseMappings.${metabaseDatabaseId}\n`

View file

@ -1,7 +1,7 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest';
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from '@ktx/context/ingest';
import { initKtxProject, ktxLocalStateDbPath, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
@ -9,6 +9,12 @@ import { runKtxConnectionMetabaseSetup } from './connection-metabase-setup.js';
const CANCEL_PROMPT = Symbol('cancel');
async function metabaseMappingRows(projectDir: string, connectionId = 'metabase') {
const project = await loadKtxProject({ projectDir });
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
return new KtxYamlMetabaseSourceStateReader(project, { discoveryCache }).listDatabaseMappings(connectionId);
}
function createTestMetabaseSetupPromptAdapter(options: {
selects?: Array<string | typeof CANCEL_PROMPT>;
multiselects?: Array<Array<unknown> | typeof CANCEL_PROMPT>;
@ -238,10 +244,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
expect(config).toContain('driver: metabase');
expect(config).toContain('api_url: http://metabase.example.test:3000');
expect(config).toContain('api_key: mb_example');
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Analytics',
@ -294,10 +297,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
{ createMetabaseClient: async () => metabaseClient as never },
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true },
]);
});
@ -369,10 +369,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
{ createMetabaseClient: async () => metabaseClient as never },
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true },
]);
});
@ -659,10 +656,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
{ createMetabaseClient: async () => metabaseClient as never },
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 1, targetConnectionId: 'orbit', syncEnabled: true },
{ metabaseDatabaseId: 2, targetConnectionId: null, syncEnabled: false },
]);
@ -785,10 +779,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(config).toContain('driver: metabase');
expect(io.stderr()).toContain(`ktx ingest run --connection-id metabase --adapter metabase --project-dir ${projectDir}`);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' },
]);
});
@ -886,10 +877,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
expect(config).toContain('driver: metabase');
expect(config).toContain('api_url: http://metabase.example.test:3000');
expect(config).toContain(`api_key: ${interactiveMetabaseCredential}`);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{
metabaseDatabaseId: 2,
targetConnectionId: 'orbit',
@ -957,10 +945,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
},
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true },
{ metabaseDatabaseId: 3, targetConnectionId: 'warehouse2', syncEnabled: false },
]);
@ -1128,9 +1113,6 @@ describe('runKtxConnectionMetabaseSetup', () => {
const afterConfig = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(afterConfig).toBe(beforeConfig);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toEqual([]);
await expect(metabaseMappingRows(projectDir)).resolves.toEqual([]);
});
});

View file

@ -16,7 +16,8 @@ import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
LocalMetabaseSourceStateReader,
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
MetabaseClient,
type MetabaseDatabase,
type MetabaseRuntimeClient,
@ -29,7 +30,9 @@ import {
type KtxProjectConnectionConfig,
ktxLocalStateDbPath,
loadKtxProject,
parseMetabaseMappingBootstrap,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import { createClackSpinner, type KtxCliSpinner } from '../clack.js';
@ -338,6 +341,33 @@ function noteMetabaseSetupSummary(options: {
);
}
function metabaseMappingsBlockForSetup(options: {
connectionId: string;
connection: KtxProjectConnectionConfig;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
}): Record<string, unknown> {
const existing = parseMetabaseMappingBootstrap(options.connectionId, options.connection);
const databaseMappings = { ...existing.databaseMappings };
const syncEnabled = { ...existing.syncEnabled };
for (const mapping of options.mappings) {
const key = String(mapping.metabaseDatabaseId);
databaseMappings[key] = mapping.targetConnectionId;
syncEnabled[key] = false;
}
for (const metabaseDatabaseId of options.syncEnabledDatabaseIds) {
syncEnabled[String(metabaseDatabaseId)] = true;
}
return {
databaseMappings,
syncEnabled,
syncMode: options.syncMode,
selections: existing.selections,
defaultTagNames: existing.defaultTagNames,
};
}
export async function runKtxConnectionMetabaseSetup(
args: KtxConnectionMetabaseSetupArgs,
io: KtxCliIo,
@ -674,54 +704,37 @@ export async function runKtxConnectionMetabaseSetup(
}
}
const finalConnectionConfig: KtxProjectConnectionConfig = {
...transientConnectionConfig,
mappings: metabaseMappingsBlockForSetup({
connectionId,
connection: transientConnectionConfig,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
syncMode: args.syncMode,
}),
};
const finalConfig = {
...configWithTransient,
connections: {
...configWithTransient.connections,
[connectionId]: finalConnectionConfig,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(configWithTransient),
serializeKtxProjectConfig(stripKtxSetupCompletedSteps(finalConfig)),
'ktx',
'ktx@example.com',
`Setup Metabase connection ${connectionId}`,
);
const updatedProject = await loadKtxProject({ projectDir: args.projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await store.refreshDiscoveredDatabases({ connectionId, discovered });
for (const mapping of resolvedMappings) {
await store.upsertDatabaseMapping({
connectionId,
metabaseDatabaseId: mapping.metabaseDatabaseId,
targetConnectionId: mapping.targetConnectionId,
syncEnabled: false,
source: 'cli',
});
}
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
await store.setMappingSyncEnabled({
connectionId,
metabaseDatabaseId,
syncEnabled: true,
});
}
const existingSyncState = await store.getSourceState(connectionId);
await store.setSyncState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(updatedProject) });
await discoveryCache.refreshDiscoveredDatabases({ connectionId, discovered });
const rows = await new KtxYamlMetabaseSourceStateReader(updatedProject, { discoveryCache }).listDatabaseMappings(
connectionId,
syncMode: args.syncMode,
defaultTagNames: existingSyncState.defaultTagNames,
selections: existingSyncState.selections,
});
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
if (unhydrated.length > 0) {
io.stderr.write(
`Sync-enabled mappings are missing discovery metadata; run ktx connection mapping refresh ${connectionId} --auto-accept\n`,
);
return 1;
}
const rows = await store.listDatabaseMappings(connectionId);
);
const physicalFailures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];

View file

@ -5,7 +5,8 @@ import { join } from 'node:path';
import { AgentRunnerService, type RunLoopParams } from '@ktx/context/agent';
import {
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
MetabaseSourceAdapter,
getLocalIngestStatus,
type ChunkResult,
@ -485,6 +486,23 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
' driver: metabase',
' api_url: https://metabase.example.test',
' api_key: literal-test-key',
' mappings:',
' databaseMappings:',
' "1": warehouse_a',
' syncEnabled:',
' "1": true',
` syncMode: ${input.syncMode}`,
' selections:',
` collections: [${input.selections
.filter((selection) => selection.selectionType === 'collection')
.map((selection) => selection.metabaseObjectId)
.join(', ')}]`,
` items: [${input.selections
.filter((selection) => selection.selectionType === 'item')
.map((selection) => selection.metabaseObjectId)
.join(', ')}]`,
' defaultTagNames:',
' - sync-mode-smoke',
' warehouse_a:',
' driver: postgres',
' url: postgresql://readonly@db.example.test/warehouse_a',
@ -499,29 +517,15 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
);
const project = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await store.replaceSourceState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
syncMode: input.syncMode,
defaultTagNames: ['sync-mode-smoke'],
selections: input.selections,
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
],
discovered: [{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_a' }],
});
const adapter = new MetabaseSourceAdapter({
clientFactory: new StaticMetabaseClientFactory(createSyncModeMetabaseClient()),
sourceStateReader: store,
sourceStateReader: new KtxYamlMetabaseSourceStateReader(project, { discoveryCache }),
});
const jobId = `metabase-sync-mode-${input.name}-child`;
const io = makeIo();

View file

@ -3,7 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
getLocalIngestStatus,
type LocalIngestResult,
type LocalMetabaseFanoutProgress,
@ -437,6 +437,16 @@ describe('runKtxIngest', () => {
' driver: metabase',
' api_url: https://metabase.example.test',
' api_key: literal-test-key',
' mappings:',
' databaseMappings:',
' "1": warehouse_a',
' "2": warehouse_b',
' syncEnabled:',
' "1": true',
' "2": true',
' syncMode: ALL',
' defaultTagNames:',
' - ktx',
' warehouse_a:',
' driver: postgres',
' url: postgresql://readonly@db.example.test/warehouse_a',
@ -453,33 +463,12 @@ describe('runKtxIngest', () => {
'utf-8',
);
const project = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await store.replaceSourceState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: [],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Warehouse B',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_b',
targetConnectionId: 'warehouse_b',
syncEnabled: true,
source: 'refresh',
},
discovered: [
{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_a' },
{ id: 2, name: 'Warehouse B', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_b' },
],
});
const adapter = new CliMetabaseSourceAdapter();

View file

@ -1124,7 +1124,6 @@ describe('setup databases step', () => {
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: [],
});
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
expect(io.stdout()).toContain('Primary source ready');
@ -1163,7 +1162,6 @@ describe('setup databases step', () => {
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: [],
});
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
});
@ -1213,7 +1211,7 @@ describe('setup databases step', () => {
expect(scanConnection).toHaveBeenCalledTimes(2);
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'analytics']);
expect(config.setup?.completed_steps).toEqual([]);
expect(config.setup?.completed_steps).toBeUndefined();
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
});

View file

@ -94,7 +94,6 @@ describe('setup project step', () => {
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: [],
});
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['llm', 'project'] });
});

View file

@ -171,7 +171,7 @@ describe('setup sources step', () => {
source_dir: '/repo/dbt',
project_name: 'analytics',
});
expect(config.setup?.completed_steps).toEqual([]);
expect(config.setup?.completed_steps).toBeUndefined();
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' });
});
@ -190,7 +190,7 @@ describe('setup sources step', () => {
source: 'metabase',
sourceConnectionId: 'prod_metabase',
sourceUrl: 'https://metabase.example.com',
sourceApiKeyRef: 'env:METABASE_API_KEY',
sourceApiKeyRef: 'env:METABASE_API_KEY', // pragma: allowlist secret
sourceWarehouseConnectionId: 'warehouse',
metabaseDatabaseId: 1,
runInitialSourceIngest: false,
@ -204,7 +204,7 @@ describe('setup sources step', () => {
expect((await readConfig()).connections.prod_metabase).toMatchObject({
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
@ -225,7 +225,7 @@ describe('setup sources step', () => {
inputMode: 'disabled',
source: 'notion',
sourceConnectionId: 'notion-main',
sourceApiKeyRef: 'env:NOTION_TOKEN',
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
notionCrawlMode: 'selected_roots',
notionRootPageIds: ['page-1'],
runInitialSourceIngest: false,
@ -256,7 +256,7 @@ describe('setup sources step', () => {
inputMode: 'disabled',
source: 'notion',
sourceConnectionId: 'notion-main',
sourceApiKeyRef: 'env:NOTION_TOKEN',
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
notionCrawlMode: 'all_accessible',
notionRootPageIds: ['page-1'],
runInitialSourceIngest: false,
@ -766,7 +766,7 @@ describe('setup sources step', () => {
connection: {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
@ -786,7 +786,7 @@ describe('setup sources step', () => {
driver: 'looker',
base_url: 'https://looker.example.com',
client_id: 'client-id',
client_secret_ref: 'env:LOOKER_CLIENT_SECRET',
client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret
mappings: { connectionMappings: { warehouse: 'warehouse' } },
},
deps: {