apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-16 08:25:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge origin/main into drop-legacy-migration-code

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 14:45:59 +02:00
commit e22c7704d8
44 changed files with 1845 additions and 1403 deletions
Download patch file Download diff file Expand all files Collapse all files

3
AGENTS.md
View file

@ -24,6 +24,9 @@ database migrations, ORPC contracts, or `python-service/` layout exist here.
- **MUST**: Keep package/public API changes intentional. Do not add compatibility
wrappers for old KTX names unless the user explicitly asks for a migration
bridge.
- **MUST**: Treat KTX as having no public users unless the user says otherwise.
Legacy support is not necessary by default; prefer clean breaking changes over
compatibility shims, migration bridges, or preserved stale behavior.
### Absolute Prohibitions

54
packages/cli/src/commands/connection-mapping.test.ts
View file

@ -1,8 +1,8 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { LocalMetabaseDiscoveryCache } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnectionMapping } from './connection-mapping.js';
@ -79,19 +79,24 @@ describe('runKtxConnectionMapping', () => {
it('sets, lists, disables, and clears local Metabase mappings', async () => {
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
const setCode = await runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
);
expect(setCode, io.stderr()).toBe(0);
let config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toMatchObject({
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
});
const listIo = makeIo();
await expect(
@ -113,6 +118,12 @@ describe('runKtxConnectionMapping', () => {
),
).resolves.toBe(0);
config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toMatchObject({
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': false },
});
await expect(
runKtxConnectionMapping(
{
@ -124,6 +135,9 @@ describe('runKtxConnectionMapping', () => {
makeIo().io,
),
).resolves.toBe(0);
config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toBeUndefined();
});
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
@ -194,9 +208,11 @@ describe('runKtxConnectionMapping', () => {
expect(io.stdout()).toContain('Discovery: 1 database');
expect(client.cleanup).toHaveBeenCalledTimes(1);
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toBeUndefined();
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(discoveryCache.listDiscoveredDatabases('prod-metabase')).resolves.toMatchObject([
{ id: 1, name: 'Analytics', engine: 'postgres' },
]);
});

239
packages/cli/src/commands/connection-mapping.ts
View file

@ -4,8 +4,9 @@ import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
KtxYamlMetabaseSourceStateReader,
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
@ -16,10 +17,18 @@ import {
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type LocalMetabaseMappingListRow,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
} from '@ktx/context/ingest';
import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import {
type KtxLocalProject,
type KtxProjectConfig,
ktxLocalStateDbPath,
loadKtxProject,
parseMetabaseMappingBootstrap,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
@ -84,6 +93,89 @@ function parseId(value: string, label: string): number {
return parsed;
}
interface MetabaseMappingsBlock {
databaseMappings: Record<string, string | null>;
syncEnabled: Record<string, boolean>;
syncMode: MetabaseSyncMode;
selections: { collections: number[]; items: number[] };
defaultTagNames: string[];
}
function currentMetabaseMappings(project: KtxLocalProject, connectionId: string): MetabaseMappingsBlock {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const bootstrap = parseMetabaseMappingBootstrap(connectionId, connection);
return {
databaseMappings: { ...bootstrap.databaseMappings },
syncEnabled: { ...bootstrap.syncEnabled },
syncMode: bootstrap.syncMode,
selections: {
collections: [...bootstrap.selections.collections],
items: [...bootstrap.selections.items],
},
defaultTagNames: [...bootstrap.defaultTagNames],
};
}
function hasMetabaseMappings(block: MetabaseMappingsBlock): boolean {
return (
Object.keys(block.databaseMappings).length > 0 ||
Object.keys(block.syncEnabled).length > 0 ||
block.syncMode !== 'ALL' ||
block.selections.collections.length > 0 ||
block.selections.items.length > 0 ||
block.defaultTagNames.length > 0
);
}
function serializeMetabaseMappingsBlock(block: MetabaseMappingsBlock): Record<string, unknown> | undefined {
if (!hasMetabaseMappings(block)) {
return undefined;
}
return {
databaseMappings: block.databaseMappings,
syncEnabled: block.syncEnabled,
syncMode: block.syncMode,
selections: block.selections,
defaultTagNames: block.defaultTagNames,
};
}
async function writeMetabaseMappings(
project: KtxLocalProject,
connectionId: string,
block: MetabaseMappingsBlock,
message: string,
): Promise<void> {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const mappings = serializeMetabaseMappingsBlock(block);
const nextConnection = { ...connection };
if (mappings) {
nextConnection.mappings = mappings;
} else {
delete nextConnection.mappings;
}
const nextConfig: KtxProjectConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: nextConnection,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
message,
);
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
@ -149,9 +241,7 @@ function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
};
}
function renderMapping(
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
): string {
function renderMapping(row: LocalMetabaseMappingListRow): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
@ -255,92 +345,78 @@ export async function runKtxConnectionMapping(
}
assertMetabaseConnection(project, args.connectionId);
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
const metabaseStateReader = new KtxYamlMetabaseSourceStateReader(project, { discoveryCache });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
const rows = await metabaseStateReader.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'databaseMappings') {
throw new Error('Metabase mapping set requires databaseMappings <metabaseDatabaseId>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
await store.upsertDatabaseMapping({
connectionId: args.connectionId,
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
targetConnectionId: args.value,
syncEnabled: true,
source: 'cli',
});
const block = currentMetabaseMappings(project, args.connectionId);
const metabaseDatabaseId = String(parseId(args.key, 'metabaseDatabaseId'));
block.databaseMappings[metabaseDatabaseId] = args.value;
block.syncEnabled[metabaseDatabaseId] = true;
await writeMetabaseMappings(project, args.connectionId, block, `Set Metabase mapping ${args.connectionId}.${metabaseDatabaseId}`);
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'apply-bulk') {
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
const existingState = await store.getSourceState(args.connectionId);
const existingRows = await store.listDatabaseMappings(args.connectionId);
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
const block = currentMetabaseMappings(project, args.connectionId);
const databaseMappings = payload.databaseMappings ?? {};
for (const targetConnectionId of Object.values(databaseMappings)) {
if (targetConnectionId) {
assertTargetConnection(project, targetConnectionId);
}
}
const mappingIds = new Set([
...existingRows.map((row) => row.metabaseDatabaseId),
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
]);
await store.replaceSourceState({
connectionId: args.connectionId,
syncMode: payload.syncMode ?? existingState.syncMode,
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
selections:
payload.selections === undefined
? existingState.selections
: [
...(payload.selections.collections ?? []).map((id) => ({
selectionType: 'collection' as const,
metabaseObjectId: id,
})),
...(payload.selections.items ?? []).map((id) => ({
selectionType: 'item' as const,
metabaseObjectId: id,
})),
],
mappings: [...mappingIds]
.sort((a, b) => a - b)
.map((id) => {
const existing = existingById.get(id);
return {
metabaseDatabaseId: id,
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
metabaseEngine: existing?.metabaseEngine ?? null,
metabaseHost: existing?.metabaseHost ?? null,
metabaseDbName: existing?.metabaseDbName ?? null,
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
source: 'cli',
};
}),
});
for (const id of Object.keys(databaseMappings)) {
parseId(id, 'metabaseDatabaseId');
block.databaseMappings[id] = databaseMappings[id] ?? null;
}
for (const [id, enabled] of Object.entries(payload.syncEnabled ?? {})) {
parseId(id, 'metabaseDatabaseId');
block.syncEnabled[id] = enabled;
}
if (payload.syncMode !== undefined) {
block.syncMode = payload.syncMode;
}
if (payload.defaultTagNames !== undefined) {
block.defaultTagNames = payload.defaultTagNames;
}
if (payload.selections !== undefined) {
block.selections = {
collections: payload.selections.collections ?? [],
items: payload.selections.items ?? [],
};
}
await writeMetabaseMappings(project, args.connectionId, block, `Apply Metabase mappings ${args.connectionId}`);
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'set-sync-enabled') {
await store.setMappingSyncEnabled({
connectionId: args.connectionId,
metabaseDatabaseId: args.metabaseDatabaseId,
syncEnabled: args.enabled,
});
const block = currentMetabaseMappings(project, args.connectionId);
block.syncEnabled[String(args.metabaseDatabaseId)] = args.enabled;
await writeMetabaseMappings(
project,
args.connectionId,
block,
`Set Metabase sync ${args.connectionId}.${args.metabaseDatabaseId}`,
);
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
return 0;
}
if (args.command === 'sync-state-get') {
const state = await store.getSourceState(args.connectionId);
const state = await metabaseStateReader.getSourceState(args.connectionId);
const payload = {
syncMode: state.syncMode,
selections: state.selections,
@ -351,15 +427,11 @@ export async function runKtxConnectionMapping(
}
if (args.command === 'sync-state-set') {
await store.setSyncState({
connectionId: args.connectionId,
syncMode: args.syncMode,
defaultTagNames: args.tagNames,
selections: [
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
],
});
const block = currentMetabaseMappings(project, args.connectionId);
block.syncMode = args.syncMode;
block.defaultTagNames = args.tagNames;
block.selections = { collections: args.collectionIds, items: args.itemIds };
await writeMetabaseMappings(project, args.connectionId, block, `Set Metabase sync state ${args.connectionId}`);
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
return 0;
}
@ -368,15 +440,11 @@ export async function runKtxConnectionMapping(
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const block = currentMetabaseMappings(project, args.connectionId);
const existing = block.databaseMappings;
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
await discoveryCache.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
@ -388,7 +456,9 @@ export async function runKtxConnectionMapping(
}
if (args.command === 'validate') {
const rows = await store.listDatabaseMappings(args.connectionId);
const rows = (await metabaseStateReader.listDatabaseMappings(args.connectionId)).filter(
(row) => row.source === 'ktx.yaml',
);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
@ -412,7 +482,18 @@ export async function runKtxConnectionMapping(
}
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
const block = currentMetabaseMappings(project, args.connectionId);
if (metabaseDatabaseId === undefined) {
block.databaseMappings = {};
block.syncEnabled = {};
block.syncMode = 'ALL';
block.selections = { collections: [], items: [] };
block.defaultTagNames = [];
} else {
delete block.databaseMappings[String(metabaseDatabaseId)];
delete block.syncEnabled[String(metabaseDatabaseId)];
}
await writeMetabaseMappings(project, args.connectionId, block, `Clear Metabase mappings ${args.connectionId}`);
io.stdout.write(
metabaseDatabaseId
? `Cleared databaseMappings.${metabaseDatabaseId}\n`

48
packages/cli/src/commands/connection-metabase-setup.test.ts
View file

@ -1,7 +1,7 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest';
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from '@ktx/context/ingest';
import { initKtxProject, ktxLocalStateDbPath, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
@ -9,6 +9,12 @@ import { runKtxConnectionMetabaseSetup } from './connection-metabase-setup.js';
const CANCEL_PROMPT = Symbol('cancel');
async function metabaseMappingRows(projectDir: string, connectionId = 'metabase') {
const project = await loadKtxProject({ projectDir });
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
return new KtxYamlMetabaseSourceStateReader(project, { discoveryCache }).listDatabaseMappings(connectionId);
}
function createTestMetabaseSetupPromptAdapter(options: {
selects?: Array<string | typeof CANCEL_PROMPT>;
multiselects?: Array<Array<unknown> | typeof CANCEL_PROMPT>;
@ -238,10 +244,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
expect(config).toContain('driver: metabase');
expect(config).toContain('api_url: http://metabase.example.test:3000');
expect(config).toContain('api_key: mb_example');
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Analytics',
@ -294,10 +297,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
{ createMetabaseClient: async () => metabaseClient as never },
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true },
]);
});
@ -369,10 +369,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
{ createMetabaseClient: async () => metabaseClient as never },
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true },
]);
});
@ -659,10 +656,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
{ createMetabaseClient: async () => metabaseClient as never },
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 1, targetConnectionId: 'orbit', syncEnabled: true },
{ metabaseDatabaseId: 2, targetConnectionId: null, syncEnabled: false },
]);
@ -785,10 +779,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(config).toContain('driver: metabase');
expect(io.stderr()).toContain(`ktx ingest run --connection-id metabase --adapter metabase --project-dir ${projectDir}`);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' },
]);
});
@ -886,10 +877,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
expect(config).toContain('driver: metabase');
expect(config).toContain('api_url: http://metabase.example.test:3000');
expect(config).toContain(`api_key: ${interactiveMetabaseCredential}`);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{
metabaseDatabaseId: 2,
targetConnectionId: 'orbit',
@ -957,10 +945,7 @@ describe('runKtxConnectionMetabaseSetup', () => {
},
),
).resolves.toBe(0);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([
await expect(metabaseMappingRows(projectDir)).resolves.toMatchObject([
{ metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true },
{ metabaseDatabaseId: 3, targetConnectionId: 'warehouse2', syncEnabled: false },
]);
@ -1128,9 +1113,6 @@ describe('runKtxConnectionMetabaseSetup', () => {
const afterConfig = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(afterConfig).toBe(beforeConfig);
const updatedProject = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await expect(store.listDatabaseMappings('metabase')).resolves.toEqual([]);
await expect(metabaseMappingRows(projectDir)).resolves.toEqual([]);
});
});

92
packages/cli/src/commands/connection-metabase-setup.ts
View file

@ -16,7 +16,8 @@ import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
LocalMetabaseSourceStateReader,
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
MetabaseClient,
type MetabaseDatabase,
type MetabaseRuntimeClient,
@ -29,6 +30,7 @@ import {
type KtxProjectConnectionConfig,
ktxLocalStateDbPath,
loadKtxProject,
parseMetabaseMappingBootstrap,
serializeKtxProjectConfig,
} from '@ktx/context/project';
@ -338,6 +340,33 @@ function noteMetabaseSetupSummary(options: {
);
}
function metabaseMappingsBlockForSetup(options: {
connectionId: string;
connection: KtxProjectConnectionConfig;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
}): Record<string, unknown> {
const existing = parseMetabaseMappingBootstrap(options.connectionId, options.connection);
const databaseMappings = { ...existing.databaseMappings };
const syncEnabled = { ...existing.syncEnabled };
for (const mapping of options.mappings) {
const key = String(mapping.metabaseDatabaseId);
databaseMappings[key] = mapping.targetConnectionId;
syncEnabled[key] = false;
}
for (const metabaseDatabaseId of options.syncEnabledDatabaseIds) {
syncEnabled[String(metabaseDatabaseId)] = true;
}
return {
databaseMappings,
syncEnabled,
syncMode: options.syncMode,
selections: existing.selections,
defaultTagNames: existing.defaultTagNames,
};
}
export async function runKtxConnectionMetabaseSetup(
args: KtxConnectionMetabaseSetupArgs,
io: KtxCliIo,
@ -674,54 +703,37 @@ export async function runKtxConnectionMetabaseSetup(
}
}
const finalConnectionConfig: KtxProjectConnectionConfig = {
...transientConnectionConfig,
mappings: metabaseMappingsBlockForSetup({
connectionId,
connection: transientConnectionConfig,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
syncMode: args.syncMode,
}),
};
const finalConfig = {
...configWithTransient,
connections: {
...configWithTransient.connections,
[connectionId]: finalConnectionConfig,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(configWithTransient),
serializeKtxProjectConfig(finalConfig),
'ktx',
'ktx@example.com',
`Setup Metabase connection ${connectionId}`,
);
const updatedProject = await loadKtxProject({ projectDir: args.projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await store.refreshDiscoveredDatabases({ connectionId, discovered });
for (const mapping of resolvedMappings) {
await store.upsertDatabaseMapping({
connectionId,
metabaseDatabaseId: mapping.metabaseDatabaseId,
targetConnectionId: mapping.targetConnectionId,
syncEnabled: false,
source: 'cli',
});
}
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
await store.setMappingSyncEnabled({
connectionId,
metabaseDatabaseId,
syncEnabled: true,
});
}
const existingSyncState = await store.getSourceState(connectionId);
await store.setSyncState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(updatedProject) });
await discoveryCache.refreshDiscoveredDatabases({ connectionId, discovered });
const rows = await new KtxYamlMetabaseSourceStateReader(updatedProject, { discoveryCache }).listDatabaseMappings(
connectionId,
syncMode: args.syncMode,
defaultTagNames: existingSyncState.defaultTagNames,
selections: existingSyncState.selections,
});
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
if (unhydrated.length > 0) {
io.stderr.write(
`Sync-enabled mappings are missing discovery metadata; run ktx connection mapping refresh ${connectionId} --auto-accept\n`,
);
return 1;
}
const rows = await store.listDatabaseMappings(connectionId);
);
const physicalFailures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];

30
packages/cli/src/commands/setup-commands.ts
View file

@ -2,6 +2,7 @@ import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-
import type { KtxCliCommandContext } from '../cli-program.js';
import { resolveCommandProjectDir } from '../cli-program.js';
import type { KtxSetupDatabaseDriver } from '../setup-databases.js';
import type { KtxSetupLlmBackend } from '../setup-models.js';
import type { KtxSetupSourceType } from '../setup-sources.js';
async function runSetupArgs(
@ -27,6 +28,13 @@ function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function llmBackend(value: string): KtxSetupLlmBackend {
if (value === 'anthropic' || value === 'vertex') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function databaseDriver(value: string): KtxSetupDatabaseDriver {
if (
value === 'sqlite' ||
@ -93,9 +101,12 @@ function shouldShowSetupEntryMenu(
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
llmBackend?: KtxSetupLlmBackend;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
anthropicModel?: string;
vertexProject?: string;
vertexLocation?: string;
skipLlm?: boolean;
embeddingBackend?: string;
embeddingApiKeyEnv?: string;
@ -165,9 +176,12 @@ function shouldShowSetupEntryMenu(
'skipAgents',
'yes',
'input',
'llmBackend',
'anthropicApiKeyEnv',
'anthropicApiKeyFile',
'anthropicModel',
'vertexProject',
'vertexLocation',
'skipLlm',
'embeddingBackend',
'embeddingApiKeyEnv',
@ -225,9 +239,12 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
.option('--no-input', 'Disable interactive terminal input')
.addOption(new Option('--llm-backend <backend>', 'LLM backend').argParser(llmBackend))
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
.option('--vertex-project <project>', 'Google Vertex AI project ID, env:NAME, or file:/path')
.option('--vertex-location <location>', 'Google Vertex AI location, env:NAME, or file:/path')
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
@ -318,6 +335,16 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
context.setExitCode(1);
return;
}
if (options.llmBackend === 'vertex' && (options.anthropicApiKeyEnv || options.anthropicApiKeyFile)) {
context.io.stderr.write('Anthropic API key flags are only valid with --llm-backend anthropic.\n');
context.setExitCode(1);
return;
}
if (options.llmBackend === 'anthropic' && (options.vertexProject || options.vertexLocation)) {
context.io.stderr.write('Vertex AI flags are only valid with --llm-backend vertex.\n');
context.setExitCode(1);
return;
}
if (options.embeddingApiKeyEnv && options.embeddingApiKeyFile) {
context.io.stderr.write(
'Choose only one embedding credential source: --embedding-api-key-env or --embedding-api-key-file.\n',
@ -356,9 +383,12 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
inputMode: options.input === false ? 'disabled' : 'auto',
yes: options.yes === true,
cliVersion: context.packageInfo.version,
...(options.llmBackend ? { llmBackend: options.llmBackend } : {}),
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}),
...(options.vertexProject ? { vertexProject: options.vertexProject } : {}),
...(options.vertexLocation ? { vertexLocation: options.vertexLocation } : {}),
skipLlm: options.skipLlm === true,
...(options.embeddingBackend ? { embeddingBackend: options.embeddingBackend } : {}),
...(options.embeddingApiKeyEnv ? { embeddingApiKeyEnv: options.embeddingApiKeyEnv } : {}),

41
packages/cli/src/index.test.ts
View file

@ -853,6 +853,47 @@ describe('runKtxCli', () => {
);
});
it('dispatches Vertex AI setup flags to the setup runner', async () => {
const setup = vi.fn(async () => 0);
const setupIo = makeIo();
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'setup',
'--no-input',
'--llm-backend',
'vertex',
'--vertex-project',
'local-gcp-project',
'--vertex-location',
'us-east5',
'--anthropic-model',
'claude-sonnet-4-6',
],
setupIo.io,
{ setup },
),
).resolves.toBe(0);
expect(setup).toHaveBeenCalledWith(
expect.objectContaining({
command: 'run',
projectDir: tempDir,
inputMode: 'disabled',
cliVersion: '0.0.0-private',
llmBackend: 'vertex',
vertexProject: 'local-gcp-project',
vertexLocation: 'us-east5',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
}),
setupIo.io,
);
});
it('rejects conflicting Anthropic credential setup flags', async () => {
const setup = vi.fn(async () => 0);
const setupIo = makeIo();

42
packages/cli/src/ingest.test-utils.ts
View file

@ -3,7 +3,8 @@ import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { AgentRunnerService, type RunLoopParams } from '@ktx/context/agent';
import {
LocalMetabaseSourceStateReader,
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
MetabaseSourceAdapter,
getLocalIngestStatus,
type ChunkResult,
@ -493,6 +494,23 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
' driver: metabase',
' api_url: https://metabase.example.test',
' api_key: literal-test-key',
' mappings:',
' databaseMappings:',
' "1": warehouse_a',
' syncEnabled:',
' "1": true',
` syncMode: ${input.syncMode}`,
' selections:',
` collections: [${input.selections
.filter((selection) => selection.selectionType === 'collection')
.map((selection) => selection.metabaseObjectId)
.join(', ')}]`,
` items: [${input.selections
.filter((selection) => selection.selectionType === 'item')
.map((selection) => selection.metabaseObjectId)
.join(', ')}]`,
' defaultTagNames:',
' - sync-mode-smoke',
' warehouse_a:',
' driver: postgres',
' url: postgresql://readonly@db.example.test/warehouse_a',
@ -507,29 +525,15 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
);
const project = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await store.replaceSourceState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
syncMode: input.syncMode,
defaultTagNames: ['sync-mode-smoke'],
selections: input.selections,
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
],
discovered: [{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_a' }],
});
const adapter = new MetabaseSourceAdapter({
clientFactory: new StaticMetabaseClientFactory(createSyncModeMetabaseClient()),
sourceStateReader: store,
sourceStateReader: new KtxYamlMetabaseSourceStateReader(project, { discoveryCache }),
});
const jobId = `metabase-sync-mode-${input.name}-child`;
const io = makeIo();

43
packages/cli/src/ingest.test.ts
View file

@ -3,7 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
type LocalIngestResult,
type LocalMetabaseFanoutProgress,
type RunLocalIngestOptions,
@ -433,6 +433,16 @@ describe('runKtxIngest', () => {
' driver: metabase',
' api_url: https://metabase.example.test',
' api_key: literal-test-key',
' mappings:',
' databaseMappings:',
' "1": warehouse_a',
' "2": warehouse_b',
' syncEnabled:',
' "1": true',
' "2": true',
' syncMode: ALL',
' defaultTagNames:',
' - ktx',
' warehouse_a:',
' driver: postgres',
' url: postgresql://readonly@db.example.test/warehouse_a',
@ -449,33 +459,12 @@ describe('runKtxIngest', () => {
'utf-8',
);
const project = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await store.replaceSourceState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: [],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Warehouse B',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_b',
targetConnectionId: 'warehouse_b',
syncEnabled: true,
source: 'refresh',
},
discovered: [
{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_a' },
{ id: 2, name: 'Warehouse B', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_b' },
],
});
const adapter = new CliMetabaseSourceAdapter();

3
packages/cli/src/setup-agents.ts
View file

@ -6,7 +6,6 @@ import {
loadKtxProject,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { withMenuOptionsSpacing, withMultiselectNavigation } from './prompt-navigation.js';
@ -364,7 +363,7 @@ async function installTarget(input: {
async function markAgentsComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'agents');
}

11
packages/cli/src/setup-context.test.ts
View file

@ -1,7 +1,7 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { readKtxSetupState } from '@ktx/context/project';
import { readKtxSetupState, writeKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
@ -40,12 +40,6 @@ async function writeReadyProject(projectDir: string) {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - llm',
' - embeddings',
' - databases',
' - sources',
'connections:',
' warehouse:',
' driver: postgres',
@ -71,6 +65,9 @@ async function writeReadyProject(projectDir: string) {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(projectDir, {
completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources'],
});
}
async function writeScanReport(

6
packages/cli/src/setup-context.ts
View file

@ -5,11 +5,9 @@ import { cancel, isCancel, select } from '@clack/prompts';
import {
type KtxLocalProject,
loadKtxProject,
ktxSetupCompletedSteps,
markKtxSetupStateStepComplete,
readKtxSetupState,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { buildPublicIngestPlan } from './public-ingest.js';
@ -470,7 +468,7 @@ async function defaultVerifyContextReady(projectDir: string): Promise<KtxSetupCo
async function markContextComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'context');
}
@ -704,7 +702,7 @@ export async function runKtxSetupContextStep(
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
const existingState = await readKtxSetupContextState(args.projectDir);
const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(args.projectDir));
const completedSteps = (await readKtxSetupState(args.projectDir)).completed_steps;
if (completedSteps.includes('context') && existingState.status === 'completed') {
return { status: 'ready', projectDir: args.projectDir, runId: existingState.runId ?? 'setup-context-completed' };
}

20
packages/cli/src/setup-databases.test.ts
View file

@ -1,7 +1,7 @@
import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
type KtxSetupDatabaseDriver,
@ -546,12 +546,11 @@ describe('setup databases step', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - databases',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({ multiselectValues: [['back']], selectValues: ['continue'] });
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
@ -588,12 +587,11 @@ describe('setup databases step', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - databases',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
selectValues: ['add', 'url', 'continue'],
multiselectValues: [['mysql']],
@ -704,12 +702,11 @@ describe('setup databases step', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - databases',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const io = makeIo();
const prompts = makePromptAdapter({
multiselectValues: [[]],
@ -1122,7 +1119,6 @@ describe('setup databases step', () => {
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: [],
});
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
expect(io.stdout()).toContain('Primary source ready');
@ -1161,7 +1157,6 @@ describe('setup databases step', () => {
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: [],
});
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
});
@ -1211,7 +1206,7 @@ describe('setup databases step', () => {
expect(scanConnection).toHaveBeenCalledTimes(2);
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.database_connection_ids).toEqual(['warehouse', 'analytics']);
expect(config.setup?.completed_steps).toEqual([]);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('databases');
});
@ -1237,7 +1232,7 @@ describe('setup databases step', () => {
expect(result.status).toBe('failed');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL' });
expect(config.setup?.completed_steps ?? []).not.toContain('databases');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(io.stderr()).toContain('Structural scan failed for warehouse.');
});
@ -1537,7 +1532,6 @@ describe('setup databases step', () => {
expect(result.status).toBe('skipped');
expect(io.stdout()).toContain('KTX cannot work until you add a primary source.');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps ?? []).not.toContain('databases');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
});
});

9
packages/cli/src/setup-databases.ts
View file

@ -7,7 +7,6 @@ import {
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
setKtxSetupDatabaseConnectionIds,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxTableListEntry } from '@ktx/context/scan';
import type { KtxCliIo } from './cli-runtime.js';
@ -1018,7 +1017,7 @@ async function writeConnectionConfig(input: {
[input.connectionId]: input.connection,
},
};
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
const historicSql =
typeof input.connection.historicSql === 'object' &&
@ -1312,7 +1311,7 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void
await writeFile(
project.configPath,
serializeKtxProjectConfig(
stripKtxSetupCompletedSteps({
{
...project.config,
ingest: {
...project.config.ingest,
@ -1322,7 +1321,7 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void
maxConcurrency,
},
},
}),
},
),
'utf-8',
);
@ -1331,7 +1330,7 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void
async function markDatabasesComplete(projectDir: string, connectionIds: string[]): Promise<void> {
const project = await loadKtxProject({ projectDir });
const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds));
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'databases');
}

15
packages/cli/src/setup-embeddings.test.ts
View file

@ -1,7 +1,7 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js';
@ -172,7 +172,7 @@ describe('setup embeddings step', () => {
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
expect(spinnerEvents).toContainEqual(
'start:Testing local sentence-transformers embeddings (all-MiniLM-L6-v2, 384 dimensions). First run may take up to 60 seconds.',
@ -251,7 +251,7 @@ describe('setup embeddings step', () => {
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
});
@ -301,7 +301,7 @@ describe('setup embeddings step', () => {
expect(result.status).toBe('failed');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps ?? []).not.toContain('embeddings');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(config.ingest.embeddings.backend).toBe('deterministic');
expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]');
expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings');
@ -413,7 +413,7 @@ describe('setup embeddings step', () => {
expect(result.status).toBe('skipped');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps ?? []).not.toContain('embeddings');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(config.ingest.embeddings.backend).toBe('deterministic');
});
@ -450,10 +450,6 @@ describe('setup embeddings step', () => {
'project: warehouse',
'setup:',
' database_connection_ids: []',
' completed_steps:',
' - project',
' - llm',
' - embeddings',
'connections: {}',
'ingest:',
' embeddings:',
@ -466,6 +462,7 @@ describe('setup embeddings step', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'llm', 'embeddings'] });
const healthCheck = vi.fn(async () => ({ ok: true as const }));
await expect(

28
packages/cli/src/setup-embeddings.ts
View file

@ -4,12 +4,10 @@ import { resolveKtxConfigReference } from '@ktx/context/core';
import {
type KtxProjectConfig,
type KtxProjectEmbeddingConfig,
ktxSetupCompletedSteps,
loadKtxProject,
markKtxSetupStateStepComplete,
readKtxSetupState,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
@ -110,7 +108,7 @@ function createPromptAdapter(): KtxSetupEmbeddingsPromptAdapter {
async function hasCompletedEmbeddings(projectDir: string, config: KtxProjectConfig): Promise<boolean> {
return (
ktxSetupCompletedSteps(config, await readKtxSetupState(projectDir)).includes('embeddings') &&
(await readKtxSetupState(projectDir)).completed_steps.includes('embeddings') &&
config.ingest.embeddings.backend !== 'none' &&
config.ingest.embeddings.backend !== 'deterministic' &&
typeof config.ingest.embeddings.model === 'string' &&
@ -184,22 +182,20 @@ function embeddingBackendDisplayName(backend: KtxSetupEmbeddingBackend): string
async function persistEmbeddingConfig(projectDir: string, embeddings: KtxProjectEmbeddingConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
const config = stripKtxSetupCompletedSteps(
{
...project.config,
ingest: {
...project.config.ingest,
const config = {
...project.config,
ingest: {
...project.config.ingest,
embeddings,
},
scan: {
...project.config.scan,
enrichment: {
...project.config.scan.enrichment,
embeddings,
},
scan: {
...project.config.scan,
enrichment: {
...project.config.scan.enrichment,
embeddings,
},
},
},
);
};
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'embeddings');
}

381
packages/cli/src/setup-models.test.ts
View file

@ -1,14 +1,16 @@
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
BUNDLED_ANTHROPIC_MODELS,
fetchAnthropicModels,
type KtxSetupModelPromptAdapter,
runKtxSetupGcloudApplicationDefaultAuth,
runKtxSetupAnthropicModelStep,
} from './setup-models.js';
import type { KtxCliIo } from './cli-runtime.js';
function makeIo() {
let stdout = '';
@ -33,6 +35,7 @@ function makeIo() {
}
function makePromptAdapter(options: {
providerChoice?: string;
selectValues?: string[];
credentialChoice?: string;
modelChoice?: string;
@ -43,8 +46,20 @@ function makePromptAdapter(options: {
const selectValues = [...(options.selectValues ?? [])];
const textValues = [...(options.textValues ?? [])];
const passwordValues = [...(options.passwordValues ?? [])];
let providerPromptCount = 0;
return {
select: vi.fn(async ({ message }) => {
if (message.includes('LLM provider')) {
providerPromptCount += 1;
const nextProviderChoice = selectValues[0];
if (nextProviderChoice === 'anthropic' || nextProviderChoice === 'vertex' || nextProviderChoice === 'back') {
return selectValues.shift() ?? nextProviderChoice;
}
if (options.credentialChoice === 'back' && providerPromptCount > 1) {
return 'back';
}
return options.providerChoice ?? 'anthropic';
}
const nextValue = selectValues.shift();
if (nextValue) {
return nextValue;
@ -55,7 +70,10 @@ function makePromptAdapter(options: {
return options.modelChoice ?? 'claude-sonnet-4-6';
}),
text: vi.fn(async () => textValues.shift() ?? ''),
password: vi.fn(async () => (passwordValues.length > 0 ? passwordValues.shift() : options.passwordValue ?? 'sk-ant-pasted')),
password: vi.fn(
async () =>
passwordValues.length > 0 ? passwordValues.shift() : options.passwordValue ?? 'sk-ant-pasted', // pragma: allowlist secret
),
cancel: vi.fn(),
};
}
@ -89,7 +107,7 @@ describe('setup Anthropic model step', () => {
),
);
await expect(fetchAnthropicModels('sk-ant-test', fetchModels)).resolves.toEqual([
await expect(fetchAnthropicModels('sk-ant-test', fetchModels)).resolves.toEqual([ // pragma: allowlist secret
{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true },
{ id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false },
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false },
@ -107,7 +125,7 @@ describe('setup Anthropic model step', () => {
makeIo().io,
{
prompts,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
listModels: vi.fn(async () => [
{ id: 'claude-sonnet-4', label: 'Claude Sonnet 4', recommended: true },
{ id: 'claude-opus-4', label: 'Claude Opus 4', recommended: false },
@ -132,19 +150,58 @@ describe('setup Anthropic model step', () => {
);
});
it('offers Vertex AI as an Anthropic model provider option', async () => {
const prompts = makePromptAdapter({ providerChoice: 'back' });
const result = await runKtxSetupAnthropicModelStep(
{ projectDir: tempDir, inputMode: 'auto', skipLlm: false },
makeIo().io,
{ prompts, env: {} },
);
expect(result.status).toBe('back');
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('Which LLM provider should KTX use?'),
options: expect.arrayContaining([
{ value: 'vertex', label: 'Google Vertex AI for Anthropic Claude' },
{ value: 'back', label: 'Back' },
]),
}),
);
});
it('returns from Anthropic credential Back to provider selection', async () => {
const prompts = makePromptAdapter({ selectValues: ['anthropic', 'back', 'back'] });
const result = await runKtxSetupAnthropicModelStep(
{ projectDir: tempDir, inputMode: 'auto', skipLlm: false },
makeIo().io,
{ prompts, env: {} },
);
expect(result.status).toBe('back');
expect(prompts.select).toHaveBeenNthCalledWith(
3,
expect.objectContaining({
message: expect.stringContaining('Which LLM provider should KTX use?'),
}),
);
});
it('configures env credentials, selected model, prompt caching, and llm completion state', async () => {
const io = makeIo();
const result = await runKtxSetupAnthropicModelStep(
{
projectDir: tempDir,
inputMode: 'disabled',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
},
io.io,
{
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
healthCheck: vi.fn(async () => ({ ok: true as const })),
},
);
@ -154,22 +211,270 @@ describe('setup Anthropic model step', () => {
expect(config.llm).toMatchObject({
provider: {
backend: 'anthropic',
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' },
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
},
models: { default: 'claude-sonnet-4-6' },
promptCaching: { enabled: true },
});
expect(config.scan.enrichment.mode).toBe('llm');
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stdout()).toContain('LLM ready: yes');
expect(io.stdout()).not.toContain('sk-ant-test');
});
it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => {
const io = makeIo();
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const result = await runKtxSetupAnthropicModelStep(
{
projectDir: tempDir,
inputMode: 'disabled',
llmBackend: 'vertex',
vertexProject: 'local-gcp-project',
vertexLocation: 'us-east5',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
},
io.io,
{ env: {}, healthCheck },
);
expect(result.status).toBe('ready');
expect(healthCheck).toHaveBeenCalledWith({
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
modelSlots: { default: 'claude-sonnet-4-6' },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.llm).toMatchObject({
provider: {
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
},
models: { default: 'claude-sonnet-4-6' },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
});
expect(config.scan.enrichment.mode).toBe('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)');
});
it('can run gcloud auth for Vertex AI and infer project and default location', async () => {
const io = makeIo();
const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] });
const runGcloudAuth = vi.fn(async () => ({ ok: true as const }));
const readGcloudProject = vi.fn(async () => 'local-gcp-project');
const listGcloudProjects = vi.fn(async () => [
{ projectId: 'local-gcp-project', name: 'Local project' },
{ projectId: 'other-gcp-project', name: 'Other project' },
]);
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const result = await runKtxSetupAnthropicModelStep(
{ projectDir: tempDir, inputMode: 'auto', skipLlm: false },
io.io,
{
prompts,
env: {},
runGcloudAuth,
readGcloudProject,
listGcloudProjects,
healthCheck,
},
);
expect(result.status).toBe('ready');
expect(runGcloudAuth).toHaveBeenCalledWith(io.io);
expect(readGcloudProject).toHaveBeenCalled();
expect(listGcloudProjects).toHaveBeenCalled();
expect(prompts.text).not.toHaveBeenCalled();
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('Which Google Cloud project should KTX use for Vertex AI?'),
options: [
{ value: 'local-gcp-project', label: 'local-gcp-project - Local project (current gcloud project)' },
{ value: 'other-gcp-project', label: 'other-gcp-project - Other project' },
{ value: 'manual', label: 'Enter a project ID manually' },
{ value: 'back', label: 'Back' },
],
}),
);
expect(healthCheck).toHaveBeenCalledWith({
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
modelSlots: { default: 'claude-sonnet-4-6' },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.llm.provider).toMatchObject({
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
});
});
it('lets users choose a different visible gcloud project for Vertex AI', async () => {
const io = makeIo();
const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'other-gcp-project', 'claude-sonnet-4-6'] });
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const result = await runKtxSetupAnthropicModelStep(
{ projectDir: tempDir, inputMode: 'auto', skipLlm: false },
io.io,
{
prompts,
env: {},
readGcloudProject: vi.fn(async () => 'current-gcp-project'),
listGcloudProjects: vi.fn(async () => [
{ projectId: 'current-gcp-project', name: 'Current project' },
{ projectId: 'other-gcp-project', name: 'Other project' },
]),
healthCheck,
},
);
expect(result.status).toBe('ready');
expect(healthCheck).toHaveBeenCalledWith({
backend: 'vertex',
vertex: { project: 'other-gcp-project', location: 'us-east5' },
modelSlots: { default: 'claude-sonnet-4-6' },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.llm.provider).toMatchObject({
backend: 'vertex',
vertex: { project: 'other-gcp-project', location: 'us-east5' },
});
});
it('allows manual Vertex AI project entry when gcloud project listing is empty', async () => {
const io = makeIo();
const prompts = makePromptAdapter({
selectValues: ['vertex', 'existing', 'manual', 'claude-sonnet-4-6'],
textValues: ['manual-gcp-project'],
});
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const result = await runKtxSetupAnthropicModelStep(
{ projectDir: tempDir, inputMode: 'auto', skipLlm: false },
io.io,
{
prompts,
env: {},
readGcloudProject: vi.fn(async () => undefined),
listGcloudProjects: vi.fn(async () => []),
healthCheck,
},
);
expect(result.status).toBe('ready');
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('Which Google Cloud project should KTX use for Vertex AI?'),
options: [
{ value: 'manual', label: 'Enter a project ID manually' },
{ value: 'back', label: 'Back' },
],
}),
);
expect(prompts.text).toHaveBeenCalledWith(
expect.objectContaining({
message: 'Google Cloud project ID\n│ Press Escape to go back.\n│',
}),
);
expect(healthCheck).toHaveBeenCalledWith(
expect.objectContaining({
vertex: { project: 'manual-gcp-project', location: 'us-east5' },
}),
);
});
it('returns from Vertex AI project selection Back to provider selection', async () => {
const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'back', 'back'] });
const result = await runKtxSetupAnthropicModelStep(
{ projectDir: tempDir, inputMode: 'auto', skipLlm: false },
makeIo().io,
{
prompts,
env: {},
readGcloudProject: vi.fn(async () => 'current-gcp-project'),
listGcloudProjects: vi.fn(async () => [{ projectId: 'current-gcp-project', name: 'Current project' }]),
},
);
expect(result.status).toBe('back');
expect(prompts.select).toHaveBeenNthCalledWith(
4,
expect.objectContaining({
message: expect.stringContaining('Which LLM provider should KTX use?'),
}),
);
});
it('runs only gcloud application-default login for Vertex AI auth', async () => {
const io = makeIo();
const runGcloud = vi.fn(async () => ({ ok: true as const }));
await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true });
expect(runGcloud).toHaveBeenCalledTimes(1);
expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything());
expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything());
expect(io.stdout()).toContain('gcloud auth application-default login');
expect(io.stdout()).not.toContain('gcloud auth login');
});
it('indents gcloud auth output inside the setup gutter', async () => {
const io = makeIo();
const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => {
commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n');
commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n');
return { ok: true as const };
});
await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true });
expect(io.stdout()).toContain('│ Your browser has been opened to visit:');
expect(io.stdout()).toContain('│ https://accounts.example/auth');
expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]');
expect(io.stdout()).not.toContain('\nYour browser has been opened');
});
it('explains common Vertex AI Forbidden health-check causes', async () => {
const io = makeIo();
const result = await runKtxSetupAnthropicModelStep(
{
projectDir: tempDir,
inputMode: 'disabled',
llmBackend: 'vertex',
vertexProject: 'kaelio-orbit-looker-20260430',
vertexLocation: 'us-east5',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
},
io.io,
{
env: {},
healthCheck: vi.fn(async () => ({ ok: false as const, message: 'Forbidden' })),
},
);
expect(result.status).toBe('failed');
expect(io.stderr()).toContain('project kaelio-orbit-looker-20260430');
expect(io.stderr()).toContain('Vertex AI API is enabled');
expect(io.stderr()).toContain('Anthropic Claude model access');
expect(io.stderr()).toContain('roles/aiplatform.user');
});
it('resolves --anthropic-api-key-file for health checks and stores a file reference', async () => {
const io = makeIo();
const secretPath = join(tempDir, 'anthropic-api-key');
await writeFile(secretPath, 'sk-ant-file', 'utf-8');
await writeFile(secretPath, 'sk-ant-file', 'utf-8'); // pragma: allowlist secret
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const result = await runKtxSetupAnthropicModelStep(
@ -187,7 +492,7 @@ describe('setup Anthropic model step', () => {
expect(result.status).toBe('ready');
expect(healthCheck).toHaveBeenCalledWith(
expect.objectContaining({
anthropic: { apiKey: 'sk-ant-file' },
anthropic: { apiKey: 'sk-ant-file' }, // pragma: allowlist secret
modelSlots: { default: 'claude-sonnet-4-6' },
}),
);
@ -195,11 +500,11 @@ describe('setup Anthropic model step', () => {
expect(config.llm).toMatchObject({
provider: {
backend: 'anthropic',
anthropic: { api_key: `file:${secretPath}` },
anthropic: { api_key: `file:${secretPath}` }, // pragma: allowlist secret
},
models: { default: 'claude-sonnet-4-6' },
});
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stdout()).not.toContain('sk-ant-file');
});
@ -249,11 +554,11 @@ describe('setup Anthropic model step', () => {
{
projectDir: tempDir,
inputMode: 'disabled',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
skipLlm: false,
},
io.io,
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, healthCheck },
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, healthCheck }, // pragma: allowlist secret
);
expect(result.status).toBe('missing-input');
@ -267,7 +572,7 @@ describe('setup Anthropic model step', () => {
const prompts = makePromptAdapter({
credentialChoice: 'paste',
modelChoice: 'claude-sonnet-4-6',
passwordValue: 'sk-ant-pasted',
passwordValue: 'sk-ant-pasted', // pragma: allowlist secret
});
const result = await runKtxSetupAnthropicModelStep(
@ -282,7 +587,7 @@ describe('setup Anthropic model step', () => {
);
expect(result.status).toBe('ready');
await expect(readFile(join(tempDir, '.ktx/secrets/anthropic-api-key'), 'utf-8')).resolves.toBe('sk-ant-pasted\n');
await expect(readFile(join(tempDir, '.ktx/secrets/anthropic-api-key'), 'utf-8')).resolves.toBe('sk-ant-pasted\n'); // pragma: allowlist secret
if (process.platform !== 'win32') {
expect((await stat(join(tempDir, '.ktx/secrets/anthropic-api-key'))).mode & 0o777).toBe(0o600);
}
@ -295,7 +600,7 @@ describe('setup Anthropic model step', () => {
it('opens pasted key entry directly and tells users Escape goes back', async () => {
const prompts = makePromptAdapter({
selectValues: ['paste', 'claude-sonnet-4-6'],
passwordValue: 'sk-ant-pasted',
passwordValue: 'sk-ant-pasted', // pragma: allowlist secret
});
const result = await runKtxSetupAnthropicModelStep(
@ -370,7 +675,7 @@ describe('setup Anthropic model step', () => {
makeIo().io,
{
prompts,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]),
},
);
@ -401,7 +706,7 @@ describe('setup Anthropic model step', () => {
io.io,
{
prompts,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
listModels: vi.fn(async () => [{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }]),
healthCheck: vi.fn(async () => ({ ok: true as const })),
},
@ -424,7 +729,7 @@ describe('setup Anthropic model step', () => {
await expect(
runKtxSetupAnthropicModelStep({ projectDir: tempDir, inputMode: 'auto', skipLlm: false }, io.io, {
prompts,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
listModels: vi.fn(async () => {
throw new Error('network unavailable');
}),
@ -444,7 +749,7 @@ describe('setup Anthropic model step', () => {
io.io,
{
prompts,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
listModels: vi.fn(async () => {
throw new Error('network unavailable');
}),
@ -504,20 +809,19 @@ describe('setup Anthropic model step', () => {
{
projectDir: tempDir,
inputMode: 'disabled',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
},
io.io,
{
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
healthCheck: vi.fn(async () => ({ ok: false as const, message: '401 invalid x-api-key [redacted]' })),
},
);
expect(result.status).toBe('failed');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps ?? []).not.toContain('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(io.stderr()).toContain('Anthropic model health check failed: 401 invalid x-api-key [redacted]');
expect(io.stderr()).not.toContain('sk-ant-test');
});
@ -537,7 +841,7 @@ describe('setup Anthropic model step', () => {
io.io,
{
prompts,
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
listModels: vi.fn(async () => [
{ id: 'claude-haiku-3-5', label: 'Claude Haiku 3.5', recommended: false },
{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true },
@ -548,12 +852,12 @@ describe('setup Anthropic model step', () => {
expect(result.status).toBe('ready');
expect(healthCheck).toHaveBeenCalledTimes(2);
expect(prompts.select).toHaveBeenCalledTimes(4);
expect(prompts.select).toHaveBeenCalledTimes(5);
expect(io.stderr()).toContain('Anthropic model health check failed: model not found');
expect(io.stderr()).toContain('Choose a different credential source or model, or Back.');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.llm.models.default).toBe('claude-sonnet-4-6');
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(io.stderr()).not.toContain('sk-ant-test');
});
@ -565,8 +869,7 @@ describe('setup Anthropic model step', () => {
);
expect(result.status).toBe('skipped');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps ?? []).not.toContain('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
});
it('returns back without writing config when Back is selected', async () => {
@ -585,7 +888,7 @@ describe('setup Anthropic model step', () => {
it('returns from model selection Back to credential selection instead of exiting setup', async () => {
const prompts = makePromptAdapter({
selectValues: ['paste', 'back', 'back'],
passwordValue: 'sk-ant-pasted',
passwordValue: 'sk-ant-pasted', // pragma: allowlist secret
});
const result = await runKtxSetupAnthropicModelStep(
@ -601,7 +904,7 @@ describe('setup Anthropic model step', () => {
expect(result.status).toBe('back');
expect(prompts.select).toHaveBeenNthCalledWith(
3,
4,
expect.objectContaining({
message: expect.stringContaining('How should KTX find your Anthropic API key?'),
}),
@ -637,7 +940,7 @@ describe('setup Anthropic model step', () => {
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.llm.provider).toMatchObject({
backend: 'anthropic',
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' },
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
});
});
@ -650,15 +953,12 @@ describe('setup Anthropic model step', () => {
'project: warehouse',
'setup:',
' database_connection_ids: []',
' completed_steps:',
' - project',
' - llm',
'connections: {}',
'llm:',
' provider:',
' backend: anthropic',
' anthropic:',
' api_key: env:ANTHROPIC_API_KEY',
' api_key: env:ANTHROPIC_API_KEY', // pragma: allowlist secret
' models:',
' default: claude-sonnet-4-6',
'ingest:',
@ -669,11 +969,12 @@ describe('setup Anthropic model step', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'llm'] });
const healthCheck = vi.fn(async () => ({ ok: true as const }));
await expect(
runKtxSetupAnthropicModelStep({ projectDir: tempDir, inputMode: 'disabled', skipLlm: false }, makeIo().io, {
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
healthCheck,
}),
).resolves.toMatchObject({ status: 'ready' });
@ -698,9 +999,6 @@ describe('setup Anthropic model step', () => {
'project: warehouse',
'setup:',
' database_connection_ids: []',
' completed_steps:',
' - project',
' - llm',
'connections: {}',
'llm:',
' provider:',
@ -715,6 +1013,7 @@ describe('setup Anthropic model step', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'llm'] });
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const io = makeIo();

611
packages/cli/src/setup-models.ts
View file

@ -1,4 +1,6 @@
import { execFile, spawn } from 'node:child_process';
import { writeFile } from 'node:fs/promises';
import { promisify } from 'node:util';
import { cancel, isCancel, password, select, text } from '@clack/prompts';
import { resolveLocalKtxLlmConfig } from '@ktx/context';
import { resolveKtxConfigReference } from '@ktx/context/core';
@ -8,7 +10,6 @@ import {
loadKtxProject,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
@ -19,9 +20,12 @@ import { envCredentialReference, writeProjectLocalSecretReference } from './setu
export interface KtxSetupModelArgs {
projectDir: string;
inputMode: 'auto' | 'disabled';
llmBackend?: KtxSetupLlmBackend;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
anthropicModel?: string;
vertexProject?: string;
vertexLocation?: string;
forcePrompt?: boolean;
showPromptInstructions?: boolean;
skipLlm: boolean;
@ -40,6 +44,8 @@ export interface AnthropicModelChoice {
recommended: boolean;
}
export type KtxSetupLlmBackend = 'anthropic' | 'vertex';
export interface KtxSetupModelPromptAdapter {
select(options: { message: string; options: Array<{ value: string; label: string }> }): Promise<string>;
text(options: { message: string; placeholder?: string }): Promise<string | undefined>;
@ -53,6 +59,9 @@ export interface KtxSetupModelDeps {
prompts?: KtxSetupModelPromptAdapter;
listModels?: (apiKey: string) => Promise<AnthropicModelChoice[]>;
healthCheck?: (config: KtxLlmConfig) => Promise<KtxLlmHealthCheckResult>;
runGcloudAuth?: (io: KtxCliIo) => Promise<GcloudAuthResult>;
readGcloudProject?: () => Promise<string | undefined>;
listGcloudProjects?: () => Promise<GcloudProjectChoice[]>;
}
export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07';
@ -79,6 +88,16 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT =
'KTX uses this as the default model for ingest agents that turn schemas, SQL, BI metadata, and docs ' +
'into semantic-layer sources and wiki context.';
const VERTEX_AUTH_PROMPT_CONTEXT =
'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' +
'gcloud browser login flow and does not store Google credentials in ktx.yaml.';
const VERTEX_PROJECT_PROMPT_CONTEXT =
'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' +
'access. Project visibility depends on the signed-in Google account and organization permissions.';
const DEFAULT_VERTEX_LOCATION = 'us-east5';
const execFileAsync = promisify(execFile);
type AnthropicModelDiscoveryErrorReason = 'authentication' | 'http' | 'empty-response';
export class AnthropicModelDiscoveryError extends Error {
@ -104,6 +123,27 @@ type ChooseModelResult =
| { status: 'ready'; model: string }
| { status: 'back' | 'missing-input' | 'invalid-credential' };
type ChooseBackendResult =
| { status: 'ready'; backend: KtxSetupLlmBackend; prompted: boolean }
| { status: 'back' };
type VertexConfigChoice =
| {
status: 'ready';
refs: { project?: string; location: string };
values: { project?: string; location: string };
}
| { status: 'back' | 'missing-input' };
type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' };
export type GcloudAuthResult = { ok: true } | { ok: false; message: string };
interface GcloudProjectChoice {
projectId: string;
name?: string;
}
type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise<GcloudAuthResult>;
function createPromptAdapter(): KtxSetupModelPromptAdapter {
return {
async select(options) {
@ -132,6 +172,122 @@ function createPromptAdapter(): KtxSetupModelPromptAdapter {
};
}
function createIndentedCommandIo(io: KtxCliIo): KtxCliIo {
const indentedWriter = (write: (chunk: string) => void) => {
let atLineStart = true;
return (chunk: string) => {
for (const char of chunk) {
if (atLineStart) {
write('│ ');
atLineStart = false;
}
write(char);
if (char === '\n') {
atLineStart = true;
}
}
};
};
return {
stdout: {
isTTY: io.stdout.isTTY,
columns: io.stdout.columns,
write: indentedWriter((chunk) => io.stdout.write(chunk)),
},
stderr: {
write: indentedWriter((chunk) => io.stderr.write(chunk)),
},
};
}
function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise<GcloudAuthResult> {
return new Promise((resolve) => {
let settled = false;
const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] });
child.stdout?.on('data', (chunk: Buffer) => {
io.stdout.write(chunk.toString('utf8'));
});
child.stderr?.on('data', (chunk: Buffer) => {
io.stderr.write(chunk.toString('utf8'));
});
child.on('error', (error: NodeJS.ErrnoException) => {
if (settled) {
return;
}
settled = true;
if (error.code === 'ENOENT') {
resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' });
return;
}
resolve({ ok: false, message: error.message });
});
child.on('close', (code, signal) => {
if (settled) {
return;
}
settled = true;
if (code === 0) {
resolve({ ok: true });
return;
}
resolve({
ok: false,
message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`,
});
});
});
}
export async function runKtxSetupGcloudApplicationDefaultAuth(
io: KtxCliIo,
runGcloud: GcloudCommandRunner = runInteractiveGcloud,
): Promise<GcloudAuthResult> {
io.stdout.write('│ Running gcloud auth application-default login...\n');
return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io));
}
async function defaultReadGcloudProject(): Promise<string | undefined> {
try {
const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' });
const value = stdout.trim();
return value && value !== '(unset)' ? value : undefined;
} catch {
return undefined;
}
}
async function defaultListGcloudProjects(): Promise<GcloudProjectChoice[]> {
try {
const { stdout } = await execFileAsync('gcloud', ['projects', 'list', '--format=json(projectId,name)'], {
encoding: 'utf8',
});
const parsed = JSON.parse(stdout.trim() || '[]') as unknown;
if (!Array.isArray(parsed)) {
return [];
}
return parsed
.map((item): GcloudProjectChoice | undefined => {
if (!item || typeof item !== 'object') {
return undefined;
}
const record = item as { projectId?: unknown; name?: unknown };
if (typeof record.projectId !== 'string' || !record.projectId.trim()) {
return undefined;
}
const name = typeof record.name === 'string' && record.name.trim() ? record.name.trim() : undefined;
return {
projectId: record.projectId.trim(),
...(name ? { name } : {}),
};
})
.filter((project): project is GcloudProjectChoice => Boolean(project));
} catch {
return [];
}
}
export async function fetchAnthropicModels(
apiKey: string,
fetchFn: typeof fetch = fetch,
@ -196,20 +352,33 @@ function hasUsableConfiguredLlm(config: KtxProjectConfig): boolean {
function buildProjectLlmConfig(
existing: KtxProjectLlmConfig,
credentialRef: string,
provider:
| { backend: 'anthropic'; credentialRef: string }
| { backend: 'vertex'; vertex: { project?: string; location: string } },
model: string,
): KtxProjectLlmConfig {
if (provider.backend === 'vertex') {
return {
provider: {
backend: 'vertex',
vertex: provider.vertex,
},
models: { ...existing.models, default: model },
promptCaching: { ...(existing.promptCaching ?? {}), enabled: true, vertexFallbackTo5m: true },
};
}
return {
provider: {
backend: 'anthropic',
anthropic: { api_key: credentialRef },
anthropic: { api_key: provider.credentialRef },
},
models: { ...existing.models, default: model },
promptCaching: { ...(existing.promptCaching ?? {}), enabled: true },
};
}
function buildHealthConfig(credentialValue: string, model: string): KtxLlmConfig {
function buildAnthropicHealthConfig(credentialValue: string, model: string): KtxLlmConfig {
return {
backend: 'anthropic',
anthropic: { apiKey: credentialValue },
@ -218,6 +387,28 @@ function buildHealthConfig(credentialValue: string, model: string): KtxLlmConfig
};
}
function buildVertexHealthConfig(vertex: { project?: string; location: string }, model: string): KtxLlmConfig {
return {
backend: 'vertex',
vertex,
modelSlots: { default: model },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
};
}
function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string {
const trimmed = message.trim() || 'unknown error';
if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) {
return trimmed;
}
return (
`${trimmed}. Check that Vertex AI API is enabled for project ${vertex.project ?? '(unknown)'}, ` +
`Anthropic Claude model access is enabled for location ${vertex.location}, and that your Application Default ` +
'Credentials principal has Vertex AI User (roles/aiplatform.user) or equivalent permissions.'
);
}
async function chooseCredentialRef(
args: KtxSetupModelArgs,
io: KtxCliIo,
@ -299,6 +490,266 @@ async function chooseCredentialRef(
}
}
function requestedBackend(args: KtxSetupModelArgs): KtxSetupLlmBackend | undefined {
if (args.llmBackend) {
return args.llmBackend;
}
if (args.vertexProject || args.vertexLocation) {
return 'vertex';
}
if (args.anthropicApiKeyEnv || args.anthropicApiKeyFile || args.anthropicModel) {
return 'anthropic';
}
return undefined;
}
async function chooseBackend(
args: KtxSetupModelArgs,
io: KtxCliIo,
deps: KtxSetupModelDeps,
): Promise<ChooseBackendResult> {
const explicit = requestedBackend(args);
if (explicit) {
return { status: 'ready', backend: explicit, prompted: false };
}
if (args.inputMode === 'disabled') {
return { status: 'ready', backend: 'anthropic', prompted: false };
}
const prompts = deps.prompts ?? createPromptAdapter();
if (args.showPromptInstructions !== false) {
io.stdout.write(
'│ Use Up/Down to move, Enter to confirm the current selection, choose Back to return to the previous step, Ctrl+C to exit.\n',
);
}
const choice = await prompts.select({
message: 'Which LLM provider should KTX use?',
options: [
{ value: 'anthropic', label: 'Anthropic API' },
{ value: 'vertex', label: 'Google Vertex AI for Anthropic Claude' },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') {
return { status: 'back' };
}
return { status: 'ready', backend: choice === 'vertex' ? 'vertex' : 'anthropic', prompted: true };
}
async function chooseVertexAuth(
args: KtxSetupModelArgs,
io: KtxCliIo,
deps: KtxSetupModelDeps,
): Promise<VertexAuthChoice> {
if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) {
return { status: 'ready' };
}
const prompts = deps.prompts ?? createPromptAdapter();
const choice = await prompts.select({
message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`,
options: [
{ value: 'gcloud', label: 'Run gcloud Application Default Credentials login' },
{ value: 'existing', label: 'Use existing gcloud/Application Default Credentials' },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') {
return { status: 'back' };
}
if (choice !== 'gcloud') {
return { status: 'ready' };
}
const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io);
if (!result.ok) {
io.stderr.write(`gcloud authentication failed: ${result.message}\n`);
return { status: 'missing-input' };
}
return { status: 'ready' };
}
function resolveProvidedVertexRef(
label: 'project' | 'location',
ref: string,
env: NodeJS.ProcessEnv,
io: KtxCliIo,
): { status: 'ready'; ref: string; value: string } | { status: 'missing-input' } {
let value: string | undefined;
try {
value = resolveKtxConfigReference(ref, env);
} catch {
value = undefined;
}
if (!value) {
io.stderr.write(`Missing Vertex AI ${label}: ${ref} could not be resolved.\n`);
return { status: 'missing-input' };
}
return { status: 'ready', ref, value };
}
function normalizeGcloudProjectId(projectId: string | undefined): string | undefined {
const trimmed = projectId?.trim();
return trimmed ? trimmed : undefined;
}
function orderGcloudProjects(projects: GcloudProjectChoice[], currentProject: string | undefined): GcloudProjectChoice[] {
const ordered: GcloudProjectChoice[] = [];
const seen = new Set<string>();
const addProject = (project: GcloudProjectChoice) => {
const projectId = normalizeGcloudProjectId(project.projectId);
if (!projectId || seen.has(projectId)) {
return;
}
seen.add(projectId);
const name = normalizeGcloudProjectId(project.name);
ordered.push({
projectId,
...(name ? { name } : {}),
});
};
if (currentProject) {
addProject(projects.find((project) => project.projectId.trim() === currentProject) ?? { projectId: currentProject });
}
for (const project of projects) {
addProject(project);
}
return ordered;
}
function formatGcloudProjectLabel(project: GcloudProjectChoice, currentProject: string | undefined): string {
const name = project.name && project.name !== project.projectId ? ` - ${project.name}` : '';
const current = project.projectId === currentProject ? ' (current gcloud project)' : '';
return `${project.projectId}${name}${current}`;
}
async function chooseInteractiveVertexProject(
currentProject: string | undefined,
io: KtxCliIo,
deps: KtxSetupModelDeps,
): Promise<{ status: 'ready'; ref: string; value: string } | { status: 'back' | 'missing-input' }> {
const prompts = deps.prompts ?? createPromptAdapter();
let projects: GcloudProjectChoice[] = [];
try {
projects = await (deps.listGcloudProjects ?? defaultListGcloudProjects)();
} catch {
io.stderr.write('Could not list Google Cloud projects with gcloud. Enter a project ID manually or choose Back.\n');
}
const orderedProjects = orderGcloudProjects(projects, currentProject);
if (orderedProjects.length === 0) {
io.stdout.write('│ gcloud did not return any visible Google Cloud projects. Enter a project ID manually or choose Back.\n');
}
const choice = await prompts.select({
message: `Which Google Cloud project should KTX use for Vertex AI?\n\n${VERTEX_PROJECT_PROMPT_CONTEXT}`,
options: [
...orderedProjects.map((project) => ({
value: project.projectId,
label: formatGcloudProjectLabel(project, currentProject),
})),
{ value: 'manual', label: 'Enter a project ID manually' },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') {
return { status: 'back' };
}
if (choice === 'manual') {
const manual = await prompts.text({
message: withTextInputNavigation('Google Cloud project ID'),
placeholder: currentProject ?? orderedProjects[0]?.projectId,
});
if (manual === undefined) {
return { status: 'back' };
}
const project = normalizeGcloudProjectId(manual);
return project ? { status: 'ready', ref: project, value: project } : { status: 'missing-input' };
}
return { status: 'ready', ref: choice, value: choice };
}
async function chooseVertexConfig(
args: KtxSetupModelArgs,
io: KtxCliIo,
deps: KtxSetupModelDeps,
): Promise<VertexConfigChoice> {
const env = deps.env ?? process.env;
let projectRef: string | undefined;
let projectValue: string | undefined;
let gcloudProject: string | undefined;
if (args.vertexProject) {
const project = resolveProvidedVertexRef('project', args.vertexProject, env, io);
if (project.status !== 'ready') {
return { status: project.status };
}
projectRef = project.ref;
projectValue = project.value;
} else if (env.GOOGLE_VERTEX_PROJECT?.trim()) {
projectRef = envCredentialReference('GOOGLE_VERTEX_PROJECT');
projectValue = env.GOOGLE_VERTEX_PROJECT.trim();
} else {
gcloudProject = normalizeGcloudProjectId(await (deps.readGcloudProject ?? defaultReadGcloudProject)());
if (args.inputMode === 'disabled') {
if (gcloudProject) {
projectRef = gcloudProject;
projectValue = gcloudProject;
}
} else {
const project = await chooseInteractiveVertexProject(gcloudProject, io, deps);
if (project.status !== 'ready') {
return { status: project.status };
}
projectRef = project.ref;
projectValue = project.value;
}
}
let locationRef: string | undefined;
let locationValue: string | undefined;
if (args.vertexLocation) {
const location = resolveProvidedVertexRef('location', args.vertexLocation, env, io);
if (location.status !== 'ready') {
return { status: location.status };
}
locationRef = location.ref;
locationValue = location.value;
} else if (env.GOOGLE_VERTEX_LOCATION?.trim()) {
locationRef = envCredentialReference('GOOGLE_VERTEX_LOCATION');
locationValue = env.GOOGLE_VERTEX_LOCATION.trim();
} else {
locationRef = DEFAULT_VERTEX_LOCATION;
locationValue = DEFAULT_VERTEX_LOCATION;
}
if (!projectRef || !projectValue) {
io.stderr.write(
'Missing Vertex AI project: run `gcloud config set project PROJECT_ID`, pass --vertex-project, or set GOOGLE_VERTEX_PROJECT.\n',
);
return { status: 'missing-input' };
}
if (!locationRef || !locationValue) {
io.stderr.write('Missing Vertex AI location: pass --vertex-location.\n');
return { status: 'missing-input' };
}
return {
status: 'ready',
refs: {
...(projectRef ? { project: projectRef } : {}),
location: locationRef,
},
values: {
...(projectValue ? { project: projectValue } : {}),
location: locationValue,
},
};
}
async function chooseModel(
args: KtxSetupModelArgs,
credentialValue: string,
@ -360,30 +811,73 @@ async function chooseModel(
return { status: 'ready', model: choice };
}
async function persistLlmConfig(projectDir: string, credentialRef: string, model: string): Promise<void> {
async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: KtxSetupModelDeps): Promise<ChooseModelResult> {
if (args.anthropicModel) {
return { status: 'ready', model: args.anthropicModel };
}
if (args.inputMode === 'disabled') {
io.stderr.write('Missing Anthropic model: pass --anthropic-model.\n');
return { status: 'missing-input' };
}
const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel);
const prompts = deps.prompts ?? createPromptAdapter();
const choice = await prompts.select({
message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`,
options: [
...selectableModels.map((model) => ({
value: model.id,
label: `${model.label || model.id}${model.recommended ? ' (recommended)' : ''}`,
})),
{ value: 'manual', label: 'Enter a model ID manually' },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') {
return { status: 'back' };
}
if (choice === 'manual') {
const manual = await prompts.text({
message: withTextInputNavigation('Anthropic model ID'),
placeholder: selectableModels.find((model) => model.recommended)?.id ?? selectableModels[0]?.id,
});
if (manual === undefined) {
return { status: 'back' };
}
return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' };
}
return { status: 'ready', model: choice };
}
async function persistLlmConfig(
projectDir: string,
provider:
| { backend: 'anthropic'; credentialRef: string }
| { backend: 'vertex'; vertex: { project?: string; location: string } },
model: string,
): Promise<void> {
const project = await loadKtxProject({ projectDir });
const config = stripKtxSetupCompletedSteps(
{
...project.config,
llm: buildProjectLlmConfig(project.config.llm, credentialRef, model),
scan: {
...project.config.scan,
enrichment: {
...project.config.scan.enrichment,
mode: 'llm',
},
const config = {
...project.config,
llm: buildProjectLlmConfig(project.config.llm, provider, model),
scan: {
...project.config.scan,
enrichment: {
...project.config.scan.enrichment,
mode: 'llm' as const,
},
},
);
};
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'llm');
}
function buildInteractiveRetryArgs(args: KtxSetupModelArgs): KtxSetupModelArgs {
function buildInteractiveRetryArgs(args: KtxSetupModelArgs, backend?: KtxSetupLlmBackend): KtxSetupModelArgs {
return {
projectDir: args.projectDir,
inputMode: args.inputMode,
...(args.showPromptInstructions !== undefined ? { showPromptInstructions: args.showPromptInstructions } : {}),
...(backend ?? args.llmBackend ? { llmBackend: backend ?? args.llmBackend } : {}),
showPromptInstructions: false,
skipLlm: args.skipLlm,
};
}
@ -402,9 +896,12 @@ export async function runKtxSetupAnthropicModelStep(
if (
args.forcePrompt !== true &&
hasUsableConfiguredLlm(project.config) &&
!args.llmBackend &&
!args.anthropicApiKeyEnv &&
!args.anthropicApiKeyFile &&
!args.anthropicModel
!args.anthropicModel &&
!args.vertexProject &&
!args.vertexLocation
) {
io.stdout.write(`│ LLM ready: yes (${project.config.llm.models.default})\n`);
return { status: 'ready', projectDir: args.projectDir };
@ -414,31 +911,91 @@ export async function runKtxSetupAnthropicModelStep(
let attemptArgs = args;
while (true) {
const credential = await chooseCredentialRef(attemptArgs, io, deps);
const backendChoice = await chooseBackend(attemptArgs, io, deps);
if (backendChoice.status !== 'ready') {
return { status: backendChoice.status, projectDir: args.projectDir };
}
const backendArgs = backendChoice.prompted
? ({ ...attemptArgs, llmBackend: backendChoice.backend, showPromptInstructions: false } satisfies KtxSetupModelArgs)
: attemptArgs;
if (backendChoice.backend === 'vertex') {
const auth = await chooseVertexAuth(backendArgs, io, deps);
if (auth.status === 'back' && backendChoice.prompted) {
attemptArgs = buildInteractiveRetryArgs(args);
continue;
}
if (auth.status !== 'ready') {
return { status: auth.status, projectDir: args.projectDir };
}
const vertex = await chooseVertexConfig(backendArgs, io, deps);
if (vertex.status === 'back' && backendChoice.prompted) {
attemptArgs = buildInteractiveRetryArgs(args);
continue;
}
if (vertex.status !== 'ready') {
return { status: vertex.status, projectDir: args.projectDir };
}
const model = await chooseVertexModel(backendArgs, io, deps);
if (model.status === 'back' && !backendArgs.vertexLocation) {
attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend);
continue;
}
if (model.status === 'invalid-credential') {
return { status: 'failed', projectDir: args.projectDir };
}
if (model.status !== 'ready') {
return { status: model.status, projectDir: args.projectDir };
}
const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model));
if (health.ok) {
await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model);
io.stdout.write(`│ LLM ready: yes (${model.model})\n`);
return { status: 'ready', projectDir: args.projectDir };
}
io.stderr.write(`Vertex AI Anthropic model health check failed: ${formatVertexHealthFailure(health.message, vertex.values)}\n`);
if (args.inputMode === 'disabled') {
return { status: 'failed', projectDir: args.projectDir };
}
io.stderr.write('Choose a different Vertex AI project, location, or model, or Back.\n');
attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend);
continue;
}
const credential = await chooseCredentialRef(backendArgs, io, deps);
if (credential.status === 'back' && backendChoice.prompted) {
attemptArgs = buildInteractiveRetryArgs(args);
continue;
}
if (credential.status !== 'ready') {
return { status: credential.status, projectDir: args.projectDir };
}
const model = await chooseModel(attemptArgs, credential.value, io, deps);
const model = await chooseModel(backendArgs, credential.value, io, deps);
if (model.status === 'invalid-credential') {
if (args.inputMode === 'disabled') {
return { status: 'failed', projectDir: args.projectDir };
}
io.stderr.write('Choose a different credential source or Back.\n');
attemptArgs = buildInteractiveRetryArgs(args);
attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend);
continue;
}
if (model.status === 'back' && !attemptArgs.anthropicApiKeyEnv && !attemptArgs.anthropicApiKeyFile) {
attemptArgs = buildInteractiveRetryArgs(args);
if (model.status === 'back' && !backendArgs.anthropicApiKeyEnv && !backendArgs.anthropicApiKeyFile) {
attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend);
continue;
}
if (model.status !== 'ready') {
return { status: model.status, projectDir: args.projectDir };
}
const health = await healthCheck(buildHealthConfig(credential.value, model.model));
const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model));
if (health.ok) {
await persistLlmConfig(args.projectDir, credential.ref, model.model);
await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model);
io.stdout.write(`│ LLM ready: yes (${model.model})\n`);
return { status: 'ready', projectDir: args.projectDir };
}
@ -448,6 +1005,6 @@ export async function runKtxSetupAnthropicModelStep(
return { status: 'failed', projectDir: args.projectDir };
}
io.stderr.write('Choose a different credential source or model, or Back.\n');
attemptArgs = buildInteractiveRetryArgs(args);
attemptArgs = buildInteractiveRetryArgs(args, backendChoice.backend);
}
}

31
packages/cli/src/setup-project.test.ts
View file

@ -3,6 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { gray } from './io/symbols.js';
import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from './setup-project.js';
function makeIo(options: { stdoutIsTty?: boolean } = {}) {
@ -37,6 +38,12 @@ function makePromptAdapter(options: { choice?: string; choices?: string[]; textV
} satisfies KtxSetupProjectPromptAdapter;
}
function defaultSubfolderLabel(parentDir: string): string {
const childName = 'ktx-project';
const childDir = join(parentDir, childName);
return `New subfolder (${gray(childDir.slice(0, -childName.length))}${childName})`;
}
describe('setup project step', () => {
let tempDir: string;
@ -59,8 +66,7 @@ describe('setup project step', () => {
expect(result.status).toBe('ready');
expect(result.projectDir).toBe(projectDir);
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] });
await expect(stat(join(projectDir, '.git'))).resolves.toBeDefined();
await expect(readFile(join(projectDir, '.ktx/.gitignore'), 'utf-8')).resolves.toContain('secrets/');
@ -68,7 +74,7 @@ describe('setup project step', () => {
expect(testIo.stderr()).toBe('');
});
it('loads an existing project with --existing and preserves existing setup metadata', async () => {
it('loads an existing project with --existing and drops config setup progress', async () => {
const projectDir = join(tempDir, 'warehouse');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeFile(
@ -94,9 +100,9 @@ describe('setup project step', () => {
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
completed_steps: [],
});
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['llm', 'project'] });
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] });
});
it('creates a missing auto-mode project only when --yes is present in no-input mode', async () => {
@ -144,16 +150,18 @@ describe('setup project step', () => {
expect.objectContaining({
message: 'Where should KTX create the project?',
options: [
expect.objectContaining({ value: 'current', label: 'Current directory' }),
expect.objectContaining({ value: 'new-default', label: 'New subfolder (./ktx-project)' }),
expect.objectContaining({ value: 'current', label: `Current directory (${projectDir})` }),
expect.objectContaining({
value: 'new-default',
label: defaultSubfolderLabel(projectDir),
}),
expect.objectContaining({ value: 'new-custom', label: 'Custom path' }),
expect.objectContaining({ value: 'exit', label: 'Exit' }),
],
}),
);
expect(prompts.text).not.toHaveBeenCalled();
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(await readKtxSetupState(projectDir)).toEqual({ completed_steps: ['project'] });
});
@ -176,7 +184,10 @@ describe('setup project step', () => {
expect.objectContaining({
message: 'Where should KTX create the project?',
options: expect.arrayContaining([
expect.objectContaining({ value: 'new-default', label: 'New subfolder (./ktx-project)' }),
expect.objectContaining({
value: 'new-default',
label: defaultSubfolderLabel(startDir),
}),
]),
}),
);

18
packages/cli/src/setup-project.ts
View file

@ -5,17 +5,14 @@ import { basename, join, resolve } from 'node:path';
import { cancel, isCancel, select, text } from '@clack/prompts';
import {
initKtxProject,
ktxSetupCompletedSteps,
type KtxLocalProject,
loadKtxProject,
markKtxSetupStateStepComplete,
mergeKtxSetupGitignoreEntries,
readKtxSetupState,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
writeKtxSetupState,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { gray } from './io/symbols.js';
import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js';
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
@ -170,10 +167,7 @@ async function normalizeSetupGitignore(projectDir: string): Promise<void> {
}
async function persistProjectStep(project: KtxLocalProject): Promise<KtxLocalProject> {
const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(project.projectDir));
const config = stripKtxSetupCompletedSteps(project.config);
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
await writeKtxSetupState(project.projectDir, { completed_steps: completedSteps });
await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8');
await markKtxSetupStateStepComplete(project.projectDir, 'project');
await normalizeSetupGitignore(project.projectDir);
return await loadKtxProject({ projectDir: project.projectDir });
@ -328,6 +322,10 @@ export async function runKtxSetupProjectStep(
const prompts = deps.prompts ?? createClackSetupProjectPromptAdapter();
const defaultProjectDir = join(projectDir, DEFAULT_NEW_PROJECT_FOLDER_NAME);
const defaultProjectDirLabel = [
gray(defaultProjectDir.slice(0, -DEFAULT_NEW_PROJECT_FOLDER_NAME.length)),
DEFAULT_NEW_PROJECT_FOLDER_NAME,
].join('');
io.stdout.write(
'│ Use Up/Down to move, Enter to confirm the current selection, choose Back to return to the previous step, Ctrl+C to exit.\n',
);
@ -335,8 +333,8 @@ export async function runKtxSetupProjectStep(
const choice = await prompts.select({
message: 'Where should KTX create the project?',
options: [
{ value: 'current', label: 'Current directory' },
{ value: 'new-default', label: 'New subfolder (./ktx-project)' },
{ value: 'current', label: `Current directory (${projectDir})` },
{ value: 'new-default', label: `New subfolder (${defaultProjectDirLabel})` },
{ value: 'new-custom', label: 'Custom path' },
...(args.allowBack ? [{ value: 'back', label: 'Back' }] : []),
...(args.allowBack ? [] : [{ value: 'exit', label: 'Exit' }]),

21
packages/cli/src/setup-sources.test.ts
View file

@ -102,7 +102,6 @@ describe('setup sources step', () => {
},
setup: {
...config.setup,
completed_steps: config.setup?.completed_steps ?? [],
database_connection_ids: ['warehouse'],
},
}),
@ -137,7 +136,7 @@ describe('setup sources step', () => {
projectDir,
});
expect((await readConfig()).setup?.completed_steps).toEqual(undefined);
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(io.stdout()).toContain('Context source setup skipped.');
});
@ -171,7 +170,7 @@ describe('setup sources step', () => {
source_dir: '/repo/dbt',
project_name: 'analytics',
});
expect(config.setup?.completed_steps).toEqual([]);
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' });
});
@ -190,7 +189,7 @@ describe('setup sources step', () => {
source: 'metabase',
sourceConnectionId: 'prod_metabase',
sourceUrl: 'https://metabase.example.com',
sourceApiKeyRef: 'env:METABASE_API_KEY',
sourceApiKeyRef: 'env:METABASE_API_KEY', // pragma: allowlist secret
sourceWarehouseConnectionId: 'warehouse',
metabaseDatabaseId: 1,
runInitialSourceIngest: false,
@ -204,7 +203,7 @@ describe('setup sources step', () => {
expect((await readConfig()).connections.prod_metabase).toMatchObject({
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
@ -225,7 +224,7 @@ describe('setup sources step', () => {
inputMode: 'disabled',
source: 'notion',
sourceConnectionId: 'notion-main',
sourceApiKeyRef: 'env:NOTION_TOKEN',
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
notionCrawlMode: 'selected_roots',
notionRootPageIds: ['page-1'],
runInitialSourceIngest: false,
@ -256,7 +255,7 @@ describe('setup sources step', () => {
inputMode: 'disabled',
source: 'notion',
sourceConnectionId: 'notion-main',
sourceApiKeyRef: 'env:NOTION_TOKEN',
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
notionCrawlMode: 'all_accessible',
notionRootPageIds: ['page-1'],
runInitialSourceIngest: false,
@ -480,7 +479,7 @@ describe('setup sources step', () => {
),
).resolves.toEqual({ status: 'failed', projectDir });
expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources');
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(io.stderr()).toContain('No LookML files found');
});
@ -766,7 +765,7 @@ describe('setup sources step', () => {
connection: {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
@ -786,7 +785,7 @@ describe('setup sources step', () => {
driver: 'looker',
base_url: 'https://looker.example.com',
client_id: 'client-id',
client_secret_ref: 'env:LOOKER_CLIENT_SECRET',
client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret
mappings: { connectionMappings: { warehouse: 'warehouse' } },
},
deps: {
@ -1032,7 +1031,7 @@ describe('setup sources step', () => {
expect(testPrompts.multiselect).not.toHaveBeenCalled();
expect(io.stdout()).toContain('Connect a primary source before adding context sources.');
expect((await readConfig()).setup?.completed_steps ?? []).not.toContain('sources');
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
});
it('auto-detects dbt_project.yml at the root of a local path', async () => {

7
packages/cli/src/setup-sources.ts
View file

@ -25,7 +25,6 @@ import {
loadKtxProject,
markKtxSetupStateStepComplete,
serializeKtxProjectConfig,
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { runKtxConnectionMapping } from './commands/connection-mapping.js';
@ -345,7 +344,7 @@ function fileRepoUrl(sourceDir: string): string {
async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
}
async function writeSourceConnection(
@ -372,7 +371,7 @@ async function writeSourceConnection(
: [...project.config.ingest.adapters, adapter],
},
};
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
return async () => {
const latest = await loadKtxProject({ projectDir });
const connections = { ...latest.config.connections };
@ -411,7 +410,7 @@ async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSo
async function markSourcesComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(stripKtxSetupCompletedSteps(project.config)), 'utf-8');
await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8');
await markKtxSetupStateStepComplete(projectDir, 'sources');
}

117
packages/cli/src/setup.test.ts
View file

@ -3,6 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { promisify } from 'node:util';
import { writeKtxSetupState } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js';
@ -72,7 +73,7 @@ describe('setup status', () => {
' provider:',
' backend: anthropic',
' anthropic:',
' api_key: env:ANTHROPIC_API_KEY',
' api_key: env:ANTHROPIC_API_KEY', // pragma: allowlist secret
' models:',
' default: claude-sonnet-4-6',
'ingest:',
@ -133,9 +134,6 @@ describe('setup status', () => {
' database_connection_ids:',
' - warehouse',
' - analytics',
' completed_steps:',
' - project',
' - databases',
'connections:',
' warehouse:',
' driver: postgres',
@ -146,10 +144,11 @@ describe('setup status', () => {
' model: text-embedding-3-small',
' dimensions: 1536',
' openai:',
' api_key: env:OPENAI_API_KEY',
' api_key: env:OPENAI_API_KEY', // pragma: allowlist secret
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] });
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
databases: [
@ -167,8 +166,6 @@ describe('setup status', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
'connections:',
' warehouse:',
' driver: postgres',
@ -178,6 +175,7 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project'] });
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
databases: [{ connectionId: 'warehouse', ready: false }],
@ -190,9 +188,6 @@ describe('setup status', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - databases',
'connections:',
' warehouse:',
' driver: postgres',
@ -202,6 +197,7 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] });
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
databases: [{ connectionId: 'warehouse', ready: true }],
@ -215,9 +211,6 @@ describe('setup status', () => {
'project: revenue',
'setup:',
' database_connection_ids: []',
' completed_steps:',
' - project',
' - sources',
'connections:',
' docs:',
' driver: notion',
@ -230,6 +223,7 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'sources'] });
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
sources: [{ connectionId: 'docs', type: 'notion', ready: true }],
@ -268,12 +262,6 @@ describe('setup status', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - llm',
' - embeddings',
' - databases',
' - sources',
'connections:',
' warehouse:',
' driver: postgres',
@ -292,6 +280,9 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, {
completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources'],
});
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-abc123',
status: 'running',
@ -324,10 +315,6 @@ describe('setup status', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - databases',
' - sources',
'connections:',
' warehouse:',
' driver: postgres',
@ -354,6 +341,7 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases', 'sources'] });
await persistLocalBundleReport(
tempDir,
localFakeBundleReport('metabase-job-1', {
@ -920,7 +908,7 @@ describe('setup status', () => {
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
skipEmbeddings: true,
@ -937,7 +925,51 @@ describe('setup status', () => {
expect.objectContaining({
projectDir: tempDir,
inputMode: 'disabled',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
}),
testIo.io,
);
});
it('passes Vertex AI model setup args after project selection succeeds', async () => {
const testIo = makeIo();
const model = vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir }));
await expect(
runKtxSetup(
{
command: 'run',
projectDir: tempDir,
mode: 'new',
agents: false,
skipAgents: true,
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
llmBackend: 'vertex',
vertexProject: 'local-gcp-project',
vertexLocation: 'us-east5',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
skipEmbeddings: true,
databaseSchemas: [],
skipDatabases: true,
skipSources: true,
},
testIo.io,
{ model },
),
).resolves.toBe(0);
expect(model).toHaveBeenCalledWith(
expect.objectContaining({
projectDir: tempDir,
inputMode: 'disabled',
llmBackend: 'vertex',
vertexProject: 'local-gcp-project',
vertexLocation: 'us-east5',
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
}),
@ -961,11 +993,11 @@ describe('setup status', () => {
inputMode: 'disabled',
yes: true,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret
skipEmbeddings: false,
databaseSchemas: [],
skipDatabases: true,
@ -983,7 +1015,7 @@ describe('setup status', () => {
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret
skipEmbeddings: false,
}),
testIo.io,
@ -1181,11 +1213,11 @@ describe('setup status', () => {
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret
skipEmbeddings: false,
databaseDrivers: ['postgres'],
databaseConnectionId: 'warehouse',
@ -1281,9 +1313,6 @@ describe('setup status', () => {
'setup:',
' database_connection_ids:',
' - warehouse',
' completed_steps:',
' - project',
' - databases',
'connections:',
' warehouse:',
' driver: postgres',
@ -1296,6 +1325,7 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] });
await expect(
runKtxSetup(
@ -1782,13 +1812,6 @@ describe('setup status', () => {
[
'project: revenue',
'setup:',
' completed_steps:',
' - project',
' - llm',
' - embeddings',
' - sources',
' - context',
' - agents',
' database_connection_ids: []',
'connections: {}',
'llm:',
@ -1805,6 +1828,9 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, {
completed_steps: ['project', 'llm', 'embeddings', 'sources', 'context', 'agents'],
});
await writeFile(
join(tempDir, '.ktx/agents/install-manifest.json'),
JSON.stringify(
@ -1893,12 +1919,6 @@ describe('setup status', () => {
[
'project: revenue',
'setup:',
' completed_steps:',
' - project',
' - llm',
' - embeddings',
' - sources',
' - context',
' database_connection_ids: []',
'connections: {}',
'llm:',
@ -1915,6 +1935,9 @@ describe('setup status', () => {
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, {
completed_steps: ['project', 'llm', 'embeddings', 'sources', 'context'],
});
await writeKtxSetupContextState(tempDir, {
runId: 'setup-context-local-ready',
status: 'completed',
@ -2041,7 +2064,7 @@ describe('setup status', () => {
inputMode: 'disabled',
yes: false,
cliVersion: '0.2.0',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY',
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
anthropicModel: 'claude-sonnet-4-6',
skipLlm: false,
skipEmbeddings: false,

16
packages/cli/src/setup.ts
View file

@ -4,7 +4,6 @@ import { cancel, isCancel, select } from '@clack/prompts';
import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest';
import {
ktxLocalStateDbPath,
ktxSetupCompletedSteps,
loadKtxProject,
readKtxSetupState,
type KtxLocalProject,
@ -25,7 +24,12 @@ import {
runKtxSetupDatabasesStep,
} from './setup-databases.js';
import { type KtxSetupEmbeddingsDeps, runKtxSetupEmbeddingsStep } from './setup-embeddings.js';
import { type KtxSetupModelDeps, isKtxSetupLlmConfigReady, runKtxSetupAnthropicModelStep } from './setup-models.js';
import {
type KtxSetupLlmBackend,
type KtxSetupModelDeps,
isKtxSetupLlmConfigReady,
runKtxSetupAnthropicModelStep,
} from './setup-models.js';
import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js';
import {
isKtxPreAgentSetupReady,
@ -66,9 +70,12 @@ export type KtxSetupArgs =
inputMode: 'auto' | 'disabled';
yes: boolean;
cliVersion: string;
llmBackend?: KtxSetupLlmBackend;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
anthropicModel?: string;
vertexProject?: string;
vertexLocation?: string;
skipLlm: boolean;
embeddingBackend?: 'openai' | 'sentence-transformers';
embeddingApiKeyEnv?: string;
@ -296,7 +303,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
};
embeddings.ready = embeddingsReady(embeddings);
const completedSteps = ktxSetupCompletedSteps(project.config, await readKtxSetupState(resolvedProjectDir));
const completedSteps = (await readKtxSetupState(resolvedProjectDir)).completed_steps;
const contextState = await readKtxSetupContextState(resolvedProjectDir);
const setupContextStatus = setupContextStatusFromState(contextState, {
completedStep: completedSteps.includes('context'),
@ -578,9 +585,12 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
{
projectDir: projectResult.projectDir,
inputMode: args.inputMode,
...(args.llmBackend ? { llmBackend: args.llmBackend } : {}),
...(args.anthropicApiKeyEnv ? { anthropicApiKeyEnv: args.anthropicApiKeyEnv } : {}),
...(args.anthropicApiKeyFile ? { anthropicApiKeyFile: args.anthropicApiKeyFile } : {}),
...(args.anthropicModel ? { anthropicModel: args.anthropicModel } : {}),
...(args.vertexProject ? { vertexProject: args.vertexProject } : {}),
...(args.vertexLocation ? { vertexLocation: args.vertexLocation } : {}),
forcePrompt: forcePromptSteps.has('models') || runOnly === 'models',
showPromptInstructions,
skipLlm: args.skipLlm || !shouldRunModels,

33
packages/context/src/ingest/adapters/metabase/fetch.test.ts
View file

@ -36,6 +36,12 @@ function makeMockClient() {
),
getCollectionTree: vi.fn().mockResolvedValue([{ id: 5, name: 'Orders Team', parent_id: null, children: [] }]),
getCollectionItems: vi.fn().mockResolvedValue([]),
getDatabase: vi.fn().mockResolvedValue({
id: 42,
name: 'Analytics',
engine: 'postgres',
details: { host: 'db.example.test', dbname: 'analytics' },
}),
cleanup: vi.fn().mockResolvedValue(undefined),
};
}
@ -253,7 +259,7 @@ describe('fetchMetabaseBundle', () => {
).rejects.toThrow(/mapping.*does not point to connection/);
});
it('throws when the matching mapping has a null metabaseDatabaseName (unhydrated)', async () => {
it('hydrates missing mapping metadata from Metabase instead of requiring a prior refresh', async () => {
sourceStateReader.getSourceState.mockResolvedValue({
syncMode: 'ALL',
selections: [],
@ -268,15 +274,22 @@ describe('fetchMetabaseBundle', () => {
],
defaultTagNames: [],
});
await expect(
fetchMetabaseBundle({
pullConfig: { metabaseConnectionId, metabaseDatabaseId: 42 },
stagedDir,
ctx: makeFetchContext(),
clientFactory,
sourceStateReader,
}),
).rejects.toThrow(/unhydrated.*ktx connection mapping refresh/);
await fetchMetabaseBundle({
pullConfig: { metabaseConnectionId, metabaseDatabaseId: 42 },
stagedDir,
ctx: makeFetchContext(),
clientFactory,
sourceStateReader,
});
expect(clientFactory.__client.getDatabase).toHaveBeenCalledWith(42);
const databaseFile = JSON.parse(await readFile(join(stagedDir, 'databases/42.json'), 'utf-8'));
expect(databaseFile).toMatchObject({
metabaseDatabaseId: 42,
metabaseDatabaseName: 'Analytics',
metabaseEngine: 'postgres',
targetConnectionId,
});
});
it('skips cards whose getResolvedSql returns null and records them in unresolved-cards.json', async () => {

17
packages/context/src/ingest/adapters/metabase/fetch.ts
View file

@ -97,15 +97,16 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr
`mapping for database ${pullConfig.metabaseDatabaseId} does not point to connection ${params.ctx.connectionId} (points to ${mapping.targetConnectionId})`,
);
}
if (mapping.metabaseDatabaseName === null) {
throw new IngestInputError(
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`,
);
}
const mappingDatabaseName: string = mapping.metabaseDatabaseName;
const client = await params.clientFactory.createClient(pullConfig, params.ctx);
try {
let mappingDatabaseName = mapping.metabaseDatabaseName;
let mappingEngine = mapping.metabaseEngine;
if (mappingDatabaseName === null) {
const database = await client.getDatabase(pullConfig.metabaseDatabaseId);
mappingDatabaseName = database.name;
mappingEngine = database.engine ?? null;
}
const stagedForScope: StagedSyncConfig = {
metabaseConnectionId: pullConfig.metabaseConnectionId,
metabaseDatabaseId: pullConfig.metabaseDatabaseId,
@ -118,7 +119,7 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr
mapping: {
metabaseDatabaseId: mapping.metabaseDatabaseId,
metabaseDatabaseName: mappingDatabaseName,
metabaseEngine: mapping.metabaseEngine,
metabaseEngine: mappingEngine,
targetConnectionId: mapping.targetConnectionId,
},
};
@ -233,7 +234,7 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr
const databaseFile: StagedDatabaseFile = {
metabaseDatabaseId: mapping.metabaseDatabaseId,
metabaseDatabaseName: mappingDatabaseName,
metabaseEngine: mapping.metabaseEngine,
metabaseEngine: mappingEngine,
targetConnectionId: mapping.targetConnectionId,
};
await writeFile(

5
packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts
View file

@ -12,7 +12,7 @@ import {
type MetabaseClientRuntimeConfig,
} from './client-port.js';
import type { MetabaseFetchLogger } from './fetch.js';
import { LocalMetabaseSourceStateReader } from './local-source-state-store.js';
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
import { MetabaseSourceAdapter } from './metabase.adapter.js';
function stringField(value: unknown): string | null {
@ -62,7 +62,8 @@ export function createLocalMetabaseSourceAdapter(
project: KtxLocalProject,
options: CreateLocalMetabaseSourceAdapterOptions = {},
): MetabaseSourceAdapter {
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
const sourceStateReader = new KtxYamlMetabaseSourceStateReader(project, { discoveryCache });
const connectionFactory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(

329
packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts
View file

@ -2,313 +2,112 @@ import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { LocalMetabaseSourceStateReader } from './local-source-state-store.js';
import { buildDefaultKtxProjectConfig } from '../../../project/index.js';
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
describe('LocalMetabaseSourceStateReader', () => {
describe('Metabase YAML source state and discovery cache', () => {
let tempDir: string;
let store: LocalMetabaseSourceStateReader;
let discoveryCache: LocalMetabaseDiscoveryCache;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-local-state-'));
store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-cache-'));
discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('round-trips hydrated source state through SQLite', async () => {
await store.replaceSourceState({
connectionId: 'prod-metabase',
function projectWithMetabaseMappings(mappings: Record<string, unknown>) {
return {
config: {
...buildDefaultKtxProjectConfig('metabase-cache-test'),
connections: {
'prod-metabase': {
driver: 'metabase',
mappings,
},
},
},
};
}
it('reads Metabase mapping intent from ktx.yaml config', async () => {
const reader = new KtxYamlMetabaseSourceStateReader(
projectWithMetabaseMappings({
databaseMappings: { '2': 'warehouse' },
syncEnabled: { '2': true },
syncMode: 'ONLY',
selections: { collections: [12], items: [99] },
defaultTagNames: ['analytics'],
}),
{ discoveryCache },
);
await expect(reader.getSourceState('prod-metabase')).resolves.toEqual({
syncMode: 'ONLY',
defaultTagNames: ['analytics', 'curated'],
defaultTagNames: ['analytics'],
selections: [
{ selectionType: 'collection', metabaseObjectId: 10 },
{ selectionType: 'collection', metabaseObjectId: 12 },
{ selectionType: 'item', metabaseObjectId: 99 },
],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Analytics',
metabaseEngine: 'postgres',
metabaseHost: 'warehouse.internal',
metabaseDbName: 'analytics',
targetConnectionId: 'warehouse',
syncEnabled: true,
source: 'cli',
},
],
});
await expect(store.getSourceState('prod-metabase')).resolves.toEqual({
syncMode: 'ONLY',
defaultTagNames: ['analytics', 'curated'],
selections: [
{ selectionType: 'collection', metabaseObjectId: 10 },
{ selectionType: 'item', metabaseObjectId: 99 },
],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Analytics',
metabaseEngine: 'postgres',
targetConnectionId: 'warehouse',
syncEnabled: true,
},
],
});
});
it('excludes unhydrated mappings from getSourceState and exposes them through the side accessor', async () => {
await store.replaceSourceState({
connectionId: 'prod-metabase',
syncMode: 'ALL',
defaultTagNames: [],
selections: [],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseId: 2,
metabaseDatabaseName: null,
metabaseEngine: null,
metabaseHost: null,
metabaseDbName: null,
targetConnectionId: 'warehouse',
syncEnabled: true,
source: 'ktx.yaml',
},
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Sandbox',
metabaseEngine: 'postgres',
metabaseHost: 'warehouse.internal',
metabaseDbName: 'sandbox',
targetConnectionId: 'warehouse',
syncEnabled: true,
source: 'refresh',
},
],
});
const state = await store.getSourceState('prod-metabase');
expect(state.mappings.map((mapping) => mapping.metabaseDatabaseId)).toEqual([2]);
await expect(store.getUnhydratedSyncEnabledMappingIds('prod-metabase')).resolves.toEqual([1]);
});
it('defaults missing sync config to ALL with no tags or selections', async () => {
await store.replaceSourceState({
it('enriches YAML mapping rows with recreatable discovery metadata', async () => {
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
mappings: [
{
metabaseDatabaseId: 3,
metabaseDatabaseName: 'Warehouse',
metabaseEngine: 'postgres',
metabaseHost: null,
metabaseDbName: null,
targetConnectionId: null,
syncEnabled: false,
source: 'refresh',
},
],
discovered: [{ id: 2, name: 'Analytics', engine: 'postgres', host: 'pg.internal', dbName: 'analytics' }],
});
const reader = new KtxYamlMetabaseSourceStateReader(
projectWithMetabaseMappings({
databaseMappings: { '2': 'warehouse' },
syncEnabled: { '2': true },
}),
{ discoveryCache },
);
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
syncMode: 'ALL',
defaultTagNames: [],
selections: [],
});
});
it('supports command-sized mapping writes and reads', async () => {
await store.upsertDatabaseMapping({
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'prod-warehouse',
syncEnabled: true,
source: 'cli',
});
await store.setSyncState({
connectionId: 'prod-metabase',
syncMode: 'ONLY',
defaultTagNames: ['analytics'],
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
});
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toEqual([
await expect(reader.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{
metabaseDatabaseId: 1,
metabaseDatabaseName: null,
metabaseEngine: null,
metabaseHost: null,
metabaseDbName: null,
targetConnectionId: 'prod-warehouse',
syncEnabled: true,
source: 'cli',
},
]);
await expect(store.getUnhydratedSyncEnabledMappingIds('prod-metabase')).resolves.toEqual([1]);
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
syncMode: 'ONLY',
defaultTagNames: ['analytics'],
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
mappings: [],
});
});
it('refreshes discovered database metadata while preserving user mapping intent', async () => {
await store.upsertDatabaseMapping({
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'prod-warehouse',
syncEnabled: true,
source: 'cli',
});
await store.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
discovered: [
{ id: 1, name: 'Analytics', engine: 'postgres', host: 'pg.internal', dbName: 'analytics' },
{ id: 2, name: 'Sandbox', engine: 'postgres', host: 'pg.internal', dbName: 'sandbox' },
],
});
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toEqual([
{
metabaseDatabaseId: 1,
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Analytics',
metabaseEngine: 'postgres',
metabaseHost: 'pg.internal',
metabaseDbName: 'analytics',
targetConnectionId: 'prod-warehouse',
targetConnectionId: 'warehouse',
syncEnabled: true,
source: 'cli',
source: 'ktx.yaml',
},
]);
});
it('lists discovered-only rows as refresh cache data without turning them into config state', async () => {
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
discovered: [{ id: 7, name: 'Unmapped', engine: 'mysql', host: 'mysql.internal', dbName: 'sales' }],
});
const reader = new KtxYamlMetabaseSourceStateReader(projectWithMetabaseMappings({}), { discoveryCache });
await expect(reader.getSourceState('prod-metabase')).resolves.toMatchObject({ mappings: [] });
await expect(reader.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Sandbox',
metabaseEngine: 'postgres',
metabaseHost: 'pg.internal',
metabaseDbName: 'sandbox',
metabaseDatabaseId: 7,
metabaseDatabaseName: 'Unmapped',
targetConnectionId: null,
syncEnabled: false,
source: 'refresh',
},
]);
});
it('updates sync-enabled, clears scoped rows, and applies bulk state in one call', async () => {
await store.replaceSourceState({
connectionId: 'prod-metabase',
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Analytics',
metabaseEngine: 'postgres',
metabaseHost: 'pg.internal',
metabaseDbName: 'analytics',
targetConnectionId: 'prod-warehouse',
syncEnabled: true,
source: 'refresh',
},
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Sandbox',
metabaseEngine: 'postgres',
metabaseHost: 'pg.internal',
metabaseDbName: 'sandbox',
targetConnectionId: 'staging-warehouse',
syncEnabled: true,
source: 'refresh',
},
],
});
await store.setMappingSyncEnabled({
connectionId: 'prod-metabase',
metabaseDatabaseId: 2,
syncEnabled: false,
});
await store.clearDatabaseMappings({ connectionId: 'prod-metabase', metabaseDatabaseId: 1 });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toEqual([
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Sandbox',
metabaseEngine: 'postgres',
metabaseHost: 'pg.internal',
metabaseDbName: 'sandbox',
targetConnectionId: 'staging-warehouse',
syncEnabled: false,
source: 'refresh',
},
]);
});
it('seeds unhydrated yaml intent without exposing it through getSourceState', async () => {
await store.applyYamlBootstrap({
connectionId: 'prod-metabase',
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
mappings: [{ metabaseDatabaseId: 1, targetConnectionId: 'prod-warehouse', syncEnabled: true }],
});
await expect(store.getUnhydratedSyncEnabledMappingIds('prod-metabase')).resolves.toEqual([1]);
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
mappings: [],
});
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{
metabaseDatabaseId: 1,
metabaseDatabaseName: null,
targetConnectionId: 'prod-warehouse',
syncEnabled: true,
source: 'ktx.yaml',
},
]);
});
it('applies yaml target intent onto refresh metadata but does not overwrite cli rows', async () => {
await store.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
discovered: [{ id: 1, name: 'Analytics', engine: 'postgres', host: 'db.test', dbName: 'analytics' }],
});
await store.upsertDatabaseMapping({
connectionId: 'prod-metabase',
metabaseDatabaseId: 2,
targetConnectionId: 'cli-warehouse',
syncEnabled: true,
source: 'cli',
});
await store.applyYamlBootstrap({
connectionId: 'prod-metabase',
syncMode: 'EXCEPT',
defaultTagNames: [],
selections: [{ selectionType: 'item', metabaseObjectId: 99 }],
mappings: [
{ metabaseDatabaseId: 1, targetConnectionId: 'yaml-warehouse', syncEnabled: true },
{ metabaseDatabaseId: 2, targetConnectionId: 'yaml-warehouse', syncEnabled: false },
],
});
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Analytics',
metabaseEngine: 'postgres',
targetConnectionId: 'yaml-warehouse',
syncEnabled: true,
source: 'ktx.yaml',
},
{
metabaseDatabaseId: 2,
targetConnectionId: 'cli-warehouse',
syncEnabled: true,
source: 'cli',
},
]);
});
});

623
packages/context/src/ingest/adapters/metabase/local-source-state-store.ts
View file

@ -1,17 +1,31 @@
import { mkdirSync } from 'node:fs';
import { dirname } from 'node:path';
import Database from 'better-sqlite3';
import {
parseMetabaseMappingBootstrap,
type KtxLocalProject,
type MetabaseMappingBootstrap,
} from '../../../project/index.js';
import type { DiscoveredMetabaseDatabase } from './mapping.js';
import type { MetabaseSourceState, MetabaseSourceStateReader, MetabaseSourceStateSelection } from './source-state-port.js';
import type { MetabaseSyncMode } from './types.js';
export type LocalMetabaseMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
export type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh';
interface LocalMetabaseSourceStateStoreOptions {
interface LocalMetabaseDiscoveryCacheOptions {
dbPath: string;
now?: () => Date;
}
export interface LocalMetabaseSourceStateMappingInput {
export interface RefreshLocalMetabaseDiscoveredDatabasesInput {
connectionId: string;
discovered: DiscoveredMetabaseDatabase[];
}
export interface LocalMetabaseDiscoveredDatabaseRow extends DiscoveredMetabaseDatabase {
updatedAt: string;
}
export interface LocalMetabaseMappingListRow {
metabaseDatabaseId: number;
metabaseDatabaseName: string | null;
metabaseEngine: string | null;
@ -22,443 +36,86 @@ export interface LocalMetabaseSourceStateMappingInput {
source: LocalMetabaseMappingSource;
}
export interface ReplaceLocalMetabaseSourceStateInput {
connectionId: string;
syncMode?: MetabaseSyncMode;
defaultTagNames?: string[];
selections?: MetabaseSourceStateSelection[];
mappings: LocalMetabaseSourceStateMappingInput[];
}
interface ApplyLocalMetabaseYamlBootstrapInput {
connectionId: string;
syncMode: MetabaseSyncMode;
defaultTagNames: string[];
selections: MetabaseSourceStateSelection[];
mappings: Array<{
metabaseDatabaseId: number;
targetConnectionId: string | null;
syncEnabled: boolean;
}>;
}
export interface LocalMetabaseMappingListRow extends LocalMetabaseSourceStateMappingInput {}
export interface UpsertLocalMetabaseDatabaseMappingInput {
connectionId: string;
metabaseDatabaseId: number;
targetConnectionId: string | null;
syncEnabled: boolean;
source: LocalMetabaseMappingSource;
}
export interface SetLocalMetabaseMappingSyncEnabledInput {
connectionId: string;
metabaseDatabaseId: number;
syncEnabled: boolean;
}
export interface SetLocalMetabaseSyncStateInput {
connectionId: string;
syncMode: MetabaseSyncMode;
defaultTagNames: string[];
selections: MetabaseSourceStateSelection[];
}
export interface RefreshLocalMetabaseDiscoveredDatabasesInput {
connectionId: string;
discovered: Array<{
id: number;
name: string;
engine: string;
host: string | null;
dbName: string | null;
}>;
}
export interface ClearLocalMetabaseMappingsInput {
connectionId: string;
metabaseDatabaseId?: number;
}
interface SelectionRow {
selection_type: 'collection' | 'item';
metabase_object_id: number;
}
interface MappingRow {
interface DiscoveryRow {
metabase_database_id: number;
metabase_database_name: string | null;
metabase_engine: string | null;
target_connection_id: string | null;
sync_enabled: number;
metabase_database_name: string;
metabase_engine: string;
metabase_host: string | null;
metabase_db_name: string | null;
updated_at: string;
}
interface SyncConfigRow {
sync_mode: MetabaseSyncMode;
default_tag_names_json: string;
function selectionState(bootstrap: MetabaseMappingBootstrap): MetabaseSourceStateSelection[] {
return [
...bootstrap.selections.collections.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...bootstrap.selections.items.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
];
}
function parseDefaultTagNames(raw: string): string[] {
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed.filter((value): value is string => typeof value === 'string') : [];
function configuredMappingIds(bootstrap: MetabaseMappingBootstrap): number[] {
return [...new Set([...Object.keys(bootstrap.databaseMappings), ...Object.keys(bootstrap.syncEnabled)].map(Number))].sort(
(left, right) => left - right,
);
}
export class LocalMetabaseSourceStateReader implements MetabaseSourceStateReader {
function discoveredRowToDatabase(row: DiscoveryRow): LocalMetabaseDiscoveredDatabaseRow {
return {
id: row.metabase_database_id,
name: row.metabase_database_name,
engine: row.metabase_engine,
host: row.metabase_host,
dbName: row.metabase_db_name,
updatedAt: row.updated_at,
};
}
function emptyMetabaseSourceState(): MetabaseSourceState {
return {
syncMode: 'ALL',
selections: [],
defaultTagNames: [],
mappings: [],
};
}
export class LocalMetabaseDiscoveryCache {
private readonly db: Database.Database;
private readonly now: () => Date;
constructor(options: LocalMetabaseSourceStateStoreOptions) {
constructor(options: LocalMetabaseDiscoveryCacheOptions) {
mkdirSync(dirname(options.dbPath), { recursive: true });
this.db = new Database(options.dbPath);
this.db.pragma('journal_mode = WAL');
this.db.pragma('foreign_keys = ON');
this.now = options.now ?? (() => new Date());
this.db.exec(`
CREATE TABLE IF NOT EXISTS local_metabase_sync_config (
metabase_connection_id TEXT PRIMARY KEY,
sync_mode TEXT NOT NULL,
default_tag_names_json TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS local_metabase_selections (
metabase_connection_id TEXT NOT NULL,
selection_type TEXT NOT NULL,
metabase_object_id INTEGER NOT NULL,
PRIMARY KEY (metabase_connection_id, selection_type, metabase_object_id)
);
CREATE TABLE IF NOT EXISTS local_metabase_database_mappings (
CREATE TABLE IF NOT EXISTS local_metabase_discovered_databases (
metabase_connection_id TEXT NOT NULL,
metabase_database_id INTEGER NOT NULL,
metabase_database_name TEXT,
metabase_engine TEXT,
metabase_database_name TEXT NOT NULL,
metabase_engine TEXT NOT NULL,
metabase_host TEXT,
metabase_db_name TEXT,
target_connection_id TEXT,
sync_enabled INTEGER NOT NULL DEFAULT 0,
source TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (metabase_connection_id, metabase_database_id)
);
`);
}
async applyYamlBootstrap(input: ApplyLocalMetabaseYamlBootstrapInput): Promise<void> {
const timestamp = this.now().toISOString();
const apply = this.db.transaction(() => {
const syncConfigExists = this.db
.prepare('SELECT 1 FROM local_metabase_sync_config WHERE metabase_connection_id = ?')
.get(input.connectionId);
if (!syncConfigExists) {
this.db
.prepare(
`
INSERT INTO local_metabase_sync_config (
metabase_connection_id,
sync_mode,
default_tag_names_json,
updated_at
)
VALUES (?, ?, ?, ?)
`,
)
.run(input.connectionId, input.syncMode, JSON.stringify(input.defaultTagNames), timestamp);
const insertSelection = this.db.prepare(`
INSERT INTO local_metabase_selections (
metabase_connection_id,
selection_type,
metabase_object_id
)
VALUES (?, ?, ?)
`);
for (const selection of input.selections) {
insertSelection.run(input.connectionId, selection.selectionType, selection.metabaseObjectId);
}
}
const existing = this.db.prepare(`
SELECT target_connection_id, source
FROM local_metabase_database_mappings
WHERE metabase_connection_id = ? AND metabase_database_id = ?
`);
const insert = this.db.prepare(`
INSERT INTO local_metabase_database_mappings (
metabase_connection_id,
metabase_database_id,
metabase_database_name,
metabase_engine,
metabase_host,
metabase_db_name,
target_connection_id,
sync_enabled,
source,
updated_at
)
VALUES (?, ?, NULL, NULL, NULL, NULL, ?, ?, 'ktx.yaml', ?)
`);
const updateRefreshRow = this.db.prepare(`
UPDATE local_metabase_database_mappings
SET target_connection_id = ?,
sync_enabled = ?,
source = 'ktx.yaml',
updated_at = ?
WHERE metabase_connection_id = ?
AND metabase_database_id = ?
AND source = 'refresh'
AND target_connection_id IS NULL
`);
for (const mapping of input.mappings) {
const row = existing.get(input.connectionId, mapping.metabaseDatabaseId) as
| { target_connection_id: string | null; source: LocalMetabaseMappingSource }
| undefined;
if (!row) {
insert.run(
input.connectionId,
mapping.metabaseDatabaseId,
mapping.targetConnectionId,
mapping.syncEnabled ? 1 : 0,
timestamp,
);
continue;
}
if (row.source === 'refresh' && row.target_connection_id === null) {
updateRefreshRow.run(
mapping.targetConnectionId,
mapping.syncEnabled ? 1 : 0,
timestamp,
input.connectionId,
mapping.metabaseDatabaseId,
);
}
}
});
apply();
}
async replaceSourceState(input: ReplaceLocalMetabaseSourceStateInput): Promise<void> {
const timestamp = this.now().toISOString();
const syncMode = input.syncMode ?? 'ALL';
const selections = input.selections ?? [];
const defaultTagNames = input.defaultTagNames ?? [];
const replace = this.db.transaction(() => {
this.db
.prepare(
`
INSERT INTO local_metabase_sync_config (
metabase_connection_id,
sync_mode,
default_tag_names_json,
updated_at
)
VALUES (?, ?, ?, ?)
ON CONFLICT(metabase_connection_id) DO UPDATE SET
sync_mode = excluded.sync_mode,
default_tag_names_json = excluded.default_tag_names_json,
updated_at = excluded.updated_at
`,
)
.run(input.connectionId, syncMode, JSON.stringify(defaultTagNames), timestamp);
this.db.prepare('DELETE FROM local_metabase_selections WHERE metabase_connection_id = ?').run(input.connectionId);
const insertSelection = this.db.prepare(`
INSERT INTO local_metabase_selections (
metabase_connection_id,
selection_type,
metabase_object_id
)
VALUES (?, ?, ?)
`);
for (const selection of selections) {
insertSelection.run(input.connectionId, selection.selectionType, selection.metabaseObjectId);
}
this.db
.prepare('DELETE FROM local_metabase_database_mappings WHERE metabase_connection_id = ?')
.run(input.connectionId);
const insertMapping = this.db.prepare(`
INSERT INTO local_metabase_database_mappings (
metabase_connection_id,
metabase_database_id,
metabase_database_name,
metabase_engine,
metabase_host,
metabase_db_name,
target_connection_id,
sync_enabled,
source,
updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
for (const mapping of input.mappings) {
insertMapping.run(
input.connectionId,
mapping.metabaseDatabaseId,
mapping.metabaseDatabaseName,
mapping.metabaseEngine,
mapping.metabaseHost,
mapping.metabaseDbName,
mapping.targetConnectionId,
mapping.syncEnabled ? 1 : 0,
mapping.source,
timestamp,
);
}
});
replace();
}
async listDatabaseMappings(connectionId: string): Promise<LocalMetabaseMappingListRow[]> {
const rows = this.db
.prepare(
`
SELECT
metabase_database_id,
metabase_database_name,
metabase_engine,
metabase_host,
metabase_db_name,
target_connection_id,
sync_enabled,
source
FROM local_metabase_database_mappings
WHERE metabase_connection_id = ?
ORDER BY metabase_database_id
`,
)
.all(connectionId) as Array<{
metabase_database_id: number;
metabase_database_name: string | null;
metabase_engine: string | null;
metabase_host: string | null;
metabase_db_name: string | null;
target_connection_id: string | null;
sync_enabled: number;
source: LocalMetabaseMappingSource;
}>;
return rows.map((row) => ({
metabaseDatabaseId: row.metabase_database_id,
metabaseDatabaseName: row.metabase_database_name,
metabaseEngine: row.metabase_engine,
metabaseHost: row.metabase_host,
metabaseDbName: row.metabase_db_name,
targetConnectionId: row.target_connection_id,
syncEnabled: row.sync_enabled === 1,
source: row.source,
}));
}
async upsertDatabaseMapping(input: UpsertLocalMetabaseDatabaseMappingInput): Promise<void> {
const timestamp = this.now().toISOString();
this.db
.prepare(
`
INSERT INTO local_metabase_database_mappings (
metabase_connection_id,
metabase_database_id,
metabase_database_name,
metabase_engine,
metabase_host,
metabase_db_name,
target_connection_id,
sync_enabled,
source,
updated_at
)
VALUES (?, ?, NULL, NULL, NULL, NULL, ?, ?, ?, ?)
ON CONFLICT(metabase_connection_id, metabase_database_id) DO UPDATE SET
target_connection_id = excluded.target_connection_id,
sync_enabled = excluded.sync_enabled,
source = excluded.source,
updated_at = excluded.updated_at
`,
)
.run(
input.connectionId,
input.metabaseDatabaseId,
input.targetConnectionId,
input.syncEnabled ? 1 : 0,
input.source,
timestamp,
);
}
async setMappingSyncEnabled(input: SetLocalMetabaseMappingSyncEnabledInput): Promise<void> {
const timestamp = this.now().toISOString();
this.db
.prepare(
`
UPDATE local_metabase_database_mappings
SET sync_enabled = ?, updated_at = ?
WHERE metabase_connection_id = ? AND metabase_database_id = ?
`,
)
.run(input.syncEnabled ? 1 : 0, timestamp, input.connectionId, input.metabaseDatabaseId);
}
async setSyncState(input: SetLocalMetabaseSyncStateInput): Promise<void> {
const timestamp = this.now().toISOString();
const write = this.db.transaction(() => {
this.db
.prepare(
`
INSERT INTO local_metabase_sync_config (
metabase_connection_id,
sync_mode,
default_tag_names_json,
updated_at
)
VALUES (?, ?, ?, ?)
ON CONFLICT(metabase_connection_id) DO UPDATE SET
sync_mode = excluded.sync_mode,
default_tag_names_json = excluded.default_tag_names_json,
updated_at = excluded.updated_at
`,
)
.run(input.connectionId, input.syncMode, JSON.stringify(input.defaultTagNames), timestamp);
this.db.prepare('DELETE FROM local_metabase_selections WHERE metabase_connection_id = ?').run(input.connectionId);
const insertSelection = this.db.prepare(`
INSERT INTO local_metabase_selections (
metabase_connection_id,
selection_type,
metabase_object_id
)
VALUES (?, ?, ?)
`);
for (const selection of input.selections) {
insertSelection.run(input.connectionId, selection.selectionType, selection.metabaseObjectId);
}
});
write();
}
async refreshDiscoveredDatabases(input: RefreshLocalMetabaseDiscoveredDatabasesInput): Promise<void> {
const timestamp = this.now().toISOString();
const refresh = this.db.transaction(() => {
const upsert = this.db.prepare(`
INSERT INTO local_metabase_database_mappings (
INSERT INTO local_metabase_discovered_databases (
metabase_connection_id,
metabase_database_id,
metabase_database_name,
metabase_engine,
metabase_host,
metabase_db_name,
target_connection_id,
sync_enabled,
source,
updated_at
)
VALUES (?, ?, ?, ?, ?, ?, NULL, 0, 'refresh', ?)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(metabase_connection_id, metabase_database_id) DO UPDATE SET
metabase_database_name = excluded.metabase_database_name,
metabase_engine = excluded.metabase_engine,
@ -483,78 +140,116 @@ export class LocalMetabaseSourceStateReader implements MetabaseSourceStateReader
refresh();
}
async clearDatabaseMappings(input: ClearLocalMetabaseMappingsInput): Promise<void> {
if (input.metabaseDatabaseId === undefined) {
this.db.prepare('DELETE FROM local_metabase_database_mappings WHERE metabase_connection_id = ?').run(input.connectionId);
return;
}
this.db
.prepare('DELETE FROM local_metabase_database_mappings WHERE metabase_connection_id = ? AND metabase_database_id = ?')
.run(input.connectionId, input.metabaseDatabaseId);
}
async getUnhydratedSyncEnabledMappingIds(connectionId: string): Promise<number[]> {
async listDiscoveredDatabases(connectionId: string): Promise<LocalMetabaseDiscoveredDatabaseRow[]> {
const rows = this.db
.prepare(
`
SELECT metabase_database_id
FROM local_metabase_database_mappings
WHERE metabase_connection_id = ?
AND sync_enabled = 1
AND target_connection_id IS NOT NULL
AND metabase_database_name IS NULL
ORDER BY metabase_database_id
`,
)
.all(connectionId) as Array<{ metabase_database_id: number }>;
return rows.map((row) => row.metabase_database_id);
}
async getSourceState(connectionId: string): Promise<MetabaseSourceState> {
const config = this.db
.prepare('SELECT sync_mode, default_tag_names_json FROM local_metabase_sync_config WHERE metabase_connection_id = ?')
.get(connectionId) as SyncConfigRow | undefined;
const selections = this.db
.prepare(
`
SELECT selection_type, metabase_object_id
FROM local_metabase_selections
WHERE metabase_connection_id = ?
ORDER BY selection_type, metabase_object_id
`,
)
.all(connectionId) as SelectionRow[];
const mappings = this.db
.prepare(
`
SELECT
metabase_database_id,
metabase_database_name,
metabase_engine,
target_connection_id,
sync_enabled
FROM local_metabase_database_mappings
metabase_host,
metabase_db_name,
updated_at
FROM local_metabase_discovered_databases
WHERE metabase_connection_id = ?
AND metabase_database_name IS NOT NULL
ORDER BY metabase_database_id
`,
)
.all(connectionId) as MappingRow[];
.all(connectionId) as DiscoveryRow[];
return rows.map(discoveredRowToDatabase);
}
return {
syncMode: config?.sync_mode ?? 'ALL',
defaultTagNames: config ? parseDefaultTagNames(config.default_tag_names_json) : [],
selections: selections.map((selection) => ({
selectionType: selection.selection_type,
metabaseObjectId: selection.metabase_object_id,
})),
mappings: mappings.map((mapping) => ({
metabaseDatabaseId: mapping.metabase_database_id,
metabaseDatabaseName: mapping.metabase_database_name,
metabaseEngine: mapping.metabase_engine,
targetConnectionId: mapping.target_connection_id,
syncEnabled: mapping.sync_enabled === 1,
})),
};
async getDiscoveredDatabase(
connectionId: string,
metabaseDatabaseId: number,
): Promise<LocalMetabaseDiscoveredDatabaseRow | null> {
const row = this.db
.prepare(
`
SELECT
metabase_database_id,
metabase_database_name,
metabase_engine,
metabase_host,
metabase_db_name,
updated_at
FROM local_metabase_discovered_databases
WHERE metabase_connection_id = ? AND metabase_database_id = ?
`,
)
.get(connectionId, metabaseDatabaseId) as DiscoveryRow | undefined;
return row ? discoveredRowToDatabase(row) : null;
}
}
export class KtxYamlMetabaseSourceStateReader implements MetabaseSourceStateReader {
constructor(
private readonly project: Pick<KtxLocalProject, 'config'>,
private readonly options: { discoveryCache?: LocalMetabaseDiscoveryCache } = {},
) {}
async getSourceState(connectionId: string): Promise<MetabaseSourceState> {
const connection = this.project.config.connections[connectionId];
if (!connection || String(connection.driver ?? '').toLowerCase() !== 'metabase') {
return emptyMetabaseSourceState();
}
const bootstrap = parseMetabaseMappingBootstrap(connectionId, connection);
const discovered = new Map(
(await this.options.discoveryCache?.listDiscoveredDatabases(connectionId))?.map((database) => [database.id, database]) ??
[],
);
return {
syncMode: bootstrap.syncMode,
selections: selectionState(bootstrap),
defaultTagNames: bootstrap.defaultTagNames,
mappings: configuredMappingIds(bootstrap).map((id) => {
const metadata = discovered.get(id);
return {
metabaseDatabaseId: id,
metabaseDatabaseName: metadata?.name ?? null,
metabaseEngine: metadata?.engine ?? null,
metabaseHost: metadata?.host ?? null,
metabaseDbName: metadata?.dbName ?? null,
targetConnectionId: bootstrap.databaseMappings[String(id)] ?? null,
syncEnabled: bootstrap.syncEnabled[String(id)] ?? false,
};
}),
};
}
async listDatabaseMappings(connectionId: string): Promise<LocalMetabaseMappingListRow[]> {
const state = await this.getSourceState(connectionId);
const configuredRows: LocalMetabaseMappingListRow[] = state.mappings.map((mapping) => ({
metabaseDatabaseId: mapping.metabaseDatabaseId,
metabaseDatabaseName: mapping.metabaseDatabaseName,
metabaseEngine: mapping.metabaseEngine,
metabaseHost: mapping.metabaseHost ?? null,
metabaseDbName: mapping.metabaseDbName ?? null,
targetConnectionId: mapping.targetConnectionId,
syncEnabled: mapping.syncEnabled,
source: 'ktx.yaml',
}));
const configuredIds = new Set(configuredRows.map((row) => row.metabaseDatabaseId));
const discoveredRows =
(await this.options.discoveryCache?.listDiscoveredDatabases(connectionId))?.filter(
(database) => !configuredIds.has(database.id),
) ?? [];
return [
...configuredRows,
...discoveredRows.map((database) => ({
metabaseDatabaseId: database.id,
metabaseDatabaseName: database.name,
metabaseEngine: database.engine,
metabaseHost: database.host,
metabaseDbName: database.dbName,
targetConnectionId: null,
syncEnabled: false,
source: 'refresh' as const,
})),
].sort((left, right) => left.metabaseDatabaseId - right.metabaseDatabaseId);
}
}

2
packages/context/src/ingest/adapters/metabase/source-state-port.ts
View file

@ -9,6 +9,8 @@ export interface MetabaseSourceStateMapping {
metabaseDatabaseId: number;
metabaseDatabaseName: string | null;
metabaseEngine: string | null;
metabaseHost?: string | null;
metabaseDbName?: string | null;
targetConnectionId: string | null;
syncEnabled: boolean;
}

12
packages/context/src/ingest/index.ts
View file

@ -240,17 +240,15 @@ export {
createLocalMetabaseSourceAdapter,
metabaseRuntimeConfigFromLocalConnection,
} from './adapters/metabase/local-metabase.adapter.js';
export { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
export {
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
} from './adapters/metabase/local-source-state-store.js';
export type {
ClearLocalMetabaseMappingsInput,
LocalMetabaseDiscoveredDatabaseRow,
LocalMetabaseMappingListRow,
LocalMetabaseMappingSource,
LocalMetabaseSourceStateMappingInput,
ReplaceLocalMetabaseSourceStateInput,
RefreshLocalMetabaseDiscoveredDatabasesInput,
SetLocalMetabaseMappingSyncEnabledInput,
SetLocalMetabaseSyncStateInput,
UpsertLocalMetabaseDatabaseMappingInput,
} from './adapters/metabase/local-source-state-store.js';
export { metabaseLocalConnectionIdSchema, metabasePullConfigSchema, parseMetabasePullConfig } from './adapters/metabase/types.js';
export type { MetabasePullConfig, MetabaseSyncMode } from './adapters/metabase/types.js';

2
packages/context/src/ingest/local-adapters.test.ts
View file

@ -482,7 +482,7 @@ describe('local ingest adapters', () => {
}),
config: {
...project.config,
setup: { database_connection_ids: ['warehouse'], completed_steps: [] },
setup: { database_connection_ids: ['warehouse'] },
connections: {
warehouse: {
driver: 'postgres',

15
packages/context/src/ingest/local-ingest.ts
View file

@ -9,10 +9,9 @@ import type { KtxSemanticLayerComputePort } from '../daemon/index.js';
import type { KtxLocalProject } from '../project/index.js';
import { ktxLocalStateDbPath } from '../project/index.js';
import { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js';
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js';
import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } from './local-adapters.js';
import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
import type { MemoryFlowEventSink } from './memory-flow/types.js';
import { buildSyncId } from './raw-sources-paths.js';
import type { IngestReportBody, IngestReportSnapshot } from './reports.js';
@ -364,16 +363,10 @@ export async function runLocalMetabaseIngest(
const metabaseConnectionId = safeSegment('metabase connection id', options.metabaseConnectionId);
assertConfigured(options.project, 'metabase', metabaseConnectionId);
await seedLocalMappingStateFromKtxYaml(options.project, metabaseConnectionId);
const adapter = findAdapter(options.adapters, 'metabase');
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(options.project) });
const unhydrated = await sourceStateReader.getUnhydratedSyncEnabledMappingIds(metabaseConnectionId);
if (unhydrated.length > 0) {
throw new Error(
`Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`ktx connection mapping refresh ${metabaseConnectionId}\` before local Metabase ingest.`,
);
}
const sourceStateReader = new KtxYamlMetabaseSourceStateReader(options.project, {
discoveryCache: new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(options.project) }),
});
const state = await sourceStateReader.getSourceState(metabaseConnectionId);
const childPlans = planMetabaseFanoutChildren({

15
packages/context/src/ingest/local-mapping-reconcile.test.ts
View file

@ -4,7 +4,6 @@ import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
describe('local mapping yaml reconciliation bridge', () => {
@ -23,7 +22,7 @@ describe('local mapping yaml reconciliation bridge', () => {
} as KtxLocalProject;
}
it('seeds Metabase local state from ktx.yaml mapping intent', async () => {
it('does not copy Metabase mapping intent into local SQLite state', async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-yaml-seed-'));
const project = projectWithConnections({
'prod-metabase': {
@ -39,17 +38,7 @@ describe('local mapping yaml reconciliation bridge', () => {
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
});
await seedLocalMappingStateFromKtxYaml(project, 'prod-metabase');
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{ metabaseDatabaseId: 1, targetConnectionId: 'prod-warehouse', syncEnabled: true, source: 'ktx.yaml' },
]);
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
syncMode: 'ONLY',
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
defaultTagNames: ['ktx'],
});
await expect(seedLocalMappingStateFromKtxYaml(project, 'prod-metabase')).resolves.toBeUndefined();
});
it('seeds Looker local mappings from ktx.yaml mapping intent', async () => {

31
packages/context/src/ingest/local-mapping-reconcile.ts
View file

@ -3,29 +3,8 @@ import {
parseConnectionMappingBootstrap,
type KtxLocalProject,
type LookerMappingBootstrap,
type MetabaseMappingBootstrap,
} from '../project/index.js';
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
function metabaseSelections(bootstrap: MetabaseMappingBootstrap) {
return [
...bootstrap.selections.collections.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...bootstrap.selections.items.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
];
}
function metabaseMappings(bootstrap: MetabaseMappingBootstrap) {
const ids = new Set([...Object.keys(bootstrap.databaseMappings), ...Object.keys(bootstrap.syncEnabled)]);
return [...ids]
.map((id) => Number(id))
.sort((a, b) => a - b)
.map((id) => ({
metabaseDatabaseId: id,
targetConnectionId: bootstrap.databaseMappings[String(id)] ?? null,
syncEnabled: bootstrap.syncEnabled[String(id)] ?? false,
}));
}
function lookerMappings(bootstrap: LookerMappingBootstrap) {
return Object.entries(bootstrap.connectionMappings)
@ -44,20 +23,12 @@ export async function seedLocalMappingStateFromKtxYaml(project: KtxLocalProject,
return;
}
const dbPath = ktxLocalStateDbPath(project);
if (bootstrap.adapter === 'metabase') {
await new LocalMetabaseSourceStateReader({ dbPath }).applyYamlBootstrap({
connectionId,
syncMode: bootstrap.syncMode,
defaultTagNames: bootstrap.defaultTagNames,
selections: metabaseSelections(bootstrap),
mappings: metabaseMappings(bootstrap),
});
return;
}
if (bootstrap.adapter === 'looker') {
await new LocalLookerRuntimeStore({ dbPath }).applyYamlBootstrap({
await new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) }).applyYamlBootstrap({
lookerConnectionId: connectionId,
mappings: lookerMappings(bootstrap),
});

143
packages/context/src/ingest/local-metabase-ingest.test.ts
View file

@ -4,7 +4,7 @@ import { join } from 'node:path';
import { AgentRunnerService } from '../agent/index.js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { initKtxProject, type KtxLocalProject } from '../project/index.js';
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
import { LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js';
import { getLocalIngestStatus, runLocalMetabaseIngest } from './local-ingest.js';
import type { ChunkResult, FetchContext, SourceAdapter } from './types.js';
@ -94,33 +94,19 @@ describe('runLocalMetabaseIngest', () => {
});
async function seedMetabaseState(): Promise<void> {
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
await store.replaceSourceState({
connectionId: 'prod-metabase',
project.config.connections['prod-metabase'].mappings = {
databaseMappings: { '1': 'warehouse_a', '2': 'warehouse_b' },
syncEnabled: { '1': true, '2': true },
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: [],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'localhost',
metabaseDbName: 'a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Warehouse B',
metabaseEngine: 'postgres',
metabaseHost: 'localhost',
metabaseDbName: 'b',
targetConnectionId: 'warehouse_b',
syncEnabled: true,
source: 'refresh',
},
selections: { collections: [], items: [] },
};
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
discovered: [
{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'localhost', dbName: 'a' },
{ id: 2, name: 'Warehouse B', engine: 'postgres', host: 'localhost', dbName: 'b' },
],
});
}
@ -151,22 +137,10 @@ describe('runLocalMetabaseIngest', () => {
});
it('throws before runner work when there are no sync-enabled mapped rows', async () => {
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
await store.replaceSourceState({
connectionId: 'prod-metabase',
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: null,
metabaseDbName: null,
targetConnectionId: null,
syncEnabled: true,
source: 'refresh',
},
],
});
project.config.connections['prod-metabase'].mappings = {
databaseMappings: { '1': null },
syncEnabled: { '1': true },
};
await expect(
runLocalMetabaseIngest({
@ -178,59 +152,28 @@ describe('runLocalMetabaseIngest', () => {
).rejects.toThrow('no sync-enabled mappings with a target connection');
});
it('throws with refresh guidance for unhydrated sync-enabled rows', async () => {
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
await store.replaceSourceState({
connectionId: 'prod-metabase',
mappings: [
{
metabaseDatabaseId: 7,
metabaseDatabaseName: null,
metabaseEngine: null,
metabaseHost: null,
metabaseDbName: null,
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'ktx.yaml',
},
],
it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => {
project.config.connections['prod-metabase'].mappings = {
databaseMappings: { '1': 'warehouse_a' },
syncEnabled: { '1': true },
};
const result = await runLocalMetabaseIngest({
project,
adapters: [new FakeMetabaseSourceAdapter()],
metabaseConnectionId: 'prod-metabase',
agentRunner: new TestAgentRunner(),
jobIdFactory: () => 'metabase-child-1',
});
await expect(
runLocalMetabaseIngest({
project,
adapters: [new FakeMetabaseSourceAdapter()],
expect(result.status).toBe('all_succeeded');
expect(result.children).toMatchObject([
{
metabaseConnectionId: 'prod-metabase',
agentRunner: new TestAgentRunner(),
}),
).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`');
});
it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => {
const project = {
projectDir: tempDir,
config: {
ingest: { adapters: ['metabase'] },
connections: {
'prod-metabase': {
driver: 'metabase',
mappings: {
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
},
},
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
},
metabaseDatabaseId: 1,
targetConnectionId: 'warehouse_a',
},
} as never;
await expect(
runLocalMetabaseIngest({
project,
adapters: [new FakeMetabaseSourceAdapter()],
metabaseConnectionId: 'prod-metabase',
}),
).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`');
]);
});
it('rejects source-dir uploads through the Metabase fan-out runner', async () => {
@ -266,15 +209,15 @@ describe('runLocalMetabaseIngest', () => {
it('captures fetch-time child failures and continues later mappings', async () => {
await seedMetabaseState();
project.config.connections.warehouse_c = { driver: 'postgres', url: 'postgres://localhost/c' };
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
await store.upsertDatabaseMapping({
connectionId: 'prod-metabase',
metabaseDatabaseId: 3,
targetConnectionId: 'warehouse_c',
syncEnabled: true,
source: 'cli',
});
await store.refreshDiscoveredDatabases({
project.config.connections['prod-metabase'].mappings = {
databaseMappings: { '1': 'warehouse_a', '2': 'warehouse_b', '3': 'warehouse_c' },
syncEnabled: { '1': true, '2': true, '3': true },
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: { collections: [], items: [] },
};
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
discovered: [
{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'localhost', dbName: 'a' },

46
packages/context/src/llm/local-config.test.ts
View file

@ -37,6 +37,52 @@ describe('local KTX LLM config', () => {
});
});
it('resolves Vertex AI env references into a KtxLlmConfig', () => {
const config: KtxProjectLlmConfig = {
provider: {
backend: 'vertex',
vertex: { project: 'env:GOOGLE_VERTEX_PROJECT', location: 'env:GOOGLE_VERTEX_LOCATION' },
},
models: { default: 'env:KTX_MODEL' },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
};
expect(
resolveLocalKtxLlmConfig(config, {
GOOGLE_VERTEX_PROJECT: 'local-gcp-project',
GOOGLE_VERTEX_LOCATION: 'us-east5',
KTX_MODEL: 'claude-sonnet-4-6',
}),
).toEqual({
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
modelSlots: { default: 'claude-sonnet-4-6' },
promptCaching: { enabled: true, vertexFallbackTo5m: true },
});
});
it('ignores inactive Vertex AI references for non-Vertex backends', () => {
const config: KtxProjectLlmConfig = {
provider: {
backend: 'anthropic',
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
vertex: { location: 'env:MISSING_VERTEX_LOCATION' },
},
models: { default: 'claude-sonnet-4-6' },
};
expect(
resolveLocalKtxLlmConfig(config, {
ANTHROPIC_API_KEY: 'sk-ant-test', // pragma: allowlist secret
}),
).toEqual({
backend: 'anthropic',
anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret
modelSlots: { default: 'claude-sonnet-4-6' },
promptCaching: undefined,
});
});
it('returns null when the local LLM backend is disabled', () => {
expect(
createLocalKtxLlmProviderFromConfig({

19
packages/context/src/llm/local-config.ts
View file

@ -67,16 +67,33 @@ function resolvedProviderConfig(
};
}
function resolvedVertexConfig(
config: { project?: string; location?: string } | undefined,
env: NodeJS.ProcessEnv,
): { project?: string; location: string } | undefined {
if (!config) {
return undefined;
}
const project = resolveOptional(config.project, env);
const location = resolveRequired(config.location, env, 'llm.provider.vertex.location is required');
return {
...(project ? { project } : {}),
location,
};
}
export function resolveLocalKtxLlmConfig(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): KtxLlmConfig | null {
if (config.provider.backend === 'none') {
return null;
}
const modelSlots = resolveModelSlots(config.models, env);
const vertex = config.provider.backend === 'vertex' ? resolvedVertexConfig(config.provider.vertex, env) : undefined;
const anthropic = resolvedProviderConfig(config.provider.anthropic, env);
const gateway = resolvedProviderConfig(config.provider.gateway, env);
return {
backend: config.provider.backend,
...(config.provider.vertex ? { vertex: config.provider.vertex } : {}),
...(vertex ? { vertex } : {}),
...(anthropic ? { anthropic } : {}),
...(gateway ? { gateway } : {}),
modelSlots,

4
packages/context/src/package-exports.test.ts
View file

@ -199,7 +199,9 @@ describe('@ktx/context package exports', () => {
expect(ingest.stagedSyncConfigSchema).toBeDefined();
expect(ingest.stagedLookerScopeFileSchema).toBeDefined();
expect(ingest.stagedLookerFetchReportSchema).toBeDefined();
expect(ingest.LocalMetabaseSourceStateReader).toBeTypeOf('function');
expect('LocalMetabaseSourceStateReader' in ingest).toBe(false);
expect(ingest.KtxYamlMetabaseSourceStateReader).toBeTypeOf('function');
expect(ingest.LocalMetabaseDiscoveryCache).toBeTypeOf('function');
expect(ingest.createLocalMetabaseSourceAdapter).toBeTypeOf('function');
expect(ingest.metabaseRuntimeConfigFromLocalConnection).toBeTypeOf('function');
expect(ingest.IngestMetabaseClientFactory).toBeTypeOf('function');

8
packages/context/src/project/config.test.ts
View file

@ -81,16 +81,13 @@ describe('KTX project config', () => {
});
});
it('parses and serializes setup wizard metadata', () => {
it('parses and serializes setup warehouse metadata without setup progress', () => {
const config = parseKtxProjectConfig(`
project: revenue
setup:
database_connection_ids:
- warehouse
- analytics
completed_steps:
- project
- llm
connections:
warehouse:
driver: postgres
@ -99,13 +96,12 @@ connections:
expect(config.setup).toEqual({
database_connection_ids: ['warehouse', 'analytics'],
completed_steps: ['project', 'llm'],
});
const serialized = serializeKtxProjectConfig(config);
expect(serialized).toContain('setup:');
expect(serialized).toContain('database_connection_ids:');
expect(serialized).toContain('completed_steps:');
expect(serialized).not.toContain('completed_steps:');
});
it('parses global direct Anthropic LLM config', () => {

2
packages/context/src/project/config.ts
View file

@ -75,7 +75,6 @@ export interface KtxProjectConnectionConfig {
export interface KtxProjectSetupConfig {
database_connection_ids: string[];
completed_steps?: string[];
}
export interface KtxProjectConfig {
@ -493,7 +492,6 @@ export function parseKtxProjectConfig(raw: string): KtxProjectConfig {
? {
setup: {
database_connection_ids: stringArray(setup.database_connection_ids, []),
completed_steps: stringArray(setup.completed_steps, []),
},
}
: {}),

2
packages/context/src/project/index.ts
View file

@ -27,12 +27,10 @@ export { initKtxProject, loadKtxProject } from './project.js';
export type { KtxSetupStep } from './setup-config.js';
export {
KTX_SETUP_STEPS,
ktxSetupCompletedSteps,
ktxSetupStatePath,
markKtxSetupStateStepComplete,
mergeKtxSetupGitignoreEntries,
readKtxSetupState,
setKtxSetupDatabaseConnectionIds,
stripKtxSetupCompletedSteps,
writeKtxSetupState,
} from './setup-config.js';

32
packages/context/src/project/setup-config.test.ts
View file

@ -4,12 +4,10 @@ import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { buildDefaultKtxProjectConfig } from './config.js';
import {
ktxSetupCompletedSteps,
markKtxSetupStateStepComplete,
mergeKtxSetupGitignoreEntries,
readKtxSetupState,
setKtxSetupDatabaseConnectionIds,
stripKtxSetupCompletedSteps,
} from './setup-config.js';
describe('KTX setup config helpers', () => {
@ -48,36 +46,6 @@ describe('KTX setup config helpers', () => {
expect(config.setup).toBeUndefined();
});
it('strips setup completed steps while preserving database connection ids', () => {
const config = {
...buildDefaultKtxProjectConfig('warehouse'),
setup: {
database_connection_ids: ['warehouse'],
completed_steps: ['project', 'databases'],
},
};
expect(stripKtxSetupCompletedSteps(config).setup).toEqual({
database_connection_ids: ['warehouse'],
});
});
it('combines config setup steps with local state for reads', () => {
const config = {
...buildDefaultKtxProjectConfig('warehouse'),
setup: {
database_connection_ids: ['warehouse'],
completed_steps: ['project', 'databases'],
},
};
expect(ktxSetupCompletedSteps(config, { completed_steps: ['databases', 'sources'] })).toEqual([
'project',
'databases',
'sources',
]);
});
it('merges setup-local gitignore entries without removing existing lines', () => {
expect(mergeKtxSetupGitignoreEntries('cache/\ndb.sqlite\n')).toBe(
['cache/', 'db.sqlite', 'db.sqlite-*', 'ingest-transcripts/', 'secrets/', 'setup/', 'agents/', ''].join('\n'),

22
packages/context/src/project/setup-config.ts
View file

@ -64,27 +64,6 @@ export async function markKtxSetupStateStepComplete(projectDir: string, step: Kt
return nextState;
}
export function ktxSetupCompletedSteps(config: KtxProjectConfig, state: KtxSetupState): KtxSetupStep[] {
return uniqueSetupSteps([...(config.setup?.completed_steps ?? []), ...state.completed_steps]);
}
export function stripKtxSetupCompletedSteps(config: KtxProjectConfig): KtxProjectConfig {
if (!config.setup) {
return config;
}
const databaseConnectionIds = config.setup.database_connection_ids ?? [];
if (databaseConnectionIds.length === 0) {
const { setup: _setup, ...withoutSetup } = config;
return withoutSetup;
}
return {
...config,
setup: {
database_connection_ids: [...databaseConnectionIds],
},
};
}
export function setKtxSetupDatabaseConnectionIds(
config: KtxProjectConfig,
connectionIds: string[],
@ -95,7 +74,6 @@ export function setKtxSetupDatabaseConnectionIds(
...config,
setup: {
database_connection_ids: uniqueConnectionIds,
...(config.setup?.completed_steps ? { completed_steps: [...config.setup.completed_steps] } : {}),
},
};
}