apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-19 08:28:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge origin/main into drop-legacy-migration-code

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 15:07:09 +02:00
commit 18a9122875
33 changed files with 1096 additions and 5342 deletions
Download patch file Download diff file Expand all files Collapse all files

3
packages/cli/src/cli-program.ts
View file

@ -178,9 +178,6 @@ function shouldSuppressProjectDirLine(path: string[], options: Record<string, un
if (commandPathKey === 'ktx ingest watch') {
return options.json !== true && options.plain !== true;
}
if (commandPathKey === 'ktx connection notion pick') {
return options.input !== false;
}
const demoIndex = path.indexOf('demo');
if (demoIndex >= 0) {
const demoCommand = path[demoIndex + 1];

4
packages/cli/src/cli-runtime.ts
View file

@ -1,7 +1,5 @@
import { createRequire } from 'node:module';
import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js';
import type { KtxConnectionNotionArgs } from './commands/connection-notion.js';
import type { KtxConnectionArgs } from './connection.js';
import type { KtxDoctorArgs } from './doctor.js';
import type { KtxIngestArgs } from './ingest.js';
@ -30,8 +28,6 @@ export interface KtxCliIo {
export interface KtxCliDeps {
setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise<number>;
connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise<number>;
connectionNotion?: (args: KtxConnectionNotionArgs, io: KtxCliIo) => Promise<number>;
connectionMetabaseSetup?: (args: KtxConnectionMetabaseSetupArgs, io: KtxCliIo) => Promise<number>;
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise<number>;
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;

25
packages/cli/src/command-schemas.ts
View file

@ -1,33 +1,8 @@
import { z } from 'zod';
const projectDirSchema = z.string().min(1);
const safeConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, 'Unsafe connection id');
const stringArraySchema = z.array(z.string());
export const connectionAddCommandSchema = z.object({
command: z.literal('add'),
projectDir: projectDirSchema,
driver: z.string().min(1),
connectionId: safeConnectionIdSchema,
url: z.string().optional(),
schemas: stringArraySchema,
readonly: z.boolean(),
force: z.boolean(),
allowLiteralCredentials: z.boolean(),
notion: z
.object({
authTokenRef: z.string().min(1),
crawlMode: z.enum(['all_accessible', 'selected_roots']),
rootPageIds: stringArraySchema,
rootDatabaseIds: stringArraySchema,
rootDataSourceIds: stringArraySchema,
maxPagesPerRun: z.number().int().positive().optional(),
maxKnowledgeCreatesPerRun: z.number().int().nonnegative().optional(),
maxKnowledgeUpdatesPerRun: z.number().int().nonnegative().optional(),
})
.optional(),
});
export const wikiWriteCommandSchema = z.object({
command: z.literal('write'),
projectDir: projectDirSchema,

308
packages/cli/src/commands/connection-commands.ts
View file

@ -1,61 +1,19 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KtxCliCommandContext,
parseBooleanStringOption,
parseNonEmptyAssignmentOption,
parseNonNegativeIntegerOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { connectionAddCommandSchema } from '../command-schemas.js';
import { type Command } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxConnectionArgs } from '../connection.js';
import { profileMark } from '../startup-profile.js';
import type { KtxConnectionMappingArgs } from './connection-mapping.js';
import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js';
import { registerConnectionNotionCommands } from './connection-notion-commands.js';
profileMark('module:commands/connection-commands');
const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const;
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const;
function parseCsvIds(value: string): number[] {
return value
.split(',')
.filter(Boolean)
.map((item) => parsePositiveIntegerOption(item));
}
function parseCsvStrings(value: string): string[] {
return value
.split(',')
.map((item) => item.trim())
.filter(Boolean);
}
function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' {
if (value === 'databaseMappings' || value === 'connectionMappings') {
return value;
}
throw new InvalidArgumentError('must be databaseMappings or connectionMappings');
}
async function runConnectionArgs(context: KtxCliCommandContext, args: KtxConnectionArgs): Promise<void> {
const runner = context.deps.connection ?? (await import('../connection.js')).runKtxConnection;
context.setExitCode(await runner(args, context.io));
}
async function runMappingArgs(context: KtxCliCommandContext, args: KtxConnectionMappingArgs): Promise<void> {
const { runKtxConnectionMapping } = await import('./connection-mapping.js');
context.setExitCode(await runKtxConnectionMapping(args, context.io));
}
export function registerConnectionCommands(program: Command, context: KtxCliCommandContext, commandName = 'connection'): void {
const connection = program
.command(commandName)
.description('Add, list, test, and map data sources')
.description('List and test configured connections')
.showHelpAfterError()
.addHelpText(
'after',
@ -83,264 +41,4 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm
connectionId,
});
});
connection
.command('add')
.description('Add or replace a configured connection')
.argument('<driver>', 'Connection driver')
.argument('<connectionId>', 'KTX connection id')
.option('--url <url>', 'Connection URL, env:NAME, or file:/path reference')
.option('--schema <schema>', 'Schema to include; repeatable', collectOption, [])
.option('--readonly', 'Mark the connection as read-only', false)
.option('--force', 'Replace an existing connection', false)
.option('--allow-literal-credentials', 'Allow writing a literal credential URL to ktx.yaml', false)
.addOption(new Option('--token-env <name>', 'Environment variable containing Notion auth token').conflicts('tokenFile'))
.addOption(new Option('--token-file <path>', 'File containing Notion auth token').conflicts('tokenEnv'))
.addOption(
new Option('--crawl-mode <mode>', 'Notion crawl mode: all_accessible or selected_roots')
.choices(CRAWL_MODE_CHOICES)
.default('selected_roots'),
)
.option('--root-page-id <id>', 'Root page to crawl; repeatable', collectOption, [])
.option('--root-database-id <id>', 'Root database to crawl; repeatable', collectOption, [])
.option('--root-data-source-id <id>', 'Root data source to crawl; repeatable', collectOption, [])
.option('--max-pages <n>', 'Maximum pages per run', parsePositiveIntegerOption)
.option('--max-knowledge-creates <n>', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption)
.option('--max-knowledge-updates <n>', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption)
.action(async (driver: string, connectionId: string, options, command) => {
const notion =
driver === 'notion'
? {
authTokenRef: options.tokenEnv
? `env:${options.tokenEnv}`
: options.tokenFile
? `file:${options.tokenFile}`
: '',
crawlMode: options.crawlMode,
rootPageIds: options.rootPageId,
rootDatabaseIds: options.rootDatabaseId,
rootDataSourceIds: options.rootDataSourceId,
maxPagesPerRun: options.maxPages,
maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates,
maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates,
}
: undefined;
if (driver === 'notion' && !notion?.authTokenRef) {
throw new Error('connection add notion requires --token-env NAME or --token-file PATH');
}
if (
driver === 'notion' &&
notion?.crawlMode === 'selected_roots' &&
notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0
) {
throw new Error('connection add notion selected_roots requires at least one root id');
}
const args = connectionAddCommandSchema.parse({
command: 'add',
projectDir: resolveCommandProjectDir(command),
driver,
connectionId,
url: options.url,
schemas: options.schema.filter(Boolean),
readonly: options.readonly === true,
force: options.force === true,
allowLiteralCredentials: options.allowLiteralCredentials === true,
notion,
});
await runConnectionArgs(context, args);
});
connection
.command('remove')
.description('Remove a configured connection from ktx.yaml')
.argument('<connectionId>', 'KTX connection id')
.option('--force', 'Remove without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'remove',
projectDir: resolveCommandProjectDir(command),
connectionId,
force: options.force === true,
...(options.input === false ? { inputMode: 'disabled' } : {}),
});
});
connection
.command('map')
.description('Refresh and validate BI-to-warehouse mappings')
.argument('<sourceConnectionId>', 'Source BI connection id')
.option('--json', 'Print JSON output', false)
.action(async (sourceConnectionId: string, options: { json?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'map',
projectDir: resolveCommandProjectDir(command),
sourceConnectionId,
json: options.json === true,
});
});
registerConnectionMappingCommands(connection, context);
registerConnectionMetabaseCommands(connection, context);
registerConnectionNotionCommands(connection, context);
}
function registerConnectionMappingCommands(connection: Command, context: KtxCliCommandContext): void {
const mapping = connection
.command('mapping')
.description('Manage Metabase warehouse mappings')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
mapping
.command('list')
.description('List Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
mapping
.command('set')
.description('Set a Metabase or Looker warehouse mapping')
.argument('<connectionId>', 'Source connection id', parseSafeConnectionIdOption)
.argument('<field>', 'Mapping field', parseMappingFieldOption)
.argument('<assignment>', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption)
.action(
async (
connectionId: string,
field: 'databaseMappings' | 'connectionMappings',
assignment: { key: string; value: string },
_options: unknown,
command,
) => {
await runMappingArgs(context, {
command: 'set',
projectDir: resolveCommandProjectDir(command),
connectionId,
field,
key: assignment.key,
value: assignment.value,
});
},
);
mapping
.command('apply-bulk')
.description('Apply mappings from JSON')
.argument('<connectionId>', 'Metabase connection id')
.requiredOption('--file <path>', 'JSON mapping file')
.action(async (connectionId: string, options: { file: string }, command) => {
await runMappingArgs(context, {
command: 'apply-bulk',
projectDir: resolveCommandProjectDir(command),
connectionId,
filePath: options.file,
});
});
mapping
.command('set-sync-enabled')
.description('Enable or disable sync for one Metabase database')
.argument('<connectionId>', 'Metabase connection id')
.argument('<metabaseDatabaseId>', 'Metabase database id', parsePositiveIntegerOption)
.requiredOption('--enabled <value>', 'true or false', parseBooleanStringOption)
.action(
async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => {
await runMappingArgs(context, {
command: 'set-sync-enabled',
projectDir: resolveCommandProjectDir(command),
connectionId,
metabaseDatabaseId,
enabled: options.enabled,
});
},
);
const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection');
syncState
.command('get')
.description('Read sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'sync-state-get',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
syncState
.command('set')
.description('Write sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.addOption(new Option('--mode <mode>', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory())
.option('--collections <ids>', 'Comma-separated collection ids', parseCsvIds, [])
.option('--items <ids>', 'Comma-separated item ids', parseCsvIds, [])
.option('--tag-names <names>', 'Comma-separated tag names', parseCsvStrings, [])
.action(async (connectionId: string, options, command) => {
await runMappingArgs(context, {
command: 'sync-state-set',
projectDir: resolveCommandProjectDir(command),
connectionId,
syncMode: options.mode,
collectionIds: options.collections,
itemIds: options.items,
tagNames: options.tagNames,
});
});
mapping
.command('refresh')
.description('Refresh Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--auto-accept', 'Accept refresh changes without prompting', false)
.action(async (connectionId: string, options: { autoAccept?: boolean }, command) => {
await runMappingArgs(context, {
command: 'refresh',
projectDir: resolveCommandProjectDir(command),
connectionId,
autoAccept: options.autoAccept === true,
});
});
mapping
.command('validate')
.description('Validate Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.action(async (connectionId: string, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'validate',
projectDir: resolveCommandProjectDir(command),
connectionId,
});
});
mapping
.command('clear')
.description('Clear Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption)
.action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'clear',
projectDir: resolveCommandProjectDir(command),
connectionId,
...(metabaseDatabaseId ? { metabaseDatabaseId } : {}),
});
});
}

345
packages/cli/src/commands/connection-mapping.test.ts
View file

@ -1,345 +0,0 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseDiscoveryCache } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnectionMapping } from './connection-mapping.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKtxConnectionMapping', () => {
let tempDir: string;
let projectDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-metabase-mapping-'));
projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'mapping' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
},
'prod-warehouse': {
driver: 'postgres',
url: 'env:WAREHOUSE_URL',
readonly: true,
},
},
}),
'ktx',
'ktx@example.com',
'Seed Metabase mapping test connections',
);
});
async function replaceConnections(connections: Record<string, { driver: string; [key: string]: unknown }>) {
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections,
}),
'ktx',
'ktx@example.com',
'Replace mapping test connections',
);
}
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('sets, lists, disables, and clears local Metabase mappings', async () => {
const io = makeIo();
const setCode = await runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
);
expect(setCode, io.stderr()).toBe(0);
let config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toMatchObject({
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
});
const listIo = makeIo();
await expect(
runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io),
).resolves.toBe(0);
expect(listIo.stdout()).toContain('1 -> prod-warehouse');
expect(listIo.stdout()).toContain('unhydrated');
await expect(
runKtxConnectionMapping(
{
command: 'set-sync-enabled',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
enabled: false,
},
makeIo().io,
),
).resolves.toBe(0);
config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toMatchObject({
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': false },
});
await expect(
runKtxConnectionMapping(
{
command: 'clear',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
},
makeIo().io,
),
).resolves.toBe(0);
config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toBeUndefined();
});
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-yaml-mapping-'));
await initKtxProject({ projectDir, projectName: 'yaml-mapping' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
mappings: {
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
},
},
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'ktx',
'ktx@example.com',
'Seed yaml mappings',
);
const io = makeIo();
await expect(
runKtxConnectionMapping(
{ command: 'list', projectDir, connectionId: 'prod-metabase', json: false },
io.io,
),
).resolves.toBe(0);
expect(io.stdout()).toContain('1 -> prod-warehouse');
expect(io.stdout()).toContain('source: ktx.yaml');
});
it('refreshes Metabase discovery metadata through the injected runtime client', async () => {
const client = {
getDatabases: vi.fn().mockResolvedValue([
{
id: 1,
name: 'Analytics',
engine: 'postgres',
details: { host: 'pg.internal', dbname: 'analytics' },
is_sample: false,
},
]),
cleanup: vi.fn(),
};
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'refresh',
projectDir,
connectionId: 'prod-metabase',
autoAccept: true,
},
io.io,
{
createMetabaseClient: async () => client as never,
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 database');
expect(client.cleanup).toHaveBeenCalledTimes(1);
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['prod-metabase']?.mappings).toBeUndefined();
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(discoveryCache.listDiscoveredDatabases('prod-metabase')).resolves.toMatchObject([
{ id: 1, name: 'Analytics', engine: 'postgres' },
]);
});
it('sets and lists Looker connection mappings', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'connectionMappings',
key: 'analytics',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
await expect(
runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('analytics -> prod-warehouse');
});
it('keeps driver-specific mapping field validation in the runner', async () => {
await replaceConnections({
'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' },
warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' },
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'databaseMappings',
key: '1',
value: 'warehouse',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Looker mapping set requires connectionMappings');
});
it('refreshes Looker mapping metadata and reports drift', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{ command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true },
io.io,
{
createLookerClient: async () => ({
listLookerConnections: async () => [
{
name: 'analytics',
host: 'db.example.test',
database: 'analytics',
schema: null,
dialect: 'postgres',
},
],
cleanup: async () => {},
}),
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 connection');
expect(io.stdout()).toContain('Unmapped discovered: 1');
});
it('validates Looker mappings through the canonical local warehouse descriptor', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-descriptor-validation-'));
await initKtxProject({ projectDir, projectName: 'descriptor-validation' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-looker': {
driver: 'looker',
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
},
'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'ktx',
'ktx@example.com',
'Seed descriptor validation',
);
const io = makeIo();
await expect(
runKtxConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('Mapping validation passed: prod-looker');
expect(io.stderr()).toBe('');
});
});

507
packages/cli/src/commands/connection-mapping.ts
View file

@ -1,507 +0,0 @@
import { readFile } from 'node:fs/promises';
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
KtxYamlMetabaseSourceStateReader,
LocalLookerRuntimeStore,
LocalMetabaseDiscoveryCache,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
discoverMetabaseDatabases,
lookerCredentialsFromLocalConnection,
metabaseRuntimeConfigFromLocalConnection,
seedLocalMappingStateFromKtxYaml,
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type LocalMetabaseMappingListRow,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
} from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConfig,
ktxLocalStateDbPath,
loadKtxProject,
parseMetabaseMappingBootstrap,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/connection-mapping');
export type KtxConnectionMappingArgs =
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'set';
projectDir: string;
connectionId: string;
field: 'databaseMappings' | 'connectionMappings';
key: string;
value: string;
}
| { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string }
| {
command: 'set-sync-enabled';
projectDir: string;
connectionId: string;
metabaseDatabaseId: number;
enabled: boolean;
}
| { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'sync-state-set';
projectDir: string;
connectionId: string;
syncMode: MetabaseSyncMode;
collectionIds: number[];
itemIds: number[];
tagNames: string[];
}
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
| { command: 'validate'; projectDir: string; connectionId: string }
| { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string };
interface KtxConnectionMappingDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
createLookerClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
}
interface MetabaseBulkMappingPayload {
databaseMappings?: Record<string, string | null>;
syncEnabled?: Record<string, boolean>;
syncMode?: MetabaseSyncMode;
selections?: { collections?: number[]; items?: number[] };
defaultTagNames?: string[];
}
function parseId(value: string, label: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer`);
}
return parsed;
}
interface MetabaseMappingsBlock {
databaseMappings: Record<string, string | null>;
syncEnabled: Record<string, boolean>;
syncMode: MetabaseSyncMode;
selections: { collections: number[]; items: number[] };
defaultTagNames: string[];
}
function currentMetabaseMappings(project: KtxLocalProject, connectionId: string): MetabaseMappingsBlock {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const bootstrap = parseMetabaseMappingBootstrap(connectionId, connection);
return {
databaseMappings: { ...bootstrap.databaseMappings },
syncEnabled: { ...bootstrap.syncEnabled },
syncMode: bootstrap.syncMode,
selections: {
collections: [...bootstrap.selections.collections],
items: [...bootstrap.selections.items],
},
defaultTagNames: [...bootstrap.defaultTagNames],
};
}
function hasMetabaseMappings(block: MetabaseMappingsBlock): boolean {
return (
Object.keys(block.databaseMappings).length > 0 ||
Object.keys(block.syncEnabled).length > 0 ||
block.syncMode !== 'ALL' ||
block.selections.collections.length > 0 ||
block.selections.items.length > 0 ||
block.defaultTagNames.length > 0
);
}
function serializeMetabaseMappingsBlock(block: MetabaseMappingsBlock): Record<string, unknown> | undefined {
if (!hasMetabaseMappings(block)) {
return undefined;
}
return {
databaseMappings: block.databaseMappings,
syncEnabled: block.syncEnabled,
syncMode: block.syncMode,
selections: block.selections,
defaultTagNames: block.defaultTagNames,
};
}
async function writeMetabaseMappings(
project: KtxLocalProject,
connectionId: string,
block: MetabaseMappingsBlock,
message: string,
): Promise<void> {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const mappings = serializeMetabaseMappingsBlock(block);
const nextConnection = { ...connection };
if (mappings) {
nextConnection.mappings = mappings;
} else {
delete nextConnection.mappings;
}
const nextConfig: KtxProjectConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: nextConnection,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
message,
);
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function createDefaultLookerClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
const factory = new DefaultLookerConnectionClientFactory({
async resolve(lookerConnectionId) {
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
},
});
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
cleanup?(): Promise<void>;
};
}
function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean {
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
}
function assertLookerConnection(project: KtxLocalProject, connectionId: string): void {
if (!isLookerConnection(project, connectionId)) {
throw new Error(`Connection "${connectionId}" is not a Looker connection`);
}
}
function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void {
const connection = project.config.connections[connectionId];
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
}
}
function assertTargetConnection(project: KtxLocalProject, connectionId: string): void {
if (!project.config.connections[connectionId]) {
throw new Error(`Target connection "${connectionId}" does not exist`);
}
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function renderMapping(row: LocalMetabaseMappingListRow): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
row.source
})`;
}
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
const target = row.ktxConnectionId ?? '[unmapped]';
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
}
export async function runKtxConnectionMapping(
args: KtxConnectionMappingArgs,
io: KtxCliIo = process,
deps: KtxConnectionMappingDeps = {},
): Promise<number> {
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
await seedLocalMappingStateFromKtxYaml(project, args.connectionId);
if (isLookerConnection(project, args.connectionId)) {
assertLookerConnection(project, args.connectionId);
const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listConnectionMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'connectionMappings') {
throw new Error('Looker mapping set requires connectionMappings <lookerConnectionName>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
await store.upsertConnectionMapping({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.key,
ktxConnectionId: args.value,
source: 'cli',
});
io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
try {
const discovered = await discoverLookerConnections(client);
const drift = computeLookerMappingDrift({
storedMappings: await store.readMappings(args.connectionId),
discovered,
});
if (args.autoAccept) {
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup?.();
}
}
if (args.command === 'validate') {
const knownKtxConnectionIds = new Set(Object.keys(project.config.connections));
const knownConnectionTypes = new Map(
Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]),
);
const validation = validateLookerMappings({
mappings: await store.readMappings(args.connectionId),
knownKtxConnectionIds,
knownConnectionTypes,
});
if (!validation.ok) {
for (const error of validation.errors) {
io.stderr.write(`${error.key}: ${error.reason}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
if (args.command === 'clear') {
await store.clearConnectionMappings({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined),
});
io.stdout.write(
args.mappingKey
? `Cleared connectionMappings.${args.mappingKey}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
}
throw new Error(`Looker connection mapping does not support ${args.command}`);
}
assertMetabaseConnection(project, args.connectionId);
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
const metabaseStateReader = new KtxYamlMetabaseSourceStateReader(project, { discoveryCache });
if (args.command === 'list') {
const rows = await metabaseStateReader.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'databaseMappings') {
throw new Error('Metabase mapping set requires databaseMappings <metabaseDatabaseId>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
const block = currentMetabaseMappings(project, args.connectionId);
const metabaseDatabaseId = String(parseId(args.key, 'metabaseDatabaseId'));
block.databaseMappings[metabaseDatabaseId] = args.value;
block.syncEnabled[metabaseDatabaseId] = true;
await writeMetabaseMappings(project, args.connectionId, block, `Set Metabase mapping ${args.connectionId}.${metabaseDatabaseId}`);
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'apply-bulk') {
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
const block = currentMetabaseMappings(project, args.connectionId);
const databaseMappings = payload.databaseMappings ?? {};
for (const targetConnectionId of Object.values(databaseMappings)) {
if (targetConnectionId) {
assertTargetConnection(project, targetConnectionId);
}
}
for (const id of Object.keys(databaseMappings)) {
parseId(id, 'metabaseDatabaseId');
block.databaseMappings[id] = databaseMappings[id] ?? null;
}
for (const [id, enabled] of Object.entries(payload.syncEnabled ?? {})) {
parseId(id, 'metabaseDatabaseId');
block.syncEnabled[id] = enabled;
}
if (payload.syncMode !== undefined) {
block.syncMode = payload.syncMode;
}
if (payload.defaultTagNames !== undefined) {
block.defaultTagNames = payload.defaultTagNames;
}
if (payload.selections !== undefined) {
block.selections = {
collections: payload.selections.collections ?? [],
items: payload.selections.items ?? [],
};
}
await writeMetabaseMappings(project, args.connectionId, block, `Apply Metabase mappings ${args.connectionId}`);
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'set-sync-enabled') {
const block = currentMetabaseMappings(project, args.connectionId);
block.syncEnabled[String(args.metabaseDatabaseId)] = args.enabled;
await writeMetabaseMappings(
project,
args.connectionId,
block,
`Set Metabase sync ${args.connectionId}.${args.metabaseDatabaseId}`,
);
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
return 0;
}
if (args.command === 'sync-state-get') {
const state = await metabaseStateReader.getSourceState(args.connectionId);
const payload = {
syncMode: state.syncMode,
selections: state.selections,
defaultTagNames: state.defaultTagNames,
};
io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`);
return 0;
}
if (args.command === 'sync-state-set') {
const block = currentMetabaseMappings(project, args.connectionId);
block.syncMode = args.syncMode;
block.defaultTagNames = args.tagNames;
block.selections = { collections: args.collectionIds, items: args.itemIds };
await writeMetabaseMappings(project, args.connectionId, block, `Set Metabase sync state ${args.connectionId}`);
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const block = currentMetabaseMappings(project, args.connectionId);
const existing = block.databaseMappings;
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await discoveryCache.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup();
}
}
if (args.command === 'validate') {
const rows = (await metabaseStateReader.listDatabaseMappings(args.connectionId)).filter(
(row) => row.source === 'ktx.yaml',
);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
project.config.connections[row.targetConnectionId]
? targetPhysicalInfo(project, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (failures.length > 0) {
for (const failure of failures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
const block = currentMetabaseMappings(project, args.connectionId);
if (metabaseDatabaseId === undefined) {
block.databaseMappings = {};
block.syncEnabled = {};
block.syncMode = 'ALL';
block.selections = { collections: [], items: [] };
block.defaultTagNames = [];
} else {
delete block.databaseMappings[String(metabaseDatabaseId)];
delete block.syncEnabled[String(metabaseDatabaseId)];
}
await writeMetabaseMappings(project, args.connectionId, block, `Clear Metabase mappings ${args.connectionId}`);
io.stdout.write(
metabaseDatabaseId
? `Cleared databaseMappings.${metabaseDatabaseId}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

132
packages/cli/src/commands/connection-metabase-commands.ts
View file

@ -1,132 +0,0 @@
import { type Command, Option } from '@commander-js/extra-typings';
import {
type KtxCliCommandContext,
parseNonEmptyAssignmentOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import {
type KtxConnectionMetabaseSetupArgs,
type MetabaseSetupMappingAssignment,
type MetabaseSetupSyncMode,
runKtxConnectionMetabaseSetup,
} from './connection-metabase-setup.js';
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[];
interface ConnectionMetabaseSetupOptions {
id?: string;
url?: string;
apiKey?: string;
mintApiKey?: boolean;
username?: string;
password?: string;
map: MetabaseSetupMappingAssignment[];
sync: number[];
syncMode: MetabaseSetupSyncMode;
runIngest?: boolean;
yes?: boolean;
input?: boolean;
}
function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] {
return [...previous, parsePositiveIntegerOption(value)];
}
function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment {
const assignment = parseNonEmptyAssignmentOption(value);
return {
metabaseDatabaseId: parsePositiveIntegerOption(assignment.key),
targetConnectionId: parseSafeConnectionIdOption(assignment.value),
};
}
function collectMappingOption(
value: string,
previous: MetabaseSetupMappingAssignment[] = [],
): MetabaseSetupMappingAssignment[] {
return [...previous, parseMappingAssignment(value)];
}
async function runMetabaseSetupArgs(
context: KtxCliCommandContext,
args: KtxConnectionMetabaseSetupArgs,
): Promise<void> {
const runner = context.deps.connectionMetabaseSetup ?? runKtxConnectionMetabaseSetup;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionMetabaseCommands(connection: Command, context: KtxCliCommandContext): void {
const metabase = connection
.command('metabase')
.description('Configure Metabase connections')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
metabase.action(() => {
metabase.outputHelp();
context.setExitCode(0);
});
metabase
.command('setup')
.description('Guided setup for a Metabase connection')
.option('--id <connectionId>', 'KTX connection id to write', parseSafeConnectionIdOption)
.option('--url <url>', 'Metabase API URL')
.addOption(new Option('--api-key <key>', 'Metabase API key').conflicts('mintApiKey'))
.option('--mint-api-key', 'Mint a Metabase API key with credentials', false)
.option('--username <email>', 'Metabase admin username for API-key minting')
.option('--password <password>', 'Metabase admin password for API-key minting')
.addHelpText(
'after',
'\nGuided equivalent of:\n' +
' ktx connection mapping refresh <connectionId> --auto-accept\n' +
' ktx connection mapping set <connectionId> databaseMappings <id>=<target>\n' +
' ktx connection mapping set-sync-enabled <connectionId> <id> --enabled true\n' +
' ktx ingest run --connection-id <connectionId> --adapter metabase\n',
)
.option(
'--map <metabaseDatabaseId=targetConnectionId>',
'Assign a Metabase database id to a warehouse connection; repeatable',
collectMappingOption,
[],
)
.option(
'--sync <metabaseDatabaseId>',
'Enable Metabase sync for a discovered database; repeatable',
collectPositiveIntegerOption,
[],
)
.addOption(
new Option('--sync-mode <mode>', 'Metabase sync selection mode')
.choices(SYNC_MODE_CHOICES)
.default('ALL' satisfies MetabaseSetupSyncMode),
)
.option('--run-ingest', 'Run ingest after setup', false)
.option('--yes', 'Confirm and apply setup changes without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.showHelpAfterError()
.action(async (options: ConnectionMetabaseSetupOptions, command) => {
await runMetabaseSetupArgs(context, {
command: 'setup',
projectDir: resolveCommandProjectDir(command),
connectionId: options.id,
url: options.url,
apiKey: options.apiKey,
mintApiKey: options.mintApiKey === true,
metabaseUsername: options.username,
metabasePassword: options.password,
mappings: options.map,
syncEnabledDatabaseIds: options.sync,
syncMode: options.syncMode ?? 'ALL',
runIngest: options.runIngest === true,
yes: options.yes === true,
inputMode: options.input === false ? 'disabled' : 'auto',
});
});
}

1118
packages/cli/src/commands/connection-metabase-setup.test.ts
View file

File diff suppressed because it is too large Load diff

798
packages/cli/src/commands/connection-metabase-setup.ts
View file

@ -1,798 +0,0 @@
import type { Option as ClackOption } from '@clack/prompts';
import {
cancel,
confirm,
intro,
isCancel,
log,
multiselect,
note,
outro,
password,
select,
text,
} from '@clack/prompts';
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
MetabaseClient,
type MetabaseDatabase,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
metabaseRuntimeConfigFromLocalConnection,
validateMappingPhysicalMatch,
} from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
ktxLocalStateDbPath,
loadKtxProject,
parseMetabaseMappingBootstrap,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import { createClackSpinner, type KtxCliSpinner } from '../clack.js';
import type { KtxCliIo } from '../cli-runtime.js';
import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
import { type KtxPublicIngestArgs, runKtxPublicIngest } from '../public-ingest.js';
export type KtxMetabaseSetupInputMode = 'auto' | 'disabled';
export type MetabaseSetupSyncMode = MetabaseSyncMode;
type MetabaseSetupPromptOption<Value> = ClackOption<Value>;
export interface MetabaseSetupLogger {
info(message: string): void;
step(message: string): void;
success(message: string): void;
warn(message: string): void;
error(message: string): void;
}
export interface MetabaseSetupPromptAdapter {
intro(title?: string): void;
outro(message?: string): void;
note(message: string, title: string): void;
log: MetabaseSetupLogger;
spinner(): KtxCliSpinner;
select<T extends string>(options: { message: string; options: Array<MetabaseSetupPromptOption<T>> }): Promise<T>;
multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]>;
text(options: { message: string; placeholder?: string }): Promise<string>;
password(options: { message: string }): Promise<string>;
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
type KtxMetabaseSetupInteractiveIo = KtxCliIo & {
stdin?: { isTTY?: boolean };
};
export interface MetabaseSetupMappingAssignment {
metabaseDatabaseId: number;
targetConnectionId: string;
}
export interface MintMetabaseApiKeyArgs {
url: string;
username: string;
password: string;
}
export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KtxCliIo) => Promise<string>;
export interface KtxConnectionMetabaseSetupArgs {
command: 'setup';
projectDir: string;
connectionId?: string;
url?: string;
apiKey?: string;
mintApiKey: boolean;
metabaseUsername?: string;
metabasePassword?: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
runIngest: boolean;
yes: boolean;
inputMode: KtxMetabaseSetupInputMode;
}
export interface KtxConnectionMetabaseSetupDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>>;
mintMetabaseApiKey?: MintMetabaseApiKey;
prompts?: MetabaseSetupPromptAdapter;
runPublicIngest?: (args: Extract<KtxPublicIngestArgs, { command: 'run' }>, io: KtxCliIo) => Promise<number>;
}
function isMetabaseConnection(connection: KtxProjectConnectionConfig | undefined): boolean {
return (
String(connection?.driver ?? '')
.trim()
.toLowerCase() === 'metabase'
);
}
function stringField(value: unknown): string | undefined {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
}
function uniqueSorted(values: number[]): number[] {
return [...new Set(values)].sort((a, b) => a - b);
}
function resolveMetabaseUrl(connection: KtxProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_url);
}
function resolveLiteralMetabaseApiKey(connection: KtxProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_key);
}
function listMetabaseConnectionIds(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([_connectionId, connection]) => isMetabaseConnection(connection))
.map(([connectionId]) => connectionId)
.sort();
}
function listWarehouseConnectionIds(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
.map(([connectionId]) => connectionId)
.sort();
}
function redactSecrets(message: string, secrets: string[]): string {
let result = message;
for (const secret of secrets) {
if (!secret) {
continue;
}
result = result.split(secret).join('[redacted]');
}
return result;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise<string> {
const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
const sessionId = await loginClient.createSession(args.username, args.password);
const sessionClient = new MetabaseClient(
{ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' },
DEFAULT_METABASE_CLIENT_CONFIG,
);
const groups = await sessionClient.getPermissionGroups();
const adminGroup = groups.find((group) => group.name === 'Administrators');
if (!adminGroup) {
throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
}
const mintedKey = await sessionClient.createApiKey({
groupId: adminGroup.id,
name: `KTX CLI ${new Date().toISOString()}`,
});
const trimmedKey = stringField(mintedKey);
if (!trimmedKey) {
throw new Error('Metabase API key minting returned an empty key');
}
return trimmedKey;
}
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<MetabaseSetupPromptAdapter, 'cancel'>): T {
if (isCancel(value)) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
return value as T;
}
export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter {
return {
intro(title?: string): void {
intro(title);
},
outro(message?: string): void {
outro(message);
},
note(message: string, title: string): void {
note(message, title);
},
log: {
info(message: string): void {
log.info(message);
},
step(message: string): void {
log.step(message);
},
success(message: string): void {
log.success(message);
},
warn(message: string): void {
log.warn(message);
},
error(message: string): void {
log.error(message);
},
},
spinner(): KtxCliSpinner {
return createClackSpinner();
},
async select<T extends string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<T>>;
}): Promise<T> {
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
},
async multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]> {
return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
},
async text(options: { message: string; placeholder?: string }): Promise<string> {
return ensureNotCancelled(await text(options), this);
},
async password(options: { message: string }): Promise<string> {
return ensureNotCancelled(await password(options), this);
},
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
return ensureNotCancelled(await confirm(options), this);
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveMetabaseSetupIo(
args: Pick<KtxConnectionMetabaseSetupArgs, 'inputMode'>,
io: KtxMetabaseSetupInteractiveIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{
id: number;
name: string;
engine: string;
host: string | null;
dbName: string | null;
}> {
return databases
.filter((database) => database.is_sample !== true)
.map((database) => ({
id: database.id,
name: database.name,
engine: stringField(database.engine) ?? 'unknown',
host: stringField(database.details?.host) ?? null,
dbName: stringField(database.details?.dbname) ?? null,
}));
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function noteMetabaseSetupSummary(options: {
prompts: MetabaseSetupPromptAdapter;
connectionId: string;
url: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
}): void {
const mappingLines = options.mappings
.map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
.join('\n');
const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n');
options.prompts.note(
[
`Connection: ${options.connectionId}`,
`URL: ${options.url}`,
'',
'Mappings:',
mappingLines || ' (none)',
'',
'Sync enabled:',
syncLines || ' (none)',
].join('\n'),
'Summary',
);
}
function metabaseMappingsBlockForSetup(options: {
connectionId: string;
connection: KtxProjectConnectionConfig;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
}): Record<string, unknown> {
const existing = parseMetabaseMappingBootstrap(options.connectionId, options.connection);
const databaseMappings = { ...existing.databaseMappings };
const syncEnabled = { ...existing.syncEnabled };
for (const mapping of options.mappings) {
const key = String(mapping.metabaseDatabaseId);
databaseMappings[key] = mapping.targetConnectionId;
syncEnabled[key] = false;
}
for (const metabaseDatabaseId of options.syncEnabledDatabaseIds) {
syncEnabled[String(metabaseDatabaseId)] = true;
}
return {
databaseMappings,
syncEnabled,
syncMode: options.syncMode,
selections: existing.selections,
defaultTagNames: existing.defaultTagNames,
};
}
export async function runKtxConnectionMetabaseSetup(
args: KtxConnectionMetabaseSetupArgs,
io: KtxCliIo,
deps: KtxConnectionMetabaseSetupDeps = {},
): Promise<number> {
let apiKeyForRedaction = args.apiKey;
let passwordForRedaction = args.metabasePassword;
const interactiveIo = io as KtxMetabaseSetupInteractiveIo;
const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
try {
if (isInteractive && prompts) {
prompts.intro('KTX Metabase setup');
}
const project = await loadKtxProject({ projectDir: args.projectDir });
const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
let connectionId: string;
if (args.connectionId) {
connectionId = args.connectionId;
} else if (existingMetabaseConnectionIds.length === 1) {
const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
if (!onlyMetabaseConnectionId) {
throw new Error('No Metabase connection id was resolved');
}
connectionId = onlyMetabaseConnectionId;
} else if (existingMetabaseConnectionIds.length > 1) {
if (!isInteractive || !prompts) {
throw new Error(
`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`,
);
}
connectionId = await prompts.select({
message: 'Select the Metabase connection to configure',
options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
});
} else {
connectionId = 'metabase';
}
const existingConnection = project.config.connections[connectionId];
const warehouseConnectionIds = listWarehouseConnectionIds(project);
if (warehouseConnectionIds.length === 0) {
throw new Error('Add a warehouse connection first');
}
let url = args.url ?? resolveMetabaseUrl(existingConnection);
let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
apiKeyForRedaction = apiKey;
if (!url && isInteractive && prompts) {
url = stringField(
await prompts.text({
message: 'Metabase API URL',
placeholder: 'http://localhost:3000',
}),
);
}
if (args.inputMode === 'disabled' && !url) {
throw new Error('missing Metabase URL');
}
if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
const reuse = await prompts.confirm({
message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
initialValue: true,
});
if (!reuse) {
apiKey = undefined;
apiKeyForRedaction = undefined;
}
}
if (args.mintApiKey) {
let username = stringField(args.metabaseUsername);
let metabasePassword = stringField(args.metabasePassword);
if (isInteractive && prompts) {
if (!username) {
username = stringField(await prompts.text({ message: 'Metabase admin username' }));
}
if (!metabasePassword) {
metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
}
}
if (!username) {
throw new Error('--mint-api-key requires --username');
}
if (!metabasePassword) {
throw new Error('--mint-api-key requires --password');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
if (!apiKey && isInteractive && prompts) {
const credentialMode = await prompts.select({
message: 'Metabase credentials',
options: [
{ value: 'paste', label: 'Paste API key' },
{ value: 'mint', label: 'Mint API key' },
],
});
if (credentialMode === 'paste') {
apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
apiKeyForRedaction = apiKey;
} else {
const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
if (!username) {
throw new Error('Metabase username is required');
}
if (!metabasePassword) {
throw new Error('Metabase password is required');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
}
if (args.inputMode === 'disabled' && !apiKey) {
throw new Error('missing Metabase API key');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
if (!apiKey) {
throw new Error('Metabase API key is required (use --api-key)');
}
const transientConnectionConfig: KtxProjectConnectionConfig = {
...(existingConnection ?? {}),
driver: 'metabase',
api_url: url,
api_key: apiKey,
};
const configWithTransient = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: transientConnectionConfig,
},
};
const discoveryProject: KtxLocalProject = { ...project, config: configWithTransient };
for (const mapping of args.mappings) {
if (!configWithTransient.connections[mapping.targetConnectionId]) {
throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
}
}
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
try {
const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
authSpinner?.start('Testing Metabase connection');
const testResult = await client.testConnection();
if (!testResult.success) {
authSpinner?.error('Metabase authentication failed');
throw new Error(
`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`,
);
}
authSpinner?.stop('Metabase reachable');
const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
discoverySpinner?.start('Discovering Metabase databases');
const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
if (isInteractive && prompts) {
prompts.log.success(
`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`,
);
}
if (discovered.length === 0) {
throw new Error('Metabase auth worked but no usable databases were returned');
}
let resolvedMappings = args.mappings;
let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
}
if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
const compatibleWarehouses = onlyDiscoveredDatabase
? warehouseConnectionIds.filter((warehouseConnectionId) => {
const mismatchReason = validateMappingPhysicalMatch(
{
metabaseEngine: onlyDiscoveredDatabase.engine,
metabaseDbName: onlyDiscoveredDatabase.dbName,
metabaseHost: onlyDiscoveredDatabase.host,
},
targetPhysicalInfo(project, warehouseConnectionId),
);
return !mismatchReason;
})
: [];
const onlyWarehouseConnectionId = compatibleWarehouses[0];
if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
if (args.yes) {
resolvedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
} else if (isInteractive && prompts) {
const proposedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: proposedMappings,
syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
resolvedMappings = proposedMappings;
resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
} else {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
} else if (isInteractive && prompts) {
const selectedDatabaseIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Select Metabase databases to configure'),
options: discovered.map((database) => ({
value: database.id,
label: `${database.id}: ${database.name}`,
hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
})),
required: true,
});
resolvedMappings = [];
for (const databaseId of selectedDatabaseIds) {
const database = discovered.find((candidate) => candidate.id === databaseId);
if (!database) {
throw new Error(`Selected database id ${databaseId} was not discovered`);
}
const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
if (existingMapping) {
resolvedMappings.push(existingMapping);
continue;
}
const targetConnectionId = await prompts.select({
message: `Map Metabase database ${database.id} ("${database.name}") to which KTX connection?`,
options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
});
resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
}
const syncIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Enable sync for which databases?'),
options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
initialValues: selectedDatabaseIds,
required: true,
});
resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
if (!args.yes) {
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: 'Write changes to ktx.yaml and enable sync?',
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
}
} else if (args.inputMode === 'disabled') {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
}
if (
args.inputMode === 'disabled' &&
resolvedMappings.length > 0 &&
resolvedSyncEnabledDatabaseIds.length === 0
) {
throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
}
const discoveredIds = new Set(discovered.map((database) => database.id));
for (const mapping of resolvedMappings) {
if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
}
}
for (const syncId of resolvedSyncEnabledDatabaseIds) {
if (!discoveredIds.has(syncId)) {
throw new Error(`Sync database id ${syncId} was not discovered`);
}
}
const finalConnectionConfig: KtxProjectConnectionConfig = {
...transientConnectionConfig,
mappings: metabaseMappingsBlockForSetup({
connectionId,
connection: transientConnectionConfig,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
syncMode: args.syncMode,
}),
};
const finalConfig = {
...configWithTransient,
connections: {
...configWithTransient.connections,
[connectionId]: finalConnectionConfig,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(finalConfig),
'ktx',
'ktx@example.com',
`Setup Metabase connection ${connectionId}`,
);
const updatedProject = await loadKtxProject({ projectDir: args.projectDir });
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(updatedProject) });
await discoveryCache.refreshDiscoveredDatabases({ connectionId, discovered });
const rows = await new KtxYamlMetabaseSourceStateReader(updatedProject, { discoveryCache }).listDatabaseMappings(
connectionId,
);
const physicalFailures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
updatedProject.config.connections[row.targetConnectionId]
? targetPhysicalInfo(updatedProject, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (physicalFailures.length > 0) {
for (const failure of physicalFailures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Connection: ${connectionId}\n`);
io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(
`Next: ktx ingest run --connection-id ${connectionId} --adapter metabase --project-dir ${args.projectDir}\n`,
);
if (args.runIngest) {
const ingestRunner = deps.runPublicIngest ?? runKtxPublicIngest;
const exitCode = await ingestRunner(
{
command: 'run',
projectDir: args.projectDir,
targetConnectionId: connectionId,
all: false,
json: false,
inputMode: 'disabled',
},
io,
);
if (exitCode !== 0) {
io.stderr.write(
`Ingest failed; re-run: ktx ingest run --connection-id ${connectionId} --adapter metabase --project-dir ${args.projectDir}\n`,
);
return 1;
}
}
if (isInteractive && prompts) {
prompts.outro('Metabase setup complete');
}
return 0;
} finally {
await client.cleanup();
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
io.stderr.write(
`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`,
);
return 1;
}
}

92
packages/cli/src/commands/connection-notion-commands.ts
View file

@ -1,92 +0,0 @@
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { collectOption, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxConnectionNotionArgs } from './connection-notion.js';
interface NotionPickOptions {
input?: boolean;
rootPageId: string[];
}
function parseSafeConnectionId(value: string): string {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
}
function uniqueInOrder(values: string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const value of values) {
if (!seen.has(value)) {
seen.add(value);
result.push(value);
}
}
return result;
}
function normalizeNotionPageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
throw new Error(`Invalid Notion page UUID: ${value}`);
}
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
}
function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KtxConnectionNotionArgs {
if (options.input !== false) {
return {
command: 'pick',
projectDir,
connectionId,
mode: 'interactive',
};
}
const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId));
if (rootPageIds.length === 0) {
throw new Error('connection notion pick --no-input requires at least one --root-page-id');
}
return {
command: 'pick',
projectDir,
connectionId,
mode: 'non-interactive',
rootPageIds,
};
}
async function runConnectionNotionArgs(context: KtxCliCommandContext, args: KtxConnectionNotionArgs): Promise<void> {
const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKtxConnectionNotion;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionNotionCommands(connect: Command, context: KtxCliCommandContext): void {
const notion = connect
.command('notion')
.description('Configure Notion source selection')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
notion.action(() => {
notion.outputHelp();
context.setExitCode(0);
});
notion
.command('pick')
.description('Pick Notion root pages for a configured Notion connection')
.argument('<connectionId>', 'Notion connection id', parseSafeConnectionId)
.option('--no-input', 'Disable interactive terminal input')
.option('--root-page-id <id>', 'Root page UUID to crawl; repeatable with --no-input', collectOption, [])
.showHelpAfterError()
.action(async (connectionId: string, options: NotionPickOptions, command) => {
await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options));
});
}

513
packages/cli/src/commands/connection-notion.test.ts
View file

@ -1,513 +0,0 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
initKtxProject,
loadKtxProject,
serializeKtxProjectConfig,
type KtxProjectConfig,
} from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
applyNotionPickerWriteback,
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
resolveNotionWorkspaceLabel,
runKtxConnectionNotion,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './connection-notion.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('runKtxConnectionNotion', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-notion-pick-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(config),
'ktx',
'ktx@example.com',
'seed test config',
);
}
it('rejects unsafe connection ids before loading a project', async () => {
const io = makeIo();
const loadProject = vi.fn(async () => {
throw new Error('loadProject should not be called');
});
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir: '/tmp/project',
connectionId: '../evil',
mode: 'interactive',
},
io.io,
{ loadProject },
),
).resolves.toBe(1);
expect(loadProject).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Unsafe connection id: ../evil');
});
it('writes selected root_page_ids while preserving every other Notion connection field', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: ['99999999-9999-9999-9999-999999999999'],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
unknown_future_field: 'keep-me',
},
},
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'non-interactive',
rootPageIds: [
'11111111-2222-3333-4444-555555555555',
'66666666-7777-8888-9999-aaaaaaaaaaaa',
],
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('root_page_ids:');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa');
expect(yaml).toContain('root_database_ids:');
expect(yaml).toContain('database-1');
expect(yaml).toContain('root_data_source_ids:');
expect(yaml).toContain('data-source-1');
expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\'');
expect(yaml).toContain('unknown_future_field: keep-me');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 2');
expect(io.stdout()).toContain('crawlMode: selected_roots');
});
it('rejects empty writeback, missing connections, and non-Notion connections', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
warehouse: {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
},
},
});
const project = await loadKtxProject({ projectDir });
await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow(
'connection notion pick requires at least one root page id',
);
await expect(
applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "missing" not found');
await expect(
applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "warehouse" is not a Notion connection');
});
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain(PAGE_IDS.engineering);
expect(yaml).not.toContain(PAGE_IDS.stale);
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 1');
});
it('uses inline Notion auth_token for interactive discovery', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token: 'ntn_inline_token',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]);
const createNotionApi = vi.fn((authToken: string) => {
expect(authToken).toBe('ntn_inline_token');
return api;
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
expect(createNotionApi).toHaveBeenCalledOnce();
expect(io.stdout()).toContain('No changes saved.');
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
expect(io.stdout()).toContain('No changes saved.');
});
it('quits interactive mode without writing when the TUI returns quit', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const before = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toBe(before);
expect(io.stdout()).toContain('No changes saved.');
});
});

541
packages/cli/src/connection.test.ts
View file

@ -1,4 +1,4 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { MetabaseRuntimeClient } from '@ktx/context/ingest';
@ -6,18 +6,13 @@ import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from
import type { KtxConnectionDriver, KtxScanConnector, KtxSchemaSnapshot } from '@ktx/context/scan';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnection } from './connection.js';
import { runKtxCli, type KtxCliIo } from './index.js';
function makeIo(options: { stdoutIsTty?: boolean; stdinIsTty?: boolean } = {}) {
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdin: {
isTTY: options.stdinIsTty,
},
stdout: {
isTTY: options.stdoutIsTty,
write: (chunk: string) => {
stdout += chunk;
},
@ -87,491 +82,49 @@ describe('runKtxConnection', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('adds and lists env-referenced connections without resolving secrets', async () => {
async function writeConnections(
projectDir: string,
connections: ReturnType<typeof parseKtxProjectConfig>['connections'],
): Promise<void> {
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(join(projectDir, 'ktx.yaml'), serializeKtxProjectConfig({ ...config, connections }), 'utf-8');
}
it('lists configured connections without resolving secrets', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
});
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'env:DATABASE_URL',
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(0);
await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0);
expect(io.stdout()).toContain('Connection: warehouse');
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('url: env:DATABASE_URL');
const listIo = makeIo();
await expect(runKtxConnection({ command: 'list', projectDir }, listIo.io)).resolves.toBe(0);
expect(listIo.stdout()).toContain('warehouse');
expect(listIo.stdout()).toContain('postgres');
});
it('removes a configured connection from ktx.yaml without deleting local artifacts when forced', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const artifactPath = join(projectDir, '.ktx', 'artifacts', 'warehouse.txt');
await mkdir(join(projectDir, '.ktx', 'artifacts'), { recursive: true });
await writeFile(artifactPath, 'keep me', 'utf-8');
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(0);
const parsed = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(parsed.connections.warehouse).toBeUndefined();
await expect(readFile(artifactPath, 'utf-8')).resolves.toBe('keep me');
expect(io.stdout()).toContain('Connection removed from ktx.yaml.');
expect(io.stdout()).toContain(
'Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.',
);
expect(io.stdout()).toContain('warehouse');
expect(io.stdout()).toContain('postgres');
expect(io.stdout()).toContain('docs');
expect(io.stdout()).toContain('notion');
expect(io.stderr()).toBe('');
});
it('requires --force when removing in non-interactive mode', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('connection remove warehouse requires --force when input is disabled or not interactive');
});
it('returns a clear error when removing an unknown connection', async () => {
it('prints an empty-state message that points at setup instead of removed connection add', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'missing',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0);
expect(io.stderr()).toContain('Connection "missing" is not configured in ktx.yaml');
});
it('asks for confirmation before removing in an interactive terminal', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo({ stdoutIsTty: true, stdinIsTty: true });
const prompts = {
confirm: vi.fn(async () => true),
cancel: vi.fn(),
};
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
},
io.io,
{ prompts },
),
).resolves.toBe(0);
expect(prompts.confirm).toHaveBeenCalledWith({
message: 'Remove connection "warehouse" from ktx.yaml? Ingested artifacts will remain in .ktx/.',
initialValue: false,
});
});
it('runs public connect map as refresh, validate, and list over the low-level mapping runner', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 database\n');
mappingIo.stdout.write('Unmapped discovered: 1\n');
mappingIo.stdout.write('Stale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-metabase\n');
return 0;
}
if (argv[0] === 'list') {
mappingIo.stdout.write('1 -> [unmapped] (Analytics, sync: on, source: refresh)\n');
return 0;
}
return 1;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(0);
expect(runMapping).toHaveBeenNthCalledWith(
1,
['refresh', 'prod-metabase', '--auto-accept', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
2,
['validate', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
3,
['list', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(io.stdout()).toContain('Mapping: prod-metabase');
expect(io.stdout()).toContain('Discovery: 1 database');
expect(io.stdout()).toContain('Mappings:');
expect(io.stdout()).toContain('1 -> [unmapped]');
expect(io.stdout()).toContain('Next:');
expect(io.stdout()).toContain('ktx ingest run --connection-id prod-metabase --adapter <adapter>');
expect(io.stdout()).toContain('ktx connection mapping');
expect(io.stderr()).toBe('');
});
it('prints stable JSON for public connect map without leaking low-level stdout', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 connection\nUnmapped discovered: 0\nStale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-looker\n');
return 0;
}
if (argv[0] === 'list') {
expect(argv).toContain('--json');
mappingIo.stdout.write(
`${JSON.stringify(
[
{
lookerConnectionName: 'analytics',
ktxConnectionId: 'prod-warehouse',
source: 'ktx.yaml',
},
],
null,
2,
)}\n`,
);
return 0;
}
return 1;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-looker', json: true },
io.io,
{ runMapping },
),
).resolves.toBe(0);
const parsed = JSON.parse(io.stdout()) as {
connectionId: string;
refresh: { ok: boolean; output: string[] };
validation: { ok: boolean; output: string[] };
mappings: Array<{ lookerConnectionName: string; ktxConnectionId: string; source: string }>;
};
expect(parsed).toEqual({
connectionId: 'prod-looker',
refresh: {
ok: true,
output: ['Discovery: 1 connection', 'Unmapped discovered: 0', 'Stale mappings: 0'],
},
validation: {
ok: true,
output: ['Mapping validation passed: prod-looker'],
},
mappings: [
{
lookerConnectionName: 'analytics',
ktxConnectionId: 'prod-warehouse',
source: 'ktx.yaml',
},
],
});
expect(io.stderr()).toBe('');
});
it('returns the refresh failure when public connect map cannot discover source metadata', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stderr.write('Metabase API key is not configured\n');
return 1;
}
return 0;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(1);
expect(runMapping).toHaveBeenCalledTimes(1);
expect(io.stdout()).toBe('');
expect(io.stderr()).toContain('Metabase API key is not configured');
});
it('rejects literal credential URLs unless explicitly allowed', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'postgres://localhost:5432/warehouse',
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Literal credential URLs require --allow-literal-credentials');
});
it('warns before writing explicitly allowed literal credential URLs without echoing the URL', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
const literalUrl = 'postgres://localhost:5432/warehouse';
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: literalUrl,
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: true,
},
io.io,
),
).resolves.toBe(0);
expect(io.stderr()).toContain(
'Warning: writing a literal credential URL to ktx.yaml for connection "warehouse". Prefer env:NAME or file:/path references.',
);
expect(io.stderr()).not.toContain(literalUrl);
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain(literalUrl);
});
it('adds a Notion connection without writing token values', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: [],
rootDataSourceIds: [],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 50');
expect(yaml).not.toContain('ntn_');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('Driver: notion');
});
it('runs connection notion pick --no-input through the public connection entrypoint', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: ['database-1'],
rootDataSourceIds: ['data-source-1'],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
makeIo().io,
);
const io = makeIo();
await expect(
runKtxCli(
[
'connection',
'notion',
'pick',
'notion-main',
'--project-dir',
projectDir,
'--no-input',
'--root-page-id',
'11111111222233334444555555555555',
],
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('database-1');
expect(yaml).toContain('data-source-1');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('No connections configured. Run `ktx setup` to add one.');
expect(io.stdout()).not.toContain('ktx connection add');
});
it('tests a configured connection through the native scan connector', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
});
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
const createScanConnector = vi.fn(async () => connector);
const io = makeIo();
@ -602,22 +155,13 @@ describe('runKtxConnection', () => {
it('tests a configured Metabase connection through the Metabase runtime client', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const projectConfig = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(
join(projectDir, 'ktx.yaml'),
serializeKtxProjectConfig({
...projectConfig,
connections: {
...projectConfig.connections,
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
},
}),
'utf-8',
);
await writeConnections(projectDir, {
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
});
const testConnection = vi.fn(async () => ({ success: true as const }));
const getDatabases = vi.fn(async () => [
{ id: 1, name: 'Analytics', engine: 'postgres', details: {}, is_sample: false },
@ -657,20 +201,9 @@ describe('runKtxConnection', () => {
it('cleans up the native scan connector when connection testing fails', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
});
const cleanup = vi.fn(async () => undefined);
const connector: KtxScanConnector = {
id: 'sqlite:warehouse',

335
packages/cli/src/connection.ts
View file

@ -1,108 +1,24 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
type MetabaseRuntimeClient,
metabaseRuntimeConfigFromLocalConnection,
} from '@ktx/context/ingest';
import { type KtxLocalProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
import type { KtxScanConnector } from '@ktx/context/scan';
import type { KtxConnectionMappingArgs } from './commands/connection-mapping.js';
import type { KtxCliIo } from './index.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { profileMark } from './startup-profile.js';
profileMark('module:connection');
interface KtxNotionConnectionCliConfig {
authTokenRef: string;
crawlMode: 'all_accessible' | 'selected_roots';
rootPageIds: string[];
rootDatabaseIds: string[];
rootDataSourceIds: string[];
maxPagesPerRun?: number;
maxKnowledgeCreatesPerRun?: number;
maxKnowledgeUpdatesPerRun?: number;
}
type KtxConnectionInputMode = 'disabled';
export type KtxConnectionArgs =
| { command: 'list'; projectDir: string }
| {
command: 'add';
projectDir: string;
driver: string;
connectionId: string;
url?: string;
schemas: string[];
readonly: boolean;
force: boolean;
allowLiteralCredentials: boolean;
notion?: KtxNotionConnectionCliConfig;
}
| { command: 'test'; projectDir: string; connectionId: string }
| {
command: 'remove';
projectDir: string;
connectionId: string;
force: boolean;
inputMode?: KtxConnectionInputMode;
}
| {
command: 'map';
projectDir: string;
sourceConnectionId: string;
json: boolean;
};
interface KtxConnectionPromptAdapter {
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
interface KtxConnectionIo extends KtxCliIo {
stdin?: { isTTY?: boolean };
}
| { command: 'test'; projectDir: string; connectionId: string };
interface KtxConnectionDeps {
createScanConnector?: typeof createKtxCliScanConnector;
createMetabaseClient?: typeof createDefaultMetabaseClient;
runMapping?: (argv: string[], io: KtxCliIo) => Promise<number>;
prompts?: KtxConnectionPromptAdapter;
}
function assertSafeConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
}
function isCredentialReference(value: string): boolean {
return value.startsWith('env:') || value.startsWith('file:');
}
function literalCredentialWarning(connectionId: string): string {
return `Warning: writing a literal credential URL to ktx.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`;
}
function createClackConnectionPromptAdapter(): KtxConnectionPromptAdapter {
return {
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
const value = await confirm(options);
return isCancel(value) ? false : value;
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveConnectionIo(
args: Extract<KtxConnectionArgs, { command: 'remove' }>,
io: KtxConnectionIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
async function cleanupConnector(connector: KtxScanConnector | null): Promise<void> {
@ -186,166 +102,17 @@ async function testMetabaseConnection(
}
}
interface BufferedIo extends KtxCliIo {
stdoutText(): string;
stderrText(): string;
}
function createBufferedIo(): BufferedIo {
let stdout = '';
let stderr = '';
return {
stdout: {
write(chunk: string) {
stdout += chunk;
},
},
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
stdoutText() {
return stdout;
},
stderrText() {
return stderr;
},
};
}
function splitOutputLines(output: string): string[] {
return output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
}
async function runLowLevelMapping(
args: KtxConnectionMappingArgs,
argv: string[],
io: KtxCliIo,
deps: KtxConnectionDeps,
): Promise<number> {
if (deps.runMapping) {
return await deps.runMapping(argv, io);
}
const { runKtxConnectionMapping } = await import('./commands/connection-mapping.js');
return await runKtxConnectionMapping(args, io);
}
function parseMappingListJson(output: string): unknown[] {
const trimmed = output.trim();
if (!trimmed) {
return [];
}
const parsed = JSON.parse(trimmed) as unknown;
return Array.isArray(parsed) ? parsed : [];
}
async function runPublicConnectionMap(
args: Extract<KtxConnectionArgs, { command: 'map' }>,
io: KtxCliIo,
deps: KtxConnectionDeps,
): Promise<number> {
const refreshIo = createBufferedIo();
const refreshArgs: KtxConnectionMappingArgs = {
command: 'refresh',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
autoAccept: true,
};
const refreshCode = await runLowLevelMapping(
refreshArgs,
['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir],
refreshIo,
deps,
);
if (refreshCode !== 0) {
io.stderr.write(
refreshIo.stderrText() ||
refreshIo.stdoutText() ||
`Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`,
);
return refreshCode;
}
const validationIo = createBufferedIo();
const validationArgs: KtxConnectionMappingArgs = {
command: 'validate',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
};
const validationCode = await runLowLevelMapping(
validationArgs,
['validate', args.sourceConnectionId, '--project-dir', args.projectDir],
validationIo,
deps,
);
if (validationCode !== 0) {
io.stderr.write(
validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`,
);
return validationCode;
}
const listIo = createBufferedIo();
const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir];
const listArgs: KtxConnectionMappingArgs = {
command: 'list',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
json: args.json,
};
const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps);
if (listCode !== 0) {
io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`);
return listCode;
}
if (args.json) {
io.stdout.write(
`${JSON.stringify(
{
connectionId: args.sourceConnectionId,
refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) },
validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) },
mappings: parseMappingListJson(listIo.stdoutText()),
},
null,
2,
)}\n`,
);
return 0;
}
io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`);
io.stdout.write(refreshIo.stdoutText());
io.stdout.write(validationIo.stdoutText());
io.stdout.write('\nMappings:\n');
io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n');
io.stdout.write('\nNext:\n');
io.stdout.write(` ktx ingest run --connection-id ${args.sourceConnectionId} --adapter <adapter>\n`);
io.stdout.write(` ktx connection mapping list ${args.sourceConnectionId}\n`);
return 0;
}
export async function runKtxConnection(
args: KtxConnectionArgs,
io: KtxConnectionIo = process,
io: KtxCliIo = process,
deps: KtxConnectionDeps = {},
): Promise<number> {
try {
if (args.command === 'map') {
return await runPublicConnectionMap(args, io, deps);
}
const project = await loadKtxProject({ projectDir: args.projectDir });
if (args.command === 'list') {
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
if (entries.length === 0) {
io.stdout.write('No connections configured. Run `ktx connection add <id> --driver <driver>` to add one.\n');
io.stdout.write('No connections configured. Run `ktx setup` to add one.\n');
return 0;
}
const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
@ -360,100 +127,6 @@ export async function runKtxConnection(
return 0;
}
if (args.command === 'add') {
assertSafeConnectionId(args.connectionId);
const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url);
if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) {
throw new Error('Literal credential URLs require --allow-literal-credentials');
}
if (hasLiteralCredentialUrl) {
io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`);
}
if (project.config.connections[args.connectionId] && !args.force) {
throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`);
}
const connectionConfig =
args.driver === 'notion' && args.notion
? {
driver: 'notion',
auth_token_ref: args.notion.authTokenRef,
crawl_mode: args.notion.crawlMode,
root_page_ids: args.notion.rootPageIds,
root_database_ids: args.notion.rootDatabaseIds,
root_data_source_ids: args.notion.rootDataSourceIds,
...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}),
...(args.notion.maxKnowledgeCreatesPerRun !== undefined
? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun }
: {}),
...(args.notion.maxKnowledgeUpdatesPerRun !== undefined
? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun }
: {}),
}
: {
driver: args.driver,
...(args.url ? { url: args.url } : {}),
...(args.schemas.length > 0 ? { schemas: args.schemas } : {}),
readonly: args.readonly,
};
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[args.connectionId]: connectionConfig,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Update KTX connection: ${args.connectionId}`,
);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`Driver: ${args.driver}\n`);
return 0;
}
if (args.command === 'remove') {
if (!project.config.connections[args.connectionId]) {
throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`);
}
if (!args.force) {
if (!isInteractiveConnectionIo(args, io)) {
throw new Error(
`connection remove ${args.connectionId} requires --force when input is disabled or not interactive`,
);
}
const prompts = deps.prompts ?? createClackConnectionPromptAdapter();
const confirmed = await prompts.confirm({
message: `Remove connection "${args.connectionId}" from ktx.yaml? Ingested artifacts will remain in .ktx/.`,
initialValue: false,
});
if (!confirmed) {
prompts.cancel('Connection removal cancelled.');
return 1;
}
}
const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections;
const nextConfig = {
...project.config,
connections,
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Remove KTX connection: ${args.connectionId}`,
);
io.stdout.write('Connection removed from ktx.yaml.\n');
io.stdout.write('Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.\n');
return 0;
}
if (normalizedConnectionDriver(project, args.connectionId) === 'metabase') {
const result = await testMetabaseConnection(
project,

386
packages/cli/src/index.test.ts
View file

@ -1291,16 +1291,9 @@ describe('runKtxCli', () => {
runKtxCli(['--project-dir', tempDir, 'connection', 'list'], makeIo().io, { connection }),
).resolves.toBe(0);
const removeIo = makeIo();
const testIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'connection', 'remove', 'warehouse', '--force', '--no-input'], removeIo.io, {
connection,
}),
).resolves.toBe(0);
const mapIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'connection', 'map', 'prod-metabase', '--json'], mapIo.io, {
runKtxCli(['--project-dir', tempDir, 'connection', 'test', 'warehouse'], testIo.io, {
connection,
}),
).resolves.toBe(0);
@ -1309,21 +1302,9 @@ describe('runKtxCli', () => {
expect(connection).toHaveBeenNthCalledWith(
2,
{
command: 'remove',
command: 'test',
projectDir: tempDir,
connectionId: 'warehouse',
force: true,
inputMode: 'disabled',
},
expect.anything(),
);
expect(connection).toHaveBeenNthCalledWith(
3,
{
command: 'map',
projectDir: tempDir,
sourceConnectionId: 'prod-metabase',
json: true,
},
expect.anything(),
);
@ -1331,168 +1312,35 @@ describe('runKtxCli', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('prints help for connection metabase setup', async () => {
it('prints only list and test in connection help', async () => {
const helpIo = makeIo();
await expect(runKtxCli(['connection', 'metabase', 'setup', '--help'], helpIo.io)).resolves.toBe(0);
await expect(runKtxCli(['connection', '--help'], helpIo.io)).resolves.toBe(0);
expect(helpIo.stdout()).toContain('Usage: ktx connection metabase setup');
for (const option of [
'--id <connectionId>',
'--url <url>',
'--api-key <key>',
'--username <email>',
'--password <password>',
'--mint-api-key',
'--map <metabaseDatabaseId=targetConnectionId>',
'--sync <metabaseDatabaseId>',
'--sync-mode <mode>',
'--run-ingest',
'--yes',
'--no-input',
]) {
expect(helpIo.stdout()).toContain(option);
}
expect(helpIo.stdout()).toContain('Guided equivalent of:');
for (const line of [
'ktx connection mapping refresh <connectionId> --auto-accept',
'ktx connection mapping set <connectionId> databaseMappings <id>=<target>',
'ktx connection mapping set-sync-enabled <connectionId> <id> --enabled true',
'ktx ingest run --connection-id <connectionId> --adapter metabase',
]) {
expect(helpIo.stdout()).toContain(line);
expect(helpIo.stdout()).toContain('Usage: ktx connection');
expect(helpIo.stdout()).toContain('list');
expect(helpIo.stdout()).toContain('test <connectionId>');
for (const removed of ['add', 'remove', 'map', 'mapping', 'metabase', 'notion']) {
expect(helpIo.stdout()).not.toMatch(new RegExp(`\\b${removed}\\b`));
}
expect(helpIo.stderr()).toBe('');
});
it('dispatches connection metabase setup through Commander', async () => {
const connectionMetabaseSetup = vi.fn(async () => 0);
const fakeMetabaseCredential = 'mb_example';
const setupIo = makeIo();
await expect(
runKtxCli(
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--id',
'metabase',
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--map',
'2=orbit',
'--sync',
'2',
'--yes',
'--no-input',
],
setupIo.io,
{ connectionMetabaseSetup },
),
).resolves.toBe(0);
expect(connectionMetabaseSetup).toHaveBeenCalledWith(
{
command: 'setup',
projectDir: tempDir,
connectionId: 'metabase',
url: 'http://metabase.example.test:3000',
apiKey: fakeMetabaseCredential,
mintApiKey: false,
mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }],
syncEnabledDatabaseIds: [2],
syncMode: 'ALL',
runIngest: false,
yes: true,
inputMode: 'disabled',
},
expect.anything(),
);
expect(setupIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('validates connection metabase setup option values before runner dispatch', async () => {
const connectionMetabaseSetup = vi.fn(async () => 0);
it('rejects removed connection subcommands', async () => {
for (const argv of [
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--map',
'nope=orbit',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--map',
'2=../orbit',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--sync',
'nope',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--sync-mode',
'BAD',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--mint-api-key',
'--api-key',
'also_bad',
],
['connection', 'add', 'postgres', 'warehouse'],
['connection', 'remove', 'warehouse'],
['connection', 'map', 'prod-metabase'],
['connection', 'mapping'],
['connection', 'metabase'],
['connection', 'notion'],
]) {
const testIo = makeIo();
await expect(runKtxCli(argv, testIo.io, { connectionMetabaseSetup })).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/map|sync|sync-mode|conflict|cannot be used|invalid|integer|choices/i);
}
expect(connectionMetabaseSetup).not.toHaveBeenCalled();
await expect(runKtxCli(argv, testIo.io)).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/unknown command|error:/);
}
});
it('rejects commands removed from the May 6 root surface', async () => {
@ -1510,153 +1358,6 @@ describe('runKtxCli', () => {
}
});
it('dispatches connection add options through Commander', async () => {
const testIo = makeIo();
const connection = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'connection',
'add',
'notion',
'notion-main',
'--project-dir',
tempDir,
'--token-env',
'NOTION_TOKEN',
'--crawl-mode',
'selected_roots',
'--root-page-id',
'page-1',
'--root-database-id',
'database-1',
'--max-pages',
'80',
],
testIo.io,
{ connection },
),
).resolves.toBe(0);
expect(connection).toHaveBeenCalledWith(
{
command: 'add',
projectDir: tempDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'selected_roots',
rootPageIds: ['page-1'],
rootDatabaseIds: ['database-1'],
rootDataSourceIds: [],
maxPagesPerRun: 80,
maxKnowledgeCreatesPerRun: undefined,
maxKnowledgeUpdatesPerRun: undefined,
},
},
testIo.io,
);
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('prints generated connection notion pick help without invoking execution', async () => {
const helpCases = [
['connection', 'notion', '--help'],
['connection', 'notion', 'pick', '--help'],
['connection', 'notion', 'pick', 'notion-main', '--help'],
];
for (const argv of helpCases) {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(runKtxCli(argv, testIo.io, { connectionNotion })).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx connection notion');
expect(testIo.stdout()).toContain('pick');
expect(testIo.stderr()).toBe('');
expect(connectionNotion).not.toHaveBeenCalled();
}
});
it('dispatches connection notion pick through Commander', async () => {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'connection',
'notion',
'pick',
'notion-main',
'--no-input',
'--root-page-id',
'11111111222233334444555555555555',
'--root-page-id',
'11111111-2222-3333-4444-555555555555',
],
testIo.io,
{ connectionNotion },
),
).resolves.toBe(0);
expect(connectionNotion).toHaveBeenCalledWith(
{
command: 'pick',
projectDir: tempDir,
connectionId: 'notion-main',
mode: 'non-interactive',
rootPageIds: ['11111111-2222-3333-4444-555555555555'],
},
testIo.io,
);
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('ignores connection notion pick root page flags in interactive mode', async () => {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(
runKtxCli(['connection', 'notion', 'pick', 'notion-main', '--root-page-id', 'not-a-uuid'], testIo.io, {
connectionNotion,
}),
).resolves.toBe(0);
expect(connectionNotion).toHaveBeenCalledWith(
{
command: 'pick',
projectDir: expect.any(String),
connectionId: 'notion-main',
mode: 'interactive',
},
testIo.io,
);
expect(testIo.stderr()).toBe('');
});
it('rejects connection notion pick no-input mode without root page ids', async () => {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(
runKtxCli(['connection', 'notion', 'pick', 'notion-main', '--no-input'], testIo.io, { connectionNotion }),
).resolves.toBe(1);
expect(connectionNotion).not.toHaveBeenCalled();
expect(testIo.stderr()).toContain('connection notion pick --no-input requires at least one --root-page-id');
});
it('writes basic debug dispatch information when --debug is set', async () => {
const testIo = makeIo();
const connection = vi.fn().mockResolvedValue(0);
@ -1817,51 +1518,6 @@ describe('runKtxCli', () => {
expect(ingest).not.toHaveBeenCalled();
});
it('rejects mutually exclusive credential and scan mode options before invoking runners', async () => {
const connection = vi.fn(async () => 0);
const scan = vi.fn(async () => 0);
const tokenIo = makeIo();
await expect(
runKtxCli(
[
'connection',
'add',
'notion',
'notion-main',
'--token-env',
'NOTION_TOKEN',
'--token-file',
'/tmp/notion-token',
'--root-page-id',
'11111111111111111111111111111111',
],
tokenIo.io,
{ connection },
),
).resolves.toBe(1);
expect(tokenIo.stderr()).toMatch(/conflict|cannot be used/i);
expect(connection).not.toHaveBeenCalled();
expect(scan).not.toHaveBeenCalled();
});
it('validates connection mapping set syntax before runner domain validation', async () => {
const badFieldIo = makeIo();
await expect(
runKtxCli(['connection', 'mapping', 'set', 'prod-metabase', 'invalidMappings', '1=warehouse'], badFieldIo.io),
).resolves.toBe(1);
expect(badFieldIo.stderr()).toContain('databaseMappings or connectionMappings');
for (const assignment of ['missing-equals', '=warehouse', '1=']) {
const testIo = makeIo();
await expect(
runKtxCli(['connection', 'mapping', 'set', 'prod-metabase', 'databaseMappings', assignment], testIo.io),
).resolves.toBe(1);
expect(testIo.stderr()).toContain('non-empty <key>=<value>');
}
});
it('does not expose root init after setup owns project creation', async () => {
const testIo = makeIo();

2
packages/cli/src/commands/connection-notion-tree.test.ts → packages/cli/src/notion-page-picker-tree.test.ts
View file

@ -14,7 +14,7 @@ import {
TRANSIENT_HINT_DURATION_MS,
visibleNodeIds,
type NotionPickerPageInput,
} from './connection-notion-tree.js';
} from './notion-page-picker-tree.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',

0
packages/cli/src/commands/connection-notion-tree.ts → packages/cli/src/notion-page-picker-tree.ts
View file

6
packages/cli/src/commands/connection-notion-tui.test.tsx → packages/cli/src/notion-page-picker-tui.test.tsx
View file

@ -2,7 +2,7 @@
import { render as renderInkTest } from 'ink-testing-library';
import { act, type ReactNode } from 'react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
NotionPickerApp,
notionPickerCommandForInkInput,
@ -13,7 +13,7 @@ import {
windowOffset,
type NotionPickerInkInstance,
type NotionPickerInkRenderOptions,
} from './connection-notion-tui.js';
} from './notion-page-picker-tui.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
@ -378,7 +378,7 @@ describe('renderNotionPickerTui', () => {
},
),
).resolves.toEqual({ kind: 'quit' });
expect(stderr).toContain('Use --no-input --root-page-id <UUID> for scripted mode');
expect(stderr).toContain('Use --no-input --notion-root-page-id <UUID> for scripted mode');
expect(stderr).not.toContain('secret');
});
});

6
packages/cli/src/commands/connection-notion-tui.tsx → packages/cli/src/notion-page-picker-tui.tsx
View file

@ -9,8 +9,8 @@ import {
visibleNodeIds,
type PickerCommand,
type PickerState,
} from './connection-notion-tree.js';
import type { KtxCliIo } from '../index.js';
} from './notion-page-picker-tree.js';
import type { KtxCliIo } from './cli-runtime.js';
const COLOR_THEME = {
text: 'white',
@ -331,7 +331,7 @@ export async function renderNotionPickerTui(
return result;
} catch (error) {
io.stderr.write(
`Notion picker requires a TTY. Use --no-input --root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
`Notion picker requires a TTY. Use --no-input --notion-root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
);
return { kind: 'quit' };
}

308
packages/cli/src/notion-page-picker.test.ts Normal file
View file

@ -0,0 +1,308 @@
import { describe, expect, it, vi } from 'vitest';
import {
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
pickNotionRootPages,
resolveNotionWorkspaceLabel,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './notion-page-picker.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: true,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('Notion page picker helpers', () => {
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
});
describe('pickNotionRootPages', () => {
it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
},
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toEqual({ kind: 'selected', rootPageIds: [PAGE_IDS.engineering] });
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toBe('');
});
it('uses inline Notion auth_token for discovery', async () => {
const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]);
const createNotionApi = vi.fn((authToken: string) => {
expect(authToken).toBe('ntn_inline_token');
return api;
});
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token: 'ntn_inline_token',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
},
},
makeIo().io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'back' });
expect(createNotionApi).toHaveBeenCalledOnce();
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
},
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toEqual({ kind: 'back' });
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
});
it('returns unavailable when discovery cannot load any pages', async () => {
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [],
},
},
makeIo().io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => ({
search: vi.fn(async () => {
throw new Error('Notion API unavailable');
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
})),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' });
});
});

200
packages/cli/src/commands/connection-notion.ts → packages/cli/src/notion-page-picker.ts
View file

@ -1,51 +1,40 @@
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from '@ktx/context/connections';
import { resolveNotionConnectionAuthToken } from '@ktx/context/connections';
import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
loadKtxProject,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
type NotionPickerTuiIo,
type PickerRenderInput,
type PickerRenderResult,
renderNotionPickerTui,
} from './connection-notion-tui.js';
} from './notion-page-picker-tui.js';
profileMark('module:commands/connection-notion');
profileMark('module:notion-page-picker');
export type KtxConnectionNotionArgs =
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'interactive';
}
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'non-interactive';
rootPageIds: string[];
};
export interface PickNotionRootPagesArgs {
connectionId: string;
connection: KtxProjectConnectionConfig;
}
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
export type { PickerRenderInput, PickerRenderResult };
interface KtxConnectionNotionDeps {
export type NotionRootPagePickResult =
| { kind: 'selected'; rootPageIds: string[] }
| { kind: 'back' }
| { kind: 'unavailable'; message: string };
export interface NotionRootPagePickerDeps {
env?: Record<string, string | undefined>;
loadProject?: typeof loadKtxProject;
createNotionApi?: (authToken: string) => NotionPickerApi;
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
}
const NOTION_PICKER_PAGE_CAP = 5000;
function assertSafeConnectionId(connectionId: string): void {
function assertSafeNotionPickerConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
@ -168,111 +157,74 @@ export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connecti
}
}
function notionConnection(project: KtxLocalProject, connectionId: string): KtxProjectConnectionConfig {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" not found`);
}
function assertNotionConnection(connection: KtxProjectConnectionConfig, connectionId: string): void {
if (connection.driver !== 'notion') {
throw new Error(`Connection "${connectionId}" is not a Notion connection`);
}
return connection;
}
export async function applyNotionPickerWriteback(
project: KtxLocalProject,
connectionId: string,
rootPageIds: string[],
): Promise<void> {
if (rootPageIds.length === 0) {
throw new Error('connection notion pick requires at least one root page id');
}
const existing = notionConnection(project, connectionId);
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: {
...existing,
crawl_mode: 'selected_roots',
root_page_ids: rootPageIds,
},
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`,
);
function stringArray(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string' && item.trim().length > 0) : [];
}
export async function runKtxConnectionNotion(
args: KtxConnectionNotionArgs,
function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessible' | 'selected_roots' {
return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
}
export async function pickNotionRootPages(
args: PickNotionRootPagesArgs,
io: KtxCliIo = process,
deps: KtxConnectionNotionDeps = {},
): Promise<number> {
deps: NotionRootPagePickerDeps = {},
): Promise<NotionRootPagePickResult> {
try {
assertSafeConnectionId(args.connectionId);
const loadProject = deps.loadProject ?? loadKtxProject;
if (args.mode === 'interactive') {
const project = await loadProject({ projectDir: args.projectDir });
const rawConnection = notionConnection(project, args.connectionId);
const notion = parseNotionConnectionConfig(rawConnection);
const authToken = await resolveNotionConnectionAuthToken(notion, { env: deps.env });
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: notion.root_page_ids,
currentCrawlMode: notion.crawl_mode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: notion.crawl_mode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
io.stdout.write('No changes saved.\n');
return 0;
}
await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
assertSafeNotionPickerConnectionId(args.connectionId);
assertNotionConnection(args.connection, args.connectionId);
const crawlMode = notionCrawlMode(args.connection);
const authToken = await resolveNotionConnectionAuthToken(
{
auth_token: typeof args.connection.auth_token === 'string' ? args.connection.auth_token : null,
auth_token_ref: typeof args.connection.auth_token_ref === 'string' ? args.connection.auth_token_ref : null,
},
{ env: deps.env },
);
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: stringArray(args.connection.root_page_ids),
currentCrawlMode: crawlMode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const project = await loadProject({ projectDir: args.projectDir });
await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: crawlMode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
return { kind: 'back' };
}
if (result.rootPageIds.length === 0) {
return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' };
}
return { kind: 'selected', rootPageIds: result.rootPageIds };
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) };
}
}

8
packages/cli/src/print-command-tree.test.ts
View file

@ -16,7 +16,13 @@ describe('renderKtxCommandTree', () => {
expect(topLevel).toContain(expected);
}
expect(output).toContain('│ ├── test <connectionId>');
expect(output).toContain('│ └── test <connectionId>');
expect(output).not.toContain('│ ├── add');
expect(output).not.toContain('│ ├── remove');
expect(output).not.toContain('│ ├── map');
expect(output).not.toContain('│ ├── mapping');
expect(output).not.toContain('│ ├── metabase');
expect(output).not.toContain('│ ├── notion');
});
it('ends with a single trailing newline', () => {

42
packages/cli/src/setup-databases.test.ts
View file

@ -1234,6 +1234,48 @@ describe('setup databases step', () => {
expect(config.connections.warehouse).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL' });
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect(io.stderr()).toContain('Structural scan failed for warehouse.');
expect(io.stderr()).toContain('│ Structural scan failed for warehouse.');
expect(io.stderr()).not.toMatch(/^Structural scan failed for warehouse\./m);
});
it('prints the native SQLite rebuild command when scanning hits a Node ABI mismatch', async () => {
const io = makeIo();
const result = await runKtxSetupDatabasesStep(
{
projectDir: tempDir,
inputMode: 'disabled',
databaseDrivers: ['postgres'],
databaseConnectionId: 'warehouse',
databaseUrl: 'env:DATABASE_URL',
databaseSchemas: [],
skipDatabases: false,
},
io.io,
{
testConnection: vi.fn(async () => 0),
scanConnection: vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => {
commandIo.stderr.write(
[
"The module '/workspace/node_modules/better-sqlite3/build/Release/better_sqlite3.node'",
'was compiled against a different Node.js version using',
'NODE_MODULE_VERSION 147. This version of Node.js requires',
'NODE_MODULE_VERSION 137. Please try re-compiling or re-installing',
'the module (for instance, using `npm rebuild` or `npm install`).',
'',
].join('\n'),
);
return 1;
}),
},
);
expect(result.status).toBe('failed');
expect(io.stderr()).toContain('Native SQLite is built for a different Node.js ABI.');
expect(io.stderr()).toContain('│ Native SQLite is built for a different Node.js ABI.');
expect(io.stderr()).toContain('Fix: pnpm run native:rebuild');
expect(io.stderr()).toContain(`Retry: ktx scan --project-dir ${tempDir} warehouse`);
expect(io.stderr()).not.toContain('npm rebuild');
expect(io.stderr()).not.toMatch(/^Native SQLite is built for a different Node.js ABI\./m);
});
it('writes Historic SQL config for supported Snowflake databases after validation succeeds', async () => {

55
packages/cli/src/setup-databases.ts
View file

@ -949,6 +949,36 @@ function flushBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo)
}
}
function writePrefixedLines(write: (chunk: string) => void, output: string): void {
for (const line of output.split(/\r?\n/)) {
if (line.length > 0) {
write(`│ ${line}\n`);
}
}
}
function flushPrefixedBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo): void {
writePrefixedLines((chunk) => io.stdout.write(chunk), bufferedIo.stdoutText());
writePrefixedLines((chunk) => io.stderr.write(chunk), bufferedIo.stderrText());
}
function nativeSqliteAbiMismatchDetail(output: string): string | null {
const mentionsBetterSqlite = /\bbetter-sqlite3\b|better_sqlite3/i.test(output);
const mentionsAbiMismatch = /compiled against a different Node\.js version|NODE_MODULE_VERSION/i.test(output);
if (!mentionsBetterSqlite || !mentionsAbiMismatch) {
return null;
}
const versionMatch = output.match(
/compiled against[\s\S]*?NODE_MODULE_VERSION\s+(\d+)[\s\S]*?requires[\s\S]*?NODE_MODULE_VERSION\s+(\d+)/i,
);
if (!versionMatch) {
return 'better-sqlite3 native module could not load for the current Node.js runtime.';
}
return `better-sqlite3 was compiled for NODE_MODULE_VERSION ${versionMatch[1]}, but this Node.js requires ${versionMatch[2]}.`;
}
function readOutputValue(output: string, label: string): string | undefined {
const escapedLabel = label.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const match = new RegExp(`^\\s*${escapedLabel}:\\s*(.+?)\\s*$`, 'im').exec(output);
@ -1443,9 +1473,28 @@ async function validateAndScanConnection(input: {
const scanIo = createBufferedCommandIo();
const scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo);
if (scanCode !== 0) {
flushBufferedCommandOutput(input.io, scanIo);
input.io.stderr.write(`Structural scan failed for ${input.connectionId}.\n`);
input.io.stderr.write(`Debug command: ktx scan --project-dir ${input.projectDir} ${input.connectionId}\n`);
const nativeSqliteDetail = nativeSqliteAbiMismatchDetail(`${scanIo.stderrText()}\n${scanIo.stdoutText()}`);
if (nativeSqliteDetail) {
writePrefixedLines(
(chunk) => input.io.stderr.write(chunk),
[
`Structural scan failed for ${input.connectionId}.`,
'Native SQLite is built for a different Node.js ABI.',
`Detail: ${nativeSqliteDetail}`,
'Fix: pnpm run native:rebuild',
`Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`,
].join('\n'),
);
} else {
flushPrefixedBufferedCommandOutput(input.io, scanIo);
writePrefixedLines(
(chunk) => input.io.stderr.write(chunk),
[
`Structural scan failed for ${input.connectionId}.`,
`Debug command: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`,
].join('\n'),
);
}
return false;
}
const scanOutput = scanIo.stdoutText();

22
packages/cli/src/setup-interrupt.test.ts
View file

@ -17,9 +17,11 @@ function makeTracker(ctrlCValues: boolean[]): SetupInterruptTracker {
describe('setup interrupt confirmation', () => {
const originalIsTTY = process.stdin.isTTY;
const originalRef = process.stdin.ref;
afterEach(() => {
Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: originalIsTTY });
Object.defineProperty(process.stdin, 'ref', { configurable: true, value: originalRef });
});
it('fails before opening a prompt when interactive setup has no tty', async () => {
@ -33,6 +35,26 @@ describe('setup interrupt confirmation', () => {
expect(prompt).not.toHaveBeenCalled();
});
it('refs stdin before opening a real interactive prompt', async () => {
const calls: string[] = [];
Object.defineProperty(process.stdin, 'isTTY', { configurable: true, value: true });
Object.defineProperty(process.stdin, 'ref', {
configurable: true,
value: vi.fn(() => {
calls.push('ref');
return process.stdin;
}),
});
const prompt = vi.fn(async () => {
calls.push('prompt');
return 'continued';
});
await expect(withSetupInterruptConfirmation(prompt)).resolves.toBe('continued');
expect(calls).toEqual(['ref', 'prompt']);
});
it('asks before exiting on Ctrl+C and reruns the active prompt when declined', async () => {
const prompt = vi.fn(async () => (prompt.mock.calls.length === 1 ? CANCEL : 'continued'));
const confirmExit = vi.fn(async () => false);

7
packages/cli/src/setup-interrupt.ts
View file

@ -23,6 +23,10 @@ interface SetupInterruptOptions {
const NON_INTERACTIVE_SETUP_MESSAGE =
'Interactive setup requires a terminal. Re-run this command in a TTY, or pass --no-input with the required options.';
function refSetupInput(input: NodeJS.ReadStream = stdin): void {
input.ref?.();
}
function createSetupInterruptTracker(input: NodeJS.ReadStream = stdin): SetupInterruptTracker {
let ctrlCPressed = false;
const onKeypress = (char: string | undefined, key: Key) => {
@ -73,6 +77,9 @@ export async function withSetupInterruptConfirmation<T>(
const confirmExit = options.confirmExit ?? defaultConfirmExit;
while (true) {
if (!options.tracker) {
refSetupInput();
}
const value = await tracker.track(prompt);
if (!isCancel(value)) {
return value;

128
packages/cli/src/setup-sources.test.ts
View file

@ -136,7 +136,6 @@ describe('setup sources step', () => {
projectDir,
});
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(io.stdout()).toContain('Context source setup skipped.');
});
@ -170,7 +169,6 @@ describe('setup sources step', () => {
source_dir: '/repo/dbt',
project_name: 'analytics',
});
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(projectDir)).completed_steps).toContain('sources');
expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' });
});
@ -178,7 +176,10 @@ describe('setup sources step', () => {
it('writes Metabase config and validates mapping through existing mapping path', async () => {
await addPrimarySource();
const validateMetabase = vi.fn(async () => ({ ok: true as const, detail: 'user=admin@example.com' }));
const runMapping = vi.fn(async () => 0);
const runMapping = vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => {
commandIo.stdout.write('Mapping validated — 1 mapping configured\n');
return 0;
});
const io = makeIo();
await expect(
@ -210,7 +211,16 @@ describe('setup sources step', () => {
syncMode: 'ALL',
},
});
expect(runMapping).toHaveBeenCalledWith(projectDir, 'prod_metabase', io.io);
expect(runMapping).toHaveBeenCalledWith(
projectDir,
'prod_metabase',
expect.objectContaining({
stdout: expect.objectContaining({ write: expect.any(Function) }),
stderr: expect.objectContaining({ write: expect.any(Function) }),
}),
);
expect(io.stdout()).toContain('│ Mapping validated — 1 mapping configured');
expect(io.stdout()).not.toMatch(/^Mapping validated — 1 mapping configured$/m);
});
it('writes Notion config with the full default knowledge create budget', async () => {
@ -273,6 +283,105 @@ describe('setup sources step', () => {
});
});
it('uses the rich Notion picker for interactive selected root setup', async () => {
await addPrimarySource();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' }));
const pickNotionRootPages = vi.fn(async (input: Parameters<NonNullable<KtxSetupSourcesDeps['pickNotionRootPages']>>[0]) => {
expect(input.connectionId).toBe('notion-main');
expect(input.connection).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [],
});
return { kind: 'selected' as const, rootPageIds: ['11111111-2222-3333-4444-555555555555'] };
});
const testPrompts = prompts({
multiselect: [['notion']],
select: ['env', 'selected_roots', 'done'],
text: ['notion-main'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{ prompts: testPrompts, validateNotion, pickNotionRootPages },
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(pickNotionRootPages).toHaveBeenCalledOnce();
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'Which Notion pages should KTX ingest?',
options: [
{ value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' },
{ value: 'all_accessible', label: 'All pages the integration can access' },
{ value: 'back', label: 'Back' },
],
});
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['11111111-2222-3333-4444-555555555555'],
});
});
it('backs out of the Notion picker without writing selected_roots when the picker quits', async () => {
await addPrimarySource();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=0' }));
const pickNotionRootPages = vi.fn(async () => ({ kind: 'back' as const }));
const testPrompts = prompts({
multiselect: [['notion']],
select: ['env', 'selected_roots', 'all_accessible', 'done'],
text: ['notion-main'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{ prompts: testPrompts, validateNotion, pickNotionRootPages },
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(pickNotionRootPages).toHaveBeenCalledOnce();
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
crawl_mode: 'all_accessible',
});
expect((await readConfig()).connections['notion-main']?.root_page_ids).toBeUndefined();
});
it('surfaces Notion picker failures and returns to the page-mode step', async () => {
await addPrimarySource();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=0' }));
const pickNotionRootPages = vi.fn(async () => ({
kind: 'unavailable' as const,
message: 'Notion picker requires a TTY',
}));
const testPrompts = prompts({
multiselect: [['notion']],
select: ['env', 'selected_roots', 'all_accessible', 'done'],
text: ['notion-main'],
});
const io = makeIo();
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
io.io,
{ prompts: testPrompts, validateNotion, pickNotionRootPages },
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(io.stderr()).toContain('Notion picker requires a TTY');
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
crawl_mode: 'all_accessible',
});
});
it('defaults interactive Metabase and Looker source setup to the only warehouse connection', async () => {
await addPrimarySource();
const cases: Array<{
@ -455,7 +564,14 @@ describe('setup sources step', () => {
),
).resolves.toEqual({ status: 'failed', projectDir });
expect(runMapping).toHaveBeenCalledWith(projectDir, 'metabase-main', io.io);
expect(runMapping).toHaveBeenCalledWith(
projectDir,
'metabase-main',
expect.objectContaining({
stdout: expect.objectContaining({ write: expect.any(Function) }),
stderr: expect.objectContaining({ write: expect.any(Function) }),
}),
);
expect(io.stderr()).toContain('1: Metabase database does not match KTX connection database');
expect(io.stderr()).not.toContain('Metabase mapping validation failed');
});
@ -479,7 +595,7 @@ describe('setup sources step', () => {
),
).resolves.toEqual({ status: 'failed', projectDir });
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(projectDir)).completed_steps).not.toContain('sources');
expect(io.stderr()).toContain('No LookML files found');
});

161
packages/cli/src/setup-sources.ts
View file

@ -27,8 +27,8 @@ import {
serializeKtxProjectConfig,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { runKtxConnectionMapping } from './commands/connection-mapping.js';
import { runKtxConnection } from './connection.js';
import { pickNotionRootPages } from './notion-page-picker.js';
import { runKtxSourceMapping } from './source-mapping.js';
import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
import { runKtxPublicIngest } from './public-ingest.js';
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
@ -94,6 +94,7 @@ export interface KtxSetupSourcesDeps {
validateLooker?: (projectDir: string, connectionId: string) => Promise<SourceValidationResult>;
validateLookml?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
validateNotion?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
pickNotionRootPages?: typeof pickNotionRootPages;
discoverMetabaseDatabases?: (args: {
sourceUrl: string;
sourceApiKeyRef: string;
@ -527,7 +528,7 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC
driver: 'notion',
auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'),
crawl_mode: crawlMode,
root_page_ids: rootPageIds,
...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 1000,
@ -613,7 +614,7 @@ async function defaultValidateMetricflow(connection: KtxProjectConnectionConfig)
}
async function defaultValidateLooker(projectDir: string, connectionId: string): Promise<SourceValidationResult> {
const code = await runKtxConnectionMapping(
const code = await runKtxSourceMapping(
{ command: 'refresh', projectDir, connectionId, autoAccept: true },
{ stdout: { write() {} }, stderr: { write() {} } },
);
@ -656,6 +657,47 @@ interface MappingJsonOutput {
mappings: unknown[];
}
function splitOutputLines(output: string): string[] {
return output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
}
function writeSetupPrefixedLines(write: (chunk: string) => void, output: string): void {
for (const line of output.split(/\r?\n/)) {
if (line.length > 0) {
write(`│ ${line}\n`);
}
}
}
function createSetupPrefixedIo(io: KtxCliIo): KtxCliIo {
return {
stdout: {
isTTY: io.stdout.isTTY,
columns: io.stdout.columns,
write(chunk: string) {
writeSetupPrefixedLines((line) => io.stdout.write(line), chunk);
},
},
stderr: {
write(chunk: string) {
writeSetupPrefixedLines((line) => io.stderr.write(line), chunk);
},
},
};
}
function parseMappingListJson(output: string): unknown[] {
const trimmed = output.trim();
if (!trimmed) {
return [];
}
const parsed = JSON.parse(trimmed) as unknown;
return Array.isArray(parsed) ? parsed : [];
}
function summarizeMappingResult(parsed: MappingJsonOutput): string {
const mappingCount = parsed.mappings.length;
const mappingNoun = mappingCount === 1 ? 'mapping' : 'mappings';
@ -663,22 +705,51 @@ function summarizeMappingResult(parsed: MappingJsonOutput): string {
}
async function defaultRunMapping(projectDir: string, connectionId: string, io: KtxCliIo): Promise<number> {
let captured = '';
const captureIo: KtxCliIo = {
stdout: { write(chunk: string) { captured += chunk; } },
stderr: io.stderr,
const outputs = {
refresh: '',
validation: '',
list: '',
};
const code = await runKtxConnection(
{ command: 'map', projectDir, sourceConnectionId: connectionId, json: true },
captureIo,
const refreshCode = await runKtxSourceMapping(
{ command: 'refresh', projectDir, connectionId, autoAccept: true },
{
stdout: { write(chunk: string) { outputs.refresh += chunk; } },
stderr: io.stderr,
},
);
if (code !== 0) return code;
try {
const parsed = JSON.parse(captured.trim()) as MappingJsonOutput;
io.stdout.write(`${summarizeMappingResult(parsed)}\n`);
} catch {
io.stdout.write(captured);
if (refreshCode !== 0) {
return refreshCode;
}
const validationCode = await runKtxSourceMapping(
{ command: 'validate', projectDir, connectionId },
{
stdout: { write(chunk: string) { outputs.validation += chunk; } },
stderr: io.stderr,
},
);
if (validationCode !== 0) {
return validationCode;
}
const listCode = await runKtxSourceMapping(
{ command: 'list', projectDir, connectionId, json: true },
{
stdout: { write(chunk: string) { outputs.list += chunk; } },
stderr: io.stderr,
},
);
if (listCode !== 0) {
return listCode;
}
const parsed: MappingJsonOutput = {
connectionId,
refresh: { ok: true, output: splitOutputLines(outputs.refresh) },
validation: { ok: true, output: splitOutputLines(outputs.validation) },
mappings: parseMappingListJson(outputs.list),
};
io.stdout.write(`${summarizeMappingResult(parsed)}\n`);
return 0;
}
@ -926,6 +997,8 @@ async function promptForInteractiveSource(
args: KtxSetupSourcesArgs,
source: KtxSetupSourceType,
prompts: KtxSetupSourcesPromptAdapter,
io: KtxCliIo,
deps: KtxSetupSourcesDeps,
defaultConnectionId = `${source}-main`,
testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection,
discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'],
@ -1197,7 +1270,7 @@ async function promptForInteractiveSource(
const crawlMode = await prompts.select({
message: 'Which Notion pages should KTX ingest?',
options: [
{ value: 'selected_roots', label: 'Specific pages and their subpages (you\'ll paste page IDs)' },
{ value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' },
{ value: 'all_accessible', label: 'All pages the integration can access' },
{ value: 'back', label: 'Back' },
],
@ -1212,15 +1285,29 @@ async function promptForInteractiveSource(
...(state.notionCrawlMode === 'selected_roots'
? [
async (currentState: SourcePromptState) => {
const roots = await promptText(prompts, {
message: 'Notion page IDs to ingest (each page includes all its subpages)',
placeholder: 'page-id-1, page-id-2',
});
if (roots === undefined) return 'back';
currentState.notionRootPageIds = roots
.split(',')
.map((root) => root.trim())
.filter(Boolean);
const connectionId = currentState.sourceConnectionId ?? 'notion-main';
const result = await (deps.pickNotionRootPages ?? pickNotionRootPages)(
{
connectionId,
connection: {
driver: 'notion',
auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'),
crawl_mode: 'selected_roots',
root_page_ids: currentState.notionRootPageIds ?? [],
root_database_ids: [],
root_data_source_ids: [],
},
},
io,
);
if (result.kind === 'back') {
return 'back';
}
if (result.kind === 'unavailable') {
io.stderr.write(`${result.message}\n`);
return 'back';
}
currentState.notionRootPageIds = result.rootPageIds;
return 'next';
},
]
@ -1258,7 +1345,9 @@ async function chooseInteractiveSourceConnection(input: {
source: KtxSetupSourceType;
connections: Record<string, KtxProjectConnectionConfig>;
prompts: KtxSetupSourcesPromptAdapter;
io: KtxCliIo;
testGitRepo?: KtxSetupSourcesDeps['testGitRepo'];
pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages'];
discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases'];
}): Promise<InteractiveSourceConnectionChoice> {
const existingIds = existingConnectionIdsBySource(input.connections, input.source);
@ -1270,6 +1359,11 @@ async function chooseInteractiveSourceConnection(input: {
input.args,
input.source,
input.prompts,
input.io,
{
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
},
defaultConnectionId,
input.testGitRepo,
input.discoverMetabaseDatabases,
@ -1302,6 +1396,11 @@ async function chooseInteractiveSourceConnection(input: {
input.args,
input.source,
input.prompts,
input.io,
{
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
},
defaultConnectionId,
input.testGitRepo,
input.discoverMetabaseDatabases,
@ -1416,7 +1515,9 @@ export async function runKtxSetupSourcesStep(
source,
connections: (await loadKtxProject({ projectDir: args.projectDir })).config.connections,
prompts,
io,
testGitRepo: deps.testGitRepo,
pickNotionRootPages: deps.pickNotionRootPages,
discoverMetabaseDatabases: deps.discoverMetabaseDatabases,
});
if (sourceChoice === 'back') {
@ -1448,7 +1549,11 @@ export async function runKtxSetupSourcesStep(
}
if (source === 'metabase' || source === 'looker') {
prompts.log?.(`Validating ${sourceLabel(source)} mapping…`);
const mappingCode = await (deps.runMapping ?? defaultRunMapping)(args.projectDir, connectionId, io);
const mappingCode = await (deps.runMapping ?? defaultRunMapping)(
args.projectDir,
connectionId,
createSetupPrefixedIo(io),
);
if (mappingCode !== 0) {
await rollback?.();
return { status: 'failed', projectDir: args.projectDir };

225
packages/cli/src/source-mapping.ts Normal file
View file

@ -0,0 +1,225 @@
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
KtxYamlMetabaseSourceStateReader,
LocalLookerRuntimeStore,
LocalMetabaseDiscoveryCache,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
discoverMetabaseDatabases,
lookerCredentialsFromLocalConnection,
metabaseRuntimeConfigFromLocalConnection,
seedLocalMappingStateFromKtxYaml,
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type LocalMetabaseMappingListRow,
type MetabaseRuntimeClient,
} from '@ktx/context/ingest';
import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
profileMark('module:source-mapping');
export type KtxSourceMappingArgs =
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
| { command: 'validate'; projectDir: string; connectionId: string };
interface KtxSourceMappingDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
createLookerClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function createDefaultLookerClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
const factory = new DefaultLookerConnectionClientFactory({
async resolve(lookerConnectionId) {
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
},
});
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
cleanup?(): Promise<void>;
};
}
function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean {
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
}
function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void {
const connection = project.config.connections[connectionId];
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
}
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function renderMapping(row: LocalMetabaseMappingListRow): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
row.source
})`;
}
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
const target = row.ktxConnectionId ?? '[unmapped]';
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
}
export async function runKtxSourceMapping(
args: KtxSourceMappingArgs,
io: KtxCliIo = process,
deps: KtxSourceMappingDeps = {},
): Promise<number> {
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
await seedLocalMappingStateFromKtxYaml(project, args.connectionId);
if (isLookerConnection(project, args.connectionId)) {
const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listConnectionMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
try {
const discovered = await discoverLookerConnections(client);
const drift = computeLookerMappingDrift({
storedMappings: await store.readMappings(args.connectionId),
discovered,
});
if (args.autoAccept) {
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup?.();
}
}
const knownKtxConnectionIds = new Set(Object.keys(project.config.connections));
const knownConnectionTypes = new Map(
Object.entries(project.config.connections).map(([id]) => [id, targetPhysicalInfo(project, id).connection_type]),
);
const validation = validateLookerMappings({
mappings: await store.readMappings(args.connectionId),
knownKtxConnectionIds,
knownConnectionTypes,
});
if (!validation.ok) {
for (const error of validation.errors) {
io.stderr.write(`${error.key}: ${error.reason}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
assertMetabaseConnection(project, args.connectionId);
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
const store = new KtxYamlMetabaseSourceStateReader(project, { discoveryCache });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await discoveryCache.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup();
}
}
const rows = await store.listDatabaseMappings(args.connectionId);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
project.config.connections[row.targetConnectionId]
? targetPhysicalInfo(project, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (failures.length > 0) {
for (const failure of failures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

20
packages/cli/src/standalone-smoke.test.ts
View file

@ -307,7 +307,7 @@ describe('standalone built ktx CLI smoke', () => {
});
});
it('adds a redacted Notion connection through the built binary', async () => {
it('rejects the removed connection add command through the built binary', async () => {
const projectDir = join(tempDir, 'notion-project');
const init = await runSetupNewProject(projectDir);
expectProjectStderr(init, projectDir);
@ -327,23 +327,17 @@ describe('standalone built ktx CLI smoke', () => {
'5',
]);
expectProjectStderr(add, projectDir);
expect(add.stdout).toContain('Connection: notion-main');
expect(add.stdout).toContain('Driver: notion');
expect(add.code).toBe(1);
expect(add.stdout).toBe('');
expect(add.stderr).toContain("unknown command 'add'");
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 5');
expect(yaml).not.toContain('driver: notion');
expect(yaml).not.toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).not.toContain('ntn_');
const parsed = parseKtxProjectConfig(yaml);
expect(parsed.connections['notion-main']).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
});
expect(parsed.connections['notion-main']).toBeUndefined();
});
});