apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-07 07:55:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(cli): clean up connection commands

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 14:21:05 +02:00
parent c22248dabf
commit 2f41fd019d
32 changed files with 906 additions and 5230 deletions
Download patch file Download diff file Expand all files Collapse all files

135
docs-site/content/docs/cli-reference/ktx-connection.mdx
View file

@ -1,9 +1,11 @@
---
title: "ktx connection"
description: "Add, list, test, and map data sources."
description: "List and test configured data sources."
---
Manage database and source connections in your KTX project. Connections define how KTX reaches your data warehouse, BI tools, and context sources.
Inspect configured connections in your KTX project. Connections define how KTX
reaches your data warehouse, BI tools, and context sources. Use `ktx setup` to
add, remove, or reconfigure connections.
## Command signature
@ -17,96 +19,23 @@ ktx connection <subcommand> [options]
|-----------|-------------|
| `list` | List configured connections |
| `test <connectionId>` | Test a configured connection |
| `add <driver> <connectionId>` | Add or replace a configured connection |
| `remove <connectionId>` | Remove a configured connection from `ktx.yaml` |
| `map <sourceConnectionId>` | Refresh and validate BI-to-warehouse mappings |
| `mapping list <connectionId>` | List Metabase database mappings |
| `mapping set <connectionId> <field> <assignment>` | Set a Metabase or Looker warehouse mapping |
| `mapping apply-bulk <connectionId>` | Apply mappings from JSON |
| `mapping set-sync-enabled <connectionId> <dbId>` | Enable or disable sync for one Metabase database |
| `mapping sync-state get <connectionId>` | Read sync-state selection |
| `mapping sync-state set <connectionId>` | Write sync-state selection |
| `mapping refresh <connectionId>` | Refresh Metabase database mappings |
| `mapping validate <connectionId>` | Validate Metabase database mappings |
| `mapping clear <connectionId> [dbId]` | Clear Metabase database mappings |
| `metabase setup` | Guided setup for a Metabase connection |
| `notion pick <connectionId>` | Pick Notion root pages for a configured Notion connection |
## Options
### `connection add`
The `connection` command has command-level options for listing and testing
existing connections.
| Flag | Description | Default |
|------|-------------|---------|
| `--url <url>` | Connection URL, `env:NAME`, or `file:/path` reference | — |
| `--schema <schema>` | Schema to include; repeatable | — |
| `--readonly` | Mark the connection as read-only | `false` |
| `--force` | Replace an existing connection | `false` |
| `--allow-literal-credentials` | Allow writing a literal credential URL to `ktx.yaml` | `false` |
#### Notion-specific options for `connection add`
| Flag | Description | Default |
|------|-------------|---------|
| `--token-env <name>` | Environment variable containing Notion auth token | — |
| `--token-file <path>` | File containing Notion auth token | — |
| `--crawl-mode <mode>` | Notion crawl mode (`all_accessible` or `selected_roots`) | `selected_roots` |
| `--root-page-id <id>` | Root page to crawl; repeatable | — |
| `--root-database-id <id>` | Root database to crawl; repeatable | — |
| `--root-data-source-id <id>` | Root data source to crawl; repeatable | — |
| `--max-pages <n>` | Maximum pages per run | — |
| `--max-knowledge-creates <n>` | Maximum knowledge creates per run | — |
| `--max-knowledge-updates <n>` | Maximum knowledge updates per run | — |
### `connection remove`
| Flag | Description | Default |
|------|-------------|---------|
| `--force` | Remove without prompting | `false` |
| `--no-input` | Disable interactive terminal input | — |
### `connection map`
### `connection list`
| Flag | Description | Default |
|------|-------------|---------|
| `--json` | Print JSON output | `false` |
### `connection mapping` subcommands
| Flag | Subcommand | Description | Default |
|------|-----------|-------------|---------|
| `--json` | `list`, `sync-state get` | Print JSON output | `false` |
| `--file <path>` | `apply-bulk` | JSON mapping file (required) | — |
| `--enabled <value>` | `set-sync-enabled` | `true` or `false` (required) | — |
| `--mode <mode>` | `sync-state set` | `ALL`, `ONLY`, or `EXCEPT` (required) | — |
| `--collections <ids>` | `sync-state set` | Comma-separated collection ids | — |
| `--items <ids>` | `sync-state set` | Comma-separated item ids | — |
| `--tag-names <names>` | `sync-state set` | Comma-separated tag names | — |
| `--auto-accept` | `refresh` | Accept refresh changes without prompting | `false` |
### `connection metabase setup`
### `connection test`
| Flag | Description | Default |
|------|-------------|---------|
| `--id <connectionId>` | KTX connection id to write | — |
| `--url <url>` | Metabase API URL | — |
| `--api-key <key>` | Metabase API key | — |
| `--mint-api-key` | Mint a Metabase API key with credentials | `false` |
| `--username <email>` | Metabase admin username for API-key minting | — |
| `--password <password>` | Metabase admin password for API-key minting | — |
| `--map <id=target>` | Assign a Metabase database id to a warehouse connection; repeatable | — |
| `--sync <metabaseDatabaseId>` | Enable sync for a discovered database; repeatable | — |
| `--sync-mode <mode>` | Metabase sync selection mode (`ALL`, `ONLY`, or `EXCEPT`) | `ALL` |
| `--run-ingest` | Run ingest after setup | `false` |
| `--yes` | Confirm and apply setup changes without prompting | `false` |
| `--no-input` | Disable interactive terminal input | — |
### `connection notion pick`
| Flag | Description | Default |
|------|-------------|---------|
| `--no-input` | Disable interactive terminal input | — |
| `--root-page-id <id>` | Root page UUID to crawl; repeatable (required with `--no-input`) | — |
| `--json` | Print JSON output | `false` |
## Examples
@ -114,43 +43,20 @@ ktx connection <subcommand> [options]
# List all configured connections
ktx connection list
# Add a Postgres connection using an environment variable
ktx connection add postgres my-warehouse --url "env:DATABASE_URL"
# Add a Postgres connection with specific schemas
ktx connection add postgres analytics --url "env:PG_URL" --schema public --schema analytics
# Add a read-only Snowflake connection
ktx connection add snowflake sf-prod --url "env:SNOWFLAKE_URL" --readonly
# Test a connection
ktx connection test my-warehouse
# Remove a connection
ktx connection remove old-warehouse
# Add a Notion source connection
ktx connection add notion my-notion \
--token-env NOTION_TOKEN \
--crawl-mode selected_roots \
--root-page-id abc123def456...
# Run guided Metabase setup
ktx connection metabase setup --url https://metabase.example.com
# Map a BI database to a warehouse connection
ktx connection mapping set metabase-prod databaseMappings 1=my-warehouse
# Refresh Metabase mappings
ktx connection mapping refresh metabase-prod --auto-accept
# Pick Notion root pages interactively
ktx connection notion pick my-notion
```
## Setup-managed connections
Run `ktx setup` when you need to add or reconfigure a connection. Interactive
setup includes the rich Notion page picker for selected root pages and the
Metabase mapping prompts for BI-to-warehouse mappings.
## Output
Interactive commands render prompts and status text. Commands with `--json` return machine-readable JSON suitable for scripts and agents.
Commands with `--json` return machine-readable JSON suitable for scripts and
agents.
```json
{
@ -168,7 +74,6 @@ Interactive commands render prompts and status text. Commands with `--json` retu
| Error | Cause | Recovery |
|-------|-------|----------|
| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx connection add ... --force` |
| Literal credentials rejected | KTX avoids writing raw secrets to `ktx.yaml` by default | Use `env:NAME` or `file:/path/to/secret`; use `--allow-literal-credentials` only for local throwaway projects |
| Mapping validation fails | BI database mappings do not point at valid warehouse connections | Run `ktx connection mapping refresh <connectionId> --auto-accept`, then set invalid mappings explicitly |
| Notion pick cannot run non-interactively | `--no-input` was used without root page or database ids | Pass `--root-page-id`, `--root-database-id`, or `--root-data-source-id` with `--no-input` |
| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection |
| Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Rerun `ktx setup` and update the source mapping selections |
| Notion page picker cannot run | The terminal is non-interactive or Notion discovery failed | Rerun interactive `ktx setup`, or use non-interactive setup flags with explicit root page ids |

2
docs-site/content/docs/getting-started/quickstart.mdx
View file

@ -240,7 +240,7 @@ Agent integration ready: yes (claude-code:project)
| LLM health check fails | Missing, invalid, or unauthorized Anthropic API key | Export `ANTHROPIC_API_KEY` or rerun `ktx setup` and choose the file-backed secret option |
| OpenAI embedding check fails | `OPENAI_API_KEY` is missing when OpenAI embeddings are selected | Export `OPENAI_API_KEY`, or rerun setup and choose local sentence-transformers embeddings |
| Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup |
| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx connection add ... --force` or rerun setup |
| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection |
| `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need |

3
packages/cli/src/cli-program.ts
View file

@ -178,9 +178,6 @@ function shouldSuppressProjectDirLine(path: string[], options: Record<string, un
if (commandPathKey === 'ktx ingest watch') {
return options.json !== true && options.plain !== true;
}
if (commandPathKey === 'ktx connection notion pick') {
return options.input !== false;
}
const demoIndex = path.indexOf('demo');
if (demoIndex >= 0) {
const demoCommand = path[demoIndex + 1];

4
packages/cli/src/cli-runtime.ts
View file

@ -1,7 +1,5 @@
import { createRequire } from 'node:module';
import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js';
import type { KtxConnectionNotionArgs } from './commands/connection-notion.js';
import type { KtxConnectionArgs } from './connection.js';
import type { KtxDoctorArgs } from './doctor.js';
import type { KtxIngestArgs } from './ingest.js';
@ -30,8 +28,6 @@ export interface KtxCliIo {
export interface KtxCliDeps {
setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise<number>;
connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise<number>;
connectionNotion?: (args: KtxConnectionNotionArgs, io: KtxCliIo) => Promise<number>;
connectionMetabaseSetup?: (args: KtxConnectionMetabaseSetupArgs, io: KtxCliIo) => Promise<number>;
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise<number>;
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;

25
packages/cli/src/command-schemas.ts
View file

@ -1,33 +1,8 @@
import { z } from 'zod';
const projectDirSchema = z.string().min(1);
const safeConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, 'Unsafe connection id');
const stringArraySchema = z.array(z.string());
export const connectionAddCommandSchema = z.object({
command: z.literal('add'),
projectDir: projectDirSchema,
driver: z.string().min(1),
connectionId: safeConnectionIdSchema,
url: z.string().optional(),
schemas: stringArraySchema,
readonly: z.boolean(),
force: z.boolean(),
allowLiteralCredentials: z.boolean(),
notion: z
.object({
authTokenRef: z.string().min(1),
crawlMode: z.enum(['all_accessible', 'selected_roots']),
rootPageIds: stringArraySchema,
rootDatabaseIds: stringArraySchema,
rootDataSourceIds: stringArraySchema,
maxPagesPerRun: z.number().int().positive().optional(),
maxKnowledgeCreatesPerRun: z.number().int().nonnegative().optional(),
maxKnowledgeUpdatesPerRun: z.number().int().nonnegative().optional(),
})
.optional(),
});
export const wikiWriteCommandSchema = z.object({
command: z.literal('write'),
projectDir: projectDirSchema,

308
packages/cli/src/commands/connection-commands.ts
View file

@ -1,61 +1,19 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KtxCliCommandContext,
parseBooleanStringOption,
parseNonEmptyAssignmentOption,
parseNonNegativeIntegerOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { connectionAddCommandSchema } from '../command-schemas.js';
import { type Command } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxConnectionArgs } from '../connection.js';
import { profileMark } from '../startup-profile.js';
import type { KtxConnectionMappingArgs } from './connection-mapping.js';
import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js';
import { registerConnectionNotionCommands } from './connection-notion-commands.js';
profileMark('module:commands/connection-commands');
const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const;
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const;
function parseCsvIds(value: string): number[] {
return value
.split(',')
.filter(Boolean)
.map((item) => parsePositiveIntegerOption(item));
}
function parseCsvStrings(value: string): string[] {
return value
.split(',')
.map((item) => item.trim())
.filter(Boolean);
}
function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' {
if (value === 'databaseMappings' || value === 'connectionMappings') {
return value;
}
throw new InvalidArgumentError('must be databaseMappings or connectionMappings');
}
async function runConnectionArgs(context: KtxCliCommandContext, args: KtxConnectionArgs): Promise<void> {
const runner = context.deps.connection ?? (await import('../connection.js')).runKtxConnection;
context.setExitCode(await runner(args, context.io));
}
async function runMappingArgs(context: KtxCliCommandContext, args: KtxConnectionMappingArgs): Promise<void> {
const { runKtxConnectionMapping } = await import('./connection-mapping.js');
context.setExitCode(await runKtxConnectionMapping(args, context.io));
}
export function registerConnectionCommands(program: Command, context: KtxCliCommandContext, commandName = 'connection'): void {
const connection = program
.command(commandName)
.description('Add, list, test, and map data sources')
.description('List and test configured connections')
.showHelpAfterError()
.addHelpText(
'after',
@ -83,264 +41,4 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm
connectionId,
});
});
connection
.command('add')
.description('Add or replace a configured connection')
.argument('<driver>', 'Connection driver')
.argument('<connectionId>', 'KTX connection id')
.option('--url <url>', 'Connection URL, env:NAME, or file:/path reference')
.option('--schema <schema>', 'Schema to include; repeatable', collectOption, [])
.option('--readonly', 'Mark the connection as read-only', false)
.option('--force', 'Replace an existing connection', false)
.option('--allow-literal-credentials', 'Allow writing a literal credential URL to ktx.yaml', false)
.addOption(new Option('--token-env <name>', 'Environment variable containing Notion auth token').conflicts('tokenFile'))
.addOption(new Option('--token-file <path>', 'File containing Notion auth token').conflicts('tokenEnv'))
.addOption(
new Option('--crawl-mode <mode>', 'Notion crawl mode: all_accessible or selected_roots')
.choices(CRAWL_MODE_CHOICES)
.default('selected_roots'),
)
.option('--root-page-id <id>', 'Root page to crawl; repeatable', collectOption, [])
.option('--root-database-id <id>', 'Root database to crawl; repeatable', collectOption, [])
.option('--root-data-source-id <id>', 'Root data source to crawl; repeatable', collectOption, [])
.option('--max-pages <n>', 'Maximum pages per run', parsePositiveIntegerOption)
.option('--max-knowledge-creates <n>', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption)
.option('--max-knowledge-updates <n>', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption)
.action(async (driver: string, connectionId: string, options, command) => {
const notion =
driver === 'notion'
? {
authTokenRef: options.tokenEnv
? `env:${options.tokenEnv}`
: options.tokenFile
? `file:${options.tokenFile}`
: '',
crawlMode: options.crawlMode,
rootPageIds: options.rootPageId,
rootDatabaseIds: options.rootDatabaseId,
rootDataSourceIds: options.rootDataSourceId,
maxPagesPerRun: options.maxPages,
maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates,
maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates,
}
: undefined;
if (driver === 'notion' && !notion?.authTokenRef) {
throw new Error('connection add notion requires --token-env NAME or --token-file PATH');
}
if (
driver === 'notion' &&
notion?.crawlMode === 'selected_roots' &&
notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0
) {
throw new Error('connection add notion selected_roots requires at least one root id');
}
const args = connectionAddCommandSchema.parse({
command: 'add',
projectDir: resolveCommandProjectDir(command),
driver,
connectionId,
url: options.url,
schemas: options.schema.filter(Boolean),
readonly: options.readonly === true,
force: options.force === true,
allowLiteralCredentials: options.allowLiteralCredentials === true,
notion,
});
await runConnectionArgs(context, args);
});
connection
.command('remove')
.description('Remove a configured connection from ktx.yaml')
.argument('<connectionId>', 'KTX connection id')
.option('--force', 'Remove without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'remove',
projectDir: resolveCommandProjectDir(command),
connectionId,
force: options.force === true,
...(options.input === false ? { inputMode: 'disabled' } : {}),
});
});
connection
.command('map')
.description('Refresh and validate BI-to-warehouse mappings')
.argument('<sourceConnectionId>', 'Source BI connection id')
.option('--json', 'Print JSON output', false)
.action(async (sourceConnectionId: string, options: { json?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'map',
projectDir: resolveCommandProjectDir(command),
sourceConnectionId,
json: options.json === true,
});
});
registerConnectionMappingCommands(connection, context);
registerConnectionMetabaseCommands(connection, context);
registerConnectionNotionCommands(connection, context);
}
function registerConnectionMappingCommands(connection: Command, context: KtxCliCommandContext): void {
const mapping = connection
.command('mapping')
.description('Manage Metabase warehouse mappings')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
mapping
.command('list')
.description('List Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
mapping
.command('set')
.description('Set a Metabase or Looker warehouse mapping')
.argument('<connectionId>', 'Source connection id', parseSafeConnectionIdOption)
.argument('<field>', 'Mapping field', parseMappingFieldOption)
.argument('<assignment>', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption)
.action(
async (
connectionId: string,
field: 'databaseMappings' | 'connectionMappings',
assignment: { key: string; value: string },
_options: unknown,
command,
) => {
await runMappingArgs(context, {
command: 'set',
projectDir: resolveCommandProjectDir(command),
connectionId,
field,
key: assignment.key,
value: assignment.value,
});
},
);
mapping
.command('apply-bulk')
.description('Apply mappings from JSON')
.argument('<connectionId>', 'Metabase connection id')
.requiredOption('--file <path>', 'JSON mapping file')
.action(async (connectionId: string, options: { file: string }, command) => {
await runMappingArgs(context, {
command: 'apply-bulk',
projectDir: resolveCommandProjectDir(command),
connectionId,
filePath: options.file,
});
});
mapping
.command('set-sync-enabled')
.description('Enable or disable sync for one Metabase database')
.argument('<connectionId>', 'Metabase connection id')
.argument('<metabaseDatabaseId>', 'Metabase database id', parsePositiveIntegerOption)
.requiredOption('--enabled <value>', 'true or false', parseBooleanStringOption)
.action(
async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => {
await runMappingArgs(context, {
command: 'set-sync-enabled',
projectDir: resolveCommandProjectDir(command),
connectionId,
metabaseDatabaseId,
enabled: options.enabled,
});
},
);
const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection');
syncState
.command('get')
.description('Read sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'sync-state-get',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
syncState
.command('set')
.description('Write sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.addOption(new Option('--mode <mode>', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory())
.option('--collections <ids>', 'Comma-separated collection ids', parseCsvIds, [])
.option('--items <ids>', 'Comma-separated item ids', parseCsvIds, [])
.option('--tag-names <names>', 'Comma-separated tag names', parseCsvStrings, [])
.action(async (connectionId: string, options, command) => {
await runMappingArgs(context, {
command: 'sync-state-set',
projectDir: resolveCommandProjectDir(command),
connectionId,
syncMode: options.mode,
collectionIds: options.collections,
itemIds: options.items,
tagNames: options.tagNames,
});
});
mapping
.command('refresh')
.description('Refresh Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--auto-accept', 'Accept refresh changes without prompting', false)
.action(async (connectionId: string, options: { autoAccept?: boolean }, command) => {
await runMappingArgs(context, {
command: 'refresh',
projectDir: resolveCommandProjectDir(command),
connectionId,
autoAccept: options.autoAccept === true,
});
});
mapping
.command('validate')
.description('Validate Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.action(async (connectionId: string, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'validate',
projectDir: resolveCommandProjectDir(command),
connectionId,
});
});
mapping
.command('clear')
.description('Clear Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption)
.action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'clear',
projectDir: resolveCommandProjectDir(command),
connectionId,
...(metabaseDatabaseId ? { metabaseDatabaseId } : {}),
});
});
}

329
packages/cli/src/commands/connection-mapping.test.ts
View file

@ -1,329 +0,0 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnectionMapping } from './connection-mapping.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKtxConnectionMapping', () => {
let tempDir: string;
let projectDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-metabase-mapping-'));
projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'mapping' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
},
'prod-warehouse': {
driver: 'postgres',
url: 'env:WAREHOUSE_URL',
readonly: true,
},
},
}),
'ktx',
'ktx@example.com',
'Seed Metabase mapping test connections',
);
});
async function replaceConnections(connections: Record<string, { driver: string; [key: string]: unknown }>) {
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections,
}),
'ktx',
'ktx@example.com',
'Replace mapping test connections',
);
}
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('sets, lists, disables, and clears local Metabase mappings', async () => {
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
const listIo = makeIo();
await expect(
runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io),
).resolves.toBe(0);
expect(listIo.stdout()).toContain('1 -> prod-warehouse');
expect(listIo.stdout()).toContain('unhydrated');
await expect(
runKtxConnectionMapping(
{
command: 'set-sync-enabled',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
enabled: false,
},
makeIo().io,
),
).resolves.toBe(0);
await expect(
runKtxConnectionMapping(
{
command: 'clear',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
},
makeIo().io,
),
).resolves.toBe(0);
});
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-yaml-mapping-'));
await initKtxProject({ projectDir, projectName: 'yaml-mapping' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
mappings: {
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
},
},
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'ktx',
'ktx@example.com',
'Seed yaml mappings',
);
const io = makeIo();
await expect(
runKtxConnectionMapping(
{ command: 'list', projectDir, connectionId: 'prod-metabase', json: false },
io.io,
),
).resolves.toBe(0);
expect(io.stdout()).toContain('1 -> prod-warehouse');
expect(io.stdout()).toContain('source: ktx.yaml');
});
it('refreshes Metabase discovery metadata through the injected runtime client', async () => {
const client = {
getDatabases: vi.fn().mockResolvedValue([
{
id: 1,
name: 'Analytics',
engine: 'postgres',
details: { host: 'pg.internal', dbname: 'analytics' },
is_sample: false,
},
]),
cleanup: vi.fn(),
};
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'refresh',
projectDir,
connectionId: 'prod-metabase',
autoAccept: true,
},
io.io,
{
createMetabaseClient: async () => client as never,
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 database');
expect(client.cleanup).toHaveBeenCalledTimes(1);
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
]);
});
it('sets and lists Looker connection mappings', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'connectionMappings',
key: 'analytics',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
await expect(
runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('analytics -> prod-warehouse');
});
it('keeps driver-specific mapping field validation in the runner', async () => {
await replaceConnections({
'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' },
warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' },
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'databaseMappings',
key: '1',
value: 'warehouse',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Looker mapping set requires connectionMappings');
});
it('refreshes Looker mapping metadata and reports drift', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{ command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true },
io.io,
{
createLookerClient: async () => ({
listLookerConnections: async () => [
{
name: 'analytics',
host: 'db.example.test',
database: 'analytics',
schema: null,
dialect: 'postgres',
},
],
cleanup: async () => {},
}),
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 connection');
expect(io.stdout()).toContain('Unmapped discovered: 1');
});
it('validates Looker mappings through the canonical local warehouse descriptor', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-descriptor-validation-'));
await initKtxProject({ projectDir, projectName: 'descriptor-validation' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-looker': {
driver: 'looker',
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
},
'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'ktx',
'ktx@example.com',
'Seed descriptor validation',
);
const io = makeIo();
await expect(
runKtxConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('Mapping validation passed: prod-looker');
expect(io.stderr()).toBe('');
});
});

426
packages/cli/src/commands/connection-mapping.ts
View file

@ -1,426 +0,0 @@
import { readFile } from 'node:fs/promises';
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
discoverMetabaseDatabases,
lookerCredentialsFromLocalConnection,
metabaseRuntimeConfigFromLocalConnection,
seedLocalMappingStateFromKtxYaml,
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
} from '@ktx/context/ingest';
import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/connection-mapping');
export type KtxConnectionMappingArgs =
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'set';
projectDir: string;
connectionId: string;
field: 'databaseMappings' | 'connectionMappings';
key: string;
value: string;
}
| { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string }
| {
command: 'set-sync-enabled';
projectDir: string;
connectionId: string;
metabaseDatabaseId: number;
enabled: boolean;
}
| { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'sync-state-set';
projectDir: string;
connectionId: string;
syncMode: MetabaseSyncMode;
collectionIds: number[];
itemIds: number[];
tagNames: string[];
}
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
| { command: 'validate'; projectDir: string; connectionId: string }
| { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string };
interface KtxConnectionMappingDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
createLookerClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
}
interface MetabaseBulkMappingPayload {
databaseMappings?: Record<string, string | null>;
syncEnabled?: Record<string, boolean>;
syncMode?: MetabaseSyncMode;
selections?: { collections?: number[]; items?: number[] };
defaultTagNames?: string[];
}
function parseId(value: string, label: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer`);
}
return parsed;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function createDefaultLookerClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
const factory = new DefaultLookerConnectionClientFactory({
async resolve(lookerConnectionId) {
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
},
});
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
cleanup?(): Promise<void>;
};
}
function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean {
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
}
function assertLookerConnection(project: KtxLocalProject, connectionId: string): void {
if (!isLookerConnection(project, connectionId)) {
throw new Error(`Connection "${connectionId}" is not a Looker connection`);
}
}
function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void {
const connection = project.config.connections[connectionId];
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
}
}
function assertTargetConnection(project: KtxLocalProject, connectionId: string): void {
if (!project.config.connections[connectionId]) {
throw new Error(`Target connection "${connectionId}" does not exist`);
}
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function renderMapping(
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
row.source
})`;
}
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
const target = row.ktxConnectionId ?? '[unmapped]';
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
}
export async function runKtxConnectionMapping(
args: KtxConnectionMappingArgs,
io: KtxCliIo = process,
deps: KtxConnectionMappingDeps = {},
): Promise<number> {
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
await seedLocalMappingStateFromKtxYaml(project, args.connectionId);
if (isLookerConnection(project, args.connectionId)) {
assertLookerConnection(project, args.connectionId);
const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listConnectionMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'connectionMappings') {
throw new Error('Looker mapping set requires connectionMappings <lookerConnectionName>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
await store.upsertConnectionMapping({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.key,
ktxConnectionId: args.value,
source: 'cli',
});
io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
try {
const discovered = await discoverLookerConnections(client);
const drift = computeLookerMappingDrift({
storedMappings: await store.readMappings(args.connectionId),
discovered,
});
if (args.autoAccept) {
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup?.();
}
}
if (args.command === 'validate') {
const knownKtxConnectionIds = new Set(Object.keys(project.config.connections));
const knownConnectionTypes = new Map(
Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]),
);
const validation = validateLookerMappings({
mappings: await store.readMappings(args.connectionId),
knownKtxConnectionIds,
knownConnectionTypes,
});
if (!validation.ok) {
for (const error of validation.errors) {
io.stderr.write(`${error.key}: ${error.reason}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
if (args.command === 'clear') {
await store.clearConnectionMappings({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined),
});
io.stdout.write(
args.mappingKey
? `Cleared connectionMappings.${args.mappingKey}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
}
throw new Error(`Looker connection mapping does not support ${args.command}`);
}
assertMetabaseConnection(project, args.connectionId);
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
assertTargetConnection(project, args.value);
await store.upsertDatabaseMapping({
connectionId: args.connectionId,
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
targetConnectionId: args.value,
syncEnabled: true,
source: 'cli',
});
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'apply-bulk') {
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
const existingState = await store.getSourceState(args.connectionId);
const existingRows = await store.listDatabaseMappings(args.connectionId);
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
const databaseMappings = payload.databaseMappings ?? {};
for (const targetConnectionId of Object.values(databaseMappings)) {
if (targetConnectionId) {
assertTargetConnection(project, targetConnectionId);
}
}
const mappingIds = new Set([
...existingRows.map((row) => row.metabaseDatabaseId),
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
]);
await store.replaceSourceState({
connectionId: args.connectionId,
syncMode: payload.syncMode ?? existingState.syncMode,
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
selections:
payload.selections === undefined
? existingState.selections
: [
...(payload.selections.collections ?? []).map((id) => ({
selectionType: 'collection' as const,
metabaseObjectId: id,
})),
...(payload.selections.items ?? []).map((id) => ({
selectionType: 'item' as const,
metabaseObjectId: id,
})),
],
mappings: [...mappingIds]
.sort((a, b) => a - b)
.map((id) => {
const existing = existingById.get(id);
return {
metabaseDatabaseId: id,
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
metabaseEngine: existing?.metabaseEngine ?? null,
metabaseHost: existing?.metabaseHost ?? null,
metabaseDbName: existing?.metabaseDbName ?? null,
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
source: 'cli',
};
}),
});
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'set-sync-enabled') {
await store.setMappingSyncEnabled({
connectionId: args.connectionId,
metabaseDatabaseId: args.metabaseDatabaseId,
syncEnabled: args.enabled,
});
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
return 0;
}
if (args.command === 'sync-state-get') {
const state = await store.getSourceState(args.connectionId);
const payload = {
syncMode: state.syncMode,
selections: state.selections,
defaultTagNames: state.defaultTagNames,
};
io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`);
return 0;
}
if (args.command === 'sync-state-set') {
await store.setSyncState({
connectionId: args.connectionId,
syncMode: args.syncMode,
defaultTagNames: args.tagNames,
selections: [
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
],
});
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup();
}
}
if (args.command === 'validate') {
const rows = await store.listDatabaseMappings(args.connectionId);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
project.config.connections[row.targetConnectionId]
? targetPhysicalInfo(project, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (failures.length > 0) {
for (const failure of failures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
io.stdout.write(
metabaseDatabaseId
? `Cleared databaseMappings.${metabaseDatabaseId}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

132
packages/cli/src/commands/connection-metabase-commands.ts
View file

@ -1,132 +0,0 @@
import { type Command, Option } from '@commander-js/extra-typings';
import {
type KtxCliCommandContext,
parseNonEmptyAssignmentOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import {
type KtxConnectionMetabaseSetupArgs,
type MetabaseSetupMappingAssignment,
type MetabaseSetupSyncMode,
runKtxConnectionMetabaseSetup,
} from './connection-metabase-setup.js';
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[];
interface ConnectionMetabaseSetupOptions {
id?: string;
url?: string;
apiKey?: string;
mintApiKey?: boolean;
username?: string;
password?: string;
map: MetabaseSetupMappingAssignment[];
sync: number[];
syncMode: MetabaseSetupSyncMode;
runIngest?: boolean;
yes?: boolean;
input?: boolean;
}
function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] {
return [...previous, parsePositiveIntegerOption(value)];
}
function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment {
const assignment = parseNonEmptyAssignmentOption(value);
return {
metabaseDatabaseId: parsePositiveIntegerOption(assignment.key),
targetConnectionId: parseSafeConnectionIdOption(assignment.value),
};
}
function collectMappingOption(
value: string,
previous: MetabaseSetupMappingAssignment[] = [],
): MetabaseSetupMappingAssignment[] {
return [...previous, parseMappingAssignment(value)];
}
async function runMetabaseSetupArgs(
context: KtxCliCommandContext,
args: KtxConnectionMetabaseSetupArgs,
): Promise<void> {
const runner = context.deps.connectionMetabaseSetup ?? runKtxConnectionMetabaseSetup;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionMetabaseCommands(connection: Command, context: KtxCliCommandContext): void {
const metabase = connection
.command('metabase')
.description('Configure Metabase connections')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
metabase.action(() => {
metabase.outputHelp();
context.setExitCode(0);
});
metabase
.command('setup')
.description('Guided setup for a Metabase connection')
.option('--id <connectionId>', 'KTX connection id to write', parseSafeConnectionIdOption)
.option('--url <url>', 'Metabase API URL')
.addOption(new Option('--api-key <key>', 'Metabase API key').conflicts('mintApiKey'))
.option('--mint-api-key', 'Mint a Metabase API key with credentials', false)
.option('--username <email>', 'Metabase admin username for API-key minting')
.option('--password <password>', 'Metabase admin password for API-key minting')
.addHelpText(
'after',
'\nGuided equivalent of:\n' +
' ktx connection mapping refresh <connectionId> --auto-accept\n' +
' ktx connection mapping set <connectionId> databaseMappings <id>=<target>\n' +
' ktx connection mapping set-sync-enabled <connectionId> <id> --enabled true\n' +
' ktx ingest run --connection-id <connectionId> --adapter metabase\n',
)
.option(
'--map <metabaseDatabaseId=targetConnectionId>',
'Assign a Metabase database id to a warehouse connection; repeatable',
collectMappingOption,
[],
)
.option(
'--sync <metabaseDatabaseId>',
'Enable Metabase sync for a discovered database; repeatable',
collectPositiveIntegerOption,
[],
)
.addOption(
new Option('--sync-mode <mode>', 'Metabase sync selection mode')
.choices(SYNC_MODE_CHOICES)
.default('ALL' satisfies MetabaseSetupSyncMode),
)
.option('--run-ingest', 'Run ingest after setup', false)
.option('--yes', 'Confirm and apply setup changes without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.showHelpAfterError()
.action(async (options: ConnectionMetabaseSetupOptions, command) => {
await runMetabaseSetupArgs(context, {
command: 'setup',
projectDir: resolveCommandProjectDir(command),
connectionId: options.id,
url: options.url,
apiKey: options.apiKey,
mintApiKey: options.mintApiKey === true,
metabaseUsername: options.username,
metabasePassword: options.password,
mappings: options.map,
syncEnabledDatabaseIds: options.sync,
syncMode: options.syncMode ?? 'ALL',
runIngest: options.runIngest === true,
yes: options.yes === true,
inputMode: options.input === false ? 'disabled' : 'auto',
});
});
}

1136
packages/cli/src/commands/connection-metabase-setup.test.ts
View file

File diff suppressed because it is too large Load diff

786
packages/cli/src/commands/connection-metabase-setup.ts
View file

@ -1,786 +0,0 @@
import type { Option as ClackOption } from '@clack/prompts';
import {
cancel,
confirm,
intro,
isCancel,
log,
multiselect,
note,
outro,
password,
select,
text,
} from '@clack/prompts';
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
LocalMetabaseSourceStateReader,
MetabaseClient,
type MetabaseDatabase,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
metabaseRuntimeConfigFromLocalConnection,
validateMappingPhysicalMatch,
} from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
ktxLocalStateDbPath,
loadKtxProject,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import { createClackSpinner, type KtxCliSpinner } from '../clack.js';
import type { KtxCliIo } from '../cli-runtime.js';
import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
import { type KtxPublicIngestArgs, runKtxPublicIngest } from '../public-ingest.js';
export type KtxMetabaseSetupInputMode = 'auto' | 'disabled';
export type MetabaseSetupSyncMode = MetabaseSyncMode;
type MetabaseSetupPromptOption<Value> = ClackOption<Value>;
export interface MetabaseSetupLogger {
info(message: string): void;
step(message: string): void;
success(message: string): void;
warn(message: string): void;
error(message: string): void;
}
export interface MetabaseSetupPromptAdapter {
intro(title?: string): void;
outro(message?: string): void;
note(message: string, title: string): void;
log: MetabaseSetupLogger;
spinner(): KtxCliSpinner;
select<T extends string>(options: { message: string; options: Array<MetabaseSetupPromptOption<T>> }): Promise<T>;
multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]>;
text(options: { message: string; placeholder?: string }): Promise<string>;
password(options: { message: string }): Promise<string>;
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
type KtxMetabaseSetupInteractiveIo = KtxCliIo & {
stdin?: { isTTY?: boolean };
};
export interface MetabaseSetupMappingAssignment {
metabaseDatabaseId: number;
targetConnectionId: string;
}
export interface MintMetabaseApiKeyArgs {
url: string;
username: string;
password: string;
}
export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KtxCliIo) => Promise<string>;
export interface KtxConnectionMetabaseSetupArgs {
command: 'setup';
projectDir: string;
connectionId?: string;
url?: string;
apiKey?: string;
mintApiKey: boolean;
metabaseUsername?: string;
metabasePassword?: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
runIngest: boolean;
yes: boolean;
inputMode: KtxMetabaseSetupInputMode;
}
export interface KtxConnectionMetabaseSetupDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>>;
mintMetabaseApiKey?: MintMetabaseApiKey;
prompts?: MetabaseSetupPromptAdapter;
runPublicIngest?: (args: Extract<KtxPublicIngestArgs, { command: 'run' }>, io: KtxCliIo) => Promise<number>;
}
function isMetabaseConnection(connection: KtxProjectConnectionConfig | undefined): boolean {
return (
String(connection?.driver ?? '')
.trim()
.toLowerCase() === 'metabase'
);
}
function stringField(value: unknown): string | undefined {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
}
function uniqueSorted(values: number[]): number[] {
return [...new Set(values)].sort((a, b) => a - b);
}
function resolveMetabaseUrl(connection: KtxProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url);
}
function resolveLiteralMetabaseApiKey(connection: KtxProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_key) ?? stringField(connection?.apiKey);
}
function listMetabaseConnectionIds(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([_connectionId, connection]) => isMetabaseConnection(connection))
.map(([connectionId]) => connectionId)
.sort();
}
function listWarehouseConnectionIds(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
.map(([connectionId]) => connectionId)
.sort();
}
function redactSecrets(message: string, secrets: string[]): string {
let result = message;
for (const secret of secrets) {
if (!secret) {
continue;
}
result = result.split(secret).join('[redacted]');
}
return result;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise<string> {
const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
const sessionId = await loginClient.createSession(args.username, args.password);
const sessionClient = new MetabaseClient(
{ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' },
DEFAULT_METABASE_CLIENT_CONFIG,
);
const groups = await sessionClient.getPermissionGroups();
const adminGroup = groups.find((group) => group.name === 'Administrators');
if (!adminGroup) {
throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
}
const mintedKey = await sessionClient.createApiKey({
groupId: adminGroup.id,
name: `KTX CLI ${new Date().toISOString()}`,
});
const trimmedKey = stringField(mintedKey);
if (!trimmedKey) {
throw new Error('Metabase API key minting returned an empty key');
}
return trimmedKey;
}
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<MetabaseSetupPromptAdapter, 'cancel'>): T {
if (isCancel(value)) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
return value as T;
}
export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter {
return {
intro(title?: string): void {
intro(title);
},
outro(message?: string): void {
outro(message);
},
note(message: string, title: string): void {
note(message, title);
},
log: {
info(message: string): void {
log.info(message);
},
step(message: string): void {
log.step(message);
},
success(message: string): void {
log.success(message);
},
warn(message: string): void {
log.warn(message);
},
error(message: string): void {
log.error(message);
},
},
spinner(): KtxCliSpinner {
return createClackSpinner();
},
async select<T extends string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<T>>;
}): Promise<T> {
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
},
async multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]> {
return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
},
async text(options: { message: string; placeholder?: string }): Promise<string> {
return ensureNotCancelled(await text(options), this);
},
async password(options: { message: string }): Promise<string> {
return ensureNotCancelled(await password(options), this);
},
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
return ensureNotCancelled(await confirm(options), this);
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveMetabaseSetupIo(
args: Pick<KtxConnectionMetabaseSetupArgs, 'inputMode'>,
io: KtxMetabaseSetupInteractiveIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{
id: number;
name: string;
engine: string;
host: string | null;
dbName: string | null;
}> {
return databases
.filter((database) => database.is_sample !== true)
.map((database) => ({
id: database.id,
name: database.name,
engine: stringField(database.engine) ?? 'unknown',
host: stringField(database.details?.host) ?? null,
dbName: stringField(database.details?.dbname) ?? null,
}));
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function noteMetabaseSetupSummary(options: {
prompts: MetabaseSetupPromptAdapter;
connectionId: string;
url: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
}): void {
const mappingLines = options.mappings
.map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
.join('\n');
const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n');
options.prompts.note(
[
`Connection: ${options.connectionId}`,
`URL: ${options.url}`,
'',
'Mappings:',
mappingLines || ' (none)',
'',
'Sync enabled:',
syncLines || ' (none)',
].join('\n'),
'Summary',
);
}
export async function runKtxConnectionMetabaseSetup(
args: KtxConnectionMetabaseSetupArgs,
io: KtxCliIo,
deps: KtxConnectionMetabaseSetupDeps = {},
): Promise<number> {
let apiKeyForRedaction = args.apiKey;
let passwordForRedaction = args.metabasePassword;
const interactiveIo = io as KtxMetabaseSetupInteractiveIo;
const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
try {
if (isInteractive && prompts) {
prompts.intro('KTX Metabase setup');
}
const project = await loadKtxProject({ projectDir: args.projectDir });
const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
let connectionId: string;
if (args.connectionId) {
connectionId = args.connectionId;
} else if (existingMetabaseConnectionIds.length === 1) {
const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
if (!onlyMetabaseConnectionId) {
throw new Error('No Metabase connection id was resolved');
}
connectionId = onlyMetabaseConnectionId;
} else if (existingMetabaseConnectionIds.length > 1) {
if (!isInteractive || !prompts) {
throw new Error(
`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`,
);
}
connectionId = await prompts.select({
message: 'Select the Metabase connection to configure',
options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
});
} else {
connectionId = 'metabase';
}
const existingConnection = project.config.connections[connectionId];
const warehouseConnectionIds = listWarehouseConnectionIds(project);
if (warehouseConnectionIds.length === 0) {
throw new Error('Add a warehouse connection first');
}
let url = args.url ?? resolveMetabaseUrl(existingConnection);
let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
apiKeyForRedaction = apiKey;
if (!url && isInteractive && prompts) {
url = stringField(
await prompts.text({
message: 'Metabase API URL',
placeholder: 'http://localhost:3000',
}),
);
}
if (args.inputMode === 'disabled' && !url) {
throw new Error('missing Metabase URL');
}
if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
const reuse = await prompts.confirm({
message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
initialValue: true,
});
if (!reuse) {
apiKey = undefined;
apiKeyForRedaction = undefined;
}
}
if (args.mintApiKey) {
let username = stringField(args.metabaseUsername);
let metabasePassword = stringField(args.metabasePassword);
if (isInteractive && prompts) {
if (!username) {
username = stringField(await prompts.text({ message: 'Metabase admin username' }));
}
if (!metabasePassword) {
metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
}
}
if (!username) {
throw new Error('--mint-api-key requires --username');
}
if (!metabasePassword) {
throw new Error('--mint-api-key requires --password');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
if (!apiKey && isInteractive && prompts) {
const credentialMode = await prompts.select({
message: 'Metabase credentials',
options: [
{ value: 'paste', label: 'Paste API key' },
{ value: 'mint', label: 'Mint API key' },
],
});
if (credentialMode === 'paste') {
apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
apiKeyForRedaction = apiKey;
} else {
const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
if (!username) {
throw new Error('Metabase username is required');
}
if (!metabasePassword) {
throw new Error('Metabase password is required');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
}
if (args.inputMode === 'disabled' && !apiKey) {
throw new Error('missing Metabase API key');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
if (!apiKey) {
throw new Error('Metabase API key is required (use --api-key)');
}
const transientConnectionConfig: KtxProjectConnectionConfig = {
...(existingConnection ?? {}),
driver: 'metabase',
api_url: url,
api_key: apiKey,
};
const configWithTransient = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: transientConnectionConfig,
},
};
const discoveryProject: KtxLocalProject = { ...project, config: configWithTransient };
for (const mapping of args.mappings) {
if (!configWithTransient.connections[mapping.targetConnectionId]) {
throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
}
}
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
try {
const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
authSpinner?.start('Testing Metabase connection');
const testResult = await client.testConnection();
if (!testResult.success) {
authSpinner?.error('Metabase authentication failed');
throw new Error(
`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`,
);
}
authSpinner?.stop('Metabase reachable');
const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
discoverySpinner?.start('Discovering Metabase databases');
const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
if (isInteractive && prompts) {
prompts.log.success(
`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`,
);
}
if (discovered.length === 0) {
throw new Error('Metabase auth worked but no usable databases were returned');
}
let resolvedMappings = args.mappings;
let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
}
if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
const compatibleWarehouses = onlyDiscoveredDatabase
? warehouseConnectionIds.filter((warehouseConnectionId) => {
const mismatchReason = validateMappingPhysicalMatch(
{
metabaseEngine: onlyDiscoveredDatabase.engine,
metabaseDbName: onlyDiscoveredDatabase.dbName,
metabaseHost: onlyDiscoveredDatabase.host,
},
targetPhysicalInfo(project, warehouseConnectionId),
);
return !mismatchReason;
})
: [];
const onlyWarehouseConnectionId = compatibleWarehouses[0];
if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
if (args.yes) {
resolvedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
} else if (isInteractive && prompts) {
const proposedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: proposedMappings,
syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
resolvedMappings = proposedMappings;
resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
} else {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
} else if (isInteractive && prompts) {
const selectedDatabaseIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Select Metabase databases to configure'),
options: discovered.map((database) => ({
value: database.id,
label: `${database.id}: ${database.name}`,
hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
})),
required: true,
});
resolvedMappings = [];
for (const databaseId of selectedDatabaseIds) {
const database = discovered.find((candidate) => candidate.id === databaseId);
if (!database) {
throw new Error(`Selected database id ${databaseId} was not discovered`);
}
const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
if (existingMapping) {
resolvedMappings.push(existingMapping);
continue;
}
const targetConnectionId = await prompts.select({
message: `Map Metabase database ${database.id} ("${database.name}") to which KTX connection?`,
options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
});
resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
}
const syncIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Enable sync for which databases?'),
options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
initialValues: selectedDatabaseIds,
required: true,
});
resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
if (!args.yes) {
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: 'Write changes to ktx.yaml and enable sync?',
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
}
} else if (args.inputMode === 'disabled') {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
}
if (
args.inputMode === 'disabled' &&
resolvedMappings.length > 0 &&
resolvedSyncEnabledDatabaseIds.length === 0
) {
throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
}
const discoveredIds = new Set(discovered.map((database) => database.id));
for (const mapping of resolvedMappings) {
if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
}
}
for (const syncId of resolvedSyncEnabledDatabaseIds) {
if (!discoveredIds.has(syncId)) {
throw new Error(`Sync database id ${syncId} was not discovered`);
}
}
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(configWithTransient),
'ktx',
'ktx@example.com',
`Setup Metabase connection ${connectionId}`,
);
const updatedProject = await loadKtxProject({ projectDir: args.projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await store.refreshDiscoveredDatabases({ connectionId, discovered });
for (const mapping of resolvedMappings) {
await store.upsertDatabaseMapping({
connectionId,
metabaseDatabaseId: mapping.metabaseDatabaseId,
targetConnectionId: mapping.targetConnectionId,
syncEnabled: false,
source: 'cli',
});
}
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
await store.setMappingSyncEnabled({
connectionId,
metabaseDatabaseId,
syncEnabled: true,
});
}
const existingSyncState = await store.getSourceState(connectionId);
await store.setSyncState({
connectionId,
syncMode: args.syncMode,
defaultTagNames: existingSyncState.defaultTagNames,
selections: existingSyncState.selections,
});
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
if (unhydrated.length > 0) {
io.stderr.write(
`Sync-enabled mappings are missing discovery metadata; run ktx connection mapping refresh ${connectionId} --auto-accept\n`,
);
return 1;
}
const rows = await store.listDatabaseMappings(connectionId);
const physicalFailures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
updatedProject.config.connections[row.targetConnectionId]
? targetPhysicalInfo(updatedProject, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (physicalFailures.length > 0) {
for (const failure of physicalFailures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Connection: ${connectionId}\n`);
io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(
`Next: ktx ingest run --connection-id ${connectionId} --adapter metabase --project-dir ${args.projectDir}\n`,
);
if (args.runIngest) {
const ingestRunner = deps.runPublicIngest ?? runKtxPublicIngest;
const exitCode = await ingestRunner(
{
command: 'run',
projectDir: args.projectDir,
targetConnectionId: connectionId,
all: false,
json: false,
inputMode: 'disabled',
},
io,
);
if (exitCode !== 0) {
io.stderr.write(
`Ingest failed; re-run: ktx ingest run --connection-id ${connectionId} --adapter metabase --project-dir ${args.projectDir}\n`,
);
return 1;
}
}
if (isInteractive && prompts) {
prompts.outro('Metabase setup complete');
}
return 0;
} finally {
await client.cleanup();
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
io.stderr.write(
`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`,
);
return 1;
}
}

92
packages/cli/src/commands/connection-notion-commands.ts
View file

@ -1,92 +0,0 @@
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { collectOption, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxConnectionNotionArgs } from './connection-notion.js';
interface NotionPickOptions {
input?: boolean;
rootPageId: string[];
}
function parseSafeConnectionId(value: string): string {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
}
function uniqueInOrder(values: string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const value of values) {
if (!seen.has(value)) {
seen.add(value);
result.push(value);
}
}
return result;
}
function normalizeNotionPageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
throw new Error(`Invalid Notion page UUID: ${value}`);
}
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
}
function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KtxConnectionNotionArgs {
if (options.input !== false) {
return {
command: 'pick',
projectDir,
connectionId,
mode: 'interactive',
};
}
const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId));
if (rootPageIds.length === 0) {
throw new Error('connection notion pick --no-input requires at least one --root-page-id');
}
return {
command: 'pick',
projectDir,
connectionId,
mode: 'non-interactive',
rootPageIds,
};
}
async function runConnectionNotionArgs(context: KtxCliCommandContext, args: KtxConnectionNotionArgs): Promise<void> {
const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKtxConnectionNotion;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionNotionCommands(connect: Command, context: KtxCliCommandContext): void {
const notion = connect
.command('notion')
.description('Configure Notion source selection')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
notion.action(() => {
notion.outputHelp();
context.setExitCode(0);
});
notion
.command('pick')
.description('Pick Notion root pages for a configured Notion connection')
.argument('<connectionId>', 'Notion connection id', parseSafeConnectionId)
.option('--no-input', 'Disable interactive terminal input')
.option('--root-page-id <id>', 'Root page UUID to crawl; repeatable with --no-input', collectOption, [])
.showHelpAfterError()
.action(async (connectionId: string, options: NotionPickOptions, command) => {
await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options));
});
}

513
packages/cli/src/commands/connection-notion.test.ts
View file

@ -1,513 +0,0 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
initKtxProject,
loadKtxProject,
serializeKtxProjectConfig,
type KtxProjectConfig,
} from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
applyNotionPickerWriteback,
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
resolveNotionWorkspaceLabel,
runKtxConnectionNotion,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './connection-notion.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('runKtxConnectionNotion', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-notion-pick-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(config),
'ktx',
'ktx@example.com',
'seed test config',
);
}
it('rejects unsafe connection ids before loading a project', async () => {
const io = makeIo();
const loadProject = vi.fn(async () => {
throw new Error('loadProject should not be called');
});
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir: '/tmp/project',
connectionId: '../evil',
mode: 'interactive',
},
io.io,
{ loadProject },
),
).resolves.toBe(1);
expect(loadProject).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Unsafe connection id: ../evil');
});
it('writes selected root_page_ids while preserving every other Notion connection field', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: ['99999999-9999-9999-9999-999999999999'],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
unknown_future_field: 'keep-me',
},
},
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'non-interactive',
rootPageIds: [
'11111111-2222-3333-4444-555555555555',
'66666666-7777-8888-9999-aaaaaaaaaaaa',
],
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('root_page_ids:');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa');
expect(yaml).toContain('root_database_ids:');
expect(yaml).toContain('database-1');
expect(yaml).toContain('root_data_source_ids:');
expect(yaml).toContain('data-source-1');
expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\'');
expect(yaml).toContain('unknown_future_field: keep-me');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 2');
expect(io.stdout()).toContain('crawlMode: selected_roots');
});
it('rejects empty writeback, missing connections, and non-Notion connections', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
warehouse: {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
},
},
});
const project = await loadKtxProject({ projectDir });
await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow(
'connection notion pick requires at least one root page id',
);
await expect(
applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "missing" not found');
await expect(
applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "warehouse" is not a Notion connection');
});
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain(PAGE_IDS.engineering);
expect(yaml).not.toContain(PAGE_IDS.stale);
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 1');
});
it('uses inline Notion auth_token for interactive discovery', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token: 'ntn_inline_token',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]);
const createNotionApi = vi.fn((authToken: string) => {
expect(authToken).toBe('ntn_inline_token');
return api;
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
expect(createNotionApi).toHaveBeenCalledOnce();
expect(io.stdout()).toContain('No changes saved.');
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
expect(io.stdout()).toContain('No changes saved.');
});
it('quits interactive mode without writing when the TUI returns quit', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const before = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toBe(before);
expect(io.stdout()).toContain('No changes saved.');
});
});

541
packages/cli/src/connection.test.ts
View file

@ -1,4 +1,4 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { MetabaseRuntimeClient } from '@ktx/context/ingest';
@ -6,18 +6,13 @@ import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from
import type { KtxConnectionDriver, KtxScanConnector, KtxSchemaSnapshot } from '@ktx/context/scan';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnection } from './connection.js';
import { runKtxCli, type KtxCliIo } from './index.js';
function makeIo(options: { stdoutIsTty?: boolean; stdinIsTty?: boolean } = {}) {
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdin: {
isTTY: options.stdinIsTty,
},
stdout: {
isTTY: options.stdoutIsTty,
write: (chunk: string) => {
stdout += chunk;
},
@ -87,491 +82,49 @@ describe('runKtxConnection', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('adds and lists env-referenced connections without resolving secrets', async () => {
async function writeConnections(
projectDir: string,
connections: ReturnType<typeof parseKtxProjectConfig>['connections'],
): Promise<void> {
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(join(projectDir, 'ktx.yaml'), serializeKtxProjectConfig({ ...config, connections }), 'utf-8');
}
it('lists configured connections without resolving secrets', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
});
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'env:DATABASE_URL',
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(0);
await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0);
expect(io.stdout()).toContain('Connection: warehouse');
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('url: env:DATABASE_URL');
const listIo = makeIo();
await expect(runKtxConnection({ command: 'list', projectDir }, listIo.io)).resolves.toBe(0);
expect(listIo.stdout()).toContain('warehouse');
expect(listIo.stdout()).toContain('postgres');
});
it('removes a configured connection from ktx.yaml without deleting local artifacts when forced', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const artifactPath = join(projectDir, '.ktx', 'artifacts', 'warehouse.txt');
await mkdir(join(projectDir, '.ktx', 'artifacts'), { recursive: true });
await writeFile(artifactPath, 'keep me', 'utf-8');
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(0);
const parsed = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(parsed.connections.warehouse).toBeUndefined();
await expect(readFile(artifactPath, 'utf-8')).resolves.toBe('keep me');
expect(io.stdout()).toContain('Connection removed from ktx.yaml.');
expect(io.stdout()).toContain(
'Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.',
);
expect(io.stdout()).toContain('warehouse');
expect(io.stdout()).toContain('postgres');
expect(io.stdout()).toContain('docs');
expect(io.stdout()).toContain('notion');
expect(io.stderr()).toBe('');
});
it('requires --force when removing in non-interactive mode', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('connection remove warehouse requires --force when input is disabled or not interactive');
});
it('returns a clear error when removing an unknown connection', async () => {
it('prints an empty-state message that points at setup instead of removed connection add', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'missing',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0);
expect(io.stderr()).toContain('Connection "missing" is not configured in ktx.yaml');
});
it('asks for confirmation before removing in an interactive terminal', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo({ stdoutIsTty: true, stdinIsTty: true });
const prompts = {
confirm: vi.fn(async () => true),
cancel: vi.fn(),
};
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
},
io.io,
{ prompts },
),
).resolves.toBe(0);
expect(prompts.confirm).toHaveBeenCalledWith({
message: 'Remove connection "warehouse" from ktx.yaml? Ingested artifacts will remain in .ktx/.',
initialValue: false,
});
});
it('runs public connect map as refresh, validate, and list over the low-level mapping runner', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 database\n');
mappingIo.stdout.write('Unmapped discovered: 1\n');
mappingIo.stdout.write('Stale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-metabase\n');
return 0;
}
if (argv[0] === 'list') {
mappingIo.stdout.write('1 -> [unmapped] (Analytics, sync: on, source: refresh)\n');
return 0;
}
return 1;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(0);
expect(runMapping).toHaveBeenNthCalledWith(
1,
['refresh', 'prod-metabase', '--auto-accept', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
2,
['validate', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
3,
['list', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(io.stdout()).toContain('Mapping: prod-metabase');
expect(io.stdout()).toContain('Discovery: 1 database');
expect(io.stdout()).toContain('Mappings:');
expect(io.stdout()).toContain('1 -> [unmapped]');
expect(io.stdout()).toContain('Next:');
expect(io.stdout()).toContain('ktx ingest run --connection-id prod-metabase --adapter <adapter>');
expect(io.stdout()).toContain('ktx connection mapping');
expect(io.stderr()).toBe('');
});
it('prints stable JSON for public connect map without leaking low-level stdout', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 connection\nUnmapped discovered: 0\nStale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-looker\n');
return 0;
}
if (argv[0] === 'list') {
expect(argv).toContain('--json');
mappingIo.stdout.write(
`${JSON.stringify(
[
{
lookerConnectionName: 'analytics',
ktxConnectionId: 'prod-warehouse',
source: 'ktx.yaml',
},
],
null,
2,
)}\n`,
);
return 0;
}
return 1;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-looker', json: true },
io.io,
{ runMapping },
),
).resolves.toBe(0);
const parsed = JSON.parse(io.stdout()) as {
connectionId: string;
refresh: { ok: boolean; output: string[] };
validation: { ok: boolean; output: string[] };
mappings: Array<{ lookerConnectionName: string; ktxConnectionId: string; source: string }>;
};
expect(parsed).toEqual({
connectionId: 'prod-looker',
refresh: {
ok: true,
output: ['Discovery: 1 connection', 'Unmapped discovered: 0', 'Stale mappings: 0'],
},
validation: {
ok: true,
output: ['Mapping validation passed: prod-looker'],
},
mappings: [
{
lookerConnectionName: 'analytics',
ktxConnectionId: 'prod-warehouse',
source: 'ktx.yaml',
},
],
});
expect(io.stderr()).toBe('');
});
it('returns the refresh failure when public connect map cannot discover source metadata', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stderr.write('Metabase API key is not configured\n');
return 1;
}
return 0;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(1);
expect(runMapping).toHaveBeenCalledTimes(1);
expect(io.stdout()).toBe('');
expect(io.stderr()).toContain('Metabase API key is not configured');
});
it('rejects literal credential URLs unless explicitly allowed', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'postgres://localhost:5432/warehouse',
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Literal credential URLs require --allow-literal-credentials');
});
it('warns before writing explicitly allowed literal credential URLs without echoing the URL', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
const literalUrl = 'postgres://localhost:5432/warehouse';
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: literalUrl,
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: true,
},
io.io,
),
).resolves.toBe(0);
expect(io.stderr()).toContain(
'Warning: writing a literal credential URL to ktx.yaml for connection "warehouse". Prefer env:NAME or file:/path references.',
);
expect(io.stderr()).not.toContain(literalUrl);
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain(literalUrl);
});
it('adds a Notion connection without writing token values', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: [],
rootDataSourceIds: [],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 50');
expect(yaml).not.toContain('ntn_');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('Driver: notion');
});
it('runs connection notion pick --no-input through the public connection entrypoint', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: ['database-1'],
rootDataSourceIds: ['data-source-1'],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
makeIo().io,
);
const io = makeIo();
await expect(
runKtxCli(
[
'connection',
'notion',
'pick',
'notion-main',
'--project-dir',
projectDir,
'--no-input',
'--root-page-id',
'11111111222233334444555555555555',
],
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('database-1');
expect(yaml).toContain('data-source-1');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('No connections configured. Run `ktx setup` to add one.');
expect(io.stdout()).not.toContain('ktx connection add');
});
it('tests a configured connection through the native scan connector', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
});
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
const createScanConnector = vi.fn(async () => connector);
const io = makeIo();
@ -602,22 +155,13 @@ describe('runKtxConnection', () => {
it('tests a configured Metabase connection through the Metabase runtime client', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const projectConfig = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(
join(projectDir, 'ktx.yaml'),
serializeKtxProjectConfig({
...projectConfig,
connections: {
...projectConfig.connections,
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
},
}),
'utf-8',
);
await writeConnections(projectDir, {
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
});
const testConnection = vi.fn(async () => ({ success: true as const }));
const getDatabases = vi.fn(async () => [
{ id: 1, name: 'Analytics', engine: 'postgres', details: {}, is_sample: false },
@ -657,20 +201,9 @@ describe('runKtxConnection', () => {
it('cleans up the native scan connector when connection testing fails', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
});
const cleanup = vi.fn(async () => undefined);
const connector: KtxScanConnector = {
id: 'sqlite:warehouse',

335
packages/cli/src/connection.ts
View file

@ -1,108 +1,24 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
type MetabaseRuntimeClient,
metabaseRuntimeConfigFromLocalConnection,
} from '@ktx/context/ingest';
import { type KtxLocalProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
import type { KtxScanConnector } from '@ktx/context/scan';
import type { KtxConnectionMappingArgs } from './commands/connection-mapping.js';
import type { KtxCliIo } from './index.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { profileMark } from './startup-profile.js';
profileMark('module:connection');
interface KtxNotionConnectionCliConfig {
authTokenRef: string;
crawlMode: 'all_accessible' | 'selected_roots';
rootPageIds: string[];
rootDatabaseIds: string[];
rootDataSourceIds: string[];
maxPagesPerRun?: number;
maxKnowledgeCreatesPerRun?: number;
maxKnowledgeUpdatesPerRun?: number;
}
type KtxConnectionInputMode = 'disabled';
export type KtxConnectionArgs =
| { command: 'list'; projectDir: string }
| {
command: 'add';
projectDir: string;
driver: string;
connectionId: string;
url?: string;
schemas: string[];
readonly: boolean;
force: boolean;
allowLiteralCredentials: boolean;
notion?: KtxNotionConnectionCliConfig;
}
| { command: 'test'; projectDir: string; connectionId: string }
| {
command: 'remove';
projectDir: string;
connectionId: string;
force: boolean;
inputMode?: KtxConnectionInputMode;
}
| {
command: 'map';
projectDir: string;
sourceConnectionId: string;
json: boolean;
};
interface KtxConnectionPromptAdapter {
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
interface KtxConnectionIo extends KtxCliIo {
stdin?: { isTTY?: boolean };
}
| { command: 'test'; projectDir: string; connectionId: string };
interface KtxConnectionDeps {
createScanConnector?: typeof createKtxCliScanConnector;
createMetabaseClient?: typeof createDefaultMetabaseClient;
runMapping?: (argv: string[], io: KtxCliIo) => Promise<number>;
prompts?: KtxConnectionPromptAdapter;
}
function assertSafeConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
}
function isCredentialReference(value: string): boolean {
return value.startsWith('env:') || value.startsWith('file:');
}
function literalCredentialWarning(connectionId: string): string {
return `Warning: writing a literal credential URL to ktx.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`;
}
function createClackConnectionPromptAdapter(): KtxConnectionPromptAdapter {
return {
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
const value = await confirm(options);
return isCancel(value) ? false : value;
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveConnectionIo(
args: Extract<KtxConnectionArgs, { command: 'remove' }>,
io: KtxConnectionIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
async function cleanupConnector(connector: KtxScanConnector | null): Promise<void> {
@ -186,166 +102,17 @@ async function testMetabaseConnection(
}
}
interface BufferedIo extends KtxCliIo {
stdoutText(): string;
stderrText(): string;
}
function createBufferedIo(): BufferedIo {
let stdout = '';
let stderr = '';
return {
stdout: {
write(chunk: string) {
stdout += chunk;
},
},
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
stdoutText() {
return stdout;
},
stderrText() {
return stderr;
},
};
}
function splitOutputLines(output: string): string[] {
return output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
}
async function runLowLevelMapping(
args: KtxConnectionMappingArgs,
argv: string[],
io: KtxCliIo,
deps: KtxConnectionDeps,
): Promise<number> {
if (deps.runMapping) {
return await deps.runMapping(argv, io);
}
const { runKtxConnectionMapping } = await import('./commands/connection-mapping.js');
return await runKtxConnectionMapping(args, io);
}
function parseMappingListJson(output: string): unknown[] {
const trimmed = output.trim();
if (!trimmed) {
return [];
}
const parsed = JSON.parse(trimmed) as unknown;
return Array.isArray(parsed) ? parsed : [];
}
async function runPublicConnectionMap(
args: Extract<KtxConnectionArgs, { command: 'map' }>,
io: KtxCliIo,
deps: KtxConnectionDeps,
): Promise<number> {
const refreshIo = createBufferedIo();
const refreshArgs: KtxConnectionMappingArgs = {
command: 'refresh',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
autoAccept: true,
};
const refreshCode = await runLowLevelMapping(
refreshArgs,
['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir],
refreshIo,
deps,
);
if (refreshCode !== 0) {
io.stderr.write(
refreshIo.stderrText() ||
refreshIo.stdoutText() ||
`Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`,
);
return refreshCode;
}
const validationIo = createBufferedIo();
const validationArgs: KtxConnectionMappingArgs = {
command: 'validate',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
};
const validationCode = await runLowLevelMapping(
validationArgs,
['validate', args.sourceConnectionId, '--project-dir', args.projectDir],
validationIo,
deps,
);
if (validationCode !== 0) {
io.stderr.write(
validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`,
);
return validationCode;
}
const listIo = createBufferedIo();
const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir];
const listArgs: KtxConnectionMappingArgs = {
command: 'list',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
json: args.json,
};
const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps);
if (listCode !== 0) {
io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`);
return listCode;
}
if (args.json) {
io.stdout.write(
`${JSON.stringify(
{
connectionId: args.sourceConnectionId,
refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) },
validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) },
mappings: parseMappingListJson(listIo.stdoutText()),
},
null,
2,
)}\n`,
);
return 0;
}
io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`);
io.stdout.write(refreshIo.stdoutText());
io.stdout.write(validationIo.stdoutText());
io.stdout.write('\nMappings:\n');
io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n');
io.stdout.write('\nNext:\n');
io.stdout.write(` ktx ingest run --connection-id ${args.sourceConnectionId} --adapter <adapter>\n`);
io.stdout.write(` ktx connection mapping list ${args.sourceConnectionId}\n`);
return 0;
}
export async function runKtxConnection(
args: KtxConnectionArgs,
io: KtxConnectionIo = process,
io: KtxCliIo = process,
deps: KtxConnectionDeps = {},
): Promise<number> {
try {
if (args.command === 'map') {
return await runPublicConnectionMap(args, io, deps);
}
const project = await loadKtxProject({ projectDir: args.projectDir });
if (args.command === 'list') {
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
if (entries.length === 0) {
io.stdout.write('No connections configured. Run `ktx connection add <id> --driver <driver>` to add one.\n');
io.stdout.write('No connections configured. Run `ktx setup` to add one.\n');
return 0;
}
const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
@ -360,100 +127,6 @@ export async function runKtxConnection(
return 0;
}
if (args.command === 'add') {
assertSafeConnectionId(args.connectionId);
const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url);
if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) {
throw new Error('Literal credential URLs require --allow-literal-credentials');
}
if (hasLiteralCredentialUrl) {
io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`);
}
if (project.config.connections[args.connectionId] && !args.force) {
throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`);
}
const connectionConfig =
args.driver === 'notion' && args.notion
? {
driver: 'notion',
auth_token_ref: args.notion.authTokenRef,
crawl_mode: args.notion.crawlMode,
root_page_ids: args.notion.rootPageIds,
root_database_ids: args.notion.rootDatabaseIds,
root_data_source_ids: args.notion.rootDataSourceIds,
...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}),
...(args.notion.maxKnowledgeCreatesPerRun !== undefined
? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun }
: {}),
...(args.notion.maxKnowledgeUpdatesPerRun !== undefined
? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun }
: {}),
}
: {
driver: args.driver,
...(args.url ? { url: args.url } : {}),
...(args.schemas.length > 0 ? { schemas: args.schemas } : {}),
readonly: args.readonly,
};
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[args.connectionId]: connectionConfig,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Update KTX connection: ${args.connectionId}`,
);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`Driver: ${args.driver}\n`);
return 0;
}
if (args.command === 'remove') {
if (!project.config.connections[args.connectionId]) {
throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`);
}
if (!args.force) {
if (!isInteractiveConnectionIo(args, io)) {
throw new Error(
`connection remove ${args.connectionId} requires --force when input is disabled or not interactive`,
);
}
const prompts = deps.prompts ?? createClackConnectionPromptAdapter();
const confirmed = await prompts.confirm({
message: `Remove connection "${args.connectionId}" from ktx.yaml? Ingested artifacts will remain in .ktx/.`,
initialValue: false,
});
if (!confirmed) {
prompts.cancel('Connection removal cancelled.');
return 1;
}
}
const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections;
const nextConfig = {
...project.config,
connections,
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Remove KTX connection: ${args.connectionId}`,
);
io.stdout.write('Connection removed from ktx.yaml.\n');
io.stdout.write('Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.\n');
return 0;
}
if (normalizedConnectionDriver(project, args.connectionId) === 'metabase') {
const result = await testMetabaseConnection(
project,

386
packages/cli/src/index.test.ts
View file

@ -1250,16 +1250,9 @@ describe('runKtxCli', () => {
runKtxCli(['--project-dir', tempDir, 'connection', 'list'], makeIo().io, { connection }),
).resolves.toBe(0);
const removeIo = makeIo();
const testIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'connection', 'remove', 'warehouse', '--force', '--no-input'], removeIo.io, {
connection,
}),
).resolves.toBe(0);
const mapIo = makeIo();
await expect(
runKtxCli(['--project-dir', tempDir, 'connection', 'map', 'prod-metabase', '--json'], mapIo.io, {
runKtxCli(['--project-dir', tempDir, 'connection', 'test', 'warehouse'], testIo.io, {
connection,
}),
).resolves.toBe(0);
@ -1268,21 +1261,9 @@ describe('runKtxCli', () => {
expect(connection).toHaveBeenNthCalledWith(
2,
{
command: 'remove',
command: 'test',
projectDir: tempDir,
connectionId: 'warehouse',
force: true,
inputMode: 'disabled',
},
expect.anything(),
);
expect(connection).toHaveBeenNthCalledWith(
3,
{
command: 'map',
projectDir: tempDir,
sourceConnectionId: 'prod-metabase',
json: true,
},
expect.anything(),
);
@ -1290,168 +1271,35 @@ describe('runKtxCli', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('prints help for connection metabase setup', async () => {
it('prints only list and test in connection help', async () => {
const helpIo = makeIo();
await expect(runKtxCli(['connection', 'metabase', 'setup', '--help'], helpIo.io)).resolves.toBe(0);
await expect(runKtxCli(['connection', '--help'], helpIo.io)).resolves.toBe(0);
expect(helpIo.stdout()).toContain('Usage: ktx connection metabase setup');
for (const option of [
'--id <connectionId>',
'--url <url>',
'--api-key <key>',
'--username <email>',
'--password <password>',
'--mint-api-key',
'--map <metabaseDatabaseId=targetConnectionId>',
'--sync <metabaseDatabaseId>',
'--sync-mode <mode>',
'--run-ingest',
'--yes',
'--no-input',
]) {
expect(helpIo.stdout()).toContain(option);
}
expect(helpIo.stdout()).toContain('Guided equivalent of:');
for (const line of [
'ktx connection mapping refresh <connectionId> --auto-accept',
'ktx connection mapping set <connectionId> databaseMappings <id>=<target>',
'ktx connection mapping set-sync-enabled <connectionId> <id> --enabled true',
'ktx ingest run --connection-id <connectionId> --adapter metabase',
]) {
expect(helpIo.stdout()).toContain(line);
expect(helpIo.stdout()).toContain('Usage: ktx connection');
expect(helpIo.stdout()).toContain('list');
expect(helpIo.stdout()).toContain('test <connectionId>');
for (const removed of ['add', 'remove', 'map', 'mapping', 'metabase', 'notion']) {
expect(helpIo.stdout()).not.toMatch(new RegExp(`\\b${removed}\\b`));
}
expect(helpIo.stderr()).toBe('');
});
it('dispatches connection metabase setup through Commander', async () => {
const connectionMetabaseSetup = vi.fn(async () => 0);
const fakeMetabaseCredential = 'mb_example';
const setupIo = makeIo();
await expect(
runKtxCli(
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--id',
'metabase',
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--map',
'2=orbit',
'--sync',
'2',
'--yes',
'--no-input',
],
setupIo.io,
{ connectionMetabaseSetup },
),
).resolves.toBe(0);
expect(connectionMetabaseSetup).toHaveBeenCalledWith(
{
command: 'setup',
projectDir: tempDir,
connectionId: 'metabase',
url: 'http://metabase.example.test:3000',
apiKey: fakeMetabaseCredential,
mintApiKey: false,
mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }],
syncEnabledDatabaseIds: [2],
syncMode: 'ALL',
runIngest: false,
yes: true,
inputMode: 'disabled',
},
expect.anything(),
);
expect(setupIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('validates connection metabase setup option values before runner dispatch', async () => {
const connectionMetabaseSetup = vi.fn(async () => 0);
it('rejects removed connection subcommands', async () => {
for (const argv of [
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--map',
'nope=orbit',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--map',
'2=../orbit',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--sync',
'nope',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--sync-mode',
'BAD',
],
[
'connection',
'metabase',
'setup',
'--project-dir',
tempDir,
'--url',
'http://metabase.example.test:3000',
'--api-key',
'mb_example',
'--mint-api-key',
'--api-key',
'also_bad',
],
['connection', 'add', 'postgres', 'warehouse'],
['connection', 'remove', 'warehouse'],
['connection', 'map', 'prod-metabase'],
['connection', 'mapping'],
['connection', 'metabase'],
['connection', 'notion'],
]) {
const testIo = makeIo();
await expect(runKtxCli(argv, testIo.io, { connectionMetabaseSetup })).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/map|sync|sync-mode|conflict|cannot be used|invalid|integer|choices/i);
}
expect(connectionMetabaseSetup).not.toHaveBeenCalled();
await expect(runKtxCli(argv, testIo.io)).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/unknown command|error:/);
}
});
it('rejects commands removed from the May 6 root surface', async () => {
@ -1469,153 +1317,6 @@ describe('runKtxCli', () => {
}
});
it('dispatches connection add options through Commander', async () => {
const testIo = makeIo();
const connection = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'connection',
'add',
'notion',
'notion-main',
'--project-dir',
tempDir,
'--token-env',
'NOTION_TOKEN',
'--crawl-mode',
'selected_roots',
'--root-page-id',
'page-1',
'--root-database-id',
'database-1',
'--max-pages',
'80',
],
testIo.io,
{ connection },
),
).resolves.toBe(0);
expect(connection).toHaveBeenCalledWith(
{
command: 'add',
projectDir: tempDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'selected_roots',
rootPageIds: ['page-1'],
rootDatabaseIds: ['database-1'],
rootDataSourceIds: [],
maxPagesPerRun: 80,
maxKnowledgeCreatesPerRun: undefined,
maxKnowledgeUpdatesPerRun: undefined,
},
},
testIo.io,
);
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('prints generated connection notion pick help without invoking execution', async () => {
const helpCases = [
['connection', 'notion', '--help'],
['connection', 'notion', 'pick', '--help'],
['connection', 'notion', 'pick', 'notion-main', '--help'],
];
for (const argv of helpCases) {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(runKtxCli(argv, testIo.io, { connectionNotion })).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx connection notion');
expect(testIo.stdout()).toContain('pick');
expect(testIo.stderr()).toBe('');
expect(connectionNotion).not.toHaveBeenCalled();
}
});
it('dispatches connection notion pick through Commander', async () => {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'connection',
'notion',
'pick',
'notion-main',
'--no-input',
'--root-page-id',
'11111111222233334444555555555555',
'--root-page-id',
'11111111-2222-3333-4444-555555555555',
],
testIo.io,
{ connectionNotion },
),
).resolves.toBe(0);
expect(connectionNotion).toHaveBeenCalledWith(
{
command: 'pick',
projectDir: tempDir,
connectionId: 'notion-main',
mode: 'non-interactive',
rootPageIds: ['11111111-2222-3333-4444-555555555555'],
},
testIo.io,
);
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('ignores connection notion pick root page flags in interactive mode', async () => {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(
runKtxCli(['connection', 'notion', 'pick', 'notion-main', '--root-page-id', 'not-a-uuid'], testIo.io, {
connectionNotion,
}),
).resolves.toBe(0);
expect(connectionNotion).toHaveBeenCalledWith(
{
command: 'pick',
projectDir: expect.any(String),
connectionId: 'notion-main',
mode: 'interactive',
},
testIo.io,
);
expect(testIo.stderr()).toBe('');
});
it('rejects connection notion pick no-input mode without root page ids', async () => {
const testIo = makeIo();
const connectionNotion = vi.fn(async () => 0);
await expect(
runKtxCli(['connection', 'notion', 'pick', 'notion-main', '--no-input'], testIo.io, { connectionNotion }),
).resolves.toBe(1);
expect(connectionNotion).not.toHaveBeenCalled();
expect(testIo.stderr()).toContain('connection notion pick --no-input requires at least one --root-page-id');
});
it('writes basic debug dispatch information when --debug is set', async () => {
const testIo = makeIo();
const connection = vi.fn().mockResolvedValue(0);
@ -1776,51 +1477,6 @@ describe('runKtxCli', () => {
expect(ingest).not.toHaveBeenCalled();
});
it('rejects mutually exclusive credential and scan mode options before invoking runners', async () => {
const connection = vi.fn(async () => 0);
const scan = vi.fn(async () => 0);
const tokenIo = makeIo();
await expect(
runKtxCli(
[
'connection',
'add',
'notion',
'notion-main',
'--token-env',
'NOTION_TOKEN',
'--token-file',
'/tmp/notion-token',
'--root-page-id',
'11111111111111111111111111111111',
],
tokenIo.io,
{ connection },
),
).resolves.toBe(1);
expect(tokenIo.stderr()).toMatch(/conflict|cannot be used/i);
expect(connection).not.toHaveBeenCalled();
expect(scan).not.toHaveBeenCalled();
});
it('validates connection mapping set syntax before runner domain validation', async () => {
const badFieldIo = makeIo();
await expect(
runKtxCli(['connection', 'mapping', 'set', 'prod-metabase', 'invalidMappings', '1=warehouse'], badFieldIo.io),
).resolves.toBe(1);
expect(badFieldIo.stderr()).toContain('databaseMappings or connectionMappings');
for (const assignment of ['missing-equals', '=warehouse', '1=']) {
const testIo = makeIo();
await expect(
runKtxCli(['connection', 'mapping', 'set', 'prod-metabase', 'databaseMappings', assignment], testIo.io),
).resolves.toBe(1);
expect(testIo.stderr()).toContain('non-empty <key>=<value>');
}
});
it('does not expose root init after setup owns project creation', async () => {
const testIo = makeIo();

2
packages/cli/src/commands/connection-notion-tree.test.ts → packages/cli/src/notion-page-picker-tree.test.ts
View file

@ -14,7 +14,7 @@ import {
TRANSIENT_HINT_DURATION_MS,
visibleNodeIds,
type NotionPickerPageInput,
} from './connection-notion-tree.js';
} from './notion-page-picker-tree.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',

0
packages/cli/src/commands/connection-notion-tree.ts → packages/cli/src/notion-page-picker-tree.ts
View file

4
packages/cli/src/commands/connection-notion-tui.test.tsx → packages/cli/src/notion-page-picker-tui.test.tsx
View file

@ -2,7 +2,7 @@
import { render as renderInkTest } from 'ink-testing-library';
import { act, type ReactNode } from 'react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
NotionPickerApp,
notionPickerCommandForInkInput,
@ -13,7 +13,7 @@ import {
windowOffset,
type NotionPickerInkInstance,
type NotionPickerInkRenderOptions,
} from './connection-notion-tui.js';
} from './notion-page-picker-tui.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',

4
packages/cli/src/commands/connection-notion-tui.tsx → packages/cli/src/notion-page-picker-tui.tsx
View file

@ -9,8 +9,8 @@ import {
visibleNodeIds,
type PickerCommand,
type PickerState,
} from './connection-notion-tree.js';
import type { KtxCliIo } from '../index.js';
} from './notion-page-picker-tree.js';
import type { KtxCliIo } from './cli-runtime.js';
const COLOR_THEME = {
text: 'white',

308
packages/cli/src/notion-page-picker.test.ts Normal file
View file

@ -0,0 +1,308 @@
import { describe, expect, it, vi } from 'vitest';
import {
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
pickNotionRootPages,
resolveNotionWorkspaceLabel,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './notion-page-picker.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: true,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('Notion page picker helpers', () => {
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
});
describe('pickNotionRootPages', () => {
it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
},
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toEqual({ kind: 'selected', rootPageIds: [PAGE_IDS.engineering] });
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toBe('');
});
it('uses inline Notion auth_token for discovery', async () => {
const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]);
const createNotionApi = vi.fn((authToken: string) => {
expect(authToken).toBe('ntn_inline_token');
return api;
});
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token: 'ntn_inline_token',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
},
},
makeIo().io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'back' });
expect(createNotionApi).toHaveBeenCalledOnce();
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
},
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toEqual({ kind: 'back' });
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
});
it('returns unavailable when discovery cannot load any pages', async () => {
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [],
},
},
makeIo().io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => ({
search: vi.fn(async () => {
throw new Error('Notion API unavailable');
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
})),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' });
});
});

200
packages/cli/src/commands/connection-notion.ts → packages/cli/src/notion-page-picker.ts
View file

@ -1,51 +1,40 @@
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from '@ktx/context/connections';
import { resolveNotionConnectionAuthToken } from '@ktx/context/connections';
import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
loadKtxProject,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
type NotionPickerTuiIo,
type PickerRenderInput,
type PickerRenderResult,
renderNotionPickerTui,
} from './connection-notion-tui.js';
} from './notion-page-picker-tui.js';
profileMark('module:commands/connection-notion');
profileMark('module:notion-page-picker');
export type KtxConnectionNotionArgs =
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'interactive';
}
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'non-interactive';
rootPageIds: string[];
};
export interface PickNotionRootPagesArgs {
connectionId: string;
connection: KtxProjectConnectionConfig;
}
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
export type { PickerRenderInput, PickerRenderResult };
interface KtxConnectionNotionDeps {
export type NotionRootPagePickResult =
| { kind: 'selected'; rootPageIds: string[] }
| { kind: 'back' }
| { kind: 'unavailable'; message: string };
export interface NotionRootPagePickerDeps {
env?: Record<string, string | undefined>;
loadProject?: typeof loadKtxProject;
createNotionApi?: (authToken: string) => NotionPickerApi;
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
}
const NOTION_PICKER_PAGE_CAP = 5000;
function assertSafeConnectionId(connectionId: string): void {
function assertSafeNotionPickerConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
@ -168,111 +157,74 @@ export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connecti
}
}
function notionConnection(project: KtxLocalProject, connectionId: string): KtxProjectConnectionConfig {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" not found`);
}
function assertNotionConnection(connection: KtxProjectConnectionConfig, connectionId: string): void {
if (connection.driver !== 'notion') {
throw new Error(`Connection "${connectionId}" is not a Notion connection`);
}
return connection;
}
export async function applyNotionPickerWriteback(
project: KtxLocalProject,
connectionId: string,
rootPageIds: string[],
): Promise<void> {
if (rootPageIds.length === 0) {
throw new Error('connection notion pick requires at least one root page id');
}
const existing = notionConnection(project, connectionId);
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: {
...existing,
crawl_mode: 'selected_roots',
root_page_ids: rootPageIds,
},
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`,
);
function stringArray(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string' && item.trim().length > 0) : [];
}
export async function runKtxConnectionNotion(
args: KtxConnectionNotionArgs,
function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessible' | 'selected_roots' {
return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
}
export async function pickNotionRootPages(
args: PickNotionRootPagesArgs,
io: KtxCliIo = process,
deps: KtxConnectionNotionDeps = {},
): Promise<number> {
deps: NotionRootPagePickerDeps = {},
): Promise<NotionRootPagePickResult> {
try {
assertSafeConnectionId(args.connectionId);
const loadProject = deps.loadProject ?? loadKtxProject;
if (args.mode === 'interactive') {
const project = await loadProject({ projectDir: args.projectDir });
const rawConnection = notionConnection(project, args.connectionId);
const notion = parseNotionConnectionConfig(rawConnection);
const authToken = await resolveNotionConnectionAuthToken(notion, { env: deps.env });
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: notion.root_page_ids,
currentCrawlMode: notion.crawl_mode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: notion.crawl_mode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
io.stdout.write('No changes saved.\n');
return 0;
}
await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
assertSafeNotionPickerConnectionId(args.connectionId);
assertNotionConnection(args.connection, args.connectionId);
const crawlMode = notionCrawlMode(args.connection);
const authToken = await resolveNotionConnectionAuthToken(
{
auth_token: typeof args.connection.auth_token === 'string' ? args.connection.auth_token : null,
auth_token_ref: typeof args.connection.auth_token_ref === 'string' ? args.connection.auth_token_ref : null,
},
{ env: deps.env },
);
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: stringArray(args.connection.root_page_ids),
currentCrawlMode: crawlMode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const project = await loadProject({ projectDir: args.projectDir });
await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: crawlMode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
return { kind: 'back' };
}
if (result.rootPageIds.length === 0) {
return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' };
}
return { kind: 'selected', rootPageIds: result.rootPageIds };
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) };
}
}

8
packages/cli/src/print-command-tree.test.ts
View file

@ -16,7 +16,13 @@ describe('renderKtxCommandTree', () => {
expect(topLevel).toContain(expected);
}
expect(output).toContain('│ ├── test <connectionId>');
expect(output).toContain('│ └── test <connectionId>');
expect(output).not.toContain('│ ├── add');
expect(output).not.toContain('│ ├── remove');
expect(output).not.toContain('│ ├── map');
expect(output).not.toContain('│ ├── mapping');
expect(output).not.toContain('│ ├── metabase');
expect(output).not.toContain('│ ├── notion');
});
it('ends with a single trailing newline', () => {

91
packages/cli/src/setup-sources.test.ts
View file

@ -274,6 +274,97 @@ describe('setup sources step', () => {
});
});
it('uses the rich Notion picker for interactive selected root setup', async () => {
await addPrimarySource();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' }));
const pickNotionRootPages = vi.fn(async (input: Parameters<NonNullable<KtxSetupSourcesDeps['pickNotionRootPages']>>[0]) => {
expect(input.connectionId).toBe('notion-main');
expect(input.connection).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [],
});
return { kind: 'selected' as const, rootPageIds: ['11111111-2222-3333-4444-555555555555'] };
});
const testPrompts = prompts({
multiselect: [['notion']],
select: ['env', 'selected_roots', 'done'],
text: ['notion-main'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{ prompts: testPrompts, validateNotion, pickNotionRootPages },
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(pickNotionRootPages).toHaveBeenCalledOnce();
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['11111111-2222-3333-4444-555555555555'],
});
});
it('backs out of the Notion picker without writing selected_roots when the picker quits', async () => {
await addPrimarySource();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=0' }));
const pickNotionRootPages = vi.fn(async () => ({ kind: 'back' as const }));
const testPrompts = prompts({
multiselect: [['notion']],
select: ['env', 'selected_roots', 'all_accessible', 'done'],
text: ['notion-main'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{ prompts: testPrompts, validateNotion, pickNotionRootPages },
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(pickNotionRootPages).toHaveBeenCalledOnce();
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
crawl_mode: 'all_accessible',
});
expect((await readConfig()).connections['notion-main']?.root_page_ids).toBeUndefined();
});
it('surfaces Notion picker failures and returns to the page-mode step', async () => {
await addPrimarySource();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=0' }));
const pickNotionRootPages = vi.fn(async () => ({
kind: 'unavailable' as const,
message: 'Notion picker requires a TTY',
}));
const testPrompts = prompts({
multiselect: [['notion']],
select: ['env', 'selected_roots', 'all_accessible', 'done'],
text: ['notion-main'],
});
const io = makeIo();
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
io.io,
{ prompts: testPrompts, validateNotion, pickNotionRootPages },
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(io.stderr()).toContain('Notion picker requires a TTY');
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
crawl_mode: 'all_accessible',
});
});
it('defaults interactive Metabase and Looker source setup to the only warehouse connection', async () => {
await addPrimarySource();
const cases: Array<{

128
packages/cli/src/setup-sources.ts
View file

@ -28,8 +28,8 @@ import {
stripKtxSetupCompletedSteps,
} from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { runKtxConnectionMapping } from './commands/connection-mapping.js';
import { runKtxConnection } from './connection.js';
import { pickNotionRootPages } from './notion-page-picker.js';
import { runKtxSourceMapping } from './source-mapping.js';
import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
import { runKtxPublicIngest } from './public-ingest.js';
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
@ -95,6 +95,7 @@ export interface KtxSetupSourcesDeps {
validateLooker?: (projectDir: string, connectionId: string) => Promise<SourceValidationResult>;
validateLookml?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
validateNotion?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
pickNotionRootPages?: typeof pickNotionRootPages;
discoverMetabaseDatabases?: (args: {
sourceUrl: string;
sourceApiKeyRef: string;
@ -528,7 +529,7 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC
driver: 'notion',
auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'),
crawl_mode: crawlMode,
root_page_ids: rootPageIds,
...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 1000,
@ -614,7 +615,7 @@ async function defaultValidateMetricflow(connection: KtxProjectConnectionConfig)
}
async function defaultValidateLooker(projectDir: string, connectionId: string): Promise<SourceValidationResult> {
const code = await runKtxConnectionMapping(
const code = await runKtxSourceMapping(
{ command: 'refresh', projectDir, connectionId, autoAccept: true },
{ stdout: { write() {} }, stderr: { write() {} } },
);
@ -657,6 +658,22 @@ interface MappingJsonOutput {
mappings: unknown[];
}
function splitOutputLines(output: string): string[] {
return output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
}
function parseMappingListJson(output: string): unknown[] {
const trimmed = output.trim();
if (!trimmed) {
return [];
}
const parsed = JSON.parse(trimmed) as unknown;
return Array.isArray(parsed) ? parsed : [];
}
function summarizeMappingResult(parsed: MappingJsonOutput): string {
const mappingCount = parsed.mappings.length;
const mappingNoun = mappingCount === 1 ? 'mapping' : 'mappings';
@ -664,22 +681,51 @@ function summarizeMappingResult(parsed: MappingJsonOutput): string {
}
async function defaultRunMapping(projectDir: string, connectionId: string, io: KtxCliIo): Promise<number> {
let captured = '';
const captureIo: KtxCliIo = {
stdout: { write(chunk: string) { captured += chunk; } },
stderr: io.stderr,
const outputs = {
refresh: '',
validation: '',
list: '',
};
const code = await runKtxConnection(
{ command: 'map', projectDir, sourceConnectionId: connectionId, json: true },
captureIo,
const refreshCode = await runKtxSourceMapping(
{ command: 'refresh', projectDir, connectionId, autoAccept: true },
{
stdout: { write(chunk: string) { outputs.refresh += chunk; } },
stderr: io.stderr,
},
);
if (code !== 0) return code;
try {
const parsed = JSON.parse(captured.trim()) as MappingJsonOutput;
io.stdout.write(`${summarizeMappingResult(parsed)}\n`);
} catch {
io.stdout.write(captured);
if (refreshCode !== 0) {
return refreshCode;
}
const validationCode = await runKtxSourceMapping(
{ command: 'validate', projectDir, connectionId },
{
stdout: { write(chunk: string) { outputs.validation += chunk; } },
stderr: io.stderr,
},
);
if (validationCode !== 0) {
return validationCode;
}
const listCode = await runKtxSourceMapping(
{ command: 'list', projectDir, connectionId, json: true },
{
stdout: { write(chunk: string) { outputs.list += chunk; } },
stderr: io.stderr,
},
);
if (listCode !== 0) {
return listCode;
}
const parsed: MappingJsonOutput = {
connectionId,
refresh: { ok: true, output: splitOutputLines(outputs.refresh) },
validation: { ok: true, output: splitOutputLines(outputs.validation) },
mappings: parseMappingListJson(outputs.list),
};
io.stdout.write(`${summarizeMappingResult(parsed)}\n`);
return 0;
}
@ -927,6 +973,8 @@ async function promptForInteractiveSource(
args: KtxSetupSourcesArgs,
source: KtxSetupSourceType,
prompts: KtxSetupSourcesPromptAdapter,
io: KtxCliIo,
deps: KtxSetupSourcesDeps,
defaultConnectionId = `${source}-main`,
testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection,
discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'],
@ -1213,15 +1261,29 @@ async function promptForInteractiveSource(
...(state.notionCrawlMode === 'selected_roots'
? [
async (currentState: SourcePromptState) => {
const roots = await promptText(prompts, {
message: 'Notion page IDs to ingest (each page includes all its subpages)',
placeholder: 'page-id-1, page-id-2',
});
if (roots === undefined) return 'back';
currentState.notionRootPageIds = roots
.split(',')
.map((root) => root.trim())
.filter(Boolean);
const connectionId = currentState.sourceConnectionId ?? 'notion-main';
const result = await (deps.pickNotionRootPages ?? pickNotionRootPages)(
{
connectionId,
connection: {
driver: 'notion',
auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'),
crawl_mode: 'selected_roots',
root_page_ids: currentState.notionRootPageIds ?? [],
root_database_ids: [],
root_data_source_ids: [],
},
},
io,
);
if (result.kind === 'back') {
return 'back';
}
if (result.kind === 'unavailable') {
io.stderr.write(`${result.message}\n`);
return 'back';
}
currentState.notionRootPageIds = result.rootPageIds;
return 'next';
},
]
@ -1259,7 +1321,9 @@ async function chooseInteractiveSourceConnection(input: {
source: KtxSetupSourceType;
connections: Record<string, KtxProjectConnectionConfig>;
prompts: KtxSetupSourcesPromptAdapter;
io: KtxCliIo;
testGitRepo?: KtxSetupSourcesDeps['testGitRepo'];
pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages'];
discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases'];
}): Promise<InteractiveSourceConnectionChoice> {
const existingIds = existingConnectionIdsBySource(input.connections, input.source);
@ -1271,6 +1335,11 @@ async function chooseInteractiveSourceConnection(input: {
input.args,
input.source,
input.prompts,
input.io,
{
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
},
defaultConnectionId,
input.testGitRepo,
input.discoverMetabaseDatabases,
@ -1303,6 +1372,11 @@ async function chooseInteractiveSourceConnection(input: {
input.args,
input.source,
input.prompts,
input.io,
{
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
},
defaultConnectionId,
input.testGitRepo,
input.discoverMetabaseDatabases,
@ -1417,7 +1491,9 @@ export async function runKtxSetupSourcesStep(
source,
connections: (await loadKtxProject({ projectDir: args.projectDir })).config.connections,
prompts,
io,
testGitRepo: deps.testGitRepo,
pickNotionRootPages: deps.pickNotionRootPages,
discoverMetabaseDatabases: deps.discoverMetabaseDatabases,
});
if (sourceChoice === 'back') {

224
packages/cli/src/source-mapping.ts Normal file
View file

@ -0,0 +1,224 @@
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
discoverMetabaseDatabases,
lookerCredentialsFromLocalConnection,
metabaseRuntimeConfigFromLocalConnection,
seedLocalMappingStateFromKtxYaml,
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type MetabaseRuntimeClient,
} from '@ktx/context/ingest';
import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
profileMark('module:source-mapping');
export type KtxSourceMappingArgs =
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
| { command: 'validate'; projectDir: string; connectionId: string };
interface KtxSourceMappingDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
createLookerClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function createDefaultLookerClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
const factory = new DefaultLookerConnectionClientFactory({
async resolve(lookerConnectionId) {
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
},
});
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
cleanup?(): Promise<void>;
};
}
function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean {
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
}
function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void {
const connection = project.config.connections[connectionId];
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
}
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function renderMapping(
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
row.source
})`;
}
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
const target = row.ktxConnectionId ?? '[unmapped]';
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
}
export async function runKtxSourceMapping(
args: KtxSourceMappingArgs,
io: KtxCliIo = process,
deps: KtxSourceMappingDeps = {},
): Promise<number> {
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
await seedLocalMappingStateFromKtxYaml(project, args.connectionId);
if (isLookerConnection(project, args.connectionId)) {
const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listConnectionMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
try {
const discovered = await discoverLookerConnections(client);
const drift = computeLookerMappingDrift({
storedMappings: await store.readMappings(args.connectionId),
discovered,
});
if (args.autoAccept) {
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup?.();
}
}
const knownKtxConnectionIds = new Set(Object.keys(project.config.connections));
const knownConnectionTypes = new Map(
Object.entries(project.config.connections).map(([id]) => [id, targetPhysicalInfo(project, id).connection_type]),
);
const validation = validateLookerMappings({
mappings: await store.readMappings(args.connectionId),
knownKtxConnectionIds,
knownConnectionTypes,
});
if (!validation.ok) {
for (const error of validation.errors) {
io.stderr.write(`${error.key}: ${error.reason}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
assertMetabaseConnection(project, args.connectionId);
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup();
}
}
const rows = await store.listDatabaseMappings(args.connectionId);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
project.config.connections[row.targetConnectionId]
? targetPhysicalInfo(project, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (failures.length > 0) {
for (const failure of failures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

2
packages/context/src/ingest/adapters/metabase/fetch.test.ts
View file

@ -276,7 +276,7 @@ describe('fetchMetabaseBundle', () => {
clientFactory,
sourceStateReader,
}),
).rejects.toThrow(/unhydrated.*ktx connection mapping refresh/);
).rejects.toThrow(/unhydrated.*ktx setup/);
});
it('skips cards whose getResolvedSql returns null and records them in unresolved-cards.json', async () => {

2
packages/context/src/ingest/adapters/metabase/fetch.ts
View file

@ -99,7 +99,7 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr
}
if (mapping.metabaseDatabaseName === null) {
throw new IngestInputError(
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`,
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx setup\` and reconfigure the Metabase source to populate metabaseDatabaseName before ingest.`,
);
}
const mappingDatabaseName: string = mapping.metabaseDatabaseName;

2
packages/context/src/ingest/local-ingest.ts
View file

@ -371,7 +371,7 @@ export async function runLocalMetabaseIngest(
const unhydrated = await sourceStateReader.getUnhydratedSyncEnabledMappingIds(metabaseConnectionId);
if (unhydrated.length > 0) {
throw new Error(
`Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`ktx connection mapping refresh ${metabaseConnectionId}\` before local Metabase ingest.`,
`Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`ktx setup\` and reconfigure ${metabaseConnectionId} before local Metabase ingest.`,
);
}

4
packages/context/src/ingest/local-metabase-ingest.test.ts
View file

@ -203,7 +203,7 @@ describe('runLocalMetabaseIngest', () => {
metabaseConnectionId: 'prod-metabase',
agentRunner: new TestAgentRunner(),
}),
).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`');
).rejects.toThrow('run `ktx setup` and reconfigure prod-metabase');
});
it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => {
@ -230,7 +230,7 @@ describe('runLocalMetabaseIngest', () => {
adapters: [new FakeMetabaseSourceAdapter()],
metabaseConnectionId: 'prod-metabase',
}),
).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`');
).rejects.toThrow('run `ktx setup` and reconfigure prod-metabase');
});
it('rejects source-dir uploads through the Metabase fan-out runner', async () => {

2
packages/context/src/ingest/memory-flow/known-errors.ts
View file

@ -23,6 +23,6 @@ export function formatNotionAuthorizationExpiredDetail(unitKey: string): string
export function notionAuthorizationFixSuggestions(connectionId: string): string[] {
return [
`Refresh the Notion token referenced by auth_token_ref for ${connectionId}. If it uses env:NAME, export a fresh token in that variable; if it uses file:/path, replace that file.`,
`Run ktx connection notion pick ${connectionId} to confirm Notion access, then rerun ktx ingest ${connectionId}.`,
`Run ktx setup and reconfigure the Notion source to confirm page access, then rerun ktx ingest ${connectionId}.`,
];
}

2
packages/context/src/ingest/memory-flow/summary.test.ts
View file

@ -85,7 +85,7 @@ describe('formatMemoryFlowFinalSummary', () => {
'- Refresh the Notion token referenced by auth_token_ref for notion-main. If it uses env:NAME, export a fresh token in that variable; if it uses file:/path, replace that file.',
);
expect(summary).toContain(
'- Run ktx connection notion pick notion-main to confirm Notion access, then rerun ktx ingest notion-main.',
'- Run ktx setup and reconfigure the Notion source to confirm page access, then rerun ktx ingest notion-main.',
);
expect(summary).not.toContain('error_uri');
});