From 2f41fd019d66262e0ef30d382d599dc3d73a788f Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 14:21:05 +0200 Subject: [PATCH] fix(cli): clean up connection commands --- .../docs/cli-reference/ktx-connection.mdx | 135 +- .../docs/getting-started/quickstart.mdx | 2 +- packages/cli/src/cli-program.ts | 3 - packages/cli/src/cli-runtime.ts | 4 - packages/cli/src/command-schemas.ts | 25 - .../cli/src/commands/connection-commands.ts | 308 +---- .../src/commands/connection-mapping.test.ts | 329 ----- .../cli/src/commands/connection-mapping.ts | 426 ------- .../commands/connection-metabase-commands.ts | 132 -- .../connection-metabase-setup.test.ts | 1136 ----------------- .../src/commands/connection-metabase-setup.ts | 786 ------------ .../commands/connection-notion-commands.ts | 92 -- .../src/commands/connection-notion.test.ts | 513 -------- packages/cli/src/connection.test.ts | 541 +------- packages/cli/src/connection.ts | 335 +---- packages/cli/src/index.test.ts | 386 +----- ...est.ts => notion-page-picker-tree.test.ts} | 2 +- ...ion-tree.ts => notion-page-picker-tree.ts} | 0 ...st.tsx => notion-page-picker-tui.test.tsx} | 4 +- ...ion-tui.tsx => notion-page-picker-tui.tsx} | 4 +- packages/cli/src/notion-page-picker.test.ts | 308 +++++ ...ection-notion.ts => notion-page-picker.ts} | 200 ++- packages/cli/src/print-command-tree.test.ts | 8 +- packages/cli/src/setup-sources.test.ts | 91 ++ packages/cli/src/setup-sources.ts | 128 +- packages/cli/src/source-mapping.ts | 224 ++++ .../ingest/adapters/metabase/fetch.test.ts | 2 +- .../src/ingest/adapters/metabase/fetch.ts | 2 +- packages/context/src/ingest/local-ingest.ts | 2 +- .../src/ingest/local-metabase-ingest.test.ts | 4 +- .../src/ingest/memory-flow/known-errors.ts | 2 +- .../src/ingest/memory-flow/summary.test.ts | 2 +- 32 files changed, 906 insertions(+), 5230 deletions(-) delete mode 100644 packages/cli/src/commands/connection-mapping.test.ts delete mode 100644 packages/cli/src/commands/connection-mapping.ts delete mode 100644 packages/cli/src/commands/connection-metabase-commands.ts delete mode 100644 packages/cli/src/commands/connection-metabase-setup.test.ts delete mode 100644 packages/cli/src/commands/connection-metabase-setup.ts delete mode 100644 packages/cli/src/commands/connection-notion-commands.ts delete mode 100644 packages/cli/src/commands/connection-notion.test.ts rename packages/cli/src/{commands/connection-notion-tree.test.ts => notion-page-picker-tree.test.ts} (99%) rename packages/cli/src/{commands/connection-notion-tree.ts => notion-page-picker-tree.ts} (100%) rename packages/cli/src/{commands/connection-notion-tui.test.tsx => notion-page-picker-tui.test.tsx} (99%) rename packages/cli/src/{commands/connection-notion-tui.tsx => notion-page-picker-tui.tsx} (99%) create mode 100644 packages/cli/src/notion-page-picker.test.ts rename packages/cli/src/{commands/connection-notion.ts => notion-page-picker.ts} (51%) create mode 100644 packages/cli/src/source-mapping.ts diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 31a79736..0cec3eae 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -1,9 +1,11 @@ --- title: "ktx connection" -description: "Add, list, test, and map data sources." +description: "List and test configured data sources." --- -Manage database and source connections in your KTX project. Connections define how KTX reaches your data warehouse, BI tools, and context sources. +Inspect configured connections in your KTX project. Connections define how KTX +reaches your data warehouse, BI tools, and context sources. Use `ktx setup` to +add, remove, or reconfigure connections. ## Command signature @@ -17,96 +19,23 @@ ktx connection [options] |-----------|-------------| | `list` | List configured connections | | `test ` | Test a configured connection | -| `add ` | Add or replace a configured connection | -| `remove ` | Remove a configured connection from `ktx.yaml` | -| `map ` | Refresh and validate BI-to-warehouse mappings | -| `mapping list ` | List Metabase database mappings | -| `mapping set ` | Set a Metabase or Looker warehouse mapping | -| `mapping apply-bulk ` | Apply mappings from JSON | -| `mapping set-sync-enabled ` | Enable or disable sync for one Metabase database | -| `mapping sync-state get ` | Read sync-state selection | -| `mapping sync-state set ` | Write sync-state selection | -| `mapping refresh ` | Refresh Metabase database mappings | -| `mapping validate ` | Validate Metabase database mappings | -| `mapping clear [dbId]` | Clear Metabase database mappings | -| `metabase setup` | Guided setup for a Metabase connection | -| `notion pick ` | Pick Notion root pages for a configured Notion connection | ## Options -### `connection add` +The `connection` command has command-level options for listing and testing +existing connections. -| Flag | Description | Default | -|------|-------------|---------| -| `--url ` | Connection URL, `env:NAME`, or `file:/path` reference | — | -| `--schema ` | Schema to include; repeatable | — | -| `--readonly` | Mark the connection as read-only | `false` | -| `--force` | Replace an existing connection | `false` | -| `--allow-literal-credentials` | Allow writing a literal credential URL to `ktx.yaml` | `false` | - -#### Notion-specific options for `connection add` - -| Flag | Description | Default | -|------|-------------|---------| -| `--token-env ` | Environment variable containing Notion auth token | — | -| `--token-file ` | File containing Notion auth token | — | -| `--crawl-mode ` | Notion crawl mode (`all_accessible` or `selected_roots`) | `selected_roots` | -| `--root-page-id ` | Root page to crawl; repeatable | — | -| `--root-database-id ` | Root database to crawl; repeatable | — | -| `--root-data-source-id ` | Root data source to crawl; repeatable | — | -| `--max-pages ` | Maximum pages per run | — | -| `--max-knowledge-creates ` | Maximum knowledge creates per run | — | -| `--max-knowledge-updates ` | Maximum knowledge updates per run | — | - -### `connection remove` - -| Flag | Description | Default | -|------|-------------|---------| -| `--force` | Remove without prompting | `false` | -| `--no-input` | Disable interactive terminal input | — | - -### `connection map` +### `connection list` | Flag | Description | Default | |------|-------------|---------| | `--json` | Print JSON output | `false` | -### `connection mapping` subcommands - -| Flag | Subcommand | Description | Default | -|------|-----------|-------------|---------| -| `--json` | `list`, `sync-state get` | Print JSON output | `false` | -| `--file ` | `apply-bulk` | JSON mapping file (required) | — | -| `--enabled ` | `set-sync-enabled` | `true` or `false` (required) | — | -| `--mode ` | `sync-state set` | `ALL`, `ONLY`, or `EXCEPT` (required) | — | -| `--collections ` | `sync-state set` | Comma-separated collection ids | — | -| `--items ` | `sync-state set` | Comma-separated item ids | — | -| `--tag-names ` | `sync-state set` | Comma-separated tag names | — | -| `--auto-accept` | `refresh` | Accept refresh changes without prompting | `false` | - -### `connection metabase setup` +### `connection test` | Flag | Description | Default | |------|-------------|---------| -| `--id ` | KTX connection id to write | — | -| `--url ` | Metabase API URL | — | -| `--api-key ` | Metabase API key | — | -| `--mint-api-key` | Mint a Metabase API key with credentials | `false` | -| `--username ` | Metabase admin username for API-key minting | — | -| `--password ` | Metabase admin password for API-key minting | — | -| `--map ` | Assign a Metabase database id to a warehouse connection; repeatable | — | -| `--sync ` | Enable sync for a discovered database; repeatable | — | -| `--sync-mode ` | Metabase sync selection mode (`ALL`, `ONLY`, or `EXCEPT`) | `ALL` | -| `--run-ingest` | Run ingest after setup | `false` | -| `--yes` | Confirm and apply setup changes without prompting | `false` | -| `--no-input` | Disable interactive terminal input | — | - -### `connection notion pick` - -| Flag | Description | Default | -|------|-------------|---------| -| `--no-input` | Disable interactive terminal input | — | -| `--root-page-id ` | Root page UUID to crawl; repeatable (required with `--no-input`) | — | +| `--json` | Print JSON output | `false` | ## Examples @@ -114,43 +43,20 @@ ktx connection [options] # List all configured connections ktx connection list -# Add a Postgres connection using an environment variable -ktx connection add postgres my-warehouse --url "env:DATABASE_URL" - -# Add a Postgres connection with specific schemas -ktx connection add postgres analytics --url "env:PG_URL" --schema public --schema analytics - -# Add a read-only Snowflake connection -ktx connection add snowflake sf-prod --url "env:SNOWFLAKE_URL" --readonly - # Test a connection ktx connection test my-warehouse - -# Remove a connection -ktx connection remove old-warehouse - -# Add a Notion source connection -ktx connection add notion my-notion \ - --token-env NOTION_TOKEN \ - --crawl-mode selected_roots \ - --root-page-id abc123def456... - -# Run guided Metabase setup -ktx connection metabase setup --url https://metabase.example.com - -# Map a BI database to a warehouse connection -ktx connection mapping set metabase-prod databaseMappings 1=my-warehouse - -# Refresh Metabase mappings -ktx connection mapping refresh metabase-prod --auto-accept - -# Pick Notion root pages interactively -ktx connection notion pick my-notion ``` +## Setup-managed connections + +Run `ktx setup` when you need to add or reconfigure a connection. Interactive +setup includes the rich Notion page picker for selected root pages and the +Metabase mapping prompts for BI-to-warehouse mappings. + ## Output -Interactive commands render prompts and status text. Commands with `--json` return machine-readable JSON suitable for scripts and agents. +Commands with `--json` return machine-readable JSON suitable for scripts and +agents. ```json { @@ -168,7 +74,6 @@ Interactive commands render prompts and status text. Commands with `--json` retu | Error | Cause | Recovery | |-------|-------|----------| -| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx connection add ... --force` | -| Literal credentials rejected | KTX avoids writing raw secrets to `ktx.yaml` by default | Use `env:NAME` or `file:/path/to/secret`; use `--allow-literal-credentials` only for local throwaway projects | -| Mapping validation fails | BI database mappings do not point at valid warehouse connections | Run `ktx connection mapping refresh --auto-accept`, then set invalid mappings explicitly | -| Notion pick cannot run non-interactively | `--no-input` was used without root page or database ids | Pass `--root-page-id`, `--root-database-id`, or `--root-data-source-id` with `--no-input` | +| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | +| Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Rerun `ktx setup` and update the source mapping selections | +| Notion page picker cannot run | The terminal is non-interactive or Notion discovery failed | Rerun interactive `ktx setup`, or use non-interactive setup flags with explicit root page ids | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 6aef2b14..d71a0754 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -240,7 +240,7 @@ Agent integration ready: yes (claude-code:project) | LLM health check fails | Missing, invalid, or unauthorized Anthropic API key | Export `ANTHROPIC_API_KEY` or rerun `ktx setup` and choose the file-backed secret option | | OpenAI embedding check fails | `OPENAI_API_KEY` is missing when OpenAI embeddings are selected | Export `OPENAI_API_KEY`, or rerun setup and choose local sentence-transformers embeddings | | Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup | -| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx connection add ... --force` or rerun setup | +| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | | `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now | | Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need | diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index 7d6a98f3..dbe73a72 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -178,9 +178,6 @@ function shouldSuppressProjectDirLine(path: string[], options: Record= 0) { const demoCommand = path[demoIndex + 1]; diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index f303309a..5e2430cf 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -1,7 +1,5 @@ import { createRequire } from 'node:module'; -import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js'; -import type { KtxConnectionNotionArgs } from './commands/connection-notion.js'; import type { KtxConnectionArgs } from './connection.js'; import type { KtxDoctorArgs } from './doctor.js'; import type { KtxIngestArgs } from './ingest.js'; @@ -30,8 +28,6 @@ export interface KtxCliIo { export interface KtxCliDeps { setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise; connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; - connectionNotion?: (args: KtxConnectionNotionArgs, io: KtxCliIo) => Promise; - connectionMetabaseSetup?: (args: KtxConnectionMetabaseSetupArgs, io: KtxCliIo) => Promise; doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise; runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/command-schemas.ts b/packages/cli/src/command-schemas.ts index cb11f2eb..5caece1f 100644 --- a/packages/cli/src/command-schemas.ts +++ b/packages/cli/src/command-schemas.ts @@ -1,33 +1,8 @@ import { z } from 'zod'; const projectDirSchema = z.string().min(1); -const safeConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, 'Unsafe connection id'); const stringArraySchema = z.array(z.string()); -export const connectionAddCommandSchema = z.object({ - command: z.literal('add'), - projectDir: projectDirSchema, - driver: z.string().min(1), - connectionId: safeConnectionIdSchema, - url: z.string().optional(), - schemas: stringArraySchema, - readonly: z.boolean(), - force: z.boolean(), - allowLiteralCredentials: z.boolean(), - notion: z - .object({ - authTokenRef: z.string().min(1), - crawlMode: z.enum(['all_accessible', 'selected_roots']), - rootPageIds: stringArraySchema, - rootDatabaseIds: stringArraySchema, - rootDataSourceIds: stringArraySchema, - maxPagesPerRun: z.number().int().positive().optional(), - maxKnowledgeCreatesPerRun: z.number().int().nonnegative().optional(), - maxKnowledgeUpdatesPerRun: z.number().int().nonnegative().optional(), - }) - .optional(), -}); - export const wikiWriteCommandSchema = z.object({ command: z.literal('write'), projectDir: projectDirSchema, diff --git a/packages/cli/src/commands/connection-commands.ts b/packages/cli/src/commands/connection-commands.ts index 4ce75057..d814ffe9 100644 --- a/packages/cli/src/commands/connection-commands.ts +++ b/packages/cli/src/commands/connection-commands.ts @@ -1,61 +1,19 @@ -import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings'; -import { - collectOption, - type KtxCliCommandContext, - parseBooleanStringOption, - parseNonEmptyAssignmentOption, - parseNonNegativeIntegerOption, - parsePositiveIntegerOption, - parseSafeConnectionIdOption, - resolveCommandProjectDir, -} from '../cli-program.js'; -import { connectionAddCommandSchema } from '../command-schemas.js'; +import { type Command } from '@commander-js/extra-typings'; +import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js'; import type { KtxConnectionArgs } from '../connection.js'; import { profileMark } from '../startup-profile.js'; -import type { KtxConnectionMappingArgs } from './connection-mapping.js'; -import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js'; -import { registerConnectionNotionCommands } from './connection-notion-commands.js'; profileMark('module:commands/connection-commands'); -const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const; -const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const; - -function parseCsvIds(value: string): number[] { - return value - .split(',') - .filter(Boolean) - .map((item) => parsePositiveIntegerOption(item)); -} - -function parseCsvStrings(value: string): string[] { - return value - .split(',') - .map((item) => item.trim()) - .filter(Boolean); -} - -function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' { - if (value === 'databaseMappings' || value === 'connectionMappings') { - return value; - } - throw new InvalidArgumentError('must be databaseMappings or connectionMappings'); -} - async function runConnectionArgs(context: KtxCliCommandContext, args: KtxConnectionArgs): Promise { const runner = context.deps.connection ?? (await import('../connection.js')).runKtxConnection; context.setExitCode(await runner(args, context.io)); } -async function runMappingArgs(context: KtxCliCommandContext, args: KtxConnectionMappingArgs): Promise { - const { runKtxConnectionMapping } = await import('./connection-mapping.js'); - context.setExitCode(await runKtxConnectionMapping(args, context.io)); -} - export function registerConnectionCommands(program: Command, context: KtxCliCommandContext, commandName = 'connection'): void { const connection = program .command(commandName) - .description('Add, list, test, and map data sources') + .description('List and test configured connections') .showHelpAfterError() .addHelpText( 'after', @@ -83,264 +41,4 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm connectionId, }); }); - - connection - .command('add') - .description('Add or replace a configured connection') - .argument('', 'Connection driver') - .argument('', 'KTX connection id') - .option('--url ', 'Connection URL, env:NAME, or file:/path reference') - .option('--schema ', 'Schema to include; repeatable', collectOption, []) - .option('--readonly', 'Mark the connection as read-only', false) - .option('--force', 'Replace an existing connection', false) - .option('--allow-literal-credentials', 'Allow writing a literal credential URL to ktx.yaml', false) - .addOption(new Option('--token-env ', 'Environment variable containing Notion auth token').conflicts('tokenFile')) - .addOption(new Option('--token-file ', 'File containing Notion auth token').conflicts('tokenEnv')) - .addOption( - new Option('--crawl-mode ', 'Notion crawl mode: all_accessible or selected_roots') - .choices(CRAWL_MODE_CHOICES) - .default('selected_roots'), - ) - .option('--root-page-id ', 'Root page to crawl; repeatable', collectOption, []) - .option('--root-database-id ', 'Root database to crawl; repeatable', collectOption, []) - .option('--root-data-source-id ', 'Root data source to crawl; repeatable', collectOption, []) - .option('--max-pages ', 'Maximum pages per run', parsePositiveIntegerOption) - .option('--max-knowledge-creates ', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption) - .option('--max-knowledge-updates ', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption) - .action(async (driver: string, connectionId: string, options, command) => { - const notion = - driver === 'notion' - ? { - authTokenRef: options.tokenEnv - ? `env:${options.tokenEnv}` - : options.tokenFile - ? `file:${options.tokenFile}` - : '', - crawlMode: options.crawlMode, - rootPageIds: options.rootPageId, - rootDatabaseIds: options.rootDatabaseId, - rootDataSourceIds: options.rootDataSourceId, - maxPagesPerRun: options.maxPages, - maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates, - maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates, - } - : undefined; - - if (driver === 'notion' && !notion?.authTokenRef) { - throw new Error('connection add notion requires --token-env NAME or --token-file PATH'); - } - if ( - driver === 'notion' && - notion?.crawlMode === 'selected_roots' && - notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0 - ) { - throw new Error('connection add notion selected_roots requires at least one root id'); - } - - const args = connectionAddCommandSchema.parse({ - command: 'add', - projectDir: resolveCommandProjectDir(command), - driver, - connectionId, - url: options.url, - schemas: options.schema.filter(Boolean), - readonly: options.readonly === true, - force: options.force === true, - allowLiteralCredentials: options.allowLiteralCredentials === true, - notion, - }); - - await runConnectionArgs(context, args); - }); - - connection - .command('remove') - .description('Remove a configured connection from ktx.yaml') - .argument('', 'KTX connection id') - .option('--force', 'Remove without prompting', false) - .option('--no-input', 'Disable interactive terminal input') - .action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => { - await runConnectionArgs(context, { - command: 'remove', - projectDir: resolveCommandProjectDir(command), - connectionId, - force: options.force === true, - ...(options.input === false ? { inputMode: 'disabled' } : {}), - }); - }); - - connection - .command('map') - .description('Refresh and validate BI-to-warehouse mappings') - .argument('', 'Source BI connection id') - .option('--json', 'Print JSON output', false) - .action(async (sourceConnectionId: string, options: { json?: boolean }, command) => { - await runConnectionArgs(context, { - command: 'map', - projectDir: resolveCommandProjectDir(command), - sourceConnectionId, - json: options.json === true, - }); - }); - - registerConnectionMappingCommands(connection, context); - registerConnectionMetabaseCommands(connection, context); - registerConnectionNotionCommands(connection, context); -} - -function registerConnectionMappingCommands(connection: Command, context: KtxCliCommandContext): void { - const mapping = connection - .command('mapping') - .description('Manage Metabase warehouse mappings') - .showHelpAfterError() - .addHelpText( - 'after', - '\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n', - ); - - mapping - .command('list') - .description('List Metabase database mappings') - .argument('', 'Metabase connection id') - .option('--json', 'Print JSON output where supported', false) - .action(async (connectionId: string, options: { json?: boolean }, command) => { - await runMappingArgs(context, { - command: 'list', - projectDir: resolveCommandProjectDir(command), - connectionId, - json: options.json === true, - }); - }); - - mapping - .command('set') - .description('Set a Metabase or Looker warehouse mapping') - .argument('', 'Source connection id', parseSafeConnectionIdOption) - .argument('', 'Mapping field', parseMappingFieldOption) - .argument('', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption) - .action( - async ( - connectionId: string, - field: 'databaseMappings' | 'connectionMappings', - assignment: { key: string; value: string }, - _options: unknown, - command, - ) => { - await runMappingArgs(context, { - command: 'set', - projectDir: resolveCommandProjectDir(command), - connectionId, - field, - key: assignment.key, - value: assignment.value, - }); - }, - ); - - mapping - .command('apply-bulk') - .description('Apply mappings from JSON') - .argument('', 'Metabase connection id') - .requiredOption('--file ', 'JSON mapping file') - .action(async (connectionId: string, options: { file: string }, command) => { - await runMappingArgs(context, { - command: 'apply-bulk', - projectDir: resolveCommandProjectDir(command), - connectionId, - filePath: options.file, - }); - }); - - mapping - .command('set-sync-enabled') - .description('Enable or disable sync for one Metabase database') - .argument('', 'Metabase connection id') - .argument('', 'Metabase database id', parsePositiveIntegerOption) - .requiredOption('--enabled ', 'true or false', parseBooleanStringOption) - .action( - async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => { - await runMappingArgs(context, { - command: 'set-sync-enabled', - projectDir: resolveCommandProjectDir(command), - connectionId, - metabaseDatabaseId, - enabled: options.enabled, - }); - }, - ); - - const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection'); - syncState - .command('get') - .description('Read sync-state selection') - .argument('', 'Metabase connection id') - .option('--json', 'Print JSON output where supported', false) - .action(async (connectionId: string, options: { json?: boolean }, command) => { - await runMappingArgs(context, { - command: 'sync-state-get', - projectDir: resolveCommandProjectDir(command), - connectionId, - json: options.json === true, - }); - }); - - syncState - .command('set') - .description('Write sync-state selection') - .argument('', 'Metabase connection id') - .addOption(new Option('--mode ', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory()) - .option('--collections ', 'Comma-separated collection ids', parseCsvIds, []) - .option('--items ', 'Comma-separated item ids', parseCsvIds, []) - .option('--tag-names ', 'Comma-separated tag names', parseCsvStrings, []) - .action(async (connectionId: string, options, command) => { - await runMappingArgs(context, { - command: 'sync-state-set', - projectDir: resolveCommandProjectDir(command), - connectionId, - syncMode: options.mode, - collectionIds: options.collections, - itemIds: options.items, - tagNames: options.tagNames, - }); - }); - - mapping - .command('refresh') - .description('Refresh Metabase database mappings') - .argument('', 'Metabase connection id') - .option('--auto-accept', 'Accept refresh changes without prompting', false) - .action(async (connectionId: string, options: { autoAccept?: boolean }, command) => { - await runMappingArgs(context, { - command: 'refresh', - projectDir: resolveCommandProjectDir(command), - connectionId, - autoAccept: options.autoAccept === true, - }); - }); - - mapping - .command('validate') - .description('Validate Metabase database mappings') - .argument('', 'Metabase connection id') - .action(async (connectionId: string, _options: unknown, command) => { - await runMappingArgs(context, { - command: 'validate', - projectDir: resolveCommandProjectDir(command), - connectionId, - }); - }); - - mapping - .command('clear') - .description('Clear Metabase database mappings') - .argument('', 'Metabase connection id') - .argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption) - .action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => { - await runMappingArgs(context, { - command: 'clear', - projectDir: resolveCommandProjectDir(command), - connectionId, - ...(metabaseDatabaseId ? { metabaseDatabaseId } : {}), - }); - }); } diff --git a/packages/cli/src/commands/connection-mapping.test.ts b/packages/cli/src/commands/connection-mapping.test.ts deleted file mode 100644 index 7d76cc9d..00000000 --- a/packages/cli/src/commands/connection-mapping.test.ts +++ /dev/null @@ -1,329 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest'; -import { initKtxProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxConnectionMapping } from './connection-mapping.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -describe('runKtxConnectionMapping', () => { - let tempDir: string; - let projectDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-metabase-mapping-')); - projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'mapping' }); - const project = await loadKtxProject({ projectDir }); - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig({ - ...project.config, - connections: { - 'prod-metabase': { - driver: 'metabase', - api_url: 'https://metabase.example.com', - api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret - }, - 'prod-warehouse': { - driver: 'postgres', - url: 'env:WAREHOUSE_URL', - readonly: true, - }, - }, - }), - 'ktx', - 'ktx@example.com', - 'Seed Metabase mapping test connections', - ); - }); - - async function replaceConnections(connections: Record) { - const project = await loadKtxProject({ projectDir }); - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig({ - ...project.config, - connections, - }), - 'ktx', - 'ktx@example.com', - 'Replace mapping test connections', - ); - } - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('sets, lists, disables, and clears local Metabase mappings', async () => { - const io = makeIo(); - await expect( - runKtxConnectionMapping( - { - command: 'set', - projectDir, - connectionId: 'prod-metabase', - field: 'databaseMappings', - key: '1', - value: 'prod-warehouse', - }, - io.io, - ), - ).resolves.toBe(0); - - const listIo = makeIo(); - await expect( - runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io), - ).resolves.toBe(0); - expect(listIo.stdout()).toContain('1 -> prod-warehouse'); - expect(listIo.stdout()).toContain('unhydrated'); - - await expect( - runKtxConnectionMapping( - { - command: 'set-sync-enabled', - projectDir, - connectionId: 'prod-metabase', - metabaseDatabaseId: 1, - enabled: false, - }, - makeIo().io, - ), - ).resolves.toBe(0); - - await expect( - runKtxConnectionMapping( - { - command: 'clear', - projectDir, - connectionId: 'prod-metabase', - metabaseDatabaseId: 1, - }, - makeIo().io, - ), - ).resolves.toBe(0); - }); - - it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-yaml-mapping-')); - await initKtxProject({ projectDir, projectName: 'yaml-mapping' }); - const project = await loadKtxProject({ projectDir }); - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig({ - ...project.config, - connections: { - 'prod-metabase': { - driver: 'metabase', - mappings: { - databaseMappings: { '1': 'prod-warehouse' }, - syncEnabled: { '1': true }, - }, - }, - 'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' }, - }, - }), - 'ktx', - 'ktx@example.com', - 'Seed yaml mappings', - ); - const io = makeIo(); - - await expect( - runKtxConnectionMapping( - { command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, - io.io, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('1 -> prod-warehouse'); - expect(io.stdout()).toContain('source: ktx.yaml'); - }); - - it('refreshes Metabase discovery metadata through the injected runtime client', async () => { - const client = { - getDatabases: vi.fn().mockResolvedValue([ - { - id: 1, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ]), - cleanup: vi.fn(), - }; - const io = makeIo(); - - await expect( - runKtxConnectionMapping( - { - command: 'refresh', - projectDir, - connectionId: 'prod-metabase', - autoAccept: true, - }, - io.io, - { - createMetabaseClient: async () => client as never, - }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Discovery: 1 database'); - expect(client.cleanup).toHaveBeenCalledTimes(1); - const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.ktx', 'db.sqlite') }); - await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([ - { metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' }, - ]); - }); - - it('sets and lists Looker connection mappings', async () => { - await replaceConnections({ - 'prod-looker': { - driver: 'looker', - base_url: 'https://looker.example.test', - client_id: 'id', - }, - 'prod-warehouse': { - driver: 'postgres', - url: 'postgresql://readonly@db.example.test/analytics', - }, - }); - const io = makeIo(); - - await expect( - runKtxConnectionMapping( - { - command: 'set', - projectDir, - connectionId: 'prod-looker', - field: 'connectionMappings', - key: 'analytics', - value: 'prod-warehouse', - }, - io.io, - ), - ).resolves.toBe(0); - await expect( - runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('analytics -> prod-warehouse'); - }); - - it('keeps driver-specific mapping field validation in the runner', async () => { - await replaceConnections({ - 'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' }, - warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' }, - }); - - const io = makeIo(); - await expect( - runKtxConnectionMapping( - { - command: 'set', - projectDir, - connectionId: 'prod-looker', - field: 'databaseMappings', - key: '1', - value: 'warehouse', - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('Looker mapping set requires connectionMappings'); - }); - - it('refreshes Looker mapping metadata and reports drift', async () => { - await replaceConnections({ - 'prod-looker': { - driver: 'looker', - base_url: 'https://looker.example.test', - client_id: 'id', - }, - 'prod-warehouse': { - driver: 'postgres', - url: 'postgresql://readonly@db.example.test/analytics', - }, - }); - const io = makeIo(); - - await expect( - runKtxConnectionMapping( - { command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true }, - io.io, - { - createLookerClient: async () => ({ - listLookerConnections: async () => [ - { - name: 'analytics', - host: 'db.example.test', - database: 'analytics', - schema: null, - dialect: 'postgres', - }, - ], - cleanup: async () => {}, - }), - }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Discovery: 1 connection'); - expect(io.stdout()).toContain('Unmapped discovered: 1'); - }); - - it('validates Looker mappings through the canonical local warehouse descriptor', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-descriptor-validation-')); - await initKtxProject({ projectDir, projectName: 'descriptor-validation' }); - const project = await loadKtxProject({ projectDir }); - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig({ - ...project.config, - connections: { - 'prod-looker': { - driver: 'looker', - mappings: { connectionMappings: { analytics: 'prod-warehouse' } }, - }, - 'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' }, - }, - }), - 'ktx', - 'ktx@example.com', - 'Seed descriptor validation', - ); - const io = makeIo(); - - await expect( - runKtxConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Mapping validation passed: prod-looker'); - expect(io.stderr()).toBe(''); - }); -}); diff --git a/packages/cli/src/commands/connection-mapping.ts b/packages/cli/src/commands/connection-mapping.ts deleted file mode 100644 index b35bf40f..00000000 --- a/packages/cli/src/commands/connection-mapping.ts +++ /dev/null @@ -1,426 +0,0 @@ -import { readFile } from 'node:fs/promises'; -import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections'; -import { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultLookerConnectionClientFactory, - DefaultMetabaseConnectionClientFactory, - LocalLookerRuntimeStore, - LocalMetabaseSourceStateReader, - computeLookerMappingDrift, - computeMetabaseMappingDrift, - discoverLookerConnections, - discoverMetabaseDatabases, - lookerCredentialsFromLocalConnection, - metabaseRuntimeConfigFromLocalConnection, - seedLocalMappingStateFromKtxYaml, - validateLookerMappings, - validateMappingPhysicalMatch, - type LookerMappingClient, - type MetabaseRuntimeClient, - type MetabaseSyncMode, -} from '@ktx/context/ingest'; -import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; -import type { KtxCliIo } from '../index.js'; -import { profileMark } from '../startup-profile.js'; - -profileMark('module:commands/connection-mapping'); - -export type KtxConnectionMappingArgs = - | { command: 'list'; projectDir: string; connectionId: string; json: boolean } - | { - command: 'set'; - projectDir: string; - connectionId: string; - field: 'databaseMappings' | 'connectionMappings'; - key: string; - value: string; - } - | { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string } - | { - command: 'set-sync-enabled'; - projectDir: string; - connectionId: string; - metabaseDatabaseId: number; - enabled: boolean; - } - | { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean } - | { - command: 'sync-state-set'; - projectDir: string; - connectionId: string; - syncMode: MetabaseSyncMode; - collectionIds: number[]; - itemIds: number[]; - tagNames: string[]; - } - | { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean } - | { command: 'validate'; projectDir: string; connectionId: string } - | { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string }; - -interface KtxConnectionMappingDeps { - createMetabaseClient?: ( - project: KtxLocalProject, - connectionId: string, - ) => Promise>; - createLookerClient?: ( - project: KtxLocalProject, - connectionId: string, - ) => Promise & { cleanup?(): Promise }>; -} - -interface MetabaseBulkMappingPayload { - databaseMappings?: Record; - syncEnabled?: Record; - syncMode?: MetabaseSyncMode; - selections?: { collections?: number[]; items?: number[] }; - defaultTagNames?: string[]; -} - -function parseId(value: string, label: string): number { - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 1) { - throw new Error(`${label} must be a positive integer`); - } - return parsed; -} - -async function createDefaultMetabaseClient( - project: KtxLocalProject, - connectionId: string, -): Promise> { - const factory = new DefaultMetabaseConnectionClientFactory( - (metabaseConnectionId) => - metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]), - DEFAULT_METABASE_CLIENT_CONFIG, - ); - return factory.createClient(connectionId); -} - -async function createDefaultLookerClient( - project: KtxLocalProject, - connectionId: string, -): Promise & { cleanup?(): Promise }> { - const factory = new DefaultLookerConnectionClientFactory({ - async resolve(lookerConnectionId) { - return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]); - }, - }); - return factory.createClient(connectionId) as unknown as Pick & { - cleanup?(): Promise; - }; -} - -function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean { - return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker'; -} - -function assertLookerConnection(project: KtxLocalProject, connectionId: string): void { - if (!isLookerConnection(project, connectionId)) { - throw new Error(`Connection "${connectionId}" is not a Looker connection`); - } -} - -function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void { - const connection = project.config.connections[connectionId]; - if (!connection || String(connection.driver).toLowerCase() !== 'metabase') { - throw new Error(`Connection "${connectionId}" is not a Metabase connection`); - } -} - -function assertTargetConnection(project: KtxLocalProject, connectionId: string): void { - if (!project.config.connections[connectionId]) { - throw new Error(`Target connection "${connectionId}" does not exist`); - } -} - -function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) { - const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]); - if (!descriptor) { - return { connection_type: 'UNKNOWN' }; - } - return { - connection_type: descriptor.connection_type, - host: descriptor.host ?? null, - database: descriptor.database ?? null, - account: descriptor.account ?? null, - project_id: descriptor.project_id ?? null, - dataset_id: descriptor.dataset_id ?? null, - ...descriptor.connection_params, - }; -} - -function renderMapping( - row: Awaited>[number], -): string { - const name = row.metabaseDatabaseName ?? 'unhydrated'; - const target = row.targetConnectionId ?? '[unmapped]'; - return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${ - row.source - })`; -} - -function renderLookerMapping(row: Awaited>[number]): string { - const target = row.ktxConnectionId ?? '[unmapped]'; - const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', '); - return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`; -} - -export async function runKtxConnectionMapping( - args: KtxConnectionMappingArgs, - io: KtxCliIo = process, - deps: KtxConnectionMappingDeps = {}, -): Promise { - try { - const project = await loadKtxProject({ projectDir: args.projectDir }); - await seedLocalMappingStateFromKtxYaml(project, args.connectionId); - if (isLookerConnection(project, args.connectionId)) { - assertLookerConnection(project, args.connectionId); - const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) }); - - if (args.command === 'list') { - const rows = await store.listConnectionMappings(args.connectionId); - io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`); - return 0; - } - - if (args.command === 'set') { - if (args.field !== 'connectionMappings') { - throw new Error('Looker mapping set requires connectionMappings ='); - } - assertTargetConnection(project, args.value); - await store.upsertConnectionMapping({ - lookerConnectionId: args.connectionId, - lookerConnectionName: args.key, - ktxConnectionId: args.value, - source: 'cli', - }); - io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`); - return 0; - } - - if (args.command === 'refresh') { - const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId); - try { - const discovered = await discoverLookerConnections(client); - const drift = computeLookerMappingDrift({ - storedMappings: await store.readMappings(args.connectionId), - discovered, - }); - if (args.autoAccept) { - await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered }); - } - io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`); - io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`); - io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`); - return 0; - } finally { - await client.cleanup?.(); - } - } - - if (args.command === 'validate') { - const knownKtxConnectionIds = new Set(Object.keys(project.config.connections)); - const knownConnectionTypes = new Map( - Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]), - ); - const validation = validateLookerMappings({ - mappings: await store.readMappings(args.connectionId), - knownKtxConnectionIds, - knownConnectionTypes, - }); - if (!validation.ok) { - for (const error of validation.errors) { - io.stderr.write(`${error.key}: ${error.reason}\n`); - } - return 1; - } - io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`); - return 0; - } - - if (args.command === 'clear') { - await store.clearConnectionMappings({ - lookerConnectionId: args.connectionId, - lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined), - }); - io.stdout.write( - args.mappingKey - ? `Cleared connectionMappings.${args.mappingKey}\n` - : `Cleared mappings for ${args.connectionId}\n`, - ); - return 0; - } - - throw new Error(`Looker connection mapping does not support ${args.command}`); - } - - assertMetabaseConnection(project, args.connectionId); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) }); - - if (args.command === 'list') { - const rows = await store.listDatabaseMappings(args.connectionId); - io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`); - return 0; - } - - if (args.command === 'set') { - assertTargetConnection(project, args.value); - await store.upsertDatabaseMapping({ - connectionId: args.connectionId, - metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'), - targetConnectionId: args.value, - syncEnabled: true, - source: 'cli', - }); - io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`); - return 0; - } - - if (args.command === 'apply-bulk') { - const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload; - const existingState = await store.getSourceState(args.connectionId); - const existingRows = await store.listDatabaseMappings(args.connectionId); - const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row])); - const databaseMappings = payload.databaseMappings ?? {}; - for (const targetConnectionId of Object.values(databaseMappings)) { - if (targetConnectionId) { - assertTargetConnection(project, targetConnectionId); - } - } - const mappingIds = new Set([ - ...existingRows.map((row) => row.metabaseDatabaseId), - ...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')), - ...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')), - ]); - await store.replaceSourceState({ - connectionId: args.connectionId, - syncMode: payload.syncMode ?? existingState.syncMode, - defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames, - selections: - payload.selections === undefined - ? existingState.selections - : [ - ...(payload.selections.collections ?? []).map((id) => ({ - selectionType: 'collection' as const, - metabaseObjectId: id, - })), - ...(payload.selections.items ?? []).map((id) => ({ - selectionType: 'item' as const, - metabaseObjectId: id, - })), - ], - mappings: [...mappingIds] - .sort((a, b) => a - b) - .map((id) => { - const existing = existingById.get(id); - return { - metabaseDatabaseId: id, - metabaseDatabaseName: existing?.metabaseDatabaseName ?? null, - metabaseEngine: existing?.metabaseEngine ?? null, - metabaseHost: existing?.metabaseHost ?? null, - metabaseDbName: existing?.metabaseDbName ?? null, - targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null, - syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false, - source: 'cli', - }; - }), - }); - io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`); - return 0; - } - - if (args.command === 'set-sync-enabled') { - await store.setMappingSyncEnabled({ - connectionId: args.connectionId, - metabaseDatabaseId: args.metabaseDatabaseId, - syncEnabled: args.enabled, - }); - io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`); - return 0; - } - - if (args.command === 'sync-state-get') { - const state = await store.getSourceState(args.connectionId); - const payload = { - syncMode: state.syncMode, - selections: state.selections, - defaultTagNames: state.defaultTagNames, - }; - io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`); - return 0; - } - - if (args.command === 'sync-state-set') { - await store.setSyncState({ - connectionId: args.connectionId, - syncMode: args.syncMode, - defaultTagNames: args.tagNames, - selections: [ - ...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })), - ...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })), - ], - }); - io.stdout.write(`Set sync state for ${args.connectionId}\n`); - return 0; - } - - if (args.command === 'refresh') { - const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId); - try { - const discovered = await discoverMetabaseDatabases(client); - const existing = Object.fromEntries( - (await store.listDatabaseMappings(args.connectionId)).map((row) => [ - String(row.metabaseDatabaseId), - row.targetConnectionId, - ]), - ); - const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered }); - if (args.autoAccept) { - await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered }); - } - io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`); - io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`); - io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`); - return 0; - } finally { - await client.cleanup(); - } - } - - if (args.command === 'validate') { - const rows = await store.listDatabaseMappings(args.connectionId); - const failures = rows.flatMap((row) => { - if (!row.targetConnectionId) { - return []; - } - const reason = validateMappingPhysicalMatch( - { metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost }, - project.config.connections[row.targetConnectionId] - ? targetPhysicalInfo(project, row.targetConnectionId) - : { connection_type: 'UNKNOWN' }, - ); - return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : []; - }); - if (failures.length > 0) { - for (const failure of failures) { - io.stderr.write(`${failure}\n`); - } - return 1; - } - io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`); - return 0; - } - - const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined); - await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId }); - io.stdout.write( - metabaseDatabaseId - ? `Cleared databaseMappings.${metabaseDatabaseId}\n` - : `Cleared mappings for ${args.connectionId}\n`, - ); - return 0; - } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - return 1; - } -} diff --git a/packages/cli/src/commands/connection-metabase-commands.ts b/packages/cli/src/commands/connection-metabase-commands.ts deleted file mode 100644 index c20b8e86..00000000 --- a/packages/cli/src/commands/connection-metabase-commands.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { type Command, Option } from '@commander-js/extra-typings'; - -import { - type KtxCliCommandContext, - parseNonEmptyAssignmentOption, - parsePositiveIntegerOption, - parseSafeConnectionIdOption, - resolveCommandProjectDir, -} from '../cli-program.js'; -import { - type KtxConnectionMetabaseSetupArgs, - type MetabaseSetupMappingAssignment, - type MetabaseSetupSyncMode, - runKtxConnectionMetabaseSetup, -} from './connection-metabase-setup.js'; - -const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[]; - -interface ConnectionMetabaseSetupOptions { - id?: string; - url?: string; - apiKey?: string; - mintApiKey?: boolean; - username?: string; - password?: string; - map: MetabaseSetupMappingAssignment[]; - sync: number[]; - syncMode: MetabaseSetupSyncMode; - runIngest?: boolean; - yes?: boolean; - input?: boolean; -} - -function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] { - return [...previous, parsePositiveIntegerOption(value)]; -} - -function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment { - const assignment = parseNonEmptyAssignmentOption(value); - return { - metabaseDatabaseId: parsePositiveIntegerOption(assignment.key), - targetConnectionId: parseSafeConnectionIdOption(assignment.value), - }; -} - -function collectMappingOption( - value: string, - previous: MetabaseSetupMappingAssignment[] = [], -): MetabaseSetupMappingAssignment[] { - return [...previous, parseMappingAssignment(value)]; -} - -async function runMetabaseSetupArgs( - context: KtxCliCommandContext, - args: KtxConnectionMetabaseSetupArgs, -): Promise { - const runner = context.deps.connectionMetabaseSetup ?? runKtxConnectionMetabaseSetup; - context.setExitCode(await runner(args, context.io)); -} - -export function registerConnectionMetabaseCommands(connection: Command, context: KtxCliCommandContext): void { - const metabase = connection - .command('metabase') - .description('Configure Metabase connections') - .showHelpAfterError() - .addHelpText( - 'after', - '\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n', - ); - - metabase.action(() => { - metabase.outputHelp(); - context.setExitCode(0); - }); - - metabase - .command('setup') - .description('Guided setup for a Metabase connection') - .option('--id ', 'KTX connection id to write', parseSafeConnectionIdOption) - .option('--url ', 'Metabase API URL') - .addOption(new Option('--api-key ', 'Metabase API key').conflicts('mintApiKey')) - .option('--mint-api-key', 'Mint a Metabase API key with credentials', false) - .option('--username ', 'Metabase admin username for API-key minting') - .option('--password ', 'Metabase admin password for API-key minting') - .addHelpText( - 'after', - '\nGuided equivalent of:\n' + - ' ktx connection mapping refresh --auto-accept\n' + - ' ktx connection mapping set databaseMappings =\n' + - ' ktx connection mapping set-sync-enabled --enabled true\n' + - ' ktx ingest run --connection-id --adapter metabase\n', - ) - .option( - '--map ', - 'Assign a Metabase database id to a warehouse connection; repeatable', - collectMappingOption, - [], - ) - .option( - '--sync ', - 'Enable Metabase sync for a discovered database; repeatable', - collectPositiveIntegerOption, - [], - ) - .addOption( - new Option('--sync-mode ', 'Metabase sync selection mode') - .choices(SYNC_MODE_CHOICES) - .default('ALL' satisfies MetabaseSetupSyncMode), - ) - .option('--run-ingest', 'Run ingest after setup', false) - .option('--yes', 'Confirm and apply setup changes without prompting', false) - .option('--no-input', 'Disable interactive terminal input') - .showHelpAfterError() - .action(async (options: ConnectionMetabaseSetupOptions, command) => { - await runMetabaseSetupArgs(context, { - command: 'setup', - projectDir: resolveCommandProjectDir(command), - connectionId: options.id, - url: options.url, - apiKey: options.apiKey, - mintApiKey: options.mintApiKey === true, - metabaseUsername: options.username, - metabasePassword: options.password, - mappings: options.map, - syncEnabledDatabaseIds: options.sync, - syncMode: options.syncMode ?? 'ALL', - runIngest: options.runIngest === true, - yes: options.yes === true, - inputMode: options.input === false ? 'disabled' : 'auto', - }); - }); -} diff --git a/packages/cli/src/commands/connection-metabase-setup.test.ts b/packages/cli/src/commands/connection-metabase-setup.test.ts deleted file mode 100644 index 9d462bbd..00000000 --- a/packages/cli/src/commands/connection-metabase-setup.test.ts +++ /dev/null @@ -1,1136 +0,0 @@ -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest'; -import { initKtxProject, ktxLocalStateDbPath, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -import { runKtxConnectionMetabaseSetup } from './connection-metabase-setup.js'; - -const CANCEL_PROMPT = Symbol('cancel'); - -function createTestMetabaseSetupPromptAdapter(options: { - selects?: Array; - multiselects?: Array | typeof CANCEL_PROMPT>; - texts?: Array; - passwords?: Array; - confirms?: Array; - events?: string[]; -}) { - const selects = [...(options.selects ?? [])]; - const multiselects = [...(options.multiselects ?? [])]; - const texts = [...(options.texts ?? [])]; - const passwords = [...(options.passwords ?? [])]; - const confirms = [...(options.confirms ?? [])]; - const events = options.events ?? []; - - const cancelWithError = () => { - throw new Error('Setup cancelled.'); - }; - - return { - intro(title?: string): void { - events.push(`intro:${title ?? ''}`); - }, - outro(message?: string): void { - events.push(`outro:${message ?? ''}`); - }, - note(message: string, title: string): void { - events.push(`note:${title}:${message}`); - }, - log: { - info(message: string): void { - events.push(`log.info:${message}`); - }, - step(message: string): void { - events.push(`log.step:${message}`); - }, - success(message: string): void { - events.push(`log.success:${message}`); - }, - warn(message: string): void { - events.push(`log.warn:${message}`); - }, - error(message: string): void { - events.push(`log.error:${message}`); - }, - }, - spinner() { - return { - start(message: string): void { - events.push(`spinner.start:${message}`); - }, - stop(message: string): void { - events.push(`spinner.stop:${message}`); - }, - error(message: string): void { - events.push(`spinner.error:${message}`); - }, - }; - }, - async select(): Promise { - const next = selects.shift(); - if (next === CANCEL_PROMPT) { - cancelWithError(); - } - return next as T; - }, - async multiselect(options?: { message: string }): Promise { - events.push(`multiselect:${options?.message ?? ''}`); - const next = multiselects.shift(); - if (next === CANCEL_PROMPT) { - cancelWithError(); - } - return (next ?? []) as Value[]; - }, - async text(): Promise { - const next = texts.shift(); - if (next === CANCEL_PROMPT) { - cancelWithError(); - } - return (next ?? '').toString(); - }, - async password(): Promise { - const next = passwords.shift(); - if (next === CANCEL_PROMPT) { - cancelWithError(); - } - return (next ?? '').toString(); - }, - async confirm(): Promise { - const next = confirms.shift(); - if (next === CANCEL_PROMPT) { - cancelWithError(); - } - return next === true; - }, - cancel(): void { - return; - }, - }; -} - -function makeIo(options: { isTTY?: boolean; stdinIsTTY?: boolean } = {}) { - let stdout = ''; - let stderr = ''; - return { - io: { - stdin: { - isTTY: options.stdinIsTTY, - }, - stdout: { - isTTY: options.isTTY, - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -describe('runKtxConnectionMetabaseSetup', () => { - const fakeMetabaseCredential = 'mb_example'; - const existingMetabaseCredential = 'mb_existing'; - const fakeAdminCredential = 'admin-secret-value-123'; - - let tempDir: string; - let projectDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-metabase-setup-')); - projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'metabase-setup' }); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - async function writeConnections(connections: Record) { - const project = await loadKtxProject({ projectDir }); - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig({ - ...project.config, - connections, - }), - 'ktx', - 'ktx@example.com', - 'Seed Metabase setup test connections', - ); - } - - function makeMetabaseClient(options: { - testConnectionSuccess: boolean; - databases: Array<{ - id: number; - name: string; - engine: string; - details?: { host?: string; dbname?: string }; - is_sample?: boolean; - }>; - }) { - return { - testConnection: vi.fn().mockResolvedValue({ success: options.testConnectionSuccess }), - getDatabases: vi.fn().mockResolvedValue(options.databases), - cleanup: vi.fn().mockResolvedValue(undefined), - }; - } - - it('covers the headless happy path', async () => { - await writeConnections({ - orbit: { - driver: 'postgres', - url: 'postgresql://readonly@pg.internal/analytics', - readonly: true, - }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [2], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Connection: metabase'); - expect(io.stdout()).toContain('Discovered 1 database'); - expect(io.stdout()).toContain(`ktx ingest run --connection-id metabase --adapter metabase --project-dir ${projectDir}`); - expect(io.stdout()).not.toContain('mb_example'); - expect(io.stderr()).not.toContain('mb_example'); - - const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(config).toContain('driver: metabase'); - expect(config).toContain('api_url: http://metabase.example.test:3000'); - expect(config).toContain('api_key: mb_example'); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { - metabaseDatabaseId: 2, - metabaseDatabaseName: 'Analytics', - targetConnectionId: 'orbit', - syncEnabled: true, - }, - ]); - }); - - it('auto-maps and enables sync in --no-input --yes when deterministic', async () => { - await writeConnections({ - orbit: { - driver: 'postgres', - url: 'postgresql://readonly@pg.internal/analytics', - readonly: true, - }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(0); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true }, - ]); - }); - - it('fails in --no-input when mapping/sync are missing and --yes is false', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [{ id: 2, name: 'Analytics', engine: 'postgres', is_sample: false }], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: false, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toMatch(/--map/i); - expect(io.stderr()).toMatch(/--sync/i); - }); - - it('enables sync for explicitly mapped databases in --no-input --yes when --sync is omitted', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [{ id: 2, name: 'Analytics', engine: 'postgres', is_sample: false }], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(0); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true }, - ]); - }); - - it('fails in no-input mode when the Metabase URL is missing', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('missing Metabase URL'); - }); - - it('fails in no-input mode when the Metabase API key is missing', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('missing Metabase API key'); - }); - - it('names missing minting flags before rejecting minting', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - - const missingUsernameIo = makeIo(); - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - mintApiKey: true, - metabasePassword: fakeAdminCredential, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - missingUsernameIo.io, - ), - ).resolves.toBe(1); - expect(missingUsernameIo.stderr()).toContain('--username'); - - const missingPasswordIo = makeIo(); - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - mintApiKey: true, - metabaseUsername: 'user', - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - missingPasswordIo.io, - ), - ).resolves.toBe(1); - expect(missingPasswordIo.stderr()).toContain('--password'); - - const mintedMetabaseCredential = 'mb_minted'; - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const createMetabaseClient = vi.fn(async () => metabaseClient as never); - const mintMetabaseApiKey = vi.fn(async () => mintedMetabaseCredential); - const mintingIo = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - mintApiKey: true, - metabaseUsername: 'user', - metabasePassword: fakeAdminCredential, - mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [2], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - mintingIo.io, - { createMetabaseClient, mintMetabaseApiKey }, - ), - ).resolves.toBe(0); - - expect(mintMetabaseApiKey).toHaveBeenCalledTimes(1); - expect(mintMetabaseApiKey).toHaveBeenCalledWith( - expect.objectContaining({ - url: 'http://metabase.example.test:3000', - username: 'user', - password: fakeAdminCredential, - }), - expect.anything(), - ); - - expect(createMetabaseClient).toHaveBeenCalledTimes(1); - expect(mintingIo.stdout()).not.toContain(mintedMetabaseCredential); - expect(mintingIo.stderr()).not.toContain(mintedMetabaseCredential); - expect(mintingIo.stdout()).not.toContain(fakeAdminCredential); - expect(mintingIo.stderr()).not.toContain(fakeAdminCredential); - - const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(config).toContain('driver: metabase'); - expect(config).toContain('api_url: http://metabase.example.test:3000'); - expect(config).toContain(`api_key: ${mintedMetabaseCredential}`); - }); - - it('requires at least one warehouse connection', async () => { - await writeConnections({}); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('Add a warehouse connection first'); - }); - - it('fails in --no-input --yes when a deterministic warehouse mapping cannot be derived', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - warehouse2: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toMatch(/--map/i); - expect(io.stderr()).toMatch(/--sync/i); - }); - - it('auto-enables sync in --no-input --yes from explicit mappings even when multiple databases are discovered', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 1, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - { - id: 2, - name: 'Finance', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'finance' }, - is_sample: false, - }, - ], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [{ metabaseDatabaseId: 1, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(0); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { metabaseDatabaseId: 1, targetConnectionId: 'orbit', syncEnabled: true }, - { metabaseDatabaseId: 2, targetConnectionId: null, syncEnabled: false }, - ]); - }); - - it('suggests updating api_key or using minting when authentication fails', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const metabaseClient = makeMetabaseClient({ testConnectionSuccess: false, databases: [] }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('connections.metabase.api_key'); - expect(io.stderr()).toContain('--mint-api-key'); - expect(io.stderr()).not.toContain('mb_example'); - }); - - it('fails when Metabase returns no usable databases', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [{ id: 1, name: 'Sample', engine: 'h2', is_sample: true }], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('no usable databases'); - }); - - it('preserves setup writes when --run-ingest fails and reports the debug command', async () => { - await writeConnections({ - orbit: { - driver: 'postgres', - url: 'postgresql://readonly@pg.internal/analytics', - readonly: true, - }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [2], - syncMode: 'ALL', - runIngest: true, - yes: true, - inputMode: 'disabled', - }, - io.io, - { - createMetabaseClient: async () => metabaseClient as never, - runPublicIngest: vi.fn(async () => 1), - }, - ), - ).resolves.toBe(1); - - const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(config).toContain('driver: metabase'); - expect(io.stderr()).toContain(`ktx ingest run --connection-id metabase --adapter metabase --project-dir ${projectDir}`); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { metabaseDatabaseId: 2, targetConnectionId: 'orbit' }, - ]); - }); - - it('reuses existing connection id and values when --id, --url, and --api-key are omitted', async () => { - await writeConnections({ - 'prod-metabase': { - driver: 'metabase', - api_url: 'http://metabase.example.test:3000', - api_key: existingMetabaseCredential, - }, - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [{ id: 2, name: 'Analytics', engine: 'postgres', is_sample: false }], - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - mintApiKey: false, - mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [2], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - { createMetabaseClient: async () => metabaseClient as never }, - ), - ).resolves.toBe(0); - - expect(io.stdout()).toContain('Connection: prod-metabase'); - expect(io.stdout()).not.toContain('mb_existing'); - expect(io.stderr()).not.toContain('mb_existing'); - }); - - it('covers interactive happy path when URL/key/mapping/sync are missing but deterministic', async () => { - await writeConnections({ - orbit: { - driver: 'postgres', - url: 'postgresql://readonly@pg.internal/analytics', - readonly: true, - }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo({ isTTY: true, stdinIsTTY: true }); - const interactiveMetabaseCredential = 'mb_interactive_fixture'; - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: false, - inputMode: 'auto', - }, - io.io, - { - createMetabaseClient: async () => metabaseClient as never, - prompts: createTestMetabaseSetupPromptAdapter({ - texts: ['http://metabase.example.test:3000'], - selects: ['paste'], - passwords: [interactiveMetabaseCredential], - confirms: [true], - }), - }, - ), - ).resolves.toBe(0); - - const config = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(config).toContain('driver: metabase'); - expect(config).toContain('api_url: http://metabase.example.test:3000'); - expect(config).toContain(`api_key: ${interactiveMetabaseCredential}`); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { - metabaseDatabaseId: 2, - targetConnectionId: 'orbit', - syncEnabled: true, - }, - ]); - - expect(io.stdout()).not.toContain(interactiveMetabaseCredential); - expect(io.stderr()).not.toContain(interactiveMetabaseCredential); - }); - - it('guides interactive setup for multiple databases and warehouses', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics', readonly: true }, - warehouse2: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/finance', readonly: true }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - { - id: 3, - name: 'Finance', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'finance' }, - is_sample: false, - }, - ], - }); - const io = makeIo({ isTTY: true, stdinIsTTY: true }); - const interactiveMetabaseCredential = 'mb_interactive_multi'; - const events: string[] = []; - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: false, - inputMode: 'auto', - }, - io.io, - { - createMetabaseClient: async () => metabaseClient as never, - prompts: createTestMetabaseSetupPromptAdapter({ - texts: ['http://metabase.example.test:3000'], - selects: ['paste', 'orbit', 'warehouse2'], - passwords: [interactiveMetabaseCredential], - multiselects: [[2, 3], [2]], - confirms: [true], - events, - }), - }, - ), - ).resolves.toBe(0); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toMatchObject([ - { metabaseDatabaseId: 2, targetConnectionId: 'orbit', syncEnabled: true }, - { metabaseDatabaseId: 3, targetConnectionId: 'warehouse2', syncEnabled: false }, - ]); - - expect(io.stdout()).not.toContain(interactiveMetabaseCredential); - expect(io.stderr()).not.toContain(interactiveMetabaseCredential); - expect(events).toContain( - 'multiselect:Select Metabase databases to configure\nUse Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.', - ); - expect(events).toContain( - 'multiselect:Enable sync for which databases?\nUse Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.', - ); - }); - - it('emits guided progress via the interaction toolkit in interactive mode', async () => { - await writeConnections({ - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics', readonly: true }, - }); - - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo({ isTTY: true, stdinIsTTY: true }); - const interactiveMetabaseCredential = 'mb_interaction_toolkit'; - const events: string[] = []; - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: false, - inputMode: 'auto', - }, - io.io, - { - createMetabaseClient: async () => metabaseClient as never, - prompts: createTestMetabaseSetupPromptAdapter({ - events, - texts: ['http://metabase.example.test:3000'], - selects: ['paste'], - passwords: [interactiveMetabaseCredential], - confirms: [true], - }), - }, - ), - ).resolves.toBe(0); - - expect(events).toContain('intro:KTX Metabase setup'); - expect(events.some((event) => event.startsWith('spinner.start:Testing Metabase connection'))).toBe(true); - expect(events.some((event) => event.startsWith('spinner.stop:Metabase reachable'))).toBe(true); - expect(events.some((event) => event.startsWith('spinner.start:Discovering Metabase databases'))).toBe(true); - expect(events.some((event) => event.startsWith('log.success:Discovered 1 database'))).toBe(true); - expect(events.some((event) => event.startsWith('note:Summary:'))).toBe(true); - expect(events).toContain('outro:Metabase setup complete'); - - expect(events.join('\n')).not.toContain(interactiveMetabaseCredential); - expect(io.stdout()).not.toContain(interactiveMetabaseCredential); - expect(io.stderr()).not.toContain(interactiveMetabaseCredential); - }); - - it('fails in --no-input when multiple Metabase connections exist and --id is omitted', async () => { - await writeConnections({ - metabase1: { - driver: 'metabase', - api_url: 'http://metabase.example.test:3000', - api_key: existingMetabaseCredential, - }, - metabase2: { - driver: 'metabase', - api_url: 'http://metabase.example.test:3000', - api_key: existingMetabaseCredential, - }, - orbit: { driver: 'postgres', url: 'postgresql://readonly@pg.internal/analytics' }, - }); - const io = makeIo(); - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toMatch(/--id/i); - }); - - it('treats prompt cancellation as a clean exit without writes', async () => { - await writeConnections({ - orbit: { - driver: 'postgres', - url: 'postgresql://readonly@pg.internal/analytics', - readonly: true, - }, - }); - - const beforeConfig = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - const metabaseClient = makeMetabaseClient({ - testConnectionSuccess: true, - databases: [ - { - id: 2, - name: 'Analytics', - engine: 'postgres', - details: { host: 'pg.internal', dbname: 'analytics' }, - is_sample: false, - }, - ], - }); - const io = makeIo({ isTTY: true, stdinIsTTY: true }); - const cancelMetabaseCredential = 'mb_cancel_fixture'; - - await expect( - runKtxConnectionMetabaseSetup( - { - command: 'setup', - projectDir, - mintApiKey: false, - mappings: [], - syncEnabledDatabaseIds: [], - syncMode: 'ALL', - runIngest: false, - yes: false, - inputMode: 'auto', - }, - io.io, - { - createMetabaseClient: async () => metabaseClient as never, - prompts: createTestMetabaseSetupPromptAdapter({ - texts: ['http://metabase.example.test:3000'], - selects: ['paste'], - passwords: [cancelMetabaseCredential], - confirms: [CANCEL_PROMPT], - }), - }, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('Setup cancelled.'); - expect(io.stderr()).not.toContain(cancelMetabaseCredential); - - const afterConfig = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(afterConfig).toBe(beforeConfig); - - const updatedProject = await loadKtxProject({ projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - await expect(store.listDatabaseMappings('metabase')).resolves.toEqual([]); - }); -}); diff --git a/packages/cli/src/commands/connection-metabase-setup.ts b/packages/cli/src/commands/connection-metabase-setup.ts deleted file mode 100644 index 2321ea3d..00000000 --- a/packages/cli/src/commands/connection-metabase-setup.ts +++ /dev/null @@ -1,786 +0,0 @@ -import type { Option as ClackOption } from '@clack/prompts'; -import { - cancel, - confirm, - intro, - isCancel, - log, - multiselect, - note, - outro, - password, - select, - text, -} from '@clack/prompts'; -import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections'; -import { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultMetabaseConnectionClientFactory, - LocalMetabaseSourceStateReader, - MetabaseClient, - type MetabaseDatabase, - type MetabaseRuntimeClient, - type MetabaseSyncMode, - metabaseRuntimeConfigFromLocalConnection, - validateMappingPhysicalMatch, -} from '@ktx/context/ingest'; -import { - type KtxLocalProject, - type KtxProjectConnectionConfig, - ktxLocalStateDbPath, - loadKtxProject, - serializeKtxProjectConfig, -} from '@ktx/context/project'; - -import { createClackSpinner, type KtxCliSpinner } from '../clack.js'; -import type { KtxCliIo } from '../cli-runtime.js'; -import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js'; -import { type KtxPublicIngestArgs, runKtxPublicIngest } from '../public-ingest.js'; - -export type KtxMetabaseSetupInputMode = 'auto' | 'disabled'; - -export type MetabaseSetupSyncMode = MetabaseSyncMode; - -type MetabaseSetupPromptOption = ClackOption; - -export interface MetabaseSetupLogger { - info(message: string): void; - step(message: string): void; - success(message: string): void; - warn(message: string): void; - error(message: string): void; -} - -export interface MetabaseSetupPromptAdapter { - intro(title?: string): void; - outro(message?: string): void; - note(message: string, title: string): void; - log: MetabaseSetupLogger; - spinner(): KtxCliSpinner; - select(options: { message: string; options: Array> }): Promise; - multiselect(options: { - message: string; - options: Array>; - initialValues?: Value[]; - required?: boolean; - maxItems?: number; - }): Promise; - text(options: { message: string; placeholder?: string }): Promise; - password(options: { message: string }): Promise; - confirm(options: { message: string; initialValue?: boolean }): Promise; - cancel(message: string): void; -} - -type KtxMetabaseSetupInteractiveIo = KtxCliIo & { - stdin?: { isTTY?: boolean }; -}; - -export interface MetabaseSetupMappingAssignment { - metabaseDatabaseId: number; - targetConnectionId: string; -} - -export interface MintMetabaseApiKeyArgs { - url: string; - username: string; - password: string; -} - -export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KtxCliIo) => Promise; - -export interface KtxConnectionMetabaseSetupArgs { - command: 'setup'; - projectDir: string; - connectionId?: string; - url?: string; - apiKey?: string; - mintApiKey: boolean; - metabaseUsername?: string; - metabasePassword?: string; - mappings: MetabaseSetupMappingAssignment[]; - syncEnabledDatabaseIds: number[]; - syncMode: MetabaseSetupSyncMode; - runIngest: boolean; - yes: boolean; - inputMode: KtxMetabaseSetupInputMode; -} - -export interface KtxConnectionMetabaseSetupDeps { - createMetabaseClient?: ( - project: KtxLocalProject, - connectionId: string, - ) => Promise>; - mintMetabaseApiKey?: MintMetabaseApiKey; - prompts?: MetabaseSetupPromptAdapter; - runPublicIngest?: (args: Extract, io: KtxCliIo) => Promise; -} - -function isMetabaseConnection(connection: KtxProjectConnectionConfig | undefined): boolean { - return ( - String(connection?.driver ?? '') - .trim() - .toLowerCase() === 'metabase' - ); -} - -function stringField(value: unknown): string | undefined { - return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; -} - -function uniqueSorted(values: number[]): number[] { - return [...new Set(values)].sort((a, b) => a - b); -} - -function resolveMetabaseUrl(connection: KtxProjectConnectionConfig | undefined): string | undefined { - return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url); -} - -function resolveLiteralMetabaseApiKey(connection: KtxProjectConnectionConfig | undefined): string | undefined { - return stringField(connection?.api_key) ?? stringField(connection?.apiKey); -} - -function listMetabaseConnectionIds(project: KtxLocalProject): string[] { - return Object.entries(project.config.connections) - .filter(([_connectionId, connection]) => isMetabaseConnection(connection)) - .map(([connectionId]) => connectionId) - .sort(); -} - -function listWarehouseConnectionIds(project: KtxLocalProject): string[] { - return Object.entries(project.config.connections) - .filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null) - .map(([connectionId]) => connectionId) - .sort(); -} - -function redactSecrets(message: string, secrets: string[]): string { - let result = message; - for (const secret of secrets) { - if (!secret) { - continue; - } - result = result.split(secret).join('[redacted]'); - } - return result; -} - -async function createDefaultMetabaseClient( - project: KtxLocalProject, - connectionId: string, -): Promise> { - const factory = new DefaultMetabaseConnectionClientFactory( - (metabaseConnectionId) => - metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]), - DEFAULT_METABASE_CLIENT_CONFIG, - ); - return factory.createClient(connectionId); -} - -async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise { - const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG); - const sessionId = await loginClient.createSession(args.username, args.password); - const sessionClient = new MetabaseClient( - { apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' }, - DEFAULT_METABASE_CLIENT_CONFIG, - ); - const groups = await sessionClient.getPermissionGroups(); - const adminGroup = groups.find((group) => group.name === 'Administrators'); - - if (!adminGroup) { - throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key'); - } - - const mintedKey = await sessionClient.createApiKey({ - groupId: adminGroup.id, - name: `KTX CLI ${new Date().toISOString()}`, - }); - const trimmedKey = stringField(mintedKey); - if (!trimmedKey) { - throw new Error('Metabase API key minting returned an empty key'); - } - return trimmedKey; -} - -function ensureNotCancelled(value: T | symbol, prompts: Pick): T { - if (isCancel(value)) { - prompts.cancel('Setup cancelled.'); - throw new Error('Setup cancelled.'); - } - return value as T; -} - -export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter { - return { - intro(title?: string): void { - intro(title); - }, - outro(message?: string): void { - outro(message); - }, - note(message: string, title: string): void { - note(message, title); - }, - log: { - info(message: string): void { - log.info(message); - }, - step(message: string): void { - log.step(message); - }, - success(message: string): void { - log.success(message); - }, - warn(message: string): void { - log.warn(message); - }, - error(message: string): void { - log.error(message); - }, - }, - spinner(): KtxCliSpinner { - return createClackSpinner(); - }, - async select(options: { - message: string; - options: Array>; - }): Promise { - return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this); - }, - async multiselect(options: { - message: string; - options: Array>; - initialValues?: Value[]; - required?: boolean; - maxItems?: number; - }): Promise { - return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this); - }, - async text(options: { message: string; placeholder?: string }): Promise { - return ensureNotCancelled(await text(options), this); - }, - async password(options: { message: string }): Promise { - return ensureNotCancelled(await password(options), this); - }, - async confirm(options: { message: string; initialValue?: boolean }): Promise { - return ensureNotCancelled(await confirm(options), this); - }, - cancel(message: string): void { - cancel(message); - }, - }; -} - -function isInteractiveMetabaseSetupIo( - args: Pick, - io: KtxMetabaseSetupInteractiveIo, -): boolean { - return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true; -} - -function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{ - id: number; - name: string; - engine: string; - host: string | null; - dbName: string | null; -}> { - return databases - .filter((database) => database.is_sample !== true) - .map((database) => ({ - id: database.id, - name: database.name, - engine: stringField(database.engine) ?? 'unknown', - host: stringField(database.details?.host) ?? null, - dbName: stringField(database.details?.dbname) ?? null, - })); -} - -function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) { - const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]); - if (!descriptor) { - return { connection_type: 'UNKNOWN' }; - } - return { - connection_type: descriptor.connection_type, - host: descriptor.host ?? null, - database: descriptor.database ?? null, - account: descriptor.account ?? null, - project_id: descriptor.project_id ?? null, - dataset_id: descriptor.dataset_id ?? null, - ...descriptor.connection_params, - }; -} - -function noteMetabaseSetupSummary(options: { - prompts: MetabaseSetupPromptAdapter; - connectionId: string; - url: string; - mappings: MetabaseSetupMappingAssignment[]; - syncEnabledDatabaseIds: number[]; -}): void { - const mappingLines = options.mappings - .map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`) - .join('\n'); - const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n'); - - options.prompts.note( - [ - `Connection: ${options.connectionId}`, - `URL: ${options.url}`, - '', - 'Mappings:', - mappingLines || ' (none)', - '', - 'Sync enabled:', - syncLines || ' (none)', - ].join('\n'), - 'Summary', - ); -} - -export async function runKtxConnectionMetabaseSetup( - args: KtxConnectionMetabaseSetupArgs, - io: KtxCliIo, - deps: KtxConnectionMetabaseSetupDeps = {}, -): Promise { - let apiKeyForRedaction = args.apiKey; - let passwordForRedaction = args.metabasePassword; - const interactiveIo = io as KtxMetabaseSetupInteractiveIo; - const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo); - const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined); - - try { - if (isInteractive && prompts) { - prompts.intro('KTX Metabase setup'); - } - - const project = await loadKtxProject({ projectDir: args.projectDir }); - const existingMetabaseConnectionIds = listMetabaseConnectionIds(project); - let connectionId: string; - - if (args.connectionId) { - connectionId = args.connectionId; - } else if (existingMetabaseConnectionIds.length === 1) { - const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0]; - if (!onlyMetabaseConnectionId) { - throw new Error('No Metabase connection id was resolved'); - } - connectionId = onlyMetabaseConnectionId; - } else if (existingMetabaseConnectionIds.length > 1) { - if (!isInteractive || !prompts) { - throw new Error( - `Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`, - ); - } - connectionId = await prompts.select({ - message: 'Select the Metabase connection to configure', - options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })), - }); - } else { - connectionId = 'metabase'; - } - - const existingConnection = project.config.connections[connectionId]; - const warehouseConnectionIds = listWarehouseConnectionIds(project); - - if (warehouseConnectionIds.length === 0) { - throw new Error('Add a warehouse connection first'); - } - - let url = args.url ?? resolveMetabaseUrl(existingConnection); - let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection); - apiKeyForRedaction = apiKey; - - if (!url && isInteractive && prompts) { - url = stringField( - await prompts.text({ - message: 'Metabase API URL', - placeholder: 'http://localhost:3000', - }), - ); - } - - if (args.inputMode === 'disabled' && !url) { - throw new Error('missing Metabase URL'); - } - - if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) { - const reuse = await prompts.confirm({ - message: `Reuse the existing Metabase API key from connections.${connectionId}?`, - initialValue: true, - }); - if (!reuse) { - apiKey = undefined; - apiKeyForRedaction = undefined; - } - } - - if (args.mintApiKey) { - let username = stringField(args.metabaseUsername); - let metabasePassword = stringField(args.metabasePassword); - - if (isInteractive && prompts) { - if (!username) { - username = stringField(await prompts.text({ message: 'Metabase admin username' })); - } - if (!metabasePassword) { - metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' })); - } - } - - if (!username) { - throw new Error('--mint-api-key requires --username'); - } - if (!metabasePassword) { - throw new Error('--mint-api-key requires --password'); - } - if (!url) { - throw new Error('Metabase URL is required (use --url)'); - } - - passwordForRedaction = metabasePassword; - apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)( - { url, username, password: metabasePassword }, - io, - ); - apiKeyForRedaction = apiKey; - } - - if (!apiKey && isInteractive && prompts) { - const credentialMode = await prompts.select({ - message: 'Metabase credentials', - options: [ - { value: 'paste', label: 'Paste API key' }, - { value: 'mint', label: 'Mint API key' }, - ], - }); - - if (credentialMode === 'paste') { - apiKey = stringField(await prompts.password({ message: 'Metabase API key' })); - apiKeyForRedaction = apiKey; - } else { - const username = stringField(await prompts.text({ message: 'Metabase admin username' })); - const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' })); - if (!username) { - throw new Error('Metabase username is required'); - } - if (!metabasePassword) { - throw new Error('Metabase password is required'); - } - if (!url) { - throw new Error('Metabase URL is required (use --url)'); - } - - passwordForRedaction = metabasePassword; - apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)( - { url, username, password: metabasePassword }, - io, - ); - apiKeyForRedaction = apiKey; - } - } - - if (args.inputMode === 'disabled' && !apiKey) { - throw new Error('missing Metabase API key'); - } - - if (!url) { - throw new Error('Metabase URL is required (use --url)'); - } - if (!apiKey) { - throw new Error('Metabase API key is required (use --api-key)'); - } - - const transientConnectionConfig: KtxProjectConnectionConfig = { - ...(existingConnection ?? {}), - driver: 'metabase', - api_url: url, - api_key: apiKey, - }; - const configWithTransient = { - ...project.config, - connections: { - ...project.config.connections, - [connectionId]: transientConnectionConfig, - }, - }; - const discoveryProject: KtxLocalProject = { ...project, config: configWithTransient }; - - for (const mapping of args.mappings) { - if (!configWithTransient.connections[mapping.targetConnectionId]) { - throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`); - } - } - - const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId); - try { - const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined; - authSpinner?.start('Testing Metabase connection'); - const testResult = await client.testConnection(); - if (!testResult.success) { - authSpinner?.error('Metabase authentication failed'); - throw new Error( - `Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`, - ); - } - authSpinner?.stop('Metabase reachable'); - - const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined; - discoverySpinner?.start('Discovering Metabase databases'); - const discovered = normalizeDiscoveredDatabases(await client.getDatabases()); - discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`); - if (isInteractive && prompts) { - prompts.log.success( - `Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`, - ); - } - if (discovered.length === 0) { - throw new Error('Metabase auth worked but no usable databases were returned'); - } - - let resolvedMappings = args.mappings; - let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds; - - if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) { - resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId)); - } - - if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) { - const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined; - const compatibleWarehouses = onlyDiscoveredDatabase - ? warehouseConnectionIds.filter((warehouseConnectionId) => { - const mismatchReason = validateMappingPhysicalMatch( - { - metabaseEngine: onlyDiscoveredDatabase.engine, - metabaseDbName: onlyDiscoveredDatabase.dbName, - metabaseHost: onlyDiscoveredDatabase.host, - }, - targetPhysicalInfo(project, warehouseConnectionId), - ); - return !mismatchReason; - }) - : []; - const onlyWarehouseConnectionId = compatibleWarehouses[0]; - - if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) { - if (args.yes) { - resolvedMappings = [ - { metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId }, - ]; - resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id]; - } else if (isInteractive && prompts) { - const proposedMappings = [ - { metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId }, - ]; - const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id]; - noteMetabaseSetupSummary({ - prompts, - connectionId, - url, - mappings: proposedMappings, - syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds, - }); - const confirmed = await prompts.confirm({ - message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`, - initialValue: true, - }); - if (!confirmed) { - prompts.cancel('Setup cancelled.'); - throw new Error('Setup cancelled.'); - } - resolvedMappings = proposedMappings; - resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds; - } else { - throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync'); - } - } else if (isInteractive && prompts) { - const selectedDatabaseIds = await prompts.multiselect({ - message: withMultiselectNavigation('Select Metabase databases to configure'), - options: discovered.map((database) => ({ - value: database.id, - label: `${database.id}: ${database.name}`, - hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '), - })), - required: true, - }); - - resolvedMappings = []; - for (const databaseId of selectedDatabaseIds) { - const database = discovered.find((candidate) => candidate.id === databaseId); - if (!database) { - throw new Error(`Selected database id ${databaseId} was not discovered`); - } - - const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId); - if (existingMapping) { - resolvedMappings.push(existingMapping); - continue; - } - - const targetConnectionId = await prompts.select({ - message: `Map Metabase database ${database.id} ("${database.name}") to which KTX connection?`, - options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })), - }); - resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId }); - } - - const syncIds = await prompts.multiselect({ - message: withMultiselectNavigation('Enable sync for which databases?'), - options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })), - initialValues: selectedDatabaseIds, - required: true, - }); - resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds); - - if (!args.yes) { - noteMetabaseSetupSummary({ - prompts, - connectionId, - url, - mappings: resolvedMappings, - syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds, - }); - const confirmed = await prompts.confirm({ - message: 'Write changes to ktx.yaml and enable sync?', - initialValue: true, - }); - if (!confirmed) { - prompts.cancel('Setup cancelled.'); - throw new Error('Setup cancelled.'); - } - } - } else if (args.inputMode === 'disabled') { - throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync'); - } - } - - if ( - args.inputMode === 'disabled' && - resolvedMappings.length > 0 && - resolvedSyncEnabledDatabaseIds.length === 0 - ) { - throw new Error('Metabase sync selection is required in --no-input mode; pass --sync '); - } - - const discoveredIds = new Set(discovered.map((database) => database.id)); - for (const mapping of resolvedMappings) { - if (!discoveredIds.has(mapping.metabaseDatabaseId)) { - throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`); - } - } - for (const syncId of resolvedSyncEnabledDatabaseIds) { - if (!discoveredIds.has(syncId)) { - throw new Error(`Sync database id ${syncId} was not discovered`); - } - } - - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig(configWithTransient), - 'ktx', - 'ktx@example.com', - `Setup Metabase connection ${connectionId}`, - ); - - const updatedProject = await loadKtxProject({ projectDir: args.projectDir }); - const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) }); - - await store.refreshDiscoveredDatabases({ connectionId, discovered }); - - for (const mapping of resolvedMappings) { - await store.upsertDatabaseMapping({ - connectionId, - metabaseDatabaseId: mapping.metabaseDatabaseId, - targetConnectionId: mapping.targetConnectionId, - syncEnabled: false, - source: 'cli', - }); - } - - for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) { - await store.setMappingSyncEnabled({ - connectionId, - metabaseDatabaseId, - syncEnabled: true, - }); - } - - const existingSyncState = await store.getSourceState(connectionId); - await store.setSyncState({ - connectionId, - syncMode: args.syncMode, - defaultTagNames: existingSyncState.defaultTagNames, - selections: existingSyncState.selections, - }); - - const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId); - if (unhydrated.length > 0) { - io.stderr.write( - `Sync-enabled mappings are missing discovery metadata; run ktx connection mapping refresh ${connectionId} --auto-accept\n`, - ); - return 1; - } - - const rows = await store.listDatabaseMappings(connectionId); - const physicalFailures = rows.flatMap((row) => { - if (!row.targetConnectionId) { - return []; - } - const reason = validateMappingPhysicalMatch( - { metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost }, - updatedProject.config.connections[row.targetConnectionId] - ? targetPhysicalInfo(updatedProject, row.targetConnectionId) - : { connection_type: 'UNKNOWN' }, - ); - return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : []; - }); - if (physicalFailures.length > 0) { - for (const failure of physicalFailures) { - io.stderr.write(`${failure}\n`); - } - return 1; - } - - io.stdout.write(`Connection: ${connectionId}\n`); - io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`); - io.stdout.write( - `Next: ktx ingest run --connection-id ${connectionId} --adapter metabase --project-dir ${args.projectDir}\n`, - ); - - if (args.runIngest) { - const ingestRunner = deps.runPublicIngest ?? runKtxPublicIngest; - const exitCode = await ingestRunner( - { - command: 'run', - projectDir: args.projectDir, - targetConnectionId: connectionId, - all: false, - json: false, - inputMode: 'disabled', - }, - io, - ); - if (exitCode !== 0) { - io.stderr.write( - `Ingest failed; re-run: ktx ingest run --connection-id ${connectionId} --adapter metabase --project-dir ${args.projectDir}\n`, - ); - return 1; - } - } - - if (isInteractive && prompts) { - prompts.outro('Metabase setup complete'); - } - - return 0; - } finally { - await client.cleanup(); - } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - io.stderr.write( - `${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`, - ); - return 1; - } -} diff --git a/packages/cli/src/commands/connection-notion-commands.ts b/packages/cli/src/commands/connection-notion-commands.ts deleted file mode 100644 index 8f021ad9..00000000 --- a/packages/cli/src/commands/connection-notion-commands.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { type Command, InvalidArgumentError } from '@commander-js/extra-typings'; -import { collectOption, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js'; -import type { KtxConnectionNotionArgs } from './connection-notion.js'; - -interface NotionPickOptions { - input?: boolean; - rootPageId: string[]; -} - -function parseSafeConnectionId(value: string): string { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - return value; -} - -function uniqueInOrder(values: string[]): string[] { - const seen = new Set(); - const result: string[] = []; - for (const value of values) { - if (!seen.has(value)) { - seen.add(value); - result.push(value); - } - } - return result; -} - -function normalizeNotionPageId(value: string): string { - const trimmed = value.trim(); - const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed; - if (!/^[0-9a-fA-F]{32}$/.test(compact)) { - throw new Error(`Invalid Notion page UUID: ${value}`); - } - const lower = compact.toLowerCase(); - return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`; -} - -function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KtxConnectionNotionArgs { - if (options.input !== false) { - return { - command: 'pick', - projectDir, - connectionId, - mode: 'interactive', - }; - } - - const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId)); - if (rootPageIds.length === 0) { - throw new Error('connection notion pick --no-input requires at least one --root-page-id'); - } - return { - command: 'pick', - projectDir, - connectionId, - mode: 'non-interactive', - rootPageIds, - }; -} - -async function runConnectionNotionArgs(context: KtxCliCommandContext, args: KtxConnectionNotionArgs): Promise { - const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKtxConnectionNotion; - context.setExitCode(await runner(args, context.io)); -} - -export function registerConnectionNotionCommands(connect: Command, context: KtxCliCommandContext): void { - const notion = connect - .command('notion') - .description('Configure Notion source selection') - .showHelpAfterError() - .addHelpText( - 'after', - '\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n', - ); - - notion.action(() => { - notion.outputHelp(); - context.setExitCode(0); - }); - - notion - .command('pick') - .description('Pick Notion root pages for a configured Notion connection') - .argument('', 'Notion connection id', parseSafeConnectionId) - .option('--no-input', 'Disable interactive terminal input') - .option('--root-page-id ', 'Root page UUID to crawl; repeatable with --no-input', collectOption, []) - .showHelpAfterError() - .action(async (connectionId: string, options: NotionPickOptions, command) => { - await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options)); - }); -} diff --git a/packages/cli/src/commands/connection-notion.test.ts b/packages/cli/src/commands/connection-notion.test.ts deleted file mode 100644 index 3315e1cc..00000000 --- a/packages/cli/src/commands/connection-notion.test.ts +++ /dev/null @@ -1,513 +0,0 @@ -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { - initKtxProject, - loadKtxProject, - serializeKtxProjectConfig, - type KtxProjectConfig, -} from '@ktx/context/project'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - applyNotionPickerWriteback, - discoverNotionPickerPages, - notionPickerPageFromSearchResult, - normalizeNotionPageId, - resolveNotionWorkspaceLabel, - runKtxConnectionNotion, - type NotionPickerApi, - type PickerRenderInput, - type PickerRenderResult, -} from './connection-notion.js'; - -function makeIo() { - let stdout = ''; - let stderr = ''; - return { - io: { - stdout: { - write: (chunk: string) => { - stdout += chunk; - }, - }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, - }, - }, - stdout: () => stdout, - stderr: () => stderr, - }; -} - -type FakeNotionSearchPage = Record & { id: string; object: 'page' }; - -const PAGE_IDS = { - engineering: '11111111-1111-1111-1111-111111111111', - architecture: '22222222-2222-2222-2222-222222222222', - stale: '99999999-9999-9999-9999-999999999999', -}; - -function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage { - return { - object: 'page', - id, - archived: false, - parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true }, - properties: { - title: { - type: 'title', - title: [{ plain_text: title }], - }, - }, - }; -} - -function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi { - return { - search: vi.fn(async (_filterValue, startCursor) => { - if (startCursor === 'page-2') { - return { results: pages.slice(2), hasMore: false, nextCursor: null }; - } - return { - results: pages.slice(0, 2), - hasMore: pages.length > 2, - nextCursor: pages.length > 2 ? 'page-2' : null, - }; - }), - retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })), - }; -} - -describe('normalizeNotionPageId', () => { - it('accepts dashed and compact UUIDs', () => { - expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe( - '11111111-2222-3333-4444-555555555555', - ); - expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe( - 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', - ); - }); -}); - -describe('runKtxConnectionNotion', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-notion-pick-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise { - const project = await loadKtxProject({ projectDir }); - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig(config), - 'ktx', - 'ktx@example.com', - 'seed test config', - ); - } - - it('rejects unsafe connection ids before loading a project', async () => { - const io = makeIo(); - const loadProject = vi.fn(async () => { - throw new Error('loadProject should not be called'); - }); - - await expect( - runKtxConnectionNotion( - { - command: 'pick', - projectDir: '/tmp/project', - connectionId: '../evil', - mode: 'interactive', - }, - io.io, - { loadProject }, - ), - ).resolves.toBe(1); - - expect(loadProject).not.toHaveBeenCalled(); - expect(io.stderr()).toContain('Unsafe connection id: ../evil'); - }); - - it('writes selected root_page_ids while preserving every other Notion connection field', async () => { - const projectDir = join(tempDir, 'project'); - const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeProjectConfig(projectDir, { - ...initialized.config, - connections: { - 'notion-main': { - driver: 'notion', - auth_token_ref: 'env:NOTION_TOKEN', - crawl_mode: 'all_accessible', - root_page_ids: ['99999999-9999-9999-9999-999999999999'], - root_database_ids: ['database-1'], - root_data_source_ids: ['data-source-1'], - max_pages_per_run: 12, - max_knowledge_creates_per_run: 2, - max_knowledge_updates_per_run: 7, - last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}', - unknown_future_field: 'keep-me', - }, - }, - }); - const io = makeIo(); - - await expect( - runKtxConnectionNotion( - { - command: 'pick', - projectDir, - connectionId: 'notion-main', - mode: 'non-interactive', - rootPageIds: [ - '11111111-2222-3333-4444-555555555555', - '66666666-7777-8888-9999-aaaaaaaaaaaa', - ], - }, - io.io, - ), - ).resolves.toBe(0); - - const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(yaml).toContain('crawl_mode: selected_roots'); - expect(yaml).toContain('root_page_ids:'); - expect(yaml).toContain('11111111-2222-3333-4444-555555555555'); - expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa'); - expect(yaml).toContain('root_database_ids:'); - expect(yaml).toContain('database-1'); - expect(yaml).toContain('root_data_source_ids:'); - expect(yaml).toContain('data-source-1'); - expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\''); - expect(yaml).toContain('unknown_future_field: keep-me'); - expect(io.stdout()).toContain('Connection: notion-main'); - expect(io.stdout()).toContain('rootPageIds: 2'); - expect(io.stdout()).toContain('crawlMode: selected_roots'); - }); - - it('rejects empty writeback, missing connections, and non-Notion connections', async () => { - const projectDir = join(tempDir, 'project'); - const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeProjectConfig(projectDir, { - ...initialized.config, - connections: { - warehouse: { - driver: 'postgres', - url: 'env:DATABASE_URL', - readonly: true, - }, - }, - }); - const project = await loadKtxProject({ projectDir }); - - await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow( - 'connection notion pick requires at least one root page id', - ); - await expect( - applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']), - ).rejects.toThrow('Connection "missing" not found'); - await expect( - applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']), - ).rejects.toThrow('Connection "warehouse" is not a Notion connection'); - }); - - it('extracts picker page inputs from Notion search results', () => { - expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering))) - .toEqual({ - id: PAGE_IDS.architecture, - title: 'Architecture', - archived: false, - parentId: PAGE_IDS.engineering, - }); - - expect( - notionPickerPageFromSearchResult({ - object: 'page', - id: PAGE_IDS.engineering.replaceAll('-', ''), - archived: true, - parent: { type: 'workspace', workspace: true }, - properties: {}, - }), - ).toEqual({ - id: PAGE_IDS.engineering, - title: 'Untitled', - archived: true, - parentId: null, - }); - }); - - it('discovers visible pages up to the cap and reports cap state', async () => { - const api = fakeNotionApi([ - notionPage(PAGE_IDS.engineering, 'Engineering'), - notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering), - notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering), - ]); - - await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({ - pages: [ - { id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }, - { id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering }, - ], - cappedAtCount: 2, - warnings: [], - }); - expect(api.search).toHaveBeenCalledTimes(1); - }); - - it('keeps partial discovery results when Notion search fails after at least one page', async () => { - const api: NotionPickerApi = { - search: vi - .fn() - .mockResolvedValueOnce({ - results: [notionPage(PAGE_IDS.engineering, 'Engineering')], - hasMore: true, - nextCursor: 'cursor-2', - }) - .mockRejectedValueOnce(new Error('rate limit after first page')), - retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })), - }; - - await expect(discoverNotionPickerPages(api)).resolves.toEqual({ - pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }], - cappedAtCount: null, - warnings: ['Notion search stopped early: rate limit after first page'], - }); - }); - - it('uses the Notion workspace name when available and falls back to the connection id', async () => { - await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace'); - await expect( - resolveNotionWorkspaceLabel( - { - search: vi.fn(), - retrieveBotUser: vi.fn(async () => { - throw new Error('users.me unavailable'); - }), - }, - 'notion-main', - ), - ).resolves.toBe('notion-main'); - }); - - it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => { - const projectDir = join(tempDir, 'project'); - const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeProjectConfig(projectDir, { - ...initialized.config, - connections: { - 'notion-main': { - driver: 'notion', - auth_token_ref: 'env:NOTION_TOKEN', - crawl_mode: 'all_accessible', - root_page_ids: [PAGE_IDS.stale], - root_database_ids: ['database-1'], - root_data_source_ids: ['data-source-1'], - max_pages_per_run: 12, - max_knowledge_creates_per_run: 2, - max_knowledge_updates_per_run: 7, - last_successful_cursor: null, - }, - }, - }); - const api = fakeNotionApi([ - notionPage(PAGE_IDS.engineering, 'Engineering'), - notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering), - ]); - const renderPicker = vi.fn(async (input): Promise => { - expect(input.connectionId).toBe('notion-main'); - expect(input.workspaceLabel).toBe('Design Workspace'); - expect(input.currentCrawlMode).toBe('all_accessible'); - expect(input.cappedAtCount).toBeNull(); - expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']); - return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] }; - }); - const io = makeIo(); - - await expect( - runKtxConnectionNotion( - { - command: 'pick', - projectDir, - connectionId: 'notion-main', - mode: 'interactive', - }, - io.io, - { - env: { NOTION_TOKEN: 'ntn_test_token' }, - createNotionApi: vi.fn(() => api), - renderPicker, - }, - ), - ).resolves.toBe(0); - - const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(yaml).toContain('crawl_mode: selected_roots'); - expect(yaml).toContain(PAGE_IDS.engineering); - expect(yaml).not.toContain(PAGE_IDS.stale); - expect(io.stderr()).toContain('1 stored root_page_ids no longer visible'); - expect(io.stdout()).toContain('Connection: notion-main'); - expect(io.stdout()).toContain('rootPageIds: 1'); - }); - - it('uses inline Notion auth_token for interactive discovery', async () => { - const projectDir = join(tempDir, 'project'); - const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeProjectConfig(projectDir, { - ...initialized.config, - connections: { - 'notion-main': { - driver: 'notion', - auth_token: 'ntn_inline_token', - crawl_mode: 'selected_roots', - root_page_ids: [PAGE_IDS.engineering], - root_database_ids: [], - root_data_source_ids: [], - max_pages_per_run: 12, - max_knowledge_creates_per_run: 2, - max_knowledge_updates_per_run: 7, - last_successful_cursor: null, - }, - }, - }); - const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]); - const createNotionApi = vi.fn((authToken: string) => { - expect(authToken).toBe('ntn_inline_token'); - return api; - }); - const io = makeIo(); - - await expect( - runKtxConnectionNotion( - { - command: 'pick', - projectDir, - connectionId: 'notion-main', - mode: 'interactive', - }, - io.io, - { - createNotionApi, - renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), - }, - ), - ).resolves.toBe(0); - - expect(createNotionApi).toHaveBeenCalledOnce(); - expect(io.stdout()).toContain('No changes saved.'); - }); - - it('passes partial-discovery warnings into the TUI banner state', async () => { - const projectDir = join(tempDir, 'project'); - const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeProjectConfig(projectDir, { - ...initialized.config, - connections: { - 'notion-main': { - driver: 'notion', - auth_token_ref: 'env:NOTION_TOKEN', - crawl_mode: 'selected_roots', - root_page_ids: [PAGE_IDS.engineering], - root_database_ids: [], - root_data_source_ids: [], - max_pages_per_run: 12, - max_knowledge_creates_per_run: 2, - max_knowledge_updates_per_run: 7, - last_successful_cursor: null, - }, - }, - }); - const api: NotionPickerApi = { - search: vi - .fn() - .mockResolvedValueOnce({ - results: [notionPage(PAGE_IDS.engineering, 'Engineering')], - hasMore: true, - nextCursor: 'cursor-2', - }) - .mockRejectedValueOnce(new Error('rate limit after first page')), - retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })), - }; - let renderInput: PickerRenderInput | undefined; - const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { - renderInput = input; - return { kind: 'quit' }; - }); - const io = makeIo(); - - await expect( - runKtxConnectionNotion( - { - command: 'pick', - projectDir, - connectionId: 'notion-main', - mode: 'interactive', - }, - io.io, - { - env: { NOTION_TOKEN: 'ntn_test_token' }, - createNotionApi: vi.fn(() => api), - renderPicker, - }, - ), - ).resolves.toBe(0); - - expect(renderPicker).toHaveBeenCalledOnce(); - if (!renderInput) { - throw new Error('renderPicker was not called'); - } - expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']); - expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']); - expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page'); - expect(io.stdout()).toContain('No changes saved.'); - }); - - it('quits interactive mode without writing when the TUI returns quit', async () => { - const projectDir = join(tempDir, 'project'); - const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' }); - await writeProjectConfig(projectDir, { - ...initialized.config, - connections: { - 'notion-main': { - driver: 'notion', - auth_token_ref: 'env:NOTION_TOKEN', - crawl_mode: 'selected_roots', - root_page_ids: [PAGE_IDS.engineering], - root_database_ids: [], - root_data_source_ids: [], - max_pages_per_run: 12, - max_knowledge_creates_per_run: 2, - max_knowledge_updates_per_run: 7, - last_successful_cursor: null, - }, - }, - }); - const before = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - const io = makeIo(); - - await expect( - runKtxConnectionNotion( - { - command: 'pick', - projectDir, - connectionId: 'notion-main', - mode: 'interactive', - }, - io.io, - { - env: { NOTION_TOKEN: 'ntn_test_token' }, - createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])), - renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), - }, - ), - ).resolves.toBe(0); - - await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toBe(before); - expect(io.stdout()).toContain('No changes saved.'); - }); -}); diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 4b6cacf1..57ed8742 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { MetabaseRuntimeClient } from '@ktx/context/ingest'; @@ -6,18 +6,13 @@ import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from import type { KtxConnectionDriver, KtxScanConnector, KtxSchemaSnapshot } from '@ktx/context/scan'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxConnection } from './connection.js'; -import { runKtxCli, type KtxCliIo } from './index.js'; -function makeIo(options: { stdoutIsTty?: boolean; stdinIsTty?: boolean } = {}) { +function makeIo() { let stdout = ''; let stderr = ''; return { io: { - stdin: { - isTTY: options.stdinIsTty, - }, stdout: { - isTTY: options.stdoutIsTty, write: (chunk: string) => { stdout += chunk; }, @@ -87,491 +82,49 @@ describe('runKtxConnection', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('adds and lists env-referenced connections without resolving secrets', async () => { + async function writeConnections( + projectDir: string, + connections: ReturnType['connections'], + ): Promise { + const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); + await writeFile(join(projectDir, 'ktx.yaml'), serializeKtxProjectConfig({ ...config, connections }), 'utf-8'); + } + + it('lists configured connections without resolving secrets', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); + await writeConnections(projectDir, { + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' }, + }); const io = makeIo(); - await expect( - runKtxConnection( - { - command: 'add', - projectDir, - driver: 'postgres', - connectionId: 'warehouse', - url: 'env:DATABASE_URL', - schemas: ['public'], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - io.io, - ), - ).resolves.toBe(0); + await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0); - expect(io.stdout()).toContain('Connection: warehouse'); - await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('url: env:DATABASE_URL'); - - const listIo = makeIo(); - await expect(runKtxConnection({ command: 'list', projectDir }, listIo.io)).resolves.toBe(0); - expect(listIo.stdout()).toContain('warehouse'); - expect(listIo.stdout()).toContain('postgres'); - }); - - it('removes a configured connection from ktx.yaml without deleting local artifacts when forced', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - await runKtxConnection( - { - command: 'add', - projectDir, - driver: 'sqlite', - connectionId: 'warehouse', - url: undefined, - schemas: [], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - makeIo().io, - ); - const artifactPath = join(projectDir, '.ktx', 'artifacts', 'warehouse.txt'); - await mkdir(join(projectDir, '.ktx', 'artifacts'), { recursive: true }); - await writeFile(artifactPath, 'keep me', 'utf-8'); - - const io = makeIo(); - - await expect( - runKtxConnection( - { - command: 'remove', - projectDir, - connectionId: 'warehouse', - force: true, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(0); - - const parsed = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); - expect(parsed.connections.warehouse).toBeUndefined(); - await expect(readFile(artifactPath, 'utf-8')).resolves.toBe('keep me'); - expect(io.stdout()).toContain('Connection removed from ktx.yaml.'); - expect(io.stdout()).toContain( - 'Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.', - ); + expect(io.stdout()).toContain('warehouse'); + expect(io.stdout()).toContain('postgres'); + expect(io.stdout()).toContain('docs'); + expect(io.stdout()).toContain('notion'); expect(io.stderr()).toBe(''); }); - it('requires --force when removing in non-interactive mode', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - await runKtxConnection( - { - command: 'add', - projectDir, - driver: 'sqlite', - connectionId: 'warehouse', - url: undefined, - schemas: [], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - makeIo().io, - ); - const io = makeIo(); - - await expect( - runKtxConnection( - { - command: 'remove', - projectDir, - connectionId: 'warehouse', - force: false, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('connection remove warehouse requires --force when input is disabled or not interactive'); - }); - - it('returns a clear error when removing an unknown connection', async () => { + it('prints an empty-state message that points at setup instead of removed connection add', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); const io = makeIo(); - await expect( - runKtxConnection( - { - command: 'remove', - projectDir, - connectionId: 'missing', - force: true, - inputMode: 'disabled', - }, - io.io, - ), - ).resolves.toBe(1); + await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0); - expect(io.stderr()).toContain('Connection "missing" is not configured in ktx.yaml'); - }); - - it('asks for confirmation before removing in an interactive terminal', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - await runKtxConnection( - { - command: 'add', - projectDir, - driver: 'sqlite', - connectionId: 'warehouse', - url: undefined, - schemas: [], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - makeIo().io, - ); - const io = makeIo({ stdoutIsTty: true, stdinIsTty: true }); - const prompts = { - confirm: vi.fn(async () => true), - cancel: vi.fn(), - }; - - await expect( - runKtxConnection( - { - command: 'remove', - projectDir, - connectionId: 'warehouse', - force: false, - }, - io.io, - { prompts }, - ), - ).resolves.toBe(0); - - expect(prompts.confirm).toHaveBeenCalledWith({ - message: 'Remove connection "warehouse" from ktx.yaml? Ingested artifacts will remain in .ktx/.', - initialValue: false, - }); - }); - - it('runs public connect map as refresh, validate, and list over the low-level mapping runner', async () => { - const io = makeIo(); - const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => { - if (argv[0] === 'refresh') { - mappingIo.stdout.write('Discovery: 1 database\n'); - mappingIo.stdout.write('Unmapped discovered: 1\n'); - mappingIo.stdout.write('Stale mappings: 0\n'); - return 0; - } - if (argv[0] === 'validate') { - mappingIo.stdout.write('Mapping validation passed: prod-metabase\n'); - return 0; - } - if (argv[0] === 'list') { - mappingIo.stdout.write('1 -> [unmapped] (Analytics, sync: on, source: refresh)\n'); - return 0; - } - return 1; - }); - - await expect( - runKtxConnection( - { command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false }, - io.io, - { runMapping }, - ), - ).resolves.toBe(0); - - expect(runMapping).toHaveBeenNthCalledWith( - 1, - ['refresh', 'prod-metabase', '--auto-accept', '--project-dir', '/tmp/project'], - expect.any(Object), - ); - expect(runMapping).toHaveBeenNthCalledWith( - 2, - ['validate', 'prod-metabase', '--project-dir', '/tmp/project'], - expect.any(Object), - ); - expect(runMapping).toHaveBeenNthCalledWith( - 3, - ['list', 'prod-metabase', '--project-dir', '/tmp/project'], - expect.any(Object), - ); - expect(io.stdout()).toContain('Mapping: prod-metabase'); - expect(io.stdout()).toContain('Discovery: 1 database'); - expect(io.stdout()).toContain('Mappings:'); - expect(io.stdout()).toContain('1 -> [unmapped]'); - expect(io.stdout()).toContain('Next:'); - expect(io.stdout()).toContain('ktx ingest run --connection-id prod-metabase --adapter '); - expect(io.stdout()).toContain('ktx connection mapping'); - expect(io.stderr()).toBe(''); - }); - - it('prints stable JSON for public connect map without leaking low-level stdout', async () => { - const io = makeIo(); - const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => { - if (argv[0] === 'refresh') { - mappingIo.stdout.write('Discovery: 1 connection\nUnmapped discovered: 0\nStale mappings: 0\n'); - return 0; - } - if (argv[0] === 'validate') { - mappingIo.stdout.write('Mapping validation passed: prod-looker\n'); - return 0; - } - if (argv[0] === 'list') { - expect(argv).toContain('--json'); - mappingIo.stdout.write( - `${JSON.stringify( - [ - { - lookerConnectionName: 'analytics', - ktxConnectionId: 'prod-warehouse', - source: 'ktx.yaml', - }, - ], - null, - 2, - )}\n`, - ); - return 0; - } - return 1; - }); - - await expect( - runKtxConnection( - { command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-looker', json: true }, - io.io, - { runMapping }, - ), - ).resolves.toBe(0); - - const parsed = JSON.parse(io.stdout()) as { - connectionId: string; - refresh: { ok: boolean; output: string[] }; - validation: { ok: boolean; output: string[] }; - mappings: Array<{ lookerConnectionName: string; ktxConnectionId: string; source: string }>; - }; - expect(parsed).toEqual({ - connectionId: 'prod-looker', - refresh: { - ok: true, - output: ['Discovery: 1 connection', 'Unmapped discovered: 0', 'Stale mappings: 0'], - }, - validation: { - ok: true, - output: ['Mapping validation passed: prod-looker'], - }, - mappings: [ - { - lookerConnectionName: 'analytics', - ktxConnectionId: 'prod-warehouse', - source: 'ktx.yaml', - }, - ], - }); - expect(io.stderr()).toBe(''); - }); - - it('returns the refresh failure when public connect map cannot discover source metadata', async () => { - const io = makeIo(); - const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => { - if (argv[0] === 'refresh') { - mappingIo.stderr.write('Metabase API key is not configured\n'); - return 1; - } - return 0; - }); - - await expect( - runKtxConnection( - { command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false }, - io.io, - { runMapping }, - ), - ).resolves.toBe(1); - - expect(runMapping).toHaveBeenCalledTimes(1); - expect(io.stdout()).toBe(''); - expect(io.stderr()).toContain('Metabase API key is not configured'); - }); - - it('rejects literal credential URLs unless explicitly allowed', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - const io = makeIo(); - - await expect( - runKtxConnection( - { - command: 'add', - projectDir, - driver: 'postgres', - connectionId: 'warehouse', - url: 'postgres://localhost:5432/warehouse', - schemas: [], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - io.io, - ), - ).resolves.toBe(1); - - expect(io.stderr()).toContain('Literal credential URLs require --allow-literal-credentials'); - }); - - it('warns before writing explicitly allowed literal credential URLs without echoing the URL', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - const io = makeIo(); - const literalUrl = 'postgres://localhost:5432/warehouse'; - - await expect( - runKtxConnection( - { - command: 'add', - projectDir, - driver: 'postgres', - connectionId: 'warehouse', - url: literalUrl, - schemas: ['public'], - readonly: true, - force: false, - allowLiteralCredentials: true, - }, - io.io, - ), - ).resolves.toBe(0); - - expect(io.stderr()).toContain( - 'Warning: writing a literal credential URL to ktx.yaml for connection "warehouse". Prefer env:NAME or file:/path references.', - ); - expect(io.stderr()).not.toContain(literalUrl); - await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain(literalUrl); - }); - - it('adds a Notion connection without writing token values', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - const io = makeIo(); - - await expect( - runKtxConnection( - { - command: 'add', - projectDir, - driver: 'notion', - connectionId: 'notion-main', - url: undefined, - schemas: [], - readonly: false, - force: false, - allowLiteralCredentials: false, - notion: { - authTokenRef: 'env:NOTION_TOKEN', - crawlMode: 'all_accessible', - rootPageIds: [], - rootDatabaseIds: [], - rootDataSourceIds: [], - maxPagesPerRun: 50, - maxKnowledgeCreatesPerRun: 4, - maxKnowledgeUpdatesPerRun: 12, - }, - }, - io.io, - ), - ).resolves.toBe(0); - - const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(yaml).toContain('driver: notion'); - expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN'); - expect(yaml).toContain('crawl_mode: all_accessible'); - expect(yaml).toContain('max_pages_per_run: 50'); - expect(yaml).not.toContain('ntn_'); - expect(io.stdout()).toContain('Connection: notion-main'); - expect(io.stdout()).toContain('Driver: notion'); - }); - - it('runs connection notion pick --no-input through the public connection entrypoint', async () => { - const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - await runKtxConnection( - { - command: 'add', - projectDir, - driver: 'notion', - connectionId: 'notion-main', - url: undefined, - schemas: [], - readonly: false, - force: false, - allowLiteralCredentials: false, - notion: { - authTokenRef: 'env:NOTION_TOKEN', - crawlMode: 'all_accessible', - rootPageIds: [], - rootDatabaseIds: ['database-1'], - rootDataSourceIds: ['data-source-1'], - maxPagesPerRun: 50, - maxKnowledgeCreatesPerRun: 4, - maxKnowledgeUpdatesPerRun: 12, - }, - }, - makeIo().io, - ); - const io = makeIo(); - - await expect( - runKtxCli( - [ - 'connection', - 'notion', - 'pick', - 'notion-main', - '--project-dir', - projectDir, - '--no-input', - '--root-page-id', - '11111111222233334444555555555555', - ], - io.io, - ), - ).resolves.toBe(0); - - const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); - expect(yaml).toContain('crawl_mode: selected_roots'); - expect(yaml).toContain('11111111-2222-3333-4444-555555555555'); - expect(yaml).toContain('database-1'); - expect(yaml).toContain('data-source-1'); - expect(io.stdout()).toContain('Connection: notion-main'); + expect(io.stdout()).toContain('No connections configured. Run `ktx setup` to add one.'); + expect(io.stdout()).not.toContain('ktx connection add'); }); it('tests a configured connection through the native scan connector', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); - await runKtxConnection( - { - command: 'add', - projectDir, - driver: 'sqlite', - connectionId: 'warehouse', - url: undefined, - schemas: [], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - makeIo().io, - ); + await writeConnections(projectDir, { + warehouse: { driver: 'sqlite', readonly: true }, + }); const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']); const createScanConnector = vi.fn(async () => connector); const io = makeIo(); @@ -602,22 +155,13 @@ describe('runKtxConnection', () => { it('tests a configured Metabase connection through the Metabase runtime client', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); - const projectConfig = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); - await writeFile( - join(projectDir, 'ktx.yaml'), - serializeKtxProjectConfig({ - ...projectConfig, - connections: { - ...projectConfig.connections, - prod_metabase: { - driver: 'metabase', - api_url: 'http://metabase.example.test', - api_key: 'mb_test', - }, - }, - }), - 'utf-8', - ); + await writeConnections(projectDir, { + prod_metabase: { + driver: 'metabase', + api_url: 'http://metabase.example.test', + api_key: 'mb_test', + }, + }); const testConnection = vi.fn(async () => ({ success: true as const })); const getDatabases = vi.fn(async () => [ { id: 1, name: 'Analytics', engine: 'postgres', details: {}, is_sample: false }, @@ -657,20 +201,9 @@ describe('runKtxConnection', () => { it('cleans up the native scan connector when connection testing fails', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); - await runKtxConnection( - { - command: 'add', - projectDir, - driver: 'sqlite', - connectionId: 'warehouse', - url: undefined, - schemas: [], - readonly: true, - force: false, - allowLiteralCredentials: false, - }, - makeIo().io, - ); + await writeConnections(projectDir, { + warehouse: { driver: 'sqlite', readonly: true }, + }); const cleanup = vi.fn(async () => undefined); const connector: KtxScanConnector = { id: 'sqlite:warehouse', diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts index b199239a..cf0b512b 100644 --- a/packages/cli/src/connection.ts +++ b/packages/cli/src/connection.ts @@ -1,108 +1,24 @@ -import { cancel, confirm, isCancel } from '@clack/prompts'; import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory, type MetabaseRuntimeClient, metabaseRuntimeConfigFromLocalConnection, } from '@ktx/context/ingest'; -import { type KtxLocalProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project'; +import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project'; import type { KtxScanConnector } from '@ktx/context/scan'; -import type { KtxConnectionMappingArgs } from './commands/connection-mapping.js'; import type { KtxCliIo } from './index.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; import { profileMark } from './startup-profile.js'; profileMark('module:connection'); -interface KtxNotionConnectionCliConfig { - authTokenRef: string; - crawlMode: 'all_accessible' | 'selected_roots'; - rootPageIds: string[]; - rootDatabaseIds: string[]; - rootDataSourceIds: string[]; - maxPagesPerRun?: number; - maxKnowledgeCreatesPerRun?: number; - maxKnowledgeUpdatesPerRun?: number; -} - -type KtxConnectionInputMode = 'disabled'; - export type KtxConnectionArgs = | { command: 'list'; projectDir: string } - | { - command: 'add'; - projectDir: string; - driver: string; - connectionId: string; - url?: string; - schemas: string[]; - readonly: boolean; - force: boolean; - allowLiteralCredentials: boolean; - notion?: KtxNotionConnectionCliConfig; - } - | { command: 'test'; projectDir: string; connectionId: string } - | { - command: 'remove'; - projectDir: string; - connectionId: string; - force: boolean; - inputMode?: KtxConnectionInputMode; - } - | { - command: 'map'; - projectDir: string; - sourceConnectionId: string; - json: boolean; - }; - -interface KtxConnectionPromptAdapter { - confirm(options: { message: string; initialValue?: boolean }): Promise; - cancel(message: string): void; -} - -interface KtxConnectionIo extends KtxCliIo { - stdin?: { isTTY?: boolean }; -} + | { command: 'test'; projectDir: string; connectionId: string }; interface KtxConnectionDeps { createScanConnector?: typeof createKtxCliScanConnector; createMetabaseClient?: typeof createDefaultMetabaseClient; - runMapping?: (argv: string[], io: KtxCliIo) => Promise; - prompts?: KtxConnectionPromptAdapter; -} - -function assertSafeConnectionId(connectionId: string): void { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) { - throw new Error(`Unsafe connection id: ${connectionId}`); - } -} - -function isCredentialReference(value: string): boolean { - return value.startsWith('env:') || value.startsWith('file:'); -} - -function literalCredentialWarning(connectionId: string): string { - return `Warning: writing a literal credential URL to ktx.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`; -} - -function createClackConnectionPromptAdapter(): KtxConnectionPromptAdapter { - return { - async confirm(options: { message: string; initialValue?: boolean }): Promise { - const value = await confirm(options); - return isCancel(value) ? false : value; - }, - cancel(message: string): void { - cancel(message); - }, - }; -} - -function isInteractiveConnectionIo( - args: Extract, - io: KtxConnectionIo, -): boolean { - return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true; } async function cleanupConnector(connector: KtxScanConnector | null): Promise { @@ -186,166 +102,17 @@ async function testMetabaseConnection( } } -interface BufferedIo extends KtxCliIo { - stdoutText(): string; - stderrText(): string; -} - -function createBufferedIo(): BufferedIo { - let stdout = ''; - let stderr = ''; - return { - stdout: { - write(chunk: string) { - stdout += chunk; - }, - }, - stderr: { - write(chunk: string) { - stderr += chunk; - }, - }, - stdoutText() { - return stdout; - }, - stderrText() { - return stderr; - }, - }; -} - -function splitOutputLines(output: string): string[] { - return output - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); -} - -async function runLowLevelMapping( - args: KtxConnectionMappingArgs, - argv: string[], - io: KtxCliIo, - deps: KtxConnectionDeps, -): Promise { - if (deps.runMapping) { - return await deps.runMapping(argv, io); - } - - const { runKtxConnectionMapping } = await import('./commands/connection-mapping.js'); - return await runKtxConnectionMapping(args, io); -} - -function parseMappingListJson(output: string): unknown[] { - const trimmed = output.trim(); - if (!trimmed) { - return []; - } - const parsed = JSON.parse(trimmed) as unknown; - return Array.isArray(parsed) ? parsed : []; -} - -async function runPublicConnectionMap( - args: Extract, - io: KtxCliIo, - deps: KtxConnectionDeps, -): Promise { - const refreshIo = createBufferedIo(); - const refreshArgs: KtxConnectionMappingArgs = { - command: 'refresh', - projectDir: args.projectDir, - connectionId: args.sourceConnectionId, - autoAccept: true, - }; - const refreshCode = await runLowLevelMapping( - refreshArgs, - ['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir], - refreshIo, - deps, - ); - if (refreshCode !== 0) { - io.stderr.write( - refreshIo.stderrText() || - refreshIo.stdoutText() || - `Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`, - ); - return refreshCode; - } - - const validationIo = createBufferedIo(); - const validationArgs: KtxConnectionMappingArgs = { - command: 'validate', - projectDir: args.projectDir, - connectionId: args.sourceConnectionId, - }; - const validationCode = await runLowLevelMapping( - validationArgs, - ['validate', args.sourceConnectionId, '--project-dir', args.projectDir], - validationIo, - deps, - ); - if (validationCode !== 0) { - io.stderr.write( - validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`, - ); - return validationCode; - } - - const listIo = createBufferedIo(); - const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir]; - const listArgs: KtxConnectionMappingArgs = { - command: 'list', - projectDir: args.projectDir, - connectionId: args.sourceConnectionId, - json: args.json, - }; - const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps); - if (listCode !== 0) { - io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`); - return listCode; - } - - if (args.json) { - io.stdout.write( - `${JSON.stringify( - { - connectionId: args.sourceConnectionId, - refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) }, - validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) }, - mappings: parseMappingListJson(listIo.stdoutText()), - }, - null, - 2, - )}\n`, - ); - return 0; - } - - io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`); - io.stdout.write(refreshIo.stdoutText()); - io.stdout.write(validationIo.stdoutText()); - io.stdout.write('\nMappings:\n'); - io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n'); - io.stdout.write('\nNext:\n'); - io.stdout.write(` ktx ingest run --connection-id ${args.sourceConnectionId} --adapter \n`); - io.stdout.write(` ktx connection mapping list ${args.sourceConnectionId}\n`); - return 0; -} - export async function runKtxConnection( args: KtxConnectionArgs, - io: KtxConnectionIo = process, + io: KtxCliIo = process, deps: KtxConnectionDeps = {}, ): Promise { try { - if (args.command === 'map') { - return await runPublicConnectionMap(args, io, deps); - } - const project = await loadKtxProject({ projectDir: args.projectDir }); if (args.command === 'list') { const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b)); if (entries.length === 0) { - io.stdout.write('No connections configured. Run `ktx connection add --driver ` to add one.\n'); + io.stdout.write('No connections configured. Run `ktx setup` to add one.\n'); return 0; } const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length)); @@ -360,100 +127,6 @@ export async function runKtxConnection( return 0; } - if (args.command === 'add') { - assertSafeConnectionId(args.connectionId); - const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url); - if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) { - throw new Error('Literal credential URLs require --allow-literal-credentials'); - } - if (hasLiteralCredentialUrl) { - io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`); - } - if (project.config.connections[args.connectionId] && !args.force) { - throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`); - } - const connectionConfig = - args.driver === 'notion' && args.notion - ? { - driver: 'notion', - auth_token_ref: args.notion.authTokenRef, - crawl_mode: args.notion.crawlMode, - root_page_ids: args.notion.rootPageIds, - root_database_ids: args.notion.rootDatabaseIds, - root_data_source_ids: args.notion.rootDataSourceIds, - ...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}), - ...(args.notion.maxKnowledgeCreatesPerRun !== undefined - ? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun } - : {}), - ...(args.notion.maxKnowledgeUpdatesPerRun !== undefined - ? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun } - : {}), - } - : { - driver: args.driver, - ...(args.url ? { url: args.url } : {}), - ...(args.schemas.length > 0 ? { schemas: args.schemas } : {}), - readonly: args.readonly, - }; - const nextConfig = { - ...project.config, - connections: { - ...project.config.connections, - [args.connectionId]: connectionConfig, - }, - }; - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig(nextConfig), - 'ktx', - 'ktx@example.com', - `Update KTX connection: ${args.connectionId}`, - ); - io.stdout.write(`Connection: ${args.connectionId}\n`); - io.stdout.write(`Driver: ${args.driver}\n`); - return 0; - } - - if (args.command === 'remove') { - if (!project.config.connections[args.connectionId]) { - throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`); - } - - if (!args.force) { - if (!isInteractiveConnectionIo(args, io)) { - throw new Error( - `connection remove ${args.connectionId} requires --force when input is disabled or not interactive`, - ); - } - - const prompts = deps.prompts ?? createClackConnectionPromptAdapter(); - const confirmed = await prompts.confirm({ - message: `Remove connection "${args.connectionId}" from ktx.yaml? Ingested artifacts will remain in .ktx/.`, - initialValue: false, - }); - if (!confirmed) { - prompts.cancel('Connection removal cancelled.'); - return 1; - } - } - - const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections; - const nextConfig = { - ...project.config, - connections, - }; - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig(nextConfig), - 'ktx', - 'ktx@example.com', - `Remove KTX connection: ${args.connectionId}`, - ); - io.stdout.write('Connection removed from ktx.yaml.\n'); - io.stdout.write('Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.\n'); - return 0; - } - if (normalizedConnectionDriver(project, args.connectionId) === 'metabase') { const result = await testMetabaseConnection( project, diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 1e69c590..bb896fc3 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -1250,16 +1250,9 @@ describe('runKtxCli', () => { runKtxCli(['--project-dir', tempDir, 'connection', 'list'], makeIo().io, { connection }), ).resolves.toBe(0); - const removeIo = makeIo(); + const testIo = makeIo(); await expect( - runKtxCli(['--project-dir', tempDir, 'connection', 'remove', 'warehouse', '--force', '--no-input'], removeIo.io, { - connection, - }), - ).resolves.toBe(0); - - const mapIo = makeIo(); - await expect( - runKtxCli(['--project-dir', tempDir, 'connection', 'map', 'prod-metabase', '--json'], mapIo.io, { + runKtxCli(['--project-dir', tempDir, 'connection', 'test', 'warehouse'], testIo.io, { connection, }), ).resolves.toBe(0); @@ -1268,21 +1261,9 @@ describe('runKtxCli', () => { expect(connection).toHaveBeenNthCalledWith( 2, { - command: 'remove', + command: 'test', projectDir: tempDir, connectionId: 'warehouse', - force: true, - inputMode: 'disabled', - }, - expect.anything(), - ); - expect(connection).toHaveBeenNthCalledWith( - 3, - { - command: 'map', - projectDir: tempDir, - sourceConnectionId: 'prod-metabase', - json: true, }, expect.anything(), ); @@ -1290,168 +1271,35 @@ describe('runKtxCli', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('prints help for connection metabase setup', async () => { + it('prints only list and test in connection help', async () => { const helpIo = makeIo(); - await expect(runKtxCli(['connection', 'metabase', 'setup', '--help'], helpIo.io)).resolves.toBe(0); + await expect(runKtxCli(['connection', '--help'], helpIo.io)).resolves.toBe(0); - expect(helpIo.stdout()).toContain('Usage: ktx connection metabase setup'); - for (const option of [ - '--id ', - '--url ', - '--api-key ', - '--username ', - '--password ', - '--mint-api-key', - '--map ', - '--sync ', - '--sync-mode ', - '--run-ingest', - '--yes', - '--no-input', - ]) { - expect(helpIo.stdout()).toContain(option); - } - expect(helpIo.stdout()).toContain('Guided equivalent of:'); - for (const line of [ - 'ktx connection mapping refresh --auto-accept', - 'ktx connection mapping set databaseMappings =', - 'ktx connection mapping set-sync-enabled --enabled true', - 'ktx ingest run --connection-id --adapter metabase', - ]) { - expect(helpIo.stdout()).toContain(line); + expect(helpIo.stdout()).toContain('Usage: ktx connection'); + expect(helpIo.stdout()).toContain('list'); + expect(helpIo.stdout()).toContain('test '); + for (const removed of ['add', 'remove', 'map', 'mapping', 'metabase', 'notion']) { + expect(helpIo.stdout()).not.toMatch(new RegExp(`\\b${removed}\\b`)); } expect(helpIo.stderr()).toBe(''); }); - it('dispatches connection metabase setup through Commander', async () => { - const connectionMetabaseSetup = vi.fn(async () => 0); - const fakeMetabaseCredential = 'mb_example'; - const setupIo = makeIo(); - - await expect( - runKtxCli( - [ - 'connection', - 'metabase', - 'setup', - '--project-dir', - tempDir, - '--id', - 'metabase', - '--url', - 'http://metabase.example.test:3000', - '--api-key', - 'mb_example', - '--map', - '2=orbit', - '--sync', - '2', - '--yes', - '--no-input', - ], - setupIo.io, - { connectionMetabaseSetup }, - ), - ).resolves.toBe(0); - - expect(connectionMetabaseSetup).toHaveBeenCalledWith( - { - command: 'setup', - projectDir: tempDir, - connectionId: 'metabase', - url: 'http://metabase.example.test:3000', - apiKey: fakeMetabaseCredential, - mintApiKey: false, - mappings: [{ metabaseDatabaseId: 2, targetConnectionId: 'orbit' }], - syncEnabledDatabaseIds: [2], - syncMode: 'ALL', - runIngest: false, - yes: true, - inputMode: 'disabled', - }, - expect.anything(), - ); - expect(setupIo.stderr()).toBe(`Project: ${tempDir}\n`); - }); - - it('validates connection metabase setup option values before runner dispatch', async () => { - const connectionMetabaseSetup = vi.fn(async () => 0); - + it('rejects removed connection subcommands', async () => { for (const argv of [ - [ - 'connection', - 'metabase', - 'setup', - '--project-dir', - tempDir, - '--url', - 'http://metabase.example.test:3000', - '--api-key', - 'mb_example', - '--map', - 'nope=orbit', - ], - [ - 'connection', - 'metabase', - 'setup', - '--project-dir', - tempDir, - '--url', - 'http://metabase.example.test:3000', - '--api-key', - 'mb_example', - '--map', - '2=../orbit', - ], - [ - 'connection', - 'metabase', - 'setup', - '--project-dir', - tempDir, - '--url', - 'http://metabase.example.test:3000', - '--api-key', - 'mb_example', - '--sync', - 'nope', - ], - [ - 'connection', - 'metabase', - 'setup', - '--project-dir', - tempDir, - '--url', - 'http://metabase.example.test:3000', - '--api-key', - 'mb_example', - '--sync-mode', - 'BAD', - ], - [ - 'connection', - 'metabase', - 'setup', - '--project-dir', - tempDir, - '--url', - 'http://metabase.example.test:3000', - '--api-key', - 'mb_example', - '--mint-api-key', - '--api-key', - 'also_bad', - ], + ['connection', 'add', 'postgres', 'warehouse'], + ['connection', 'remove', 'warehouse'], + ['connection', 'map', 'prod-metabase'], + ['connection', 'mapping'], + ['connection', 'metabase'], + ['connection', 'notion'], ]) { const testIo = makeIo(); - await expect(runKtxCli(argv, testIo.io, { connectionMetabaseSetup })).resolves.toBe(1); - expect(testIo.stderr()).toMatch(/map|sync|sync-mode|conflict|cannot be used|invalid|integer|choices/i); - } - expect(connectionMetabaseSetup).not.toHaveBeenCalled(); + await expect(runKtxCli(argv, testIo.io)).resolves.toBe(1); + + expect(testIo.stderr()).toMatch(/unknown command|error:/); + } }); it('rejects commands removed from the May 6 root surface', async () => { @@ -1469,153 +1317,6 @@ describe('runKtxCli', () => { } }); - it('dispatches connection add options through Commander', async () => { - const testIo = makeIo(); - const connection = vi.fn(async () => 0); - - await expect( - runKtxCli( - [ - 'connection', - 'add', - 'notion', - 'notion-main', - '--project-dir', - tempDir, - '--token-env', - 'NOTION_TOKEN', - '--crawl-mode', - 'selected_roots', - '--root-page-id', - 'page-1', - '--root-database-id', - 'database-1', - '--max-pages', - '80', - ], - testIo.io, - { connection }, - ), - ).resolves.toBe(0); - - expect(connection).toHaveBeenCalledWith( - { - command: 'add', - projectDir: tempDir, - driver: 'notion', - connectionId: 'notion-main', - url: undefined, - schemas: [], - readonly: false, - force: false, - allowLiteralCredentials: false, - notion: { - authTokenRef: 'env:NOTION_TOKEN', - crawlMode: 'selected_roots', - rootPageIds: ['page-1'], - rootDatabaseIds: ['database-1'], - rootDataSourceIds: [], - maxPagesPerRun: 80, - maxKnowledgeCreatesPerRun: undefined, - maxKnowledgeUpdatesPerRun: undefined, - }, - }, - testIo.io, - ); - expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`); - }); - - it('prints generated connection notion pick help without invoking execution', async () => { - const helpCases = [ - ['connection', 'notion', '--help'], - ['connection', 'notion', 'pick', '--help'], - ['connection', 'notion', 'pick', 'notion-main', '--help'], - ]; - - for (const argv of helpCases) { - const testIo = makeIo(); - const connectionNotion = vi.fn(async () => 0); - - await expect(runKtxCli(argv, testIo.io, { connectionNotion })).resolves.toBe(0); - - expect(testIo.stdout()).toContain('Usage: ktx connection notion'); - expect(testIo.stdout()).toContain('pick'); - expect(testIo.stderr()).toBe(''); - expect(connectionNotion).not.toHaveBeenCalled(); - } - }); - - it('dispatches connection notion pick through Commander', async () => { - const testIo = makeIo(); - const connectionNotion = vi.fn(async () => 0); - - await expect( - runKtxCli( - [ - '--project-dir', - tempDir, - 'connection', - 'notion', - 'pick', - 'notion-main', - '--no-input', - '--root-page-id', - '11111111222233334444555555555555', - '--root-page-id', - '11111111-2222-3333-4444-555555555555', - ], - testIo.io, - { connectionNotion }, - ), - ).resolves.toBe(0); - - expect(connectionNotion).toHaveBeenCalledWith( - { - command: 'pick', - projectDir: tempDir, - connectionId: 'notion-main', - mode: 'non-interactive', - rootPageIds: ['11111111-2222-3333-4444-555555555555'], - }, - testIo.io, - ); - expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`); - }); - - it('ignores connection notion pick root page flags in interactive mode', async () => { - const testIo = makeIo(); - const connectionNotion = vi.fn(async () => 0); - - await expect( - runKtxCli(['connection', 'notion', 'pick', 'notion-main', '--root-page-id', 'not-a-uuid'], testIo.io, { - connectionNotion, - }), - ).resolves.toBe(0); - - expect(connectionNotion).toHaveBeenCalledWith( - { - command: 'pick', - projectDir: expect.any(String), - connectionId: 'notion-main', - mode: 'interactive', - }, - testIo.io, - ); - expect(testIo.stderr()).toBe(''); - }); - - it('rejects connection notion pick no-input mode without root page ids', async () => { - const testIo = makeIo(); - const connectionNotion = vi.fn(async () => 0); - - await expect( - runKtxCli(['connection', 'notion', 'pick', 'notion-main', '--no-input'], testIo.io, { connectionNotion }), - ).resolves.toBe(1); - - expect(connectionNotion).not.toHaveBeenCalled(); - expect(testIo.stderr()).toContain('connection notion pick --no-input requires at least one --root-page-id'); - }); - it('writes basic debug dispatch information when --debug is set', async () => { const testIo = makeIo(); const connection = vi.fn().mockResolvedValue(0); @@ -1776,51 +1477,6 @@ describe('runKtxCli', () => { expect(ingest).not.toHaveBeenCalled(); }); - it('rejects mutually exclusive credential and scan mode options before invoking runners', async () => { - const connection = vi.fn(async () => 0); - const scan = vi.fn(async () => 0); - - const tokenIo = makeIo(); - await expect( - runKtxCli( - [ - 'connection', - 'add', - 'notion', - 'notion-main', - '--token-env', - 'NOTION_TOKEN', - '--token-file', - '/tmp/notion-token', - '--root-page-id', - '11111111111111111111111111111111', - ], - tokenIo.io, - { connection }, - ), - ).resolves.toBe(1); - expect(tokenIo.stderr()).toMatch(/conflict|cannot be used/i); - - expect(connection).not.toHaveBeenCalled(); - expect(scan).not.toHaveBeenCalled(); - }); - - it('validates connection mapping set syntax before runner domain validation', async () => { - const badFieldIo = makeIo(); - await expect( - runKtxCli(['connection', 'mapping', 'set', 'prod-metabase', 'invalidMappings', '1=warehouse'], badFieldIo.io), - ).resolves.toBe(1); - expect(badFieldIo.stderr()).toContain('databaseMappings or connectionMappings'); - - for (const assignment of ['missing-equals', '=warehouse', '1=']) { - const testIo = makeIo(); - await expect( - runKtxCli(['connection', 'mapping', 'set', 'prod-metabase', 'databaseMappings', assignment], testIo.io), - ).resolves.toBe(1); - expect(testIo.stderr()).toContain('non-empty ='); - } - }); - it('does not expose root init after setup owns project creation', async () => { const testIo = makeIo(); diff --git a/packages/cli/src/commands/connection-notion-tree.test.ts b/packages/cli/src/notion-page-picker-tree.test.ts similarity index 99% rename from packages/cli/src/commands/connection-notion-tree.test.ts rename to packages/cli/src/notion-page-picker-tree.test.ts index ed1126d4..94b46b57 100644 --- a/packages/cli/src/commands/connection-notion-tree.test.ts +++ b/packages/cli/src/notion-page-picker-tree.test.ts @@ -14,7 +14,7 @@ import { TRANSIENT_HINT_DURATION_MS, visibleNodeIds, type NotionPickerPageInput, -} from './connection-notion-tree.js'; +} from './notion-page-picker-tree.js'; const IDS = { engineering: '11111111-1111-1111-1111-111111111111', diff --git a/packages/cli/src/commands/connection-notion-tree.ts b/packages/cli/src/notion-page-picker-tree.ts similarity index 100% rename from packages/cli/src/commands/connection-notion-tree.ts rename to packages/cli/src/notion-page-picker-tree.ts diff --git a/packages/cli/src/commands/connection-notion-tui.test.tsx b/packages/cli/src/notion-page-picker-tui.test.tsx similarity index 99% rename from packages/cli/src/commands/connection-notion-tui.test.tsx rename to packages/cli/src/notion-page-picker-tui.test.tsx index dc394688..ae39d39e 100644 --- a/packages/cli/src/commands/connection-notion-tui.test.tsx +++ b/packages/cli/src/notion-page-picker-tui.test.tsx @@ -2,7 +2,7 @@ import { render as renderInkTest } from 'ink-testing-library'; import { act, type ReactNode } from 'react'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js'; +import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { NotionPickerApp, notionPickerCommandForInkInput, @@ -13,7 +13,7 @@ import { windowOffset, type NotionPickerInkInstance, type NotionPickerInkRenderOptions, -} from './connection-notion-tui.js'; +} from './notion-page-picker-tui.js'; const IDS = { engineering: '11111111-1111-1111-1111-111111111111', diff --git a/packages/cli/src/commands/connection-notion-tui.tsx b/packages/cli/src/notion-page-picker-tui.tsx similarity index 99% rename from packages/cli/src/commands/connection-notion-tui.tsx rename to packages/cli/src/notion-page-picker-tui.tsx index b2a47036..fac7f339 100644 --- a/packages/cli/src/commands/connection-notion-tui.tsx +++ b/packages/cli/src/notion-page-picker-tui.tsx @@ -9,8 +9,8 @@ import { visibleNodeIds, type PickerCommand, type PickerState, -} from './connection-notion-tree.js'; -import type { KtxCliIo } from '../index.js'; +} from './notion-page-picker-tree.js'; +import type { KtxCliIo } from './cli-runtime.js'; const COLOR_THEME = { text: 'white', diff --git a/packages/cli/src/notion-page-picker.test.ts b/packages/cli/src/notion-page-picker.test.ts new file mode 100644 index 00000000..77710716 --- /dev/null +++ b/packages/cli/src/notion-page-picker.test.ts @@ -0,0 +1,308 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + discoverNotionPickerPages, + notionPickerPageFromSearchResult, + normalizeNotionPageId, + pickNotionRootPages, + resolveNotionWorkspaceLabel, + type NotionPickerApi, + type PickerRenderInput, + type PickerRenderResult, +} from './notion-page-picker.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: true, + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +type FakeNotionSearchPage = Record & { id: string; object: 'page' }; + +const PAGE_IDS = { + engineering: '11111111-1111-1111-1111-111111111111', + architecture: '22222222-2222-2222-2222-222222222222', + stale: '99999999-9999-9999-9999-999999999999', +}; + +function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage { + return { + object: 'page', + id, + archived: false, + parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true }, + properties: { + title: { + type: 'title', + title: [{ plain_text: title }], + }, + }, + }; +} + +function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi { + return { + search: vi.fn(async (_filterValue, startCursor) => { + if (startCursor === 'page-2') { + return { results: pages.slice(2), hasMore: false, nextCursor: null }; + } + return { + results: pages.slice(0, 2), + hasMore: pages.length > 2, + nextCursor: pages.length > 2 ? 'page-2' : null, + }; + }), + retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })), + }; +} + +describe('normalizeNotionPageId', () => { + it('accepts dashed and compact UUIDs', () => { + expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe( + '11111111-2222-3333-4444-555555555555', + ); + expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe( + 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + ); + }); +}); + +describe('Notion page picker helpers', () => { + it('extracts picker page inputs from Notion search results', () => { + expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering))) + .toEqual({ + id: PAGE_IDS.architecture, + title: 'Architecture', + archived: false, + parentId: PAGE_IDS.engineering, + }); + + expect( + notionPickerPageFromSearchResult({ + object: 'page', + id: PAGE_IDS.engineering.replaceAll('-', ''), + archived: true, + parent: { type: 'workspace', workspace: true }, + properties: {}, + }), + ).toEqual({ + id: PAGE_IDS.engineering, + title: 'Untitled', + archived: true, + parentId: null, + }); + }); + + it('discovers visible pages up to the cap and reports cap state', async () => { + const api = fakeNotionApi([ + notionPage(PAGE_IDS.engineering, 'Engineering'), + notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering), + notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering), + ]); + + await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({ + pages: [ + { id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }, + { id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering }, + ], + cappedAtCount: 2, + warnings: [], + }); + expect(api.search).toHaveBeenCalledTimes(1); + }); + + it('keeps partial discovery results when Notion search fails after at least one page', async () => { + const api: NotionPickerApi = { + search: vi + .fn() + .mockResolvedValueOnce({ + results: [notionPage(PAGE_IDS.engineering, 'Engineering')], + hasMore: true, + nextCursor: 'cursor-2', + }) + .mockRejectedValueOnce(new Error('rate limit after first page')), + retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })), + }; + + await expect(discoverNotionPickerPages(api)).resolves.toEqual({ + pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }], + cappedAtCount: null, + warnings: ['Notion search stopped early: rate limit after first page'], + }); + }); + + it('uses the Notion workspace name when available and falls back to the connection id', async () => { + await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace'); + await expect( + resolveNotionWorkspaceLabel( + { + search: vi.fn(), + retrieveBotUser: vi.fn(async () => { + throw new Error('users.me unavailable'); + }), + }, + 'notion-main', + ), + ).resolves.toBe('notion-main'); + }); +}); + +describe('pickNotionRootPages', () => { + it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => { + const api = fakeNotionApi([ + notionPage(PAGE_IDS.engineering, 'Engineering'), + notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering), + ]); + const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { + expect(input.connectionId).toBe('notion-main'); + expect(input.workspaceLabel).toBe('Design Workspace'); + expect(input.currentCrawlMode).toBe('all_accessible'); + expect(input.cappedAtCount).toBeNull(); + expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']); + return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] }; + }); + const io = makeIo(); + + await expect( + pickNotionRootPages( + { + connectionId: 'notion-main', + connection: { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'all_accessible', + root_page_ids: [PAGE_IDS.stale], + }, + }, + io.io, + { + env: { NOTION_TOKEN: 'ntn_test_token' }, + createNotionApi: vi.fn(() => api), + renderPicker, + }, + ), + ).resolves.toEqual({ kind: 'selected', rootPageIds: [PAGE_IDS.engineering] }); + + expect(io.stderr()).toContain('1 stored root_page_ids no longer visible'); + expect(io.stdout()).toBe(''); + }); + + it('uses inline Notion auth_token for discovery', async () => { + const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]); + const createNotionApi = vi.fn((authToken: string) => { + expect(authToken).toBe('ntn_inline_token'); + return api; + }); + + await expect( + pickNotionRootPages( + { + connectionId: 'notion-main', + connection: { + driver: 'notion', + auth_token: 'ntn_inline_token', + crawl_mode: 'selected_roots', + root_page_ids: [PAGE_IDS.engineering], + }, + }, + makeIo().io, + { + createNotionApi, + renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), + }, + ), + ).resolves.toEqual({ kind: 'back' }); + + expect(createNotionApi).toHaveBeenCalledOnce(); + }); + + it('passes partial-discovery warnings into the TUI banner state', async () => { + const api: NotionPickerApi = { + search: vi + .fn() + .mockResolvedValueOnce({ + results: [notionPage(PAGE_IDS.engineering, 'Engineering')], + hasMore: true, + nextCursor: 'cursor-2', + }) + .mockRejectedValueOnce(new Error('rate limit after first page')), + retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })), + }; + let renderInput: PickerRenderInput | undefined; + const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { + renderInput = input; + return { kind: 'quit' }; + }); + const io = makeIo(); + + await expect( + pickNotionRootPages( + { + connectionId: 'notion-main', + connection: { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: [PAGE_IDS.engineering], + }, + }, + io.io, + { + env: { NOTION_TOKEN: 'ntn_test_token' }, + createNotionApi: vi.fn(() => api), + renderPicker, + }, + ), + ).resolves.toEqual({ kind: 'back' }); + + expect(renderPicker).toHaveBeenCalledOnce(); + if (!renderInput) { + throw new Error('renderPicker was not called'); + } + expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']); + expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']); + expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page'); + }); + + it('returns unavailable when discovery cannot load any pages', async () => { + await expect( + pickNotionRootPages( + { + connectionId: 'notion-main', + connection: { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: [], + }, + }, + makeIo().io, + { + env: { NOTION_TOKEN: 'ntn_test_token' }, + createNotionApi: vi.fn(() => ({ + search: vi.fn(async () => { + throw new Error('Notion API unavailable'); + }), + retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })), + })), + renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), + }, + ), + ).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' }); + }); +}); diff --git a/packages/cli/src/commands/connection-notion.ts b/packages/cli/src/notion-page-picker.ts similarity index 51% rename from packages/cli/src/commands/connection-notion.ts rename to packages/cli/src/notion-page-picker.ts index e0f68c0b..807c0fc0 100644 --- a/packages/cli/src/commands/connection-notion.ts +++ b/packages/cli/src/notion-page-picker.ts @@ -1,51 +1,40 @@ -import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from '@ktx/context/connections'; +import { resolveNotionConnectionAuthToken } from '@ktx/context/connections'; import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/ingest'; -import { - type KtxLocalProject, - type KtxProjectConnectionConfig, - loadKtxProject, - serializeKtxProjectConfig, -} from '@ktx/context/project'; -import type { KtxCliIo } from '../index.js'; -import { profileMark } from '../startup-profile.js'; -import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js'; +import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { KtxCliIo } from './cli-runtime.js'; +import { profileMark } from './startup-profile.js'; +import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { type NotionPickerTuiIo, type PickerRenderInput, type PickerRenderResult, renderNotionPickerTui, -} from './connection-notion-tui.js'; +} from './notion-page-picker-tui.js'; -profileMark('module:commands/connection-notion'); +profileMark('module:notion-page-picker'); -export type KtxConnectionNotionArgs = - | { - command: 'pick'; - projectDir: string; - connectionId: string; - mode: 'interactive'; - } - | { - command: 'pick'; - projectDir: string; - connectionId: string; - mode: 'non-interactive'; - rootPageIds: string[]; - }; +export interface PickNotionRootPagesArgs { + connectionId: string; + connection: KtxProjectConnectionConfig; +} export type NotionPickerApi = Pick; export type { PickerRenderInput, PickerRenderResult }; -interface KtxConnectionNotionDeps { +export type NotionRootPagePickResult = + | { kind: 'selected'; rootPageIds: string[] } + | { kind: 'back' } + | { kind: 'unavailable'; message: string }; + +export interface NotionRootPagePickerDeps { env?: Record; - loadProject?: typeof loadKtxProject; createNotionApi?: (authToken: string) => NotionPickerApi; renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise; } const NOTION_PICKER_PAGE_CAP = 5000; -function assertSafeConnectionId(connectionId: string): void { +function assertSafeNotionPickerConnectionId(connectionId: string): void { if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) { throw new Error(`Unsafe connection id: ${connectionId}`); } @@ -168,111 +157,74 @@ export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connecti } } -function notionConnection(project: KtxLocalProject, connectionId: string): KtxProjectConnectionConfig { - const connection = project.config.connections[connectionId]; - if (!connection) { - throw new Error(`Connection "${connectionId}" not found`); - } +function assertNotionConnection(connection: KtxProjectConnectionConfig, connectionId: string): void { if (connection.driver !== 'notion') { throw new Error(`Connection "${connectionId}" is not a Notion connection`); } - return connection; } -export async function applyNotionPickerWriteback( - project: KtxLocalProject, - connectionId: string, - rootPageIds: string[], -): Promise { - if (rootPageIds.length === 0) { - throw new Error('connection notion pick requires at least one root page id'); - } - - const existing = notionConnection(project, connectionId); - const nextConfig = { - ...project.config, - connections: { - ...project.config.connections, - [connectionId]: { - ...existing, - crawl_mode: 'selected_roots', - root_page_ids: rootPageIds, - }, - }, - }; - - await project.fileStore.writeFile( - 'ktx.yaml', - serializeKtxProjectConfig(nextConfig), - 'ktx', - 'ktx@example.com', - `Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`, - ); +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string' && item.trim().length > 0) : []; } -export async function runKtxConnectionNotion( - args: KtxConnectionNotionArgs, +function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessible' | 'selected_roots' { + return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; +} + +export async function pickNotionRootPages( + args: PickNotionRootPagesArgs, io: KtxCliIo = process, - deps: KtxConnectionNotionDeps = {}, -): Promise { + deps: NotionRootPagePickerDeps = {}, +): Promise { try { - assertSafeConnectionId(args.connectionId); - const loadProject = deps.loadProject ?? loadKtxProject; - - if (args.mode === 'interactive') { - const project = await loadProject({ projectDir: args.projectDir }); - const rawConnection = notionConnection(project, args.connectionId); - const notion = parseNotionConnectionConfig(rawConnection); - const authToken = await resolveNotionConnectionAuthToken(notion, { env: deps.env }); - const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken); - const discovery = await discoverNotionPickerPages(api); - const tree = buildPickerTree(discovery.pages); - const initialState = buildInitialState({ - tree, - existingRootPageIds: notion.root_page_ids, - currentCrawlMode: notion.crawl_mode, - }); - const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings]; - const renderState = - preLoadWarnings.length > 0 - ? { - ...initialState, - preLoadWarnings, - } - : initialState; - for (const warning of preLoadWarnings) { - io.stderr.write(`${warning}\n`); - } - const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId); - const result = await (deps.renderPicker ?? renderNotionPickerTui)( - { - initialState: renderState, - connectionId: args.connectionId, - workspaceLabel, - cappedAtCount: discovery.cappedAtCount, - currentCrawlMode: notion.crawl_mode, - }, - io as NotionPickerTuiIo, - ); - if (result.kind === 'quit') { - io.stdout.write('No changes saved.\n'); - return 0; - } - await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds); - io.stdout.write(`Connection: ${args.connectionId}\n`); - io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`); - io.stdout.write('crawlMode: selected_roots\n'); - return 0; + assertSafeNotionPickerConnectionId(args.connectionId); + assertNotionConnection(args.connection, args.connectionId); + const crawlMode = notionCrawlMode(args.connection); + const authToken = await resolveNotionConnectionAuthToken( + { + auth_token: typeof args.connection.auth_token === 'string' ? args.connection.auth_token : null, + auth_token_ref: typeof args.connection.auth_token_ref === 'string' ? args.connection.auth_token_ref : null, + }, + { env: deps.env }, + ); + const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken); + const discovery = await discoverNotionPickerPages(api); + const tree = buildPickerTree(discovery.pages); + const initialState = buildInitialState({ + tree, + existingRootPageIds: stringArray(args.connection.root_page_ids), + currentCrawlMode: crawlMode, + }); + const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings]; + const renderState = + preLoadWarnings.length > 0 + ? { + ...initialState, + preLoadWarnings, + } + : initialState; + for (const warning of preLoadWarnings) { + io.stderr.write(`${warning}\n`); } - - const project = await loadProject({ projectDir: args.projectDir }); - await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds); - io.stdout.write(`Connection: ${args.connectionId}\n`); - io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`); - io.stdout.write('crawlMode: selected_roots\n'); - return 0; + const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId); + const result = await (deps.renderPicker ?? renderNotionPickerTui)( + { + initialState: renderState, + connectionId: args.connectionId, + workspaceLabel, + cappedAtCount: discovery.cappedAtCount, + currentCrawlMode: crawlMode, + }, + io as NotionPickerTuiIo, + ); + if (result.kind === 'quit') { + return { kind: 'back' }; + } + if (result.rootPageIds.length === 0) { + return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' }; + } + return { kind: 'selected', rootPageIds: result.rootPageIds }; } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - return 1; + return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) }; } } diff --git a/packages/cli/src/print-command-tree.test.ts b/packages/cli/src/print-command-tree.test.ts index c50ee9a3..1385d37d 100644 --- a/packages/cli/src/print-command-tree.test.ts +++ b/packages/cli/src/print-command-tree.test.ts @@ -16,7 +16,13 @@ describe('renderKtxCommandTree', () => { expect(topLevel).toContain(expected); } - expect(output).toContain('│ ├── test '); + expect(output).toContain('│ └── test '); + expect(output).not.toContain('│ ├── add'); + expect(output).not.toContain('│ ├── remove'); + expect(output).not.toContain('│ ├── map'); + expect(output).not.toContain('│ ├── mapping'); + expect(output).not.toContain('│ ├── metabase'); + expect(output).not.toContain('│ ├── notion'); }); it('ends with a single trailing newline', () => { diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index 76ba5d0f..3ee15b20 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -274,6 +274,97 @@ describe('setup sources step', () => { }); }); + it('uses the rich Notion picker for interactive selected root setup', async () => { + await addPrimarySource(); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' })); + const pickNotionRootPages = vi.fn(async (input: Parameters>[0]) => { + expect(input.connectionId).toBe('notion-main'); + expect(input.connection).toMatchObject({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: [], + }); + return { kind: 'selected' as const, rootPageIds: ['11111111-2222-3333-4444-555555555555'] }; + }); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['env', 'selected_roots', 'done'], + text: ['notion-main'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { prompts: testPrompts, validateNotion, pickNotionRootPages }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(pickNotionRootPages).toHaveBeenCalledOnce(); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['11111111-2222-3333-4444-555555555555'], + }); + }); + + it('backs out of the Notion picker without writing selected_roots when the picker quits', async () => { + await addPrimarySource(); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })); + const pickNotionRootPages = vi.fn(async () => ({ kind: 'back' as const })); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['env', 'selected_roots', 'all_accessible', 'done'], + text: ['notion-main'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { prompts: testPrompts, validateNotion, pickNotionRootPages }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(pickNotionRootPages).toHaveBeenCalledOnce(); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + crawl_mode: 'all_accessible', + }); + expect((await readConfig()).connections['notion-main']?.root_page_ids).toBeUndefined(); + }); + + it('surfaces Notion picker failures and returns to the page-mode step', async () => { + await addPrimarySource(); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=0' })); + const pickNotionRootPages = vi.fn(async () => ({ + kind: 'unavailable' as const, + message: 'Notion picker requires a TTY', + })); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['env', 'selected_roots', 'all_accessible', 'done'], + text: ['notion-main'], + }); + const io = makeIo(); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { prompts: testPrompts, validateNotion, pickNotionRootPages }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(io.stderr()).toContain('Notion picker requires a TTY'); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + crawl_mode: 'all_accessible', + }); + }); + it('defaults interactive Metabase and Looker source setup to the only warehouse connection', async () => { await addPrimarySource(); const cases: Array<{ diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 6674ef75..51043510 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -28,8 +28,8 @@ import { stripKtxSetupCompletedSteps, } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; -import { runKtxConnectionMapping } from './commands/connection-mapping.js'; -import { runKtxConnection } from './connection.js'; +import { pickNotionRootPages } from './notion-page-picker.js'; +import { runKtxSourceMapping } from './source-mapping.js'; import { withMenuOptionsSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; import { runKtxPublicIngest } from './public-ingest.js'; import { withSetupInterruptConfirmation } from './setup-interrupt.js'; @@ -95,6 +95,7 @@ export interface KtxSetupSourcesDeps { validateLooker?: (projectDir: string, connectionId: string) => Promise; validateLookml?: (connection: KtxProjectConnectionConfig) => Promise; validateNotion?: (connection: KtxProjectConnectionConfig) => Promise; + pickNotionRootPages?: typeof pickNotionRootPages; discoverMetabaseDatabases?: (args: { sourceUrl: string; sourceApiKeyRef: string; @@ -528,7 +529,7 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC driver: 'notion', auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'), crawl_mode: crawlMode, - root_page_ids: rootPageIds, + ...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}), root_database_ids: [], root_data_source_ids: [], max_pages_per_run: 1000, @@ -614,7 +615,7 @@ async function defaultValidateMetricflow(connection: KtxProjectConnectionConfig) } async function defaultValidateLooker(projectDir: string, connectionId: string): Promise { - const code = await runKtxConnectionMapping( + const code = await runKtxSourceMapping( { command: 'refresh', projectDir, connectionId, autoAccept: true }, { stdout: { write() {} }, stderr: { write() {} } }, ); @@ -657,6 +658,22 @@ interface MappingJsonOutput { mappings: unknown[]; } +function splitOutputLines(output: string): string[] { + return output + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); +} + +function parseMappingListJson(output: string): unknown[] { + const trimmed = output.trim(); + if (!trimmed) { + return []; + } + const parsed = JSON.parse(trimmed) as unknown; + return Array.isArray(parsed) ? parsed : []; +} + function summarizeMappingResult(parsed: MappingJsonOutput): string { const mappingCount = parsed.mappings.length; const mappingNoun = mappingCount === 1 ? 'mapping' : 'mappings'; @@ -664,22 +681,51 @@ function summarizeMappingResult(parsed: MappingJsonOutput): string { } async function defaultRunMapping(projectDir: string, connectionId: string, io: KtxCliIo): Promise { - let captured = ''; - const captureIo: KtxCliIo = { - stdout: { write(chunk: string) { captured += chunk; } }, - stderr: io.stderr, + const outputs = { + refresh: '', + validation: '', + list: '', }; - const code = await runKtxConnection( - { command: 'map', projectDir, sourceConnectionId: connectionId, json: true }, - captureIo, + const refreshCode = await runKtxSourceMapping( + { command: 'refresh', projectDir, connectionId, autoAccept: true }, + { + stdout: { write(chunk: string) { outputs.refresh += chunk; } }, + stderr: io.stderr, + }, ); - if (code !== 0) return code; - try { - const parsed = JSON.parse(captured.trim()) as MappingJsonOutput; - io.stdout.write(`${summarizeMappingResult(parsed)}\n`); - } catch { - io.stdout.write(captured); + if (refreshCode !== 0) { + return refreshCode; } + + const validationCode = await runKtxSourceMapping( + { command: 'validate', projectDir, connectionId }, + { + stdout: { write(chunk: string) { outputs.validation += chunk; } }, + stderr: io.stderr, + }, + ); + if (validationCode !== 0) { + return validationCode; + } + + const listCode = await runKtxSourceMapping( + { command: 'list', projectDir, connectionId, json: true }, + { + stdout: { write(chunk: string) { outputs.list += chunk; } }, + stderr: io.stderr, + }, + ); + if (listCode !== 0) { + return listCode; + } + + const parsed: MappingJsonOutput = { + connectionId, + refresh: { ok: true, output: splitOutputLines(outputs.refresh) }, + validation: { ok: true, output: splitOutputLines(outputs.validation) }, + mappings: parseMappingListJson(outputs.list), + }; + io.stdout.write(`${summarizeMappingResult(parsed)}\n`); return 0; } @@ -927,6 +973,8 @@ async function promptForInteractiveSource( args: KtxSetupSourcesArgs, source: KtxSetupSourceType, prompts: KtxSetupSourcesPromptAdapter, + io: KtxCliIo, + deps: KtxSetupSourcesDeps, defaultConnectionId = `${source}-main`, testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection, discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'], @@ -1213,15 +1261,29 @@ async function promptForInteractiveSource( ...(state.notionCrawlMode === 'selected_roots' ? [ async (currentState: SourcePromptState) => { - const roots = await promptText(prompts, { - message: 'Notion page IDs to ingest (each page includes all its subpages)', - placeholder: 'page-id-1, page-id-2', - }); - if (roots === undefined) return 'back'; - currentState.notionRootPageIds = roots - .split(',') - .map((root) => root.trim()) - .filter(Boolean); + const connectionId = currentState.sourceConnectionId ?? 'notion-main'; + const result = await (deps.pickNotionRootPages ?? pickNotionRootPages)( + { + connectionId, + connection: { + driver: 'notion', + auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'), + crawl_mode: 'selected_roots', + root_page_ids: currentState.notionRootPageIds ?? [], + root_database_ids: [], + root_data_source_ids: [], + }, + }, + io, + ); + if (result.kind === 'back') { + return 'back'; + } + if (result.kind === 'unavailable') { + io.stderr.write(`${result.message}\n`); + return 'back'; + } + currentState.notionRootPageIds = result.rootPageIds; return 'next'; }, ] @@ -1259,7 +1321,9 @@ async function chooseInteractiveSourceConnection(input: { source: KtxSetupSourceType; connections: Record; prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; testGitRepo?: KtxSetupSourcesDeps['testGitRepo']; + pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages']; discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases']; }): Promise { const existingIds = existingConnectionIdsBySource(input.connections, input.source); @@ -1271,6 +1335,11 @@ async function chooseInteractiveSourceConnection(input: { input.args, input.source, input.prompts, + input.io, + { + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }, defaultConnectionId, input.testGitRepo, input.discoverMetabaseDatabases, @@ -1303,6 +1372,11 @@ async function chooseInteractiveSourceConnection(input: { input.args, input.source, input.prompts, + input.io, + { + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }, defaultConnectionId, input.testGitRepo, input.discoverMetabaseDatabases, @@ -1417,7 +1491,9 @@ export async function runKtxSetupSourcesStep( source, connections: (await loadKtxProject({ projectDir: args.projectDir })).config.connections, prompts, + io, testGitRepo: deps.testGitRepo, + pickNotionRootPages: deps.pickNotionRootPages, discoverMetabaseDatabases: deps.discoverMetabaseDatabases, }); if (sourceChoice === 'back') { diff --git a/packages/cli/src/source-mapping.ts b/packages/cli/src/source-mapping.ts new file mode 100644 index 00000000..ade7fa4d --- /dev/null +++ b/packages/cli/src/source-mapping.ts @@ -0,0 +1,224 @@ +import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections'; +import { + DEFAULT_METABASE_CLIENT_CONFIG, + DefaultLookerConnectionClientFactory, + DefaultMetabaseConnectionClientFactory, + LocalLookerRuntimeStore, + LocalMetabaseSourceStateReader, + computeLookerMappingDrift, + computeMetabaseMappingDrift, + discoverLookerConnections, + discoverMetabaseDatabases, + lookerCredentialsFromLocalConnection, + metabaseRuntimeConfigFromLocalConnection, + seedLocalMappingStateFromKtxYaml, + validateLookerMappings, + validateMappingPhysicalMatch, + type LookerMappingClient, + type MetabaseRuntimeClient, +} from '@ktx/context/ingest'; +import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; +import type { KtxCliIo } from './cli-runtime.js'; +import { profileMark } from './startup-profile.js'; + +profileMark('module:source-mapping'); + +export type KtxSourceMappingArgs = + | { command: 'list'; projectDir: string; connectionId: string; json: boolean } + | { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean } + | { command: 'validate'; projectDir: string; connectionId: string }; + +interface KtxSourceMappingDeps { + createMetabaseClient?: ( + project: KtxLocalProject, + connectionId: string, + ) => Promise>; + createLookerClient?: ( + project: KtxLocalProject, + connectionId: string, + ) => Promise & { cleanup?(): Promise }>; +} + +async function createDefaultMetabaseClient( + project: KtxLocalProject, + connectionId: string, +): Promise> { + const factory = new DefaultMetabaseConnectionClientFactory( + (metabaseConnectionId) => + metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]), + DEFAULT_METABASE_CLIENT_CONFIG, + ); + return factory.createClient(connectionId); +} + +async function createDefaultLookerClient( + project: KtxLocalProject, + connectionId: string, +): Promise & { cleanup?(): Promise }> { + const factory = new DefaultLookerConnectionClientFactory({ + async resolve(lookerConnectionId) { + return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]); + }, + }); + return factory.createClient(connectionId) as unknown as Pick & { + cleanup?(): Promise; + }; +} + +function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean { + return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker'; +} + +function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void { + const connection = project.config.connections[connectionId]; + if (!connection || String(connection.driver).toLowerCase() !== 'metabase') { + throw new Error(`Connection "${connectionId}" is not a Metabase connection`); + } +} + +function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) { + const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]); + if (!descriptor) { + return { connection_type: 'UNKNOWN' }; + } + return { + connection_type: descriptor.connection_type, + host: descriptor.host ?? null, + database: descriptor.database ?? null, + account: descriptor.account ?? null, + project_id: descriptor.project_id ?? null, + dataset_id: descriptor.dataset_id ?? null, + ...descriptor.connection_params, + }; +} + +function renderMapping( + row: Awaited>[number], +): string { + const name = row.metabaseDatabaseName ?? 'unhydrated'; + const target = row.targetConnectionId ?? '[unmapped]'; + return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${ + row.source + })`; +} + +function renderLookerMapping(row: Awaited>[number]): string { + const target = row.ktxConnectionId ?? '[unmapped]'; + const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', '); + return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`; +} + +export async function runKtxSourceMapping( + args: KtxSourceMappingArgs, + io: KtxCliIo = process, + deps: KtxSourceMappingDeps = {}, +): Promise { + try { + const project = await loadKtxProject({ projectDir: args.projectDir }); + await seedLocalMappingStateFromKtxYaml(project, args.connectionId); + if (isLookerConnection(project, args.connectionId)) { + const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) }); + + if (args.command === 'list') { + const rows = await store.listConnectionMappings(args.connectionId); + io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`); + return 0; + } + + if (args.command === 'refresh') { + const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId); + try { + const discovered = await discoverLookerConnections(client); + const drift = computeLookerMappingDrift({ + storedMappings: await store.readMappings(args.connectionId), + discovered, + }); + if (args.autoAccept) { + await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered }); + } + io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`); + io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`); + io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`); + return 0; + } finally { + await client.cleanup?.(); + } + } + + const knownKtxConnectionIds = new Set(Object.keys(project.config.connections)); + const knownConnectionTypes = new Map( + Object.entries(project.config.connections).map(([id]) => [id, targetPhysicalInfo(project, id).connection_type]), + ); + const validation = validateLookerMappings({ + mappings: await store.readMappings(args.connectionId), + knownKtxConnectionIds, + knownConnectionTypes, + }); + if (!validation.ok) { + for (const error of validation.errors) { + io.stderr.write(`${error.key}: ${error.reason}\n`); + } + return 1; + } + io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`); + return 0; + } + + assertMetabaseConnection(project, args.connectionId); + const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) }); + + if (args.command === 'list') { + const rows = await store.listDatabaseMappings(args.connectionId); + io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`); + return 0; + } + + if (args.command === 'refresh') { + const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId); + try { + const discovered = await discoverMetabaseDatabases(client); + const existing = Object.fromEntries( + (await store.listDatabaseMappings(args.connectionId)).map((row) => [ + String(row.metabaseDatabaseId), + row.targetConnectionId, + ]), + ); + const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered }); + if (args.autoAccept) { + await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered }); + } + io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`); + io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`); + io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`); + return 0; + } finally { + await client.cleanup(); + } + } + + const rows = await store.listDatabaseMappings(args.connectionId); + const failures = rows.flatMap((row) => { + if (!row.targetConnectionId) { + return []; + } + const reason = validateMappingPhysicalMatch( + { metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost }, + project.config.connections[row.targetConnectionId] + ? targetPhysicalInfo(project, row.targetConnectionId) + : { connection_type: 'UNKNOWN' }, + ); + return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : []; + }); + if (failures.length > 0) { + for (const failure of failures) { + io.stderr.write(`${failure}\n`); + } + return 1; + } + io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`); + return 0; + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } +} diff --git a/packages/context/src/ingest/adapters/metabase/fetch.test.ts b/packages/context/src/ingest/adapters/metabase/fetch.test.ts index c8d4f4fb..45973c2b 100644 --- a/packages/context/src/ingest/adapters/metabase/fetch.test.ts +++ b/packages/context/src/ingest/adapters/metabase/fetch.test.ts @@ -276,7 +276,7 @@ describe('fetchMetabaseBundle', () => { clientFactory, sourceStateReader, }), - ).rejects.toThrow(/unhydrated.*ktx connection mapping refresh/); + ).rejects.toThrow(/unhydrated.*ktx setup/); }); it('skips cards whose getResolvedSql returns null and records them in unresolved-cards.json', async () => { diff --git a/packages/context/src/ingest/adapters/metabase/fetch.ts b/packages/context/src/ingest/adapters/metabase/fetch.ts index 9ccb2be6..52cd1b1b 100644 --- a/packages/context/src/ingest/adapters/metabase/fetch.ts +++ b/packages/context/src/ingest/adapters/metabase/fetch.ts @@ -99,7 +99,7 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr } if (mapping.metabaseDatabaseName === null) { throw new IngestInputError( - `mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`, + `mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx setup\` and reconfigure the Metabase source to populate metabaseDatabaseName before ingest.`, ); } const mappingDatabaseName: string = mapping.metabaseDatabaseName; diff --git a/packages/context/src/ingest/local-ingest.ts b/packages/context/src/ingest/local-ingest.ts index 2ec13184..a9beb5de 100644 --- a/packages/context/src/ingest/local-ingest.ts +++ b/packages/context/src/ingest/local-ingest.ts @@ -371,7 +371,7 @@ export async function runLocalMetabaseIngest( const unhydrated = await sourceStateReader.getUnhydratedSyncEnabledMappingIds(metabaseConnectionId); if (unhydrated.length > 0) { throw new Error( - `Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`ktx connection mapping refresh ${metabaseConnectionId}\` before local Metabase ingest.`, + `Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`ktx setup\` and reconfigure ${metabaseConnectionId} before local Metabase ingest.`, ); } diff --git a/packages/context/src/ingest/local-metabase-ingest.test.ts b/packages/context/src/ingest/local-metabase-ingest.test.ts index da00c7ec..dcc81e27 100644 --- a/packages/context/src/ingest/local-metabase-ingest.test.ts +++ b/packages/context/src/ingest/local-metabase-ingest.test.ts @@ -203,7 +203,7 @@ describe('runLocalMetabaseIngest', () => { metabaseConnectionId: 'prod-metabase', agentRunner: new TestAgentRunner(), }), - ).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`'); + ).rejects.toThrow('run `ktx setup` and reconfigure prod-metabase'); }); it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => { @@ -230,7 +230,7 @@ describe('runLocalMetabaseIngest', () => { adapters: [new FakeMetabaseSourceAdapter()], metabaseConnectionId: 'prod-metabase', }), - ).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`'); + ).rejects.toThrow('run `ktx setup` and reconfigure prod-metabase'); }); it('rejects source-dir uploads through the Metabase fan-out runner', async () => { diff --git a/packages/context/src/ingest/memory-flow/known-errors.ts b/packages/context/src/ingest/memory-flow/known-errors.ts index 8273ed86..f9998f89 100644 --- a/packages/context/src/ingest/memory-flow/known-errors.ts +++ b/packages/context/src/ingest/memory-flow/known-errors.ts @@ -23,6 +23,6 @@ export function formatNotionAuthorizationExpiredDetail(unitKey: string): string export function notionAuthorizationFixSuggestions(connectionId: string): string[] { return [ `Refresh the Notion token referenced by auth_token_ref for ${connectionId}. If it uses env:NAME, export a fresh token in that variable; if it uses file:/path, replace that file.`, - `Run ktx connection notion pick ${connectionId} to confirm Notion access, then rerun ktx ingest ${connectionId}.`, + `Run ktx setup and reconfigure the Notion source to confirm page access, then rerun ktx ingest ${connectionId}.`, ]; } diff --git a/packages/context/src/ingest/memory-flow/summary.test.ts b/packages/context/src/ingest/memory-flow/summary.test.ts index 0720dbae..a22ca1ff 100644 --- a/packages/context/src/ingest/memory-flow/summary.test.ts +++ b/packages/context/src/ingest/memory-flow/summary.test.ts @@ -85,7 +85,7 @@ describe('formatMemoryFlowFinalSummary', () => { '- Refresh the Notion token referenced by auth_token_ref for notion-main. If it uses env:NAME, export a fresh token in that variable; if it uses file:/path, replace that file.', ); expect(summary).toContain( - '- Run ktx connection notion pick notion-main to confirm Notion access, then rerun ktx ingest notion-main.', + '- Run ktx setup and reconfigure the Notion source to confirm page access, then rerun ktx ingest notion-main.', ); expect(summary).not.toContain('error_uri'); });