mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
Initial open-source release
This commit is contained in:
commit
1a42152e6f
1199 changed files with 257054 additions and 0 deletions
137
packages/cli/src/commands/agent-commands.ts
Normal file
137
packages/cli/src/commands/agent-commands.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import { Option, type Command } from '@commander-js/extra-typings';
|
||||
import type { KloAgentArgs } from '../agent.js';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
|
||||
|
||||
async function runAgent(context: KloCliCommandContext, args: KloAgentArgs): Promise<void> {
|
||||
const runner = context.deps.agent ?? (await import('../agent.js')).runKloAgent;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
function jsonOption(): Option {
|
||||
return new Option('--json', 'Print JSON output').makeOptionMandatory();
|
||||
}
|
||||
|
||||
export function registerAgentCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const agent = program
|
||||
.command('agent', { hidden: true })
|
||||
.description('Machine-readable KLO commands for coding agents')
|
||||
.showHelpAfterError();
|
||||
|
||||
agent.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('agent', actionCommand);
|
||||
});
|
||||
|
||||
agent
|
||||
.command('tools')
|
||||
.description('Print available agent-facing KLO tools')
|
||||
.addOption(jsonOption())
|
||||
.action(async (_options, command) => {
|
||||
await runAgent(context, { command: 'tools', projectDir: resolveCommandProjectDir(command), json: true });
|
||||
});
|
||||
|
||||
agent
|
||||
.command('context')
|
||||
.description('Print project context for agent planning')
|
||||
.addOption(jsonOption())
|
||||
.action(async (_options, command) => {
|
||||
await runAgent(context, { command: 'context', projectDir: resolveCommandProjectDir(command), json: true });
|
||||
});
|
||||
|
||||
const sl = agent.command('sl').description('Semantic-layer agent commands');
|
||||
sl.command('list')
|
||||
.description('List semantic-layer sources')
|
||||
.addOption(jsonOption())
|
||||
.option('--connection-id <id>', 'Filter by connection id')
|
||||
.option('--query <text>', 'Search source names and descriptions')
|
||||
.action(async (options: { connectionId?: string; query?: string }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'sl-list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
...(options.query ? { query: options.query } : {}),
|
||||
});
|
||||
});
|
||||
sl.command('read')
|
||||
.description('Read one semantic-layer source')
|
||||
.argument('<sourceName>')
|
||||
.addOption(jsonOption())
|
||||
.option('--connection-id <id>', 'Connection id containing the source')
|
||||
.action(async (sourceName: string, options: { connectionId?: string }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'sl-read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
sourceName,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
});
|
||||
});
|
||||
sl.command('query')
|
||||
.description('Run a semantic-layer query JSON file')
|
||||
.addOption(jsonOption())
|
||||
.requiredOption('--connection-id <id>', 'Connection id for execution')
|
||||
.requiredOption('--query-file <path>', 'JSON semantic-layer query file')
|
||||
.option('--execute', 'Execute the compiled query against the connection', false)
|
||||
.option('--max-rows <number>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
|
||||
.action(
|
||||
async (
|
||||
options: { connectionId: string; queryFile: string; execute: boolean; maxRows?: number },
|
||||
command,
|
||||
) => {
|
||||
await runAgent(context, {
|
||||
command: 'sl-query',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
connectionId: options.connectionId,
|
||||
queryFile: options.queryFile,
|
||||
execute: options.execute,
|
||||
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
const wiki = agent.command('wiki').description('KLO wiki agent commands');
|
||||
wiki
|
||||
.command('search')
|
||||
.description('Search KLO wiki pages')
|
||||
.argument('<query>')
|
||||
.addOption(jsonOption())
|
||||
.option('--limit <number>', 'Maximum search results', parsePositiveIntegerOption, 10)
|
||||
.action(async (query: string, options: { limit: number }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'wiki-search',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
query,
|
||||
limit: options.limit,
|
||||
});
|
||||
});
|
||||
wiki
|
||||
.command('read')
|
||||
.description('Read one KLO wiki page')
|
||||
.argument('<pageId>')
|
||||
.addOption(jsonOption())
|
||||
.action(async (pageId: string, _options, command) => {
|
||||
await runAgent(context, { command: 'wiki-read', projectDir: resolveCommandProjectDir(command), json: true, pageId });
|
||||
});
|
||||
|
||||
const sql = agent.command('sql').description('Safe SQL execution commands');
|
||||
sql
|
||||
.command('execute')
|
||||
.description('Execute read-only SQL with a row limit')
|
||||
.addOption(jsonOption())
|
||||
.requiredOption('--connection-id <id>', 'Connection id for execution')
|
||||
.requiredOption('--sql-file <path>', 'SQL file to execute')
|
||||
.requiredOption('--max-rows <number>', 'Maximum rows to return', parsePositiveIntegerOption)
|
||||
.action(async (options: { connectionId: string; sqlFile: string; maxRows: number }, command) => {
|
||||
await runAgent(context, {
|
||||
command: 'sql-execute',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: true,
|
||||
connectionId: options.connectionId,
|
||||
sqlFile: options.sqlFile,
|
||||
maxRows: options.maxRows,
|
||||
});
|
||||
});
|
||||
}
|
||||
47
packages/cli/src/commands/completion-commands.ts
Normal file
47
packages/cli/src/commands/completion-commands.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import type { CommandUnknownOpts } from '@commander-js/extra-typings';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { completeCommanderInput, installZshCompletion, zshCompletionScript } from '../completion.js';
|
||||
|
||||
export function registerCompletionCommands(
|
||||
program: CommandUnknownOpts,
|
||||
context: KloCliCommandContext,
|
||||
completionRoot: CommandUnknownOpts = program,
|
||||
): void {
|
||||
program
|
||||
.command('completion')
|
||||
.description('Generate shell completion scripts')
|
||||
.command('zsh')
|
||||
.description('Generate zsh completion script')
|
||||
.option('--install', 'Install zsh completion into ~/.zfunc and update ~/.zshrc', false)
|
||||
.action(async (options: { install?: boolean }) => {
|
||||
if (options.install === true) {
|
||||
const result = await installZshCompletion();
|
||||
context.io.stdout.write(`Installed zsh completion: ${result.completionPath}\n`);
|
||||
context.io.stdout.write(`Updated zsh config: ${result.zshrcPath}\n`);
|
||||
context.io.stdout.write('Restart your shell or run: source ~/.zshrc\n');
|
||||
context.setExitCode(0);
|
||||
return;
|
||||
}
|
||||
context.io.stdout.write(zshCompletionScript());
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
program
|
||||
.command('__complete', { hidden: true })
|
||||
.description('Internal shell completion endpoint')
|
||||
.requiredOption('--shell <shell>', 'Shell requesting completions')
|
||||
.requiredOption('--position <position>', 'Current shell word position', (value) => Number(value))
|
||||
.argument('[words...]', 'Current shell words')
|
||||
.allowUnknownOption()
|
||||
.allowExcessArguments()
|
||||
.action((words: string[], options: { shell: string; position: number }) => {
|
||||
if (options.shell !== 'zsh') {
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
for (const completion of completeCommanderInput(completionRoot, { position: options.position, words })) {
|
||||
context.io.stdout.write(`${completion}\n`);
|
||||
}
|
||||
context.setExitCode(0);
|
||||
});
|
||||
}
|
||||
346
packages/cli/src/commands/connection-commands.ts
Normal file
346
packages/cli/src/commands/connection-commands.ts
Normal file
|
|
@ -0,0 +1,346 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import {
|
||||
collectOption,
|
||||
type KloCliCommandContext,
|
||||
parseBooleanStringOption,
|
||||
parseNonEmptyAssignmentOption,
|
||||
parseNonNegativeIntegerOption,
|
||||
parsePositiveIntegerOption,
|
||||
parseSafeConnectionIdOption,
|
||||
resolveCommandProjectDir,
|
||||
} from '../cli-program.js';
|
||||
import { connectionAddCommandSchema } from '../command-schemas.js';
|
||||
import type { KloConnectionArgs } from '../connection.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
import type { KloConnectionMappingArgs } from './connection-mapping.js';
|
||||
import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js';
|
||||
import { registerConnectionNotionCommands } from './connection-notion-commands.js';
|
||||
|
||||
profileMark('module:commands/connection-commands');
|
||||
|
||||
const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const;
|
||||
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const;
|
||||
|
||||
function parseCsvIds(value: string): number[] {
|
||||
return value
|
||||
.split(',')
|
||||
.filter(Boolean)
|
||||
.map((item) => parsePositiveIntegerOption(item));
|
||||
}
|
||||
|
||||
function parseCsvStrings(value: string): string[] {
|
||||
return value
|
||||
.split(',')
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' {
|
||||
if (value === 'databaseMappings' || value === 'connectionMappings') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('must be databaseMappings or connectionMappings');
|
||||
}
|
||||
|
||||
async function runConnectionArgs(context: KloCliCommandContext, args: KloConnectionArgs): Promise<void> {
|
||||
const runner = context.deps.connection ?? (await import('../connection.js')).runKloConnection;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
async function runMappingArgs(context: KloCliCommandContext, args: KloConnectionMappingArgs): Promise<void> {
|
||||
const { runKloConnectionMapping } = await import('./connection-mapping.js');
|
||||
context.setExitCode(await runKloConnectionMapping(args, context.io));
|
||||
}
|
||||
|
||||
export function registerConnectionCommands(program: Command, context: KloCliCommandContext, commandName = 'connection'): void {
|
||||
const connection = program
|
||||
.command(commandName)
|
||||
.description('Add, list, test, and map data sources')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the nearest klo.yaml or current working directory.\n',
|
||||
);
|
||||
connection.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.(commandName, actionCommand);
|
||||
});
|
||||
|
||||
connection
|
||||
.command('list')
|
||||
.description('List configured connections')
|
||||
.action(async (_options: unknown, command) => {
|
||||
await runConnectionArgs(context, { command: 'list', projectDir: resolveCommandProjectDir(command) });
|
||||
});
|
||||
|
||||
connection
|
||||
.command('test')
|
||||
.description('Test a configured connection')
|
||||
.argument('<connectionId>', 'KLO connection id')
|
||||
.action(async (connectionId: string, _options: unknown, command) => {
|
||||
await runConnectionArgs(context, {
|
||||
command: 'test',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
});
|
||||
});
|
||||
|
||||
connection
|
||||
.command('add')
|
||||
.description('Add or replace a configured connection')
|
||||
.argument('<driver>', 'Connection driver')
|
||||
.argument('<connectionId>', 'KLO connection id')
|
||||
.option('--url <url>', 'Connection URL, env:NAME, or file:/path reference')
|
||||
.option('--schema <schema>', 'Schema to include; repeatable', collectOption, [])
|
||||
.option('--readonly', 'Mark the connection as read-only', false)
|
||||
.option('--force', 'Replace an existing connection', false)
|
||||
.option('--allow-literal-credentials', 'Allow writing a literal credential URL to klo.yaml', false)
|
||||
.addOption(new Option('--token-env <name>', 'Environment variable containing Notion auth token').conflicts('tokenFile'))
|
||||
.addOption(new Option('--token-file <path>', 'File containing Notion auth token').conflicts('tokenEnv'))
|
||||
.addOption(
|
||||
new Option('--crawl-mode <mode>', 'Notion crawl mode: all_accessible or selected_roots')
|
||||
.choices(CRAWL_MODE_CHOICES)
|
||||
.default('selected_roots'),
|
||||
)
|
||||
.option('--root-page-id <id>', 'Root page to crawl; repeatable', collectOption, [])
|
||||
.option('--root-database-id <id>', 'Root database to crawl; repeatable', collectOption, [])
|
||||
.option('--root-data-source-id <id>', 'Root data source to crawl; repeatable', collectOption, [])
|
||||
.option('--max-pages <n>', 'Maximum pages per run', parsePositiveIntegerOption)
|
||||
.option('--max-knowledge-creates <n>', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption)
|
||||
.option('--max-knowledge-updates <n>', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption)
|
||||
.action(async (driver: string, connectionId: string, options, command) => {
|
||||
const notion =
|
||||
driver === 'notion'
|
||||
? {
|
||||
authTokenRef: options.tokenEnv
|
||||
? `env:${options.tokenEnv}`
|
||||
: options.tokenFile
|
||||
? `file:${options.tokenFile}`
|
||||
: '',
|
||||
crawlMode: options.crawlMode,
|
||||
rootPageIds: options.rootPageId,
|
||||
rootDatabaseIds: options.rootDatabaseId,
|
||||
rootDataSourceIds: options.rootDataSourceId,
|
||||
maxPagesPerRun: options.maxPages,
|
||||
maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates,
|
||||
maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
if (driver === 'notion' && !notion?.authTokenRef) {
|
||||
throw new Error('connection add notion requires --token-env NAME or --token-file PATH');
|
||||
}
|
||||
if (
|
||||
driver === 'notion' &&
|
||||
notion?.crawlMode === 'selected_roots' &&
|
||||
notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0
|
||||
) {
|
||||
throw new Error('connection add notion selected_roots requires at least one root id');
|
||||
}
|
||||
|
||||
const args = connectionAddCommandSchema.parse({
|
||||
command: 'add',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
driver,
|
||||
connectionId,
|
||||
url: options.url,
|
||||
schemas: options.schema.filter(Boolean),
|
||||
readonly: options.readonly === true,
|
||||
force: options.force === true,
|
||||
allowLiteralCredentials: options.allowLiteralCredentials === true,
|
||||
notion,
|
||||
});
|
||||
|
||||
await runConnectionArgs(context, args);
|
||||
});
|
||||
|
||||
connection
|
||||
.command('remove')
|
||||
.description('Remove a configured connection from klo.yaml')
|
||||
.argument('<connectionId>', 'KLO connection id')
|
||||
.option('--force', 'Remove without prompting', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => {
|
||||
await runConnectionArgs(context, {
|
||||
command: 'remove',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
force: options.force === true,
|
||||
...(options.input === false ? { inputMode: 'disabled' } : {}),
|
||||
});
|
||||
});
|
||||
|
||||
connection
|
||||
.command('map')
|
||||
.description('Refresh and validate BI-to-warehouse mappings')
|
||||
.argument('<sourceConnectionId>', 'Source BI connection id')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (sourceConnectionId: string, options: { json?: boolean }, command) => {
|
||||
await runConnectionArgs(context, {
|
||||
command: 'map',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
sourceConnectionId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
registerConnectionMappingCommands(connection, context);
|
||||
registerConnectionMetabaseCommands(connection, context);
|
||||
registerConnectionNotionCommands(connection, context);
|
||||
}
|
||||
|
||||
export function registerConnectionMappingCommands(connection: Command, context: KloCliCommandContext): void {
|
||||
const mapping = connection
|
||||
.command('mapping')
|
||||
.description('Manage Metabase warehouse mappings')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
mapping
|
||||
.command('list')
|
||||
.description('List Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.option('--json', 'Print JSON output where supported', false)
|
||||
.action(async (connectionId: string, options: { json?: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('set')
|
||||
.description('Set a Metabase or Looker warehouse mapping')
|
||||
.argument('<connectionId>', 'Source connection id', parseSafeConnectionIdOption)
|
||||
.argument('<field>', 'Mapping field', parseMappingFieldOption)
|
||||
.argument('<assignment>', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption)
|
||||
.action(
|
||||
async (
|
||||
connectionId: string,
|
||||
field: 'databaseMappings' | 'connectionMappings',
|
||||
assignment: { key: string; value: string },
|
||||
_options: unknown,
|
||||
command,
|
||||
) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'set',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
field,
|
||||
key: assignment.key,
|
||||
value: assignment.value,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
mapping
|
||||
.command('apply-bulk')
|
||||
.description('Apply mappings from JSON')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.requiredOption('--file <path>', 'JSON mapping file')
|
||||
.action(async (connectionId: string, options: { file: string }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'apply-bulk',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
filePath: options.file,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('set-sync-enabled')
|
||||
.description('Enable or disable sync for one Metabase database')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.argument('<metabaseDatabaseId>', 'Metabase database id', parsePositiveIntegerOption)
|
||||
.requiredOption('--enabled <value>', 'true or false', parseBooleanStringOption)
|
||||
.action(
|
||||
async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'set-sync-enabled',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
metabaseDatabaseId,
|
||||
enabled: options.enabled,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection');
|
||||
syncState
|
||||
.command('get')
|
||||
.description('Read sync-state selection')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.option('--json', 'Print JSON output where supported', false)
|
||||
.action(async (connectionId: string, options: { json?: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'sync-state-get',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
syncState
|
||||
.command('set')
|
||||
.description('Write sync-state selection')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.addOption(new Option('--mode <mode>', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory())
|
||||
.option('--collections <ids>', 'Comma-separated collection ids', parseCsvIds, [])
|
||||
.option('--items <ids>', 'Comma-separated item ids', parseCsvIds, [])
|
||||
.option('--tag-names <names>', 'Comma-separated tag names', parseCsvStrings, [])
|
||||
.action(async (connectionId: string, options, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'sync-state-set',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
syncMode: options.mode,
|
||||
collectionIds: options.collections,
|
||||
itemIds: options.items,
|
||||
tagNames: options.tagNames,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('refresh')
|
||||
.description('Refresh Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.option('--auto-accept', 'Accept refresh changes without prompting', false)
|
||||
.action(async (connectionId: string, options: { autoAccept?: boolean }, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'refresh',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
autoAccept: options.autoAccept === true,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('validate')
|
||||
.description('Validate Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.action(async (connectionId: string, _options: unknown, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'validate',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
});
|
||||
});
|
||||
|
||||
mapping
|
||||
.command('clear')
|
||||
.description('Clear Metabase database mappings')
|
||||
.argument('<connectionId>', 'Metabase connection id')
|
||||
.argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption)
|
||||
.action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => {
|
||||
await runMappingArgs(context, {
|
||||
command: 'clear',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
...(metabaseDatabaseId ? { metabaseDatabaseId } : {}),
|
||||
});
|
||||
});
|
||||
}
|
||||
329
packages/cli/src/commands/connection-mapping.test.ts
Normal file
329
packages/cli/src/commands/connection-mapping.test.ts
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { LocalMetabaseSourceStateReader } from '@klo/context/ingest';
|
||||
import { initKloProject, loadKloProject, serializeKloProjectConfig } from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKloConnectionMapping } from './connection-mapping.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
describe('runKloConnectionMapping', () => {
|
||||
let tempDir: string;
|
||||
let projectDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-metabase-mapping-'));
|
||||
projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'mapping' });
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections: {
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||
},
|
||||
'prod-warehouse': {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_URL',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Seed Metabase mapping test connections',
|
||||
);
|
||||
});
|
||||
|
||||
async function replaceConnections(connections: Record<string, { driver: string; [key: string]: unknown }>) {
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections,
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Replace mapping test connections',
|
||||
);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('sets, lists, disables, and clears local Metabase mappings', async () => {
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
field: 'databaseMappings',
|
||||
key: '1',
|
||||
value: 'prod-warehouse',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const listIo = makeIo();
|
||||
await expect(
|
||||
runKloConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io),
|
||||
).resolves.toBe(0);
|
||||
expect(listIo.stdout()).toContain('1 -> prod-warehouse');
|
||||
expect(listIo.stdout()).toContain('unhydrated');
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set-sync-enabled',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
metabaseDatabaseId: 1,
|
||||
enabled: false,
|
||||
},
|
||||
makeIo().io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'clear',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
metabaseDatabaseId: 1,
|
||||
},
|
||||
makeIo().io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-cli-yaml-mapping-'));
|
||||
await initKloProject({ projectDir, projectName: 'yaml-mapping' });
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections: {
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
mappings: {
|
||||
databaseMappings: { '1': 'prod-warehouse' },
|
||||
syncEnabled: { '1': true },
|
||||
},
|
||||
},
|
||||
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
|
||||
},
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Seed yaml mappings',
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{ command: 'list', projectDir, connectionId: 'prod-metabase', json: false },
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('1 -> prod-warehouse');
|
||||
expect(io.stdout()).toContain('source: klo.yaml');
|
||||
});
|
||||
|
||||
it('refreshes Metabase discovery metadata through the injected runtime client', async () => {
|
||||
const client = {
|
||||
getDatabases: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: 1,
|
||||
name: 'Analytics',
|
||||
engine: 'postgres',
|
||||
details: { host: 'pg.internal', dbname: 'analytics' },
|
||||
is_sample: false,
|
||||
},
|
||||
]),
|
||||
cleanup: vi.fn(),
|
||||
};
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'refresh',
|
||||
projectDir,
|
||||
connectionId: 'prod-metabase',
|
||||
autoAccept: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
createMetabaseClient: async () => client as never,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Discovery: 1 database');
|
||||
expect(client.cleanup).toHaveBeenCalledTimes(1);
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.klo', 'db.sqlite') });
|
||||
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
|
||||
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('sets and lists Looker connection mappings', async () => {
|
||||
await replaceConnections({
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.test',
|
||||
client_id: 'id',
|
||||
},
|
||||
'prod-warehouse': {
|
||||
driver: 'postgres',
|
||||
url: 'postgresql://readonly@db.example.test/analytics',
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set',
|
||||
projectDir,
|
||||
connectionId: 'prod-looker',
|
||||
field: 'connectionMappings',
|
||||
key: 'analytics',
|
||||
value: 'prod-warehouse',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
await expect(
|
||||
runKloConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('analytics -> prod-warehouse');
|
||||
});
|
||||
|
||||
it('keeps driver-specific mapping field validation in the runner', async () => {
|
||||
await replaceConnections({
|
||||
'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' },
|
||||
warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' },
|
||||
});
|
||||
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{
|
||||
command: 'set',
|
||||
projectDir,
|
||||
connectionId: 'prod-looker',
|
||||
field: 'databaseMappings',
|
||||
key: '1',
|
||||
value: 'warehouse',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('Looker mapping set requires connectionMappings');
|
||||
});
|
||||
|
||||
it('refreshes Looker mapping metadata and reports drift', async () => {
|
||||
await replaceConnections({
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.test',
|
||||
client_id: 'id',
|
||||
},
|
||||
'prod-warehouse': {
|
||||
driver: 'postgres',
|
||||
url: 'postgresql://readonly@db.example.test/analytics',
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping(
|
||||
{ command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true },
|
||||
io.io,
|
||||
{
|
||||
createLookerClient: async () => ({
|
||||
listLookerConnections: async () => [
|
||||
{
|
||||
name: 'analytics',
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
schema: null,
|
||||
dialect: 'postgres',
|
||||
},
|
||||
],
|
||||
cleanup: async () => {},
|
||||
}),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Discovery: 1 connection');
|
||||
expect(io.stdout()).toContain('Unmapped discovered: 1');
|
||||
});
|
||||
|
||||
it('validates Looker mappings through the canonical local warehouse descriptor', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-cli-descriptor-validation-'));
|
||||
await initKloProject({ projectDir, projectName: 'descriptor-validation' });
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig({
|
||||
...project.config,
|
||||
connections: {
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
|
||||
},
|
||||
'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' },
|
||||
},
|
||||
}),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'Seed descriptor validation',
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(io.stdout()).toContain('Mapping validation passed: prod-looker');
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
});
|
||||
426
packages/cli/src/commands/connection-mapping.ts
Normal file
426
packages/cli/src/commands/connection-mapping.ts
Normal file
|
|
@ -0,0 +1,426 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import { localConnectionToWarehouseDescriptor } from '@klo/context/connections';
|
||||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
LocalLookerRuntimeStore,
|
||||
LocalMetabaseSourceStateReader,
|
||||
computeLookerMappingDrift,
|
||||
computeMetabaseMappingDrift,
|
||||
discoverLookerConnections,
|
||||
discoverMetabaseDatabases,
|
||||
lookerCredentialsFromLocalConnection,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
seedLocalMappingStateFromKloYaml,
|
||||
validateLookerMappings,
|
||||
validateMappingPhysicalMatch,
|
||||
type LookerMappingClient,
|
||||
type MetabaseRuntimeClient,
|
||||
type MetabaseSyncMode,
|
||||
} from '@klo/context/ingest';
|
||||
import { type KloLocalProject, kloLocalStateDbPath, loadKloProject } from '@klo/context/project';
|
||||
import type { KloCliIo } from '../index.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/connection-mapping');
|
||||
|
||||
export type KloConnectionMappingArgs =
|
||||
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
|
||||
| {
|
||||
command: 'set';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
field: 'databaseMappings' | 'connectionMappings';
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
| { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string }
|
||||
| {
|
||||
command: 'set-sync-enabled';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
metabaseDatabaseId: number;
|
||||
enabled: boolean;
|
||||
}
|
||||
| { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean }
|
||||
| {
|
||||
command: 'sync-state-set';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
syncMode: MetabaseSyncMode;
|
||||
collectionIds: number[];
|
||||
itemIds: number[];
|
||||
tagNames: string[];
|
||||
}
|
||||
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
|
||||
| { command: 'validate'; projectDir: string; connectionId: string }
|
||||
| { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string };
|
||||
|
||||
interface KloConnectionMappingDeps {
|
||||
createMetabaseClient?: (
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
|
||||
createLookerClient?: (
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
|
||||
}
|
||||
|
||||
interface MetabaseBulkMappingPayload {
|
||||
databaseMappings?: Record<string, string | null>;
|
||||
syncEnabled?: Record<string, boolean>;
|
||||
syncMode?: MetabaseSyncMode;
|
||||
selections?: { collections?: number[]; items?: number[] };
|
||||
defaultTagNames?: string[];
|
||||
}
|
||||
|
||||
function parseId(value: string, label: string): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function createDefaultMetabaseClient(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
|
||||
const factory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
return factory.createClient(connectionId);
|
||||
}
|
||||
|
||||
async function createDefaultLookerClient(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
|
||||
const factory = new DefaultLookerConnectionClientFactory({
|
||||
async resolve(lookerConnectionId) {
|
||||
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
|
||||
},
|
||||
});
|
||||
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
|
||||
cleanup?(): Promise<void>;
|
||||
};
|
||||
}
|
||||
|
||||
function isLookerConnection(project: KloLocalProject, connectionId: string): boolean {
|
||||
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
|
||||
}
|
||||
|
||||
function assertLookerConnection(project: KloLocalProject, connectionId: string): void {
|
||||
if (!isLookerConnection(project, connectionId)) {
|
||||
throw new Error(`Connection "${connectionId}" is not a Looker connection`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertMetabaseConnection(project: KloLocalProject, connectionId: string): void {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
|
||||
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertTargetConnection(project: KloLocalProject, connectionId: string): void {
|
||||
if (!project.config.connections[connectionId]) {
|
||||
throw new Error(`Target connection "${connectionId}" does not exist`);
|
||||
}
|
||||
}
|
||||
|
||||
function targetPhysicalInfo(project: KloLocalProject, connectionId: string) {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
|
||||
if (!descriptor) {
|
||||
return { connection_type: 'UNKNOWN' };
|
||||
}
|
||||
return {
|
||||
connection_type: descriptor.connection_type,
|
||||
host: descriptor.host ?? null,
|
||||
database: descriptor.database ?? null,
|
||||
account: descriptor.account ?? null,
|
||||
project_id: descriptor.project_id ?? null,
|
||||
dataset_id: descriptor.dataset_id ?? null,
|
||||
...descriptor.connection_params,
|
||||
};
|
||||
}
|
||||
|
||||
function renderMapping(
|
||||
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
|
||||
): string {
|
||||
const name = row.metabaseDatabaseName ?? 'unhydrated';
|
||||
const target = row.targetConnectionId ?? '[unmapped]';
|
||||
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
|
||||
row.source
|
||||
})`;
|
||||
}
|
||||
|
||||
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
|
||||
const target = row.kloConnectionId ?? '[unmapped]';
|
||||
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
|
||||
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
|
||||
}
|
||||
|
||||
export async function runKloConnectionMapping(
|
||||
args: KloConnectionMappingArgs,
|
||||
io: KloCliIo = process,
|
||||
deps: KloConnectionMappingDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
await seedLocalMappingStateFromKloYaml(project, args.connectionId);
|
||||
if (isLookerConnection(project, args.connectionId)) {
|
||||
assertLookerConnection(project, args.connectionId);
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: kloLocalStateDbPath(project) });
|
||||
|
||||
if (args.command === 'list') {
|
||||
const rows = await store.listConnectionMappings(args.connectionId);
|
||||
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'set') {
|
||||
if (args.field !== 'connectionMappings') {
|
||||
throw new Error('Looker mapping set requires connectionMappings <lookerConnectionName>=<targetConnectionId>');
|
||||
}
|
||||
assertTargetConnection(project, args.value);
|
||||
await store.upsertConnectionMapping({
|
||||
lookerConnectionId: args.connectionId,
|
||||
lookerConnectionName: args.key,
|
||||
kloConnectionId: args.value,
|
||||
source: 'cli',
|
||||
});
|
||||
io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'refresh') {
|
||||
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
|
||||
try {
|
||||
const discovered = await discoverLookerConnections(client);
|
||||
const drift = computeLookerMappingDrift({
|
||||
storedMappings: await store.readMappings(args.connectionId),
|
||||
discovered,
|
||||
});
|
||||
if (args.autoAccept) {
|
||||
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
|
||||
}
|
||||
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
|
||||
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
|
||||
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
|
||||
return 0;
|
||||
} finally {
|
||||
await client.cleanup?.();
|
||||
}
|
||||
}
|
||||
|
||||
if (args.command === 'validate') {
|
||||
const knownKloConnectionIds = new Set(Object.keys(project.config.connections));
|
||||
const knownConnectionTypes = new Map(
|
||||
Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]),
|
||||
);
|
||||
const validation = validateLookerMappings({
|
||||
mappings: await store.readMappings(args.connectionId),
|
||||
knownKloConnectionIds,
|
||||
knownConnectionTypes,
|
||||
});
|
||||
if (!validation.ok) {
|
||||
for (const error of validation.errors) {
|
||||
io.stderr.write(`${error.key}: ${error.reason}\n`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'clear') {
|
||||
await store.clearConnectionMappings({
|
||||
lookerConnectionId: args.connectionId,
|
||||
lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined),
|
||||
});
|
||||
io.stdout.write(
|
||||
args.mappingKey
|
||||
? `Cleared connectionMappings.${args.mappingKey}\n`
|
||||
: `Cleared mappings for ${args.connectionId}\n`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
throw new Error(`Looker connection mapping does not support ${args.command}`);
|
||||
}
|
||||
|
||||
assertMetabaseConnection(project, args.connectionId);
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(project) });
|
||||
|
||||
if (args.command === 'list') {
|
||||
const rows = await store.listDatabaseMappings(args.connectionId);
|
||||
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'set') {
|
||||
assertTargetConnection(project, args.value);
|
||||
await store.upsertDatabaseMapping({
|
||||
connectionId: args.connectionId,
|
||||
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
|
||||
targetConnectionId: args.value,
|
||||
syncEnabled: true,
|
||||
source: 'cli',
|
||||
});
|
||||
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'apply-bulk') {
|
||||
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
|
||||
const existingState = await store.getSourceState(args.connectionId);
|
||||
const existingRows = await store.listDatabaseMappings(args.connectionId);
|
||||
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
|
||||
const databaseMappings = payload.databaseMappings ?? {};
|
||||
for (const targetConnectionId of Object.values(databaseMappings)) {
|
||||
if (targetConnectionId) {
|
||||
assertTargetConnection(project, targetConnectionId);
|
||||
}
|
||||
}
|
||||
const mappingIds = new Set([
|
||||
...existingRows.map((row) => row.metabaseDatabaseId),
|
||||
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
|
||||
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
|
||||
]);
|
||||
await store.replaceSourceState({
|
||||
connectionId: args.connectionId,
|
||||
syncMode: payload.syncMode ?? existingState.syncMode,
|
||||
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
|
||||
selections:
|
||||
payload.selections === undefined
|
||||
? existingState.selections
|
||||
: [
|
||||
...(payload.selections.collections ?? []).map((id) => ({
|
||||
selectionType: 'collection' as const,
|
||||
metabaseObjectId: id,
|
||||
})),
|
||||
...(payload.selections.items ?? []).map((id) => ({
|
||||
selectionType: 'item' as const,
|
||||
metabaseObjectId: id,
|
||||
})),
|
||||
],
|
||||
mappings: [...mappingIds]
|
||||
.sort((a, b) => a - b)
|
||||
.map((id) => {
|
||||
const existing = existingById.get(id);
|
||||
return {
|
||||
metabaseDatabaseId: id,
|
||||
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
|
||||
metabaseEngine: existing?.metabaseEngine ?? null,
|
||||
metabaseHost: existing?.metabaseHost ?? null,
|
||||
metabaseDbName: existing?.metabaseDbName ?? null,
|
||||
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
|
||||
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
|
||||
source: 'cli',
|
||||
};
|
||||
}),
|
||||
});
|
||||
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'set-sync-enabled') {
|
||||
await store.setMappingSyncEnabled({
|
||||
connectionId: args.connectionId,
|
||||
metabaseDatabaseId: args.metabaseDatabaseId,
|
||||
syncEnabled: args.enabled,
|
||||
});
|
||||
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sync-state-get') {
|
||||
const state = await store.getSourceState(args.connectionId);
|
||||
const payload = {
|
||||
syncMode: state.syncMode,
|
||||
selections: state.selections,
|
||||
defaultTagNames: state.defaultTagNames,
|
||||
};
|
||||
io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'sync-state-set') {
|
||||
await store.setSyncState({
|
||||
connectionId: args.connectionId,
|
||||
syncMode: args.syncMode,
|
||||
defaultTagNames: args.tagNames,
|
||||
selections: [
|
||||
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
|
||||
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
|
||||
],
|
||||
});
|
||||
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (args.command === 'refresh') {
|
||||
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
|
||||
try {
|
||||
const discovered = await discoverMetabaseDatabases(client);
|
||||
const existing = Object.fromEntries(
|
||||
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
|
||||
String(row.metabaseDatabaseId),
|
||||
row.targetConnectionId,
|
||||
]),
|
||||
);
|
||||
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
|
||||
if (args.autoAccept) {
|
||||
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
|
||||
}
|
||||
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
|
||||
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
|
||||
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
|
||||
return 0;
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
if (args.command === 'validate') {
|
||||
const rows = await store.listDatabaseMappings(args.connectionId);
|
||||
const failures = rows.flatMap((row) => {
|
||||
if (!row.targetConnectionId) {
|
||||
return [];
|
||||
}
|
||||
const reason = validateMappingPhysicalMatch(
|
||||
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
|
||||
project.config.connections[row.targetConnectionId]
|
||||
? targetPhysicalInfo(project, row.targetConnectionId)
|
||||
: { connection_type: 'UNKNOWN' },
|
||||
);
|
||||
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
|
||||
});
|
||||
if (failures.length > 0) {
|
||||
for (const failure of failures) {
|
||||
io.stderr.write(`${failure}\n`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
|
||||
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
|
||||
io.stdout.write(
|
||||
metabaseDatabaseId
|
||||
? `Cleared databaseMappings.${metabaseDatabaseId}\n`
|
||||
: `Cleared mappings for ${args.connectionId}\n`,
|
||||
);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
132
packages/cli/src/commands/connection-metabase-commands.ts
Normal file
132
packages/cli/src/commands/connection-metabase-commands.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
|
||||
import {
|
||||
type KloCliCommandContext,
|
||||
parseNonEmptyAssignmentOption,
|
||||
parsePositiveIntegerOption,
|
||||
parseSafeConnectionIdOption,
|
||||
resolveCommandProjectDir,
|
||||
} from '../cli-program.js';
|
||||
import {
|
||||
type KloConnectionMetabaseSetupArgs,
|
||||
type MetabaseSetupMappingAssignment,
|
||||
type MetabaseSetupSyncMode,
|
||||
runKloConnectionMetabaseSetup,
|
||||
} from './connection-metabase-setup.js';
|
||||
|
||||
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[];
|
||||
|
||||
interface ConnectionMetabaseSetupOptions {
|
||||
id?: string;
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
mintApiKey?: boolean;
|
||||
username?: string;
|
||||
password?: string;
|
||||
map: MetabaseSetupMappingAssignment[];
|
||||
sync: number[];
|
||||
syncMode: MetabaseSetupSyncMode;
|
||||
runIngest?: boolean;
|
||||
yes?: boolean;
|
||||
input?: boolean;
|
||||
}
|
||||
|
||||
function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] {
|
||||
return [...previous, parsePositiveIntegerOption(value)];
|
||||
}
|
||||
|
||||
function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment {
|
||||
const assignment = parseNonEmptyAssignmentOption(value);
|
||||
return {
|
||||
metabaseDatabaseId: parsePositiveIntegerOption(assignment.key),
|
||||
targetConnectionId: parseSafeConnectionIdOption(assignment.value),
|
||||
};
|
||||
}
|
||||
|
||||
function collectMappingOption(
|
||||
value: string,
|
||||
previous: MetabaseSetupMappingAssignment[] = [],
|
||||
): MetabaseSetupMappingAssignment[] {
|
||||
return [...previous, parseMappingAssignment(value)];
|
||||
}
|
||||
|
||||
async function runMetabaseSetupArgs(
|
||||
context: KloCliCommandContext,
|
||||
args: KloConnectionMetabaseSetupArgs,
|
||||
): Promise<void> {
|
||||
const runner = context.deps.connectionMetabaseSetup ?? runKloConnectionMetabaseSetup;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerConnectionMetabaseCommands(connection: Command, context: KloCliCommandContext): void {
|
||||
const metabase = connection
|
||||
.command('metabase')
|
||||
.description('Configure Metabase connections')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
metabase.action(() => {
|
||||
metabase.outputHelp();
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
metabase
|
||||
.command('setup')
|
||||
.description('Guided setup for a Metabase connection')
|
||||
.option('--id <connectionId>', 'KLO connection id to write', parseSafeConnectionIdOption)
|
||||
.option('--url <url>', 'Metabase API URL')
|
||||
.addOption(new Option('--api-key <key>', 'Metabase API key').conflicts('mintApiKey'))
|
||||
.option('--mint-api-key', 'Mint a Metabase API key with credentials', false)
|
||||
.option('--username <email>', 'Metabase admin username for API-key minting')
|
||||
.option('--password <password>', 'Metabase admin password for API-key minting')
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nGuided equivalent of:\n' +
|
||||
' klo connection mapping refresh <connectionId> --auto-accept\n' +
|
||||
' klo connection mapping set <connectionId> databaseMappings <id>=<target>\n' +
|
||||
' klo connection mapping set-sync-enabled <connectionId> <id> --enabled true\n' +
|
||||
' klo ingest <connectionId>\n',
|
||||
)
|
||||
.option(
|
||||
'--map <metabaseDatabaseId=targetConnectionId>',
|
||||
'Assign a Metabase database id to a warehouse connection; repeatable',
|
||||
collectMappingOption,
|
||||
[],
|
||||
)
|
||||
.option(
|
||||
'--sync <metabaseDatabaseId>',
|
||||
'Enable Metabase sync for a discovered database; repeatable',
|
||||
collectPositiveIntegerOption,
|
||||
[],
|
||||
)
|
||||
.addOption(
|
||||
new Option('--sync-mode <mode>', 'Metabase sync selection mode')
|
||||
.choices(SYNC_MODE_CHOICES)
|
||||
.default('ALL' satisfies MetabaseSetupSyncMode),
|
||||
)
|
||||
.option('--run-ingest', 'Run ingest after setup', false)
|
||||
.option('--yes', 'Confirm and apply setup changes without prompting', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.showHelpAfterError()
|
||||
.action(async (options: ConnectionMetabaseSetupOptions, command) => {
|
||||
await runMetabaseSetupArgs(context, {
|
||||
command: 'setup',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.id,
|
||||
url: options.url,
|
||||
apiKey: options.apiKey,
|
||||
mintApiKey: options.mintApiKey === true,
|
||||
metabaseUsername: options.username,
|
||||
metabasePassword: options.password,
|
||||
mappings: options.map,
|
||||
syncEnabledDatabaseIds: options.sync,
|
||||
syncMode: options.syncMode ?? 'ALL',
|
||||
runIngest: options.runIngest === true,
|
||||
yes: options.yes === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
});
|
||||
});
|
||||
}
|
||||
1136
packages/cli/src/commands/connection-metabase-setup.test.ts
Normal file
1136
packages/cli/src/commands/connection-metabase-setup.test.ts
Normal file
File diff suppressed because it is too large
Load diff
782
packages/cli/src/commands/connection-metabase-setup.ts
Normal file
782
packages/cli/src/commands/connection-metabase-setup.ts
Normal file
|
|
@ -0,0 +1,782 @@
|
|||
import type { Option as ClackOption } from '@clack/prompts';
|
||||
import {
|
||||
cancel,
|
||||
confirm,
|
||||
intro,
|
||||
isCancel,
|
||||
log,
|
||||
multiselect,
|
||||
note,
|
||||
outro,
|
||||
password,
|
||||
select,
|
||||
text,
|
||||
} from '@clack/prompts';
|
||||
import { localConnectionToWarehouseDescriptor } from '@klo/context/connections';
|
||||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
LocalMetabaseSourceStateReader,
|
||||
MetabaseClient,
|
||||
type MetabaseDatabase,
|
||||
type MetabaseRuntimeClient,
|
||||
type MetabaseSyncMode,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
validateMappingPhysicalMatch,
|
||||
} from '@klo/context/ingest';
|
||||
import {
|
||||
type KloLocalProject,
|
||||
type KloProjectConnectionConfig,
|
||||
kloLocalStateDbPath,
|
||||
loadKloProject,
|
||||
serializeKloProjectConfig,
|
||||
} from '@klo/context/project';
|
||||
|
||||
import { createClackSpinner, type KloCliSpinner } from '../clack.js';
|
||||
import type { KloCliIo } from '../cli-runtime.js';
|
||||
import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
|
||||
import { type KloPublicIngestArgs, runKloPublicIngest } from '../public-ingest.js';
|
||||
|
||||
export type KloMetabaseSetupInputMode = 'auto' | 'disabled';
|
||||
|
||||
export type MetabaseSetupSyncMode = MetabaseSyncMode;
|
||||
|
||||
type MetabaseSetupPromptOption<Value> = ClackOption<Value>;
|
||||
|
||||
export interface MetabaseSetupLogger {
|
||||
info(message: string): void;
|
||||
step(message: string): void;
|
||||
success(message: string): void;
|
||||
warn(message: string): void;
|
||||
error(message: string): void;
|
||||
}
|
||||
|
||||
export interface MetabaseSetupPromptAdapter {
|
||||
intro(title?: string): void;
|
||||
outro(message?: string): void;
|
||||
note(message: string, title: string): void;
|
||||
log: MetabaseSetupLogger;
|
||||
spinner(): KloCliSpinner;
|
||||
select<T extends string>(options: { message: string; options: Array<MetabaseSetupPromptOption<T>> }): Promise<T>;
|
||||
multiselect<Value extends number | string>(options: {
|
||||
message: string;
|
||||
options: Array<MetabaseSetupPromptOption<Value>>;
|
||||
initialValues?: Value[];
|
||||
required?: boolean;
|
||||
maxItems?: number;
|
||||
}): Promise<Value[]>;
|
||||
text(options: { message: string; placeholder?: string }): Promise<string>;
|
||||
password(options: { message: string }): Promise<string>;
|
||||
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
|
||||
cancel(message: string): void;
|
||||
}
|
||||
|
||||
type KloMetabaseSetupInteractiveIo = KloCliIo & {
|
||||
stdin?: { isTTY?: boolean };
|
||||
};
|
||||
|
||||
export interface MetabaseSetupMappingAssignment {
|
||||
metabaseDatabaseId: number;
|
||||
targetConnectionId: string;
|
||||
}
|
||||
|
||||
export interface MintMetabaseApiKeyArgs {
|
||||
url: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KloCliIo) => Promise<string>;
|
||||
|
||||
export interface KloConnectionMetabaseSetupArgs {
|
||||
command: 'setup';
|
||||
projectDir: string;
|
||||
connectionId?: string;
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
mintApiKey: boolean;
|
||||
metabaseUsername?: string;
|
||||
metabasePassword?: string;
|
||||
mappings: MetabaseSetupMappingAssignment[];
|
||||
syncEnabledDatabaseIds: number[];
|
||||
syncMode: MetabaseSetupSyncMode;
|
||||
runIngest: boolean;
|
||||
yes: boolean;
|
||||
inputMode: KloMetabaseSetupInputMode;
|
||||
}
|
||||
|
||||
export interface KloConnectionMetabaseSetupDeps {
|
||||
createMetabaseClient?: (
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
) => Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>>;
|
||||
mintMetabaseApiKey?: MintMetabaseApiKey;
|
||||
prompts?: MetabaseSetupPromptAdapter;
|
||||
runPublicIngest?: (args: Extract<KloPublicIngestArgs, { command: 'run' }>, io: KloCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
function isMetabaseConnection(connection: KloProjectConnectionConfig | undefined): boolean {
|
||||
return (
|
||||
String(connection?.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase() === 'metabase'
|
||||
);
|
||||
}
|
||||
|
||||
function stringField(value: unknown): string | undefined {
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function uniqueSorted(values: number[]): number[] {
|
||||
return [...new Set(values)].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function resolveMetabaseUrl(connection: KloProjectConnectionConfig | undefined): string | undefined {
|
||||
return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url);
|
||||
}
|
||||
|
||||
function resolveLiteralMetabaseApiKey(connection: KloProjectConnectionConfig | undefined): string | undefined {
|
||||
return stringField(connection?.api_key) ?? stringField(connection?.apiKey);
|
||||
}
|
||||
|
||||
function listMetabaseConnectionIds(project: KloLocalProject): string[] {
|
||||
return Object.entries(project.config.connections)
|
||||
.filter(([_connectionId, connection]) => isMetabaseConnection(connection))
|
||||
.map(([connectionId]) => connectionId)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function listWarehouseConnectionIds(project: KloLocalProject): string[] {
|
||||
return Object.entries(project.config.connections)
|
||||
.filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
|
||||
.map(([connectionId]) => connectionId)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function redactSecrets(message: string, secrets: string[]): string {
|
||||
let result = message;
|
||||
for (const secret of secrets) {
|
||||
if (!secret) {
|
||||
continue;
|
||||
}
|
||||
result = result.split(secret).join('[redacted]');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function createDefaultMetabaseClient(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
|
||||
const factory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
return factory.createClient(connectionId);
|
||||
}
|
||||
|
||||
async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise<string> {
|
||||
const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
|
||||
const sessionId = await loginClient.createSession(args.username, args.password);
|
||||
const sessionClient = new MetabaseClient(
|
||||
{ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' },
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
);
|
||||
const groups = await sessionClient.getPermissionGroups();
|
||||
const adminGroup = groups.find((group) => group.name === 'Administrators');
|
||||
|
||||
if (!adminGroup) {
|
||||
throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
|
||||
}
|
||||
|
||||
const mintedKey = await sessionClient.createApiKey({
|
||||
groupId: adminGroup.id,
|
||||
name: `KLO CLI ${new Date().toISOString()}`,
|
||||
});
|
||||
const trimmedKey = stringField(mintedKey);
|
||||
if (!trimmedKey) {
|
||||
throw new Error('Metabase API key minting returned an empty key');
|
||||
}
|
||||
return trimmedKey;
|
||||
}
|
||||
|
||||
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<MetabaseSetupPromptAdapter, 'cancel'>): T {
|
||||
if (isCancel(value)) {
|
||||
prompts.cancel('Setup cancelled.');
|
||||
throw new Error('Setup cancelled.');
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter {
|
||||
return {
|
||||
intro(title?: string): void {
|
||||
intro(title);
|
||||
},
|
||||
outro(message?: string): void {
|
||||
outro(message);
|
||||
},
|
||||
note(message: string, title: string): void {
|
||||
note(message, title);
|
||||
},
|
||||
log: {
|
||||
info(message: string): void {
|
||||
log.info(message);
|
||||
},
|
||||
step(message: string): void {
|
||||
log.step(message);
|
||||
},
|
||||
success(message: string): void {
|
||||
log.success(message);
|
||||
},
|
||||
warn(message: string): void {
|
||||
log.warn(message);
|
||||
},
|
||||
error(message: string): void {
|
||||
log.error(message);
|
||||
},
|
||||
},
|
||||
spinner(): KloCliSpinner {
|
||||
return createClackSpinner();
|
||||
},
|
||||
async select<T extends string>(options: {
|
||||
message: string;
|
||||
options: Array<MetabaseSetupPromptOption<T>>;
|
||||
}): Promise<T> {
|
||||
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
|
||||
},
|
||||
async multiselect<Value extends number | string>(options: {
|
||||
message: string;
|
||||
options: Array<MetabaseSetupPromptOption<Value>>;
|
||||
initialValues?: Value[];
|
||||
required?: boolean;
|
||||
maxItems?: number;
|
||||
}): Promise<Value[]> {
|
||||
return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
|
||||
},
|
||||
async text(options: { message: string; placeholder?: string }): Promise<string> {
|
||||
return ensureNotCancelled(await text(options), this);
|
||||
},
|
||||
async password(options: { message: string }): Promise<string> {
|
||||
return ensureNotCancelled(await password(options), this);
|
||||
},
|
||||
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
|
||||
return ensureNotCancelled(await confirm(options), this);
|
||||
},
|
||||
cancel(message: string): void {
|
||||
cancel(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function isInteractiveMetabaseSetupIo(
|
||||
args: Pick<KloConnectionMetabaseSetupArgs, 'inputMode'>,
|
||||
io: KloMetabaseSetupInteractiveIo,
|
||||
): boolean {
|
||||
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
|
||||
}
|
||||
|
||||
function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{
|
||||
id: number;
|
||||
name: string;
|
||||
engine: string;
|
||||
host: string | null;
|
||||
dbName: string | null;
|
||||
}> {
|
||||
return databases
|
||||
.filter((database) => database.is_sample !== true)
|
||||
.map((database) => ({
|
||||
id: database.id,
|
||||
name: database.name,
|
||||
engine: stringField(database.engine) ?? 'unknown',
|
||||
host: stringField(database.details?.host) ?? null,
|
||||
dbName: stringField(database.details?.dbname) ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function targetPhysicalInfo(project: KloLocalProject, connectionId: string) {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
|
||||
if (!descriptor) {
|
||||
return { connection_type: 'UNKNOWN' };
|
||||
}
|
||||
return {
|
||||
connection_type: descriptor.connection_type,
|
||||
host: descriptor.host ?? null,
|
||||
database: descriptor.database ?? null,
|
||||
account: descriptor.account ?? null,
|
||||
project_id: descriptor.project_id ?? null,
|
||||
dataset_id: descriptor.dataset_id ?? null,
|
||||
...descriptor.connection_params,
|
||||
};
|
||||
}
|
||||
|
||||
function noteMetabaseSetupSummary(options: {
|
||||
prompts: MetabaseSetupPromptAdapter;
|
||||
connectionId: string;
|
||||
url: string;
|
||||
mappings: MetabaseSetupMappingAssignment[];
|
||||
syncEnabledDatabaseIds: number[];
|
||||
}): void {
|
||||
const mappingLines = options.mappings
|
||||
.map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
|
||||
.join('\n');
|
||||
const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n');
|
||||
|
||||
options.prompts.note(
|
||||
[
|
||||
`Connection: ${options.connectionId}`,
|
||||
`URL: ${options.url}`,
|
||||
'',
|
||||
'Mappings:',
|
||||
mappingLines || ' (none)',
|
||||
'',
|
||||
'Sync enabled:',
|
||||
syncLines || ' (none)',
|
||||
].join('\n'),
|
||||
'Summary',
|
||||
);
|
||||
}
|
||||
|
||||
export async function runKloConnectionMetabaseSetup(
|
||||
args: KloConnectionMetabaseSetupArgs,
|
||||
io: KloCliIo,
|
||||
deps: KloConnectionMetabaseSetupDeps = {},
|
||||
): Promise<number> {
|
||||
let apiKeyForRedaction = args.apiKey;
|
||||
let passwordForRedaction = args.metabasePassword;
|
||||
const interactiveIo = io as KloMetabaseSetupInteractiveIo;
|
||||
const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
|
||||
const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
|
||||
|
||||
try {
|
||||
if (isInteractive && prompts) {
|
||||
prompts.intro('KLO Metabase setup');
|
||||
}
|
||||
|
||||
const project = await loadKloProject({ projectDir: args.projectDir });
|
||||
const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
|
||||
let connectionId: string;
|
||||
|
||||
if (args.connectionId) {
|
||||
connectionId = args.connectionId;
|
||||
} else if (existingMetabaseConnectionIds.length === 1) {
|
||||
const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
|
||||
if (!onlyMetabaseConnectionId) {
|
||||
throw new Error('No Metabase connection id was resolved');
|
||||
}
|
||||
connectionId = onlyMetabaseConnectionId;
|
||||
} else if (existingMetabaseConnectionIds.length > 1) {
|
||||
if (!isInteractive || !prompts) {
|
||||
throw new Error(
|
||||
`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`,
|
||||
);
|
||||
}
|
||||
connectionId = await prompts.select({
|
||||
message: 'Select the Metabase connection to configure',
|
||||
options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
|
||||
});
|
||||
} else {
|
||||
connectionId = 'metabase';
|
||||
}
|
||||
|
||||
const existingConnection = project.config.connections[connectionId];
|
||||
const warehouseConnectionIds = listWarehouseConnectionIds(project);
|
||||
|
||||
if (warehouseConnectionIds.length === 0) {
|
||||
throw new Error('Add a warehouse connection first');
|
||||
}
|
||||
|
||||
let url = args.url ?? resolveMetabaseUrl(existingConnection);
|
||||
let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
|
||||
apiKeyForRedaction = apiKey;
|
||||
|
||||
if (!url && isInteractive && prompts) {
|
||||
url = stringField(
|
||||
await prompts.text({
|
||||
message: 'Metabase API URL',
|
||||
placeholder: 'http://localhost:3000',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (args.inputMode === 'disabled' && !url) {
|
||||
throw new Error('missing Metabase URL');
|
||||
}
|
||||
|
||||
if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
|
||||
const reuse = await prompts.confirm({
|
||||
message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
|
||||
initialValue: true,
|
||||
});
|
||||
if (!reuse) {
|
||||
apiKey = undefined;
|
||||
apiKeyForRedaction = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (args.mintApiKey) {
|
||||
let username = stringField(args.metabaseUsername);
|
||||
let metabasePassword = stringField(args.metabasePassword);
|
||||
|
||||
if (isInteractive && prompts) {
|
||||
if (!username) {
|
||||
username = stringField(await prompts.text({ message: 'Metabase admin username' }));
|
||||
}
|
||||
if (!metabasePassword) {
|
||||
metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
|
||||
}
|
||||
}
|
||||
|
||||
if (!username) {
|
||||
throw new Error('--mint-api-key requires --username');
|
||||
}
|
||||
if (!metabasePassword) {
|
||||
throw new Error('--mint-api-key requires --password');
|
||||
}
|
||||
if (!url) {
|
||||
throw new Error('Metabase URL is required (use --url)');
|
||||
}
|
||||
|
||||
passwordForRedaction = metabasePassword;
|
||||
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
|
||||
{ url, username, password: metabasePassword },
|
||||
io,
|
||||
);
|
||||
apiKeyForRedaction = apiKey;
|
||||
}
|
||||
|
||||
if (!apiKey && isInteractive && prompts) {
|
||||
const credentialMode = await prompts.select({
|
||||
message: 'Metabase credentials',
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste API key' },
|
||||
{ value: 'mint', label: 'Mint API key' },
|
||||
],
|
||||
});
|
||||
|
||||
if (credentialMode === 'paste') {
|
||||
apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
|
||||
apiKeyForRedaction = apiKey;
|
||||
} else {
|
||||
const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
|
||||
const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
|
||||
if (!username) {
|
||||
throw new Error('Metabase username is required');
|
||||
}
|
||||
if (!metabasePassword) {
|
||||
throw new Error('Metabase password is required');
|
||||
}
|
||||
if (!url) {
|
||||
throw new Error('Metabase URL is required (use --url)');
|
||||
}
|
||||
|
||||
passwordForRedaction = metabasePassword;
|
||||
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
|
||||
{ url, username, password: metabasePassword },
|
||||
io,
|
||||
);
|
||||
apiKeyForRedaction = apiKey;
|
||||
}
|
||||
}
|
||||
|
||||
if (args.inputMode === 'disabled' && !apiKey) {
|
||||
throw new Error('missing Metabase API key');
|
||||
}
|
||||
|
||||
if (!url) {
|
||||
throw new Error('Metabase URL is required (use --url)');
|
||||
}
|
||||
if (!apiKey) {
|
||||
throw new Error('Metabase API key is required (use --api-key)');
|
||||
}
|
||||
|
||||
const transientConnectionConfig: KloProjectConnectionConfig = {
|
||||
...(existingConnection ?? {}),
|
||||
driver: 'metabase',
|
||||
api_url: url,
|
||||
api_key: apiKey,
|
||||
};
|
||||
const configWithTransient = {
|
||||
...project.config,
|
||||
connections: {
|
||||
...project.config.connections,
|
||||
[connectionId]: transientConnectionConfig,
|
||||
},
|
||||
};
|
||||
const discoveryProject: KloLocalProject = { ...project, config: configWithTransient };
|
||||
|
||||
for (const mapping of args.mappings) {
|
||||
if (!configWithTransient.connections[mapping.targetConnectionId]) {
|
||||
throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
|
||||
}
|
||||
}
|
||||
|
||||
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
|
||||
try {
|
||||
const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
|
||||
authSpinner?.start('Testing Metabase connection');
|
||||
const testResult = await client.testConnection();
|
||||
if (!testResult.success) {
|
||||
authSpinner?.error('Metabase authentication failed');
|
||||
throw new Error(
|
||||
`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`,
|
||||
);
|
||||
}
|
||||
authSpinner?.stop('Metabase reachable');
|
||||
|
||||
const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
|
||||
discoverySpinner?.start('Discovering Metabase databases');
|
||||
const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
|
||||
discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
|
||||
if (isInteractive && prompts) {
|
||||
prompts.log.success(
|
||||
`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`,
|
||||
);
|
||||
}
|
||||
if (discovered.length === 0) {
|
||||
throw new Error('Metabase auth worked but no usable databases were returned');
|
||||
}
|
||||
|
||||
let resolvedMappings = args.mappings;
|
||||
let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
|
||||
|
||||
if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
|
||||
resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
|
||||
}
|
||||
|
||||
if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
|
||||
const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
|
||||
const compatibleWarehouses = onlyDiscoveredDatabase
|
||||
? warehouseConnectionIds.filter((warehouseConnectionId) => {
|
||||
const mismatchReason = validateMappingPhysicalMatch(
|
||||
{
|
||||
metabaseEngine: onlyDiscoveredDatabase.engine,
|
||||
metabaseDbName: onlyDiscoveredDatabase.dbName,
|
||||
metabaseHost: onlyDiscoveredDatabase.host,
|
||||
},
|
||||
targetPhysicalInfo(project, warehouseConnectionId),
|
||||
);
|
||||
return !mismatchReason;
|
||||
})
|
||||
: [];
|
||||
const onlyWarehouseConnectionId = compatibleWarehouses[0];
|
||||
|
||||
if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
|
||||
if (args.yes) {
|
||||
resolvedMappings = [
|
||||
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
|
||||
];
|
||||
resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
|
||||
} else if (isInteractive && prompts) {
|
||||
const proposedMappings = [
|
||||
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
|
||||
];
|
||||
const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
|
||||
noteMetabaseSetupSummary({
|
||||
prompts,
|
||||
connectionId,
|
||||
url,
|
||||
mappings: proposedMappings,
|
||||
syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
|
||||
});
|
||||
const confirmed = await prompts.confirm({
|
||||
message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
|
||||
initialValue: true,
|
||||
});
|
||||
if (!confirmed) {
|
||||
prompts.cancel('Setup cancelled.');
|
||||
throw new Error('Setup cancelled.');
|
||||
}
|
||||
resolvedMappings = proposedMappings;
|
||||
resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
|
||||
} else {
|
||||
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
|
||||
}
|
||||
} else if (isInteractive && prompts) {
|
||||
const selectedDatabaseIds = await prompts.multiselect<number>({
|
||||
message: withMultiselectNavigation('Select Metabase databases to configure'),
|
||||
options: discovered.map((database) => ({
|
||||
value: database.id,
|
||||
label: `${database.id}: ${database.name}`,
|
||||
hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
|
||||
})),
|
||||
required: true,
|
||||
});
|
||||
|
||||
resolvedMappings = [];
|
||||
for (const databaseId of selectedDatabaseIds) {
|
||||
const database = discovered.find((candidate) => candidate.id === databaseId);
|
||||
if (!database) {
|
||||
throw new Error(`Selected database id ${databaseId} was not discovered`);
|
||||
}
|
||||
|
||||
const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
|
||||
if (existingMapping) {
|
||||
resolvedMappings.push(existingMapping);
|
||||
continue;
|
||||
}
|
||||
|
||||
const targetConnectionId = await prompts.select({
|
||||
message: `Map Metabase database ${database.id} ("${database.name}") to which KLO connection?`,
|
||||
options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
|
||||
});
|
||||
resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
|
||||
}
|
||||
|
||||
const syncIds = await prompts.multiselect<number>({
|
||||
message: withMultiselectNavigation('Enable sync for which databases?'),
|
||||
options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
|
||||
initialValues: selectedDatabaseIds,
|
||||
required: true,
|
||||
});
|
||||
resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
|
||||
|
||||
if (!args.yes) {
|
||||
noteMetabaseSetupSummary({
|
||||
prompts,
|
||||
connectionId,
|
||||
url,
|
||||
mappings: resolvedMappings,
|
||||
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
|
||||
});
|
||||
const confirmed = await prompts.confirm({
|
||||
message: 'Write changes to klo.yaml and enable sync?',
|
||||
initialValue: true,
|
||||
});
|
||||
if (!confirmed) {
|
||||
prompts.cancel('Setup cancelled.');
|
||||
throw new Error('Setup cancelled.');
|
||||
}
|
||||
}
|
||||
} else if (args.inputMode === 'disabled') {
|
||||
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
args.inputMode === 'disabled' &&
|
||||
resolvedMappings.length > 0 &&
|
||||
resolvedSyncEnabledDatabaseIds.length === 0
|
||||
) {
|
||||
throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
|
||||
}
|
||||
|
||||
const discoveredIds = new Set(discovered.map((database) => database.id));
|
||||
for (const mapping of resolvedMappings) {
|
||||
if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
|
||||
throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
|
||||
}
|
||||
}
|
||||
for (const syncId of resolvedSyncEnabledDatabaseIds) {
|
||||
if (!discoveredIds.has(syncId)) {
|
||||
throw new Error(`Sync database id ${syncId} was not discovered`);
|
||||
}
|
||||
}
|
||||
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(configWithTransient),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
`Setup Metabase connection ${connectionId}`,
|
||||
);
|
||||
|
||||
const updatedProject = await loadKloProject({ projectDir: args.projectDir });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(updatedProject) });
|
||||
|
||||
await store.refreshDiscoveredDatabases({ connectionId, discovered });
|
||||
|
||||
for (const mapping of resolvedMappings) {
|
||||
await store.upsertDatabaseMapping({
|
||||
connectionId,
|
||||
metabaseDatabaseId: mapping.metabaseDatabaseId,
|
||||
targetConnectionId: mapping.targetConnectionId,
|
||||
syncEnabled: false,
|
||||
source: 'cli',
|
||||
});
|
||||
}
|
||||
|
||||
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
|
||||
await store.setMappingSyncEnabled({
|
||||
connectionId,
|
||||
metabaseDatabaseId,
|
||||
syncEnabled: true,
|
||||
});
|
||||
}
|
||||
|
||||
const existingSyncState = await store.getSourceState(connectionId);
|
||||
await store.setSyncState({
|
||||
connectionId,
|
||||
syncMode: args.syncMode,
|
||||
defaultTagNames: existingSyncState.defaultTagNames,
|
||||
selections: existingSyncState.selections,
|
||||
});
|
||||
|
||||
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
|
||||
if (unhydrated.length > 0) {
|
||||
io.stderr.write(
|
||||
`Sync-enabled mappings are missing discovery metadata; run klo connection mapping refresh ${connectionId} --auto-accept\n`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const rows = await store.listDatabaseMappings(connectionId);
|
||||
const physicalFailures = rows.flatMap((row) => {
|
||||
if (!row.targetConnectionId) {
|
||||
return [];
|
||||
}
|
||||
const reason = validateMappingPhysicalMatch(
|
||||
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
|
||||
updatedProject.config.connections[row.targetConnectionId]
|
||||
? targetPhysicalInfo(updatedProject, row.targetConnectionId)
|
||||
: { connection_type: 'UNKNOWN' },
|
||||
);
|
||||
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
|
||||
});
|
||||
if (physicalFailures.length > 0) {
|
||||
for (const failure of physicalFailures) {
|
||||
io.stderr.write(`${failure}\n`);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
io.stdout.write(`Connection: ${connectionId}\n`);
|
||||
io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
|
||||
io.stdout.write(`Next: klo ingest ${connectionId} --project-dir ${args.projectDir}\n`);
|
||||
|
||||
if (args.runIngest) {
|
||||
const ingestRunner = deps.runPublicIngest ?? runKloPublicIngest;
|
||||
const exitCode = await ingestRunner(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: args.projectDir,
|
||||
targetConnectionId: connectionId,
|
||||
all: false,
|
||||
json: false,
|
||||
inputMode: 'disabled',
|
||||
},
|
||||
io,
|
||||
);
|
||||
if (exitCode !== 0) {
|
||||
io.stderr.write(`Ingest failed; re-run: klo ingest ${connectionId} --project-dir ${args.projectDir}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (isInteractive && prompts) {
|
||||
prompts.outro('Metabase setup complete');
|
||||
}
|
||||
|
||||
return 0;
|
||||
} finally {
|
||||
await client.cleanup();
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
io.stderr.write(
|
||||
`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
92
packages/cli/src/commands/connection-notion-commands.ts
Normal file
92
packages/cli/src/commands/connection-notion-commands.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
|
||||
import { collectOption, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloConnectionNotionArgs } from './connection-notion.js';
|
||||
|
||||
interface NotionPickOptions {
|
||||
input?: boolean;
|
||||
rootPageId: string[];
|
||||
}
|
||||
|
||||
function parseSafeConnectionId(value: string): string {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
|
||||
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function uniqueInOrder(values: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (const value of values) {
|
||||
if (!seen.has(value)) {
|
||||
seen.add(value);
|
||||
result.push(value);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeNotionPageId(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
|
||||
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
|
||||
throw new Error(`Invalid Notion page UUID: ${value}`);
|
||||
}
|
||||
const lower = compact.toLowerCase();
|
||||
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
|
||||
}
|
||||
|
||||
function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KloConnectionNotionArgs {
|
||||
if (options.input !== false) {
|
||||
return {
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId,
|
||||
mode: 'interactive',
|
||||
};
|
||||
}
|
||||
|
||||
const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId));
|
||||
if (rootPageIds.length === 0) {
|
||||
throw new Error('connection notion pick --no-input requires at least one --root-page-id');
|
||||
}
|
||||
return {
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId,
|
||||
mode: 'non-interactive',
|
||||
rootPageIds,
|
||||
};
|
||||
}
|
||||
|
||||
async function runConnectionNotionArgs(context: KloCliCommandContext, args: KloConnectionNotionArgs): Promise<void> {
|
||||
const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKloConnectionNotion;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerConnectionNotionCommands(connect: Command, context: KloCliCommandContext): void {
|
||||
const notion = connect
|
||||
.command('notion')
|
||||
.description('Configure Notion source selection')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
notion.action(() => {
|
||||
notion.outputHelp();
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
notion
|
||||
.command('pick')
|
||||
.description('Pick Notion root pages for a configured Notion connection')
|
||||
.argument('<connectionId>', 'Notion connection id', parseSafeConnectionId)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.option('--root-page-id <id>', 'Root page UUID to crawl; repeatable with --no-input', collectOption, [])
|
||||
.showHelpAfterError()
|
||||
.action(async (connectionId: string, options: NotionPickOptions, command) => {
|
||||
await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options));
|
||||
});
|
||||
}
|
||||
283
packages/cli/src/commands/connection-notion-tree.test.ts
Normal file
283
packages/cli/src/commands/connection-notion-tree.test.ts
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildInitialState,
|
||||
buildPickerTree,
|
||||
canToggle,
|
||||
clearExpiredTransientHint,
|
||||
filterTree,
|
||||
flattenSelection,
|
||||
moveCursor,
|
||||
reducer,
|
||||
selectAllVisible,
|
||||
selectNone,
|
||||
toggleChecked,
|
||||
TRANSIENT_HINT_DURATION_MS,
|
||||
visibleNodeIds,
|
||||
type NotionPickerPageInput,
|
||||
} from './connection-notion-tree.js';
|
||||
|
||||
const IDS = {
|
||||
engineering: '11111111-1111-1111-1111-111111111111',
|
||||
architecture: '22222222-2222-2222-2222-222222222222',
|
||||
onboarding: '33333333-3333-3333-3333-333333333333',
|
||||
marketing: '44444444-4444-4444-4444-444444444444',
|
||||
journal: '55555555-5555-5555-5555-555555555555',
|
||||
orphan: '66666666-6666-6666-6666-666666666666',
|
||||
duplicate: '77777777-7777-7777-7777-777777777777',
|
||||
cycleA: '88888888-8888-8888-8888-888888888888',
|
||||
cycleB: '99999999-9999-9999-9999-999999999999',
|
||||
};
|
||||
|
||||
function pages(): NotionPickerPageInput[] {
|
||||
return [
|
||||
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
|
||||
{ id: IDS.onboarding, title: 'Onboarding', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
|
||||
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.journal, title: 'Daily journal', archived: true, parentId: IDS.marketing },
|
||||
{ id: IDS.orphan, title: '', archived: false, parentId: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' },
|
||||
{ id: IDS.duplicate, title: 'Original duplicate', archived: false, parentId: null },
|
||||
{ id: IDS.duplicate, title: 'Ignored duplicate', archived: true, parentId: IDS.marketing },
|
||||
{ id: IDS.cycleA, title: 'Cycle A', archived: false, parentId: IDS.cycleB },
|
||||
{ id: IDS.cycleB, title: 'Cycle B', archived: false, parentId: IDS.cycleA },
|
||||
];
|
||||
}
|
||||
|
||||
describe('buildPickerTree', () => {
|
||||
it('deduplicates pages, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => {
|
||||
const tree = buildPickerTree(pages());
|
||||
const byId = new Map(tree.map((node) => [node.id, node]));
|
||||
|
||||
expect(tree.map((node) => node.title)).toEqual([
|
||||
'Cycle A',
|
||||
'Cycle B',
|
||||
'Engineering Docs',
|
||||
'Architecture',
|
||||
'Onboarding',
|
||||
'Marketing',
|
||||
'Daily journal',
|
||||
'Original duplicate',
|
||||
'Untitled',
|
||||
]);
|
||||
expect(byId.get(IDS.engineering)?.childIds).toEqual([IDS.architecture, IDS.onboarding]);
|
||||
expect(byId.get(IDS.architecture)).toMatchObject({
|
||||
depth: 1,
|
||||
parentId: IDS.engineering,
|
||||
path: 'Engineering Docs / Architecture',
|
||||
});
|
||||
expect(byId.get(IDS.journal)).toMatchObject({
|
||||
archived: true,
|
||||
depth: 1,
|
||||
path: 'Marketing / Daily journal',
|
||||
});
|
||||
expect(byId.get(IDS.orphan)).toMatchObject({
|
||||
title: 'Untitled',
|
||||
parentId: null,
|
||||
depth: 0,
|
||||
path: 'Untitled',
|
||||
});
|
||||
expect(byId.get(IDS.duplicate)).toMatchObject({
|
||||
title: 'Original duplicate',
|
||||
archived: false,
|
||||
parentId: null,
|
||||
});
|
||||
expect(byId.get(IDS.cycleA)?.parentId).toBeNull();
|
||||
expect(byId.get(IDS.cycleB)?.parentId).toBe(IDS.cycleA);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selection invariants', () => {
|
||||
it('checking a parent locks descendants and keeps checked ids minimal', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
const checkedParent = toggleChecked(state, IDS.engineering, 1000);
|
||||
expect([...checkedParent.checked]).toEqual([IDS.engineering]);
|
||||
expect(canToggle(IDS.architecture, checkedParent)).toEqual({
|
||||
ok: false,
|
||||
reason: "Locked by 'Engineering Docs' - uncheck parent first",
|
||||
});
|
||||
|
||||
const lockedChildAttempt = toggleChecked(checkedParent, IDS.architecture, 2000);
|
||||
expect([...lockedChildAttempt.checked]).toEqual([IDS.engineering]);
|
||||
expect(lockedChildAttempt.transientHint).toEqual({
|
||||
text: "Locked by 'Engineering Docs' - uncheck parent first",
|
||||
expiresAt: 4500,
|
||||
});
|
||||
|
||||
const uncheckedParent = toggleChecked(lockedChildAttempt, IDS.engineering, 3000);
|
||||
expect([...uncheckedParent.checked]).toEqual([]);
|
||||
expect(canToggle(IDS.architecture, uncheckedParent)).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('normalizes stored roots, reports stale roots, expands checked ancestors, and flattens descendants', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [
|
||||
IDS.engineering.replaceAll('-', ''),
|
||||
IDS.architecture,
|
||||
'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
|
||||
],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
expect([...state.checked]).toEqual([IDS.engineering]);
|
||||
expect([...state.expanded]).toEqual([]);
|
||||
expect(state.cursorId).toBe(IDS.cycleA);
|
||||
expect(state.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
|
||||
expect(flattenSelection(new Set([IDS.engineering, IDS.architecture]), state.byId)).toEqual([IDS.engineering]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('search and cursor movement', () => {
|
||||
it('filters by title and path while deriving auto-expanded ancestors', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
const searching = {
|
||||
...state,
|
||||
search: { editing: false, query: 'architecture' },
|
||||
};
|
||||
|
||||
expect(filterTree(searching)).toEqual({
|
||||
visibleIds: new Set([IDS.engineering, IDS.architecture]),
|
||||
autoExpand: new Set([IDS.engineering]),
|
||||
});
|
||||
expect(visibleNodeIds(searching)).toEqual([IDS.engineering, IDS.architecture]);
|
||||
});
|
||||
|
||||
it('moves the cursor through visible nodes and implements left/right tree semantics', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
const atEngineering = {
|
||||
...state,
|
||||
cursorId: IDS.engineering,
|
||||
expanded: new Set([IDS.engineering]),
|
||||
};
|
||||
expect(moveCursor(atEngineering, 'down').cursorId).toBe(IDS.architecture);
|
||||
expect(moveCursor({ ...atEngineering, cursorId: IDS.architecture }, 'up').cursorId).toBe(IDS.engineering);
|
||||
expect(moveCursor(atEngineering, 'right').cursorId).toBe(IDS.architecture);
|
||||
expect(moveCursor({ ...atEngineering, cursorId: IDS.architecture }, 'left').cursorId).toBe(IDS.engineering);
|
||||
expect([...moveCursor(atEngineering, 'left').expanded]).toEqual([]);
|
||||
expect([...moveCursor({ ...state, cursorId: IDS.marketing }, 'right').expanded]).toContain(IDS.marketing);
|
||||
});
|
||||
});
|
||||
|
||||
describe('bulk actions and reducer effects', () => {
|
||||
it('selects only matching visible roots under search and clears selection', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [IDS.marketing],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
const searching = {
|
||||
...state,
|
||||
search: { editing: false, query: 'architecture' },
|
||||
};
|
||||
|
||||
const selected = selectAllVisible(searching);
|
||||
expect(flattenSelection(selected.checked, selected.byId)).toEqual([IDS.architecture, IDS.marketing]);
|
||||
expect([...selectNone(selected).checked]).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns save immediately for selected_roots and requires confirmation for all_accessible', () => {
|
||||
const selectedRoots = toggleChecked(
|
||||
buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
}),
|
||||
IDS.marketing,
|
||||
1000,
|
||||
);
|
||||
expect(reducer(selectedRoots, 'save-request')).toEqual({
|
||||
next: selectedRoots,
|
||||
effect: 'save',
|
||||
});
|
||||
|
||||
const allAccessible = {
|
||||
...selectedRoots,
|
||||
currentCrawlMode: 'all_accessible' as const,
|
||||
};
|
||||
const confirm = reducer(allAccessible, 'save-request');
|
||||
expect(confirm).toEqual({
|
||||
next: { ...allAccessible, pendingConfirm: 'mode-switch' },
|
||||
effect: null,
|
||||
});
|
||||
expect(reducer(confirm.next, 'save-cancel')).toEqual({
|
||||
next: { ...allAccessible, pendingConfirm: null },
|
||||
effect: null,
|
||||
});
|
||||
expect(reducer(confirm.next, 'save-confirm')).toEqual({
|
||||
next: { ...allAccessible, pendingConfirm: null },
|
||||
effect: 'save',
|
||||
});
|
||||
});
|
||||
|
||||
it('blocks empty saves, updates search state, and quits without saving', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
|
||||
const blockedSave = reducer(state, 'save-request', 9000);
|
||||
expect(blockedSave).toEqual({
|
||||
next: {
|
||||
...state,
|
||||
transientHint: {
|
||||
text: 'Select at least one page or press q to quit',
|
||||
expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS,
|
||||
},
|
||||
},
|
||||
effect: null,
|
||||
});
|
||||
expect(
|
||||
reducer(
|
||||
reducer(reducer(state, 'search-start').next, { type: 'search-input', value: 'a' }).next,
|
||||
'search-submit',
|
||||
).next.search,
|
||||
).toEqual({ editing: false, query: 'a' });
|
||||
expect(reducer(state, 'quit')).toEqual({
|
||||
next: state,
|
||||
effect: 'quit-without-save',
|
||||
});
|
||||
});
|
||||
|
||||
it('clears transient hints only when their expiry time has passed', () => {
|
||||
const state = buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
const withHint = {
|
||||
...state,
|
||||
transientHint: {
|
||||
text: 'Select at least one page or press q to quit',
|
||||
expiresAt: 11500,
|
||||
},
|
||||
};
|
||||
|
||||
expect(clearExpiredTransientHint(withHint, 11499)).toBe(withHint);
|
||||
expect(clearExpiredTransientHint(withHint, 11500)).toEqual({
|
||||
...withHint,
|
||||
transientHint: null,
|
||||
});
|
||||
expect(reducer(withHint, 'clear-transient-hint', 11501)).toEqual({
|
||||
next: {
|
||||
...withHint,
|
||||
transientHint: null,
|
||||
},
|
||||
effect: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
529
packages/cli/src/commands/connection-notion-tree.ts
Normal file
529
packages/cli/src/commands/connection-notion-tree.ts
Normal file
|
|
@ -0,0 +1,529 @@
|
|||
export interface NotionPickerPageInput {
|
||||
id: string;
|
||||
title?: string | null;
|
||||
archived?: boolean;
|
||||
parentId?: string | null;
|
||||
}
|
||||
|
||||
interface NotionPickerNode {
|
||||
id: string;
|
||||
title: string;
|
||||
archived: boolean;
|
||||
parentId: string | null;
|
||||
depth: number;
|
||||
childIds: string[];
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface PickerState {
|
||||
tree: NotionPickerNode[];
|
||||
byId: Map<string, NotionPickerNode>;
|
||||
expanded: Set<string>;
|
||||
checked: Set<string>;
|
||||
cursorId: string;
|
||||
search: { editing: boolean; query: string };
|
||||
pendingConfirm: 'mode-switch' | null;
|
||||
preLoadWarnings: string[];
|
||||
transientHint: { text: string; expiresAt: number } | null;
|
||||
currentCrawlMode: 'all_accessible' | 'selected_roots';
|
||||
}
|
||||
|
||||
export type PickerCommand =
|
||||
| 'cursor-up'
|
||||
| 'cursor-down'
|
||||
| 'cursor-left'
|
||||
| 'cursor-right'
|
||||
| 'expand'
|
||||
| 'collapse'
|
||||
| 'expand-all'
|
||||
| 'collapse-all'
|
||||
| 'toggle-check'
|
||||
| 'select-all-visible'
|
||||
| 'select-none'
|
||||
| 'clear-transient-hint'
|
||||
| 'search-start'
|
||||
| 'search-cancel'
|
||||
| 'search-submit'
|
||||
| 'search-backspace'
|
||||
| { type: 'search-input'; value: string }
|
||||
| 'save-request'
|
||||
| 'save-confirm'
|
||||
| 'save-cancel'
|
||||
| 'quit';
|
||||
|
||||
type PickerEffect = null | 'save' | 'quit-without-save';
|
||||
|
||||
interface MutableNode {
|
||||
id: string;
|
||||
title: string;
|
||||
archived: boolean;
|
||||
parentId: string | null;
|
||||
childIds: string[];
|
||||
}
|
||||
|
||||
export const TRANSIENT_HINT_DURATION_MS = 2500;
|
||||
|
||||
const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true });
|
||||
|
||||
function normalizePageId(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
const compact = trimmed.replace(/-/g, '');
|
||||
if (/^[0-9a-fA-F]{32}$/.test(compact)) {
|
||||
const lower = compact.toLowerCase();
|
||||
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(
|
||||
16,
|
||||
20,
|
||||
)}-${lower.slice(20)}`;
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function titleValue(value: string | null | undefined): string {
|
||||
const trimmed = value?.trim() ?? '';
|
||||
return trimmed.length > 0 ? trimmed : 'Untitled';
|
||||
}
|
||||
|
||||
function sortedNodeIds(ids: string[], nodes: Map<string, MutableNode | NotionPickerNode>): string[] {
|
||||
return [...ids].sort((leftId, rightId) => {
|
||||
const left = nodes.get(leftId);
|
||||
const right = nodes.get(rightId);
|
||||
const byTitle = collator.compare(left?.title ?? '', right?.title ?? '');
|
||||
return byTitle === 0 ? leftId.localeCompare(rightId) : byTitle;
|
||||
});
|
||||
}
|
||||
|
||||
function cloneState(state: PickerState, patch: Partial<PickerState>): PickerState {
|
||||
return { ...state, ...patch };
|
||||
}
|
||||
|
||||
function transientHint(text: string, now: number): PickerState['transientHint'] {
|
||||
return { text, expiresAt: now + TRANSIENT_HINT_DURATION_MS };
|
||||
}
|
||||
|
||||
export function clearExpiredTransientHint(state: PickerState, now = Date.now()): PickerState {
|
||||
if (!state.transientHint || state.transientHint.expiresAt > now) {
|
||||
return state;
|
||||
}
|
||||
return cloneState(state, { transientHint: null });
|
||||
}
|
||||
|
||||
function ancestorsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
|
||||
const ancestors: string[] = [];
|
||||
let parentId = byId.get(nodeId)?.parentId ?? null;
|
||||
const seen = new Set<string>();
|
||||
while (parentId && !seen.has(parentId)) {
|
||||
ancestors.push(parentId);
|
||||
seen.add(parentId);
|
||||
parentId = byId.get(parentId)?.parentId ?? null;
|
||||
}
|
||||
return ancestors;
|
||||
}
|
||||
|
||||
function descendantsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
|
||||
const result: string[] = [];
|
||||
const stack = [...(byId.get(nodeId)?.childIds ?? [])].reverse();
|
||||
while (stack.length > 0) {
|
||||
const id = stack.pop();
|
||||
if (!id) {
|
||||
continue;
|
||||
}
|
||||
result.push(id);
|
||||
const node = byId.get(id);
|
||||
if (node) {
|
||||
stack.push(...[...node.childIds].reverse());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function matchingIds(state: PickerState): Set<string> {
|
||||
const query = state.search.query.trim().toLocaleLowerCase();
|
||||
if (!query) {
|
||||
return new Set(state.tree.map((node) => node.id));
|
||||
}
|
||||
return new Set(
|
||||
state.tree
|
||||
.filter((node) => {
|
||||
const title = node.title.toLocaleLowerCase();
|
||||
const path = node.path.toLocaleLowerCase();
|
||||
return title.includes(query) || path.includes(query);
|
||||
})
|
||||
.map((node) => node.id),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionPickerNode[] {
|
||||
const nodes = new Map<string, MutableNode>();
|
||||
for (const result of searchResults) {
|
||||
const id = normalizePageId(result.id);
|
||||
if (nodes.has(id)) {
|
||||
continue;
|
||||
}
|
||||
nodes.set(id, {
|
||||
id,
|
||||
title: titleValue(result.title),
|
||||
archived: result.archived === true,
|
||||
parentId: result.parentId ? normalizePageId(result.parentId) : null,
|
||||
childIds: [],
|
||||
});
|
||||
}
|
||||
|
||||
for (const node of nodes.values()) {
|
||||
if (!node.parentId || node.parentId === node.id || !nodes.has(node.parentId)) {
|
||||
node.parentId = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
const seen = new Set([node.id]);
|
||||
let cursor: string | null = node.parentId;
|
||||
while (cursor) {
|
||||
if (seen.has(cursor)) {
|
||||
node.parentId = null;
|
||||
break;
|
||||
}
|
||||
seen.add(cursor);
|
||||
cursor = nodes.get(cursor)?.parentId ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
for (const node of nodes.values()) {
|
||||
node.childIds = [];
|
||||
}
|
||||
for (const node of nodes.values()) {
|
||||
if (node.parentId) {
|
||||
nodes.get(node.parentId)?.childIds.push(node.id);
|
||||
}
|
||||
}
|
||||
for (const node of nodes.values()) {
|
||||
node.childIds = sortedNodeIds(node.childIds, nodes);
|
||||
}
|
||||
|
||||
const roots = sortedNodeIds(
|
||||
[...nodes.values()].filter((node) => node.parentId === null).map((node) => node.id),
|
||||
nodes,
|
||||
);
|
||||
const tree: NotionPickerNode[] = [];
|
||||
|
||||
function visit(nodeId: string, depth: number, pathPrefix: string[]): void {
|
||||
const raw = nodes.get(nodeId);
|
||||
if (!raw) {
|
||||
return;
|
||||
}
|
||||
const path = [...pathPrefix, raw.title].join(' / ');
|
||||
const node: NotionPickerNode = {
|
||||
id: raw.id,
|
||||
title: raw.title,
|
||||
archived: raw.archived,
|
||||
parentId: raw.parentId,
|
||||
depth,
|
||||
childIds: raw.childIds,
|
||||
path,
|
||||
};
|
||||
tree.push(node);
|
||||
for (const childId of raw.childIds) {
|
||||
visit(childId, depth + 1, [...pathPrefix, raw.title]);
|
||||
}
|
||||
}
|
||||
|
||||
for (const rootId of roots) {
|
||||
visit(rootId, 0, []);
|
||||
}
|
||||
|
||||
return tree;
|
||||
}
|
||||
|
||||
export function isAncestorChecked(nodeId: string, checked: Set<string>, byId: Map<string, NotionPickerNode>): boolean {
|
||||
return ancestorsOf(nodeId, byId).some((ancestorId) => checked.has(ancestorId));
|
||||
}
|
||||
|
||||
function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | null {
|
||||
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
|
||||
if (state.checked.has(ancestorId)) {
|
||||
return state.byId.get(ancestorId) ?? null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } {
|
||||
if (!state.byId.has(nodeId)) {
|
||||
return { ok: false, reason: 'Page not found' };
|
||||
}
|
||||
const ancestor = checkedAncestor(nodeId, state);
|
||||
if (ancestor) {
|
||||
return { ok: false, reason: `Locked by '${ancestor.title}' - uncheck parent first` };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export function toggleChecked(state: PickerState, nodeId: string, now = Date.now()): PickerState {
|
||||
const toggle = canToggle(nodeId, state);
|
||||
if (!toggle.ok) {
|
||||
return cloneState(state, {
|
||||
transientHint: transientHint(toggle.reason, now),
|
||||
});
|
||||
}
|
||||
|
||||
const checked = new Set(state.checked);
|
||||
if (checked.has(nodeId)) {
|
||||
checked.delete(nodeId);
|
||||
} else {
|
||||
checked.add(nodeId);
|
||||
for (const descendantId of descendantsOf(nodeId, state.byId)) {
|
||||
checked.delete(descendantId);
|
||||
}
|
||||
}
|
||||
return cloneState(state, { checked, transientHint: null });
|
||||
}
|
||||
|
||||
export function flattenSelection(checked: Set<string>, byId: Map<string, NotionPickerNode>): string[] {
|
||||
const result: string[] = [];
|
||||
for (const node of byId.values()) {
|
||||
if (checked.has(node.id) && !isAncestorChecked(node.id, checked, byId)) {
|
||||
result.push(node.id);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function filterTree(state: PickerState): { visibleIds: Set<string>; autoExpand: Set<string> } {
|
||||
const matches = matchingIds(state);
|
||||
if (state.search.query.trim().length === 0) {
|
||||
return { visibleIds: matches, autoExpand: new Set() };
|
||||
}
|
||||
|
||||
const visibleIds = new Set<string>();
|
||||
const autoExpand = new Set<string>();
|
||||
for (const matchId of matches) {
|
||||
visibleIds.add(matchId);
|
||||
for (const ancestorId of ancestorsOf(matchId, state.byId)) {
|
||||
visibleIds.add(ancestorId);
|
||||
autoExpand.add(ancestorId);
|
||||
}
|
||||
}
|
||||
return { visibleIds, autoExpand };
|
||||
}
|
||||
|
||||
export function visibleNodeIds(state: PickerState): string[] {
|
||||
const { visibleIds, autoExpand } = filterTree(state);
|
||||
const result: string[] = [];
|
||||
const roots = state.tree.filter((node) => node.parentId === null).map((node) => node.id);
|
||||
|
||||
function visit(nodeId: string): void {
|
||||
if (!visibleIds.has(nodeId)) {
|
||||
return;
|
||||
}
|
||||
result.push(nodeId);
|
||||
const node = state.byId.get(nodeId);
|
||||
if (!node) {
|
||||
return;
|
||||
}
|
||||
if (state.expanded.has(nodeId) || autoExpand.has(nodeId)) {
|
||||
for (const childId of node.childIds) {
|
||||
visit(childId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const rootId of roots) {
|
||||
visit(rootId);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function selectAllVisible(state: PickerState): PickerState {
|
||||
const candidates = state.search.query.trim().length > 0 ? matchingIds(state) : new Set(visibleNodeIds(state));
|
||||
const checked = new Set(state.checked);
|
||||
|
||||
for (const node of state.tree) {
|
||||
if (!candidates.has(node.id)) {
|
||||
continue;
|
||||
}
|
||||
const hasCandidateAncestor = ancestorsOf(node.id, state.byId).some((ancestorId) => candidates.has(ancestorId));
|
||||
if (!hasCandidateAncestor && !isAncestorChecked(node.id, checked, state.byId)) {
|
||||
checked.add(node.id);
|
||||
for (const descendantId of descendantsOf(node.id, state.byId)) {
|
||||
checked.delete(descendantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cloneState(state, {
|
||||
checked: new Set(flattenSelection(checked, state.byId)),
|
||||
transientHint: null,
|
||||
});
|
||||
}
|
||||
|
||||
export function selectNone(state: PickerState): PickerState {
|
||||
return cloneState(state, { checked: new Set(), transientHint: null });
|
||||
}
|
||||
|
||||
function setExpanded(state: PickerState, nodeId: string, value: boolean | 'toggle'): PickerState {
|
||||
const expanded = new Set(state.expanded);
|
||||
const nextValue = value === 'toggle' ? !expanded.has(nodeId) : value;
|
||||
if (nextValue) {
|
||||
expanded.add(nodeId);
|
||||
} else {
|
||||
expanded.delete(nodeId);
|
||||
}
|
||||
return cloneState(state, { expanded });
|
||||
}
|
||||
|
||||
function expandPath(state: PickerState, nodeId: string): PickerState {
|
||||
const expanded = new Set(state.expanded);
|
||||
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
|
||||
expanded.add(ancestorId);
|
||||
}
|
||||
return cloneState(state, { expanded });
|
||||
}
|
||||
|
||||
export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'right'): PickerState {
|
||||
const node = state.byId.get(state.cursorId);
|
||||
if (!node) {
|
||||
return state;
|
||||
}
|
||||
|
||||
if (dir === 'left') {
|
||||
if (node.childIds.length > 0 && state.expanded.has(node.id)) {
|
||||
return setExpanded(state, node.id, false);
|
||||
}
|
||||
return node.parentId ? cloneState(state, { cursorId: node.parentId }) : state;
|
||||
}
|
||||
|
||||
if (dir === 'right') {
|
||||
if (node.childIds.length === 0) {
|
||||
return state;
|
||||
}
|
||||
if (!state.expanded.has(node.id)) {
|
||||
return setExpanded(state, node.id, true);
|
||||
}
|
||||
return cloneState(state, { cursorId: node.childIds[0] ?? node.id });
|
||||
}
|
||||
|
||||
const ids = visibleNodeIds(state);
|
||||
const index = ids.indexOf(state.cursorId);
|
||||
if (index === -1) {
|
||||
return ids[0] ? cloneState(state, { cursorId: ids[0] }) : state;
|
||||
}
|
||||
const nextIndex = dir === 'up' ? Math.max(0, index - 1) : Math.min(ids.length - 1, index + 1);
|
||||
return cloneState(state, { cursorId: ids[nextIndex] ?? state.cursorId });
|
||||
}
|
||||
|
||||
export function buildInitialState(args: {
|
||||
tree: NotionPickerNode[];
|
||||
existingRootPageIds: string[];
|
||||
currentCrawlMode?: 'all_accessible' | 'selected_roots';
|
||||
}): PickerState {
|
||||
const byId = new Map(args.tree.map((node) => [node.id, node]));
|
||||
const checked = new Set<string>();
|
||||
let staleCount = 0;
|
||||
|
||||
for (const rawId of args.existingRootPageIds) {
|
||||
const id = normalizePageId(rawId);
|
||||
if (byId.has(id)) {
|
||||
checked.add(id);
|
||||
} else {
|
||||
staleCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const minimalChecked = new Set(flattenSelection(checked, byId));
|
||||
const expanded = new Set<string>();
|
||||
for (const checkedId of minimalChecked) {
|
||||
for (const ancestorId of ancestorsOf(checkedId, byId)) {
|
||||
expanded.add(ancestorId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
tree: args.tree,
|
||||
byId,
|
||||
expanded,
|
||||
checked: minimalChecked,
|
||||
cursorId: args.tree[0]?.id ?? '',
|
||||
search: { editing: false, query: '' },
|
||||
pendingConfirm: null,
|
||||
preLoadWarnings: staleCount > 0 ? [`${staleCount} stored root_page_ids no longer visible`] : [],
|
||||
transientHint: null,
|
||||
currentCrawlMode: args.currentCrawlMode ?? 'selected_roots',
|
||||
};
|
||||
}
|
||||
|
||||
export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } {
|
||||
if (state.pendingConfirm) {
|
||||
if (cmd === 'save-confirm') {
|
||||
return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' };
|
||||
}
|
||||
if (cmd === 'save-cancel') {
|
||||
return { next: cloneState(state, { pendingConfirm: null }), effect: null };
|
||||
}
|
||||
if (cmd === 'quit') {
|
||||
return { next: state, effect: 'quit-without-save' };
|
||||
}
|
||||
return { next: state, effect: null };
|
||||
}
|
||||
|
||||
switch (cmd) {
|
||||
case 'cursor-up':
|
||||
return { next: moveCursor(state, 'up'), effect: null };
|
||||
case 'cursor-down':
|
||||
return { next: moveCursor(state, 'down'), effect: null };
|
||||
case 'cursor-left':
|
||||
return { next: moveCursor(state, 'left'), effect: null };
|
||||
case 'cursor-right':
|
||||
return { next: moveCursor(state, 'right'), effect: null };
|
||||
case 'expand':
|
||||
return { next: setExpanded(state, state.cursorId, 'toggle'), effect: null };
|
||||
case 'collapse':
|
||||
return { next: setExpanded(state, state.cursorId, false), effect: null };
|
||||
case 'expand-all':
|
||||
return {
|
||||
next: cloneState(state, {
|
||||
expanded: new Set(state.tree.filter((node) => node.childIds.length > 0).map((node) => node.id)),
|
||||
}),
|
||||
effect: null,
|
||||
};
|
||||
case 'collapse-all':
|
||||
return { next: cloneState(state, { expanded: new Set() }), effect: null };
|
||||
case 'toggle-check':
|
||||
return { next: toggleChecked(state, state.cursorId, now), effect: null };
|
||||
case 'select-all-visible':
|
||||
return { next: selectAllVisible(state), effect: null };
|
||||
case 'select-none':
|
||||
return { next: selectNone(state), effect: null };
|
||||
case 'clear-transient-hint':
|
||||
return { next: clearExpiredTransientHint(state, now), effect: null };
|
||||
case 'search-start':
|
||||
return { next: cloneState(state, { search: { ...state.search, editing: true } }), effect: null };
|
||||
case 'search-cancel':
|
||||
return { next: cloneState(state, { search: { editing: false, query: '' } }), effect: null };
|
||||
case 'search-submit':
|
||||
return { next: cloneState(state, { search: { ...state.search, editing: false } }), effect: null };
|
||||
case 'search-backspace':
|
||||
return {
|
||||
next: cloneState(state, { search: { ...state.search, query: state.search.query.slice(0, -1) } }),
|
||||
effect: null,
|
||||
};
|
||||
case 'save-request':
|
||||
if (state.checked.size === 0) {
|
||||
return {
|
||||
next: cloneState(state, {
|
||||
transientHint: transientHint('Select at least one page or press q to quit', now),
|
||||
}),
|
||||
effect: null,
|
||||
};
|
||||
}
|
||||
if (state.currentCrawlMode === 'all_accessible') {
|
||||
return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null };
|
||||
}
|
||||
return { next: state, effect: 'save' };
|
||||
case 'save-confirm':
|
||||
return { next: state, effect: 'save' };
|
||||
case 'save-cancel':
|
||||
return { next: state, effect: null };
|
||||
case 'quit':
|
||||
return { next: state, effect: 'quit-without-save' };
|
||||
default:
|
||||
return { next: cloneState(state, { search: { ...state.search, query: state.search.query + cmd.value } }), effect: null };
|
||||
}
|
||||
}
|
||||
384
packages/cli/src/commands/connection-notion-tui.test.tsx
Normal file
384
packages/cli/src/commands/connection-notion-tui.test.tsx
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
/* @jsxImportSource react */
|
||||
import { render as renderInkTest } from 'ink-testing-library';
|
||||
import React, { act, type ReactNode } from 'react';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
|
||||
import {
|
||||
NotionPickerApp,
|
||||
notionPickerCommandForInkInput,
|
||||
renderNotionPickerTui,
|
||||
resolveNotionPickerWidth,
|
||||
sanitizeNotionPickerTuiError,
|
||||
windowItems,
|
||||
windowOffset,
|
||||
type NotionPickerInkInstance,
|
||||
type NotionPickerInkRenderOptions,
|
||||
} from './connection-notion-tui.js';
|
||||
|
||||
const IDS = {
|
||||
engineering: '11111111-1111-1111-1111-111111111111',
|
||||
architecture: '22222222-2222-2222-2222-222222222222',
|
||||
marketing: '33333333-3333-3333-3333-333333333333',
|
||||
finance: '44444444-4444-4444-4444-444444444444',
|
||||
ops: '55555555-5555-5555-5555-555555555555',
|
||||
sales: '66666666-6666-6666-6666-666666666666',
|
||||
support: '77777777-7777-7777-7777-777777777777',
|
||||
product: '88888888-8888-8888-8888-888888888888',
|
||||
design: '99999999-9999-9999-9999-999999999999',
|
||||
};
|
||||
|
||||
function pages(): NotionPickerPageInput[] {
|
||||
return [
|
||||
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
|
||||
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
|
||||
];
|
||||
}
|
||||
|
||||
function manyPages(): NotionPickerPageInput[] {
|
||||
return [
|
||||
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
|
||||
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
|
||||
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
|
||||
{ id: IDS.finance, title: 'Finance', archived: false, parentId: null },
|
||||
{ id: IDS.ops, title: 'Operations', archived: false, parentId: null },
|
||||
{ id: IDS.sales, title: 'Sales', archived: false, parentId: null },
|
||||
{ id: IDS.support, title: 'Support', archived: false, parentId: null },
|
||||
{ id: IDS.product, title: 'Product', archived: false, parentId: null },
|
||||
{ id: IDS.design, title: 'Design', archived: false, parentId: null },
|
||||
];
|
||||
}
|
||||
|
||||
function state(mode: 'all_accessible' | 'selected_roots' = 'selected_roots') {
|
||||
return buildInitialState({
|
||||
tree: buildPickerTree(pages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: mode,
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForInkInput(): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
function fakeInkInstance(): NotionPickerInkInstance {
|
||||
return {
|
||||
rerender: vi.fn(),
|
||||
unmount: vi.fn(),
|
||||
waitUntilExit: vi.fn(async () => undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeFrameWrap(frame: string | undefined): string {
|
||||
return frame?.replace(/\n/g, ' ') ?? '';
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('notionPickerCommandForInkInput', () => {
|
||||
it('maps browse, search, and confirm input to reducer commands', () => {
|
||||
expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down');
|
||||
expect(notionPickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up');
|
||||
expect(notionPickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right');
|
||||
expect(notionPickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left');
|
||||
expect(notionPickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check');
|
||||
expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start');
|
||||
expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible');
|
||||
expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none');
|
||||
expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request');
|
||||
expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit');
|
||||
expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit');
|
||||
|
||||
expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({
|
||||
type: 'search-input',
|
||||
value: 'x',
|
||||
});
|
||||
expect(notionPickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe(
|
||||
'search-backspace',
|
||||
);
|
||||
expect(notionPickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe(
|
||||
'search-submit',
|
||||
);
|
||||
expect(notionPickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe(
|
||||
'search-cancel',
|
||||
);
|
||||
|
||||
expect(notionPickerCommandForInkInput('y', {}, state().search, 'mode-switch')).toBe('save-confirm');
|
||||
expect(notionPickerCommandForInkInput('', { return: true }, state().search, 'mode-switch')).toBe('save-confirm');
|
||||
expect(notionPickerCommandForInkInput('n', {}, state().search, 'mode-switch')).toBe('save-cancel');
|
||||
});
|
||||
});
|
||||
|
||||
describe('window helpers', () => {
|
||||
it('centers the selected row and returns the visible slice', () => {
|
||||
expect(windowOffset(20, 10, 5)).toBe(8);
|
||||
expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 });
|
||||
});
|
||||
|
||||
it('clamps picker width to the design rule', () => {
|
||||
expect(resolveNotionPickerWidth(200)).toBe(120);
|
||||
expect(resolveNotionPickerWidth(100)).toBe(96);
|
||||
expect(resolveNotionPickerWidth(50)).toBe(60);
|
||||
expect(resolveNotionPickerWidth(undefined)).toBe(96);
|
||||
});
|
||||
});
|
||||
|
||||
describe('NotionPickerApp', () => {
|
||||
it('renders spec banners, row glyphs, search visibility, and hint text', () => {
|
||||
const initialState = {
|
||||
...state('all_accessible'),
|
||||
preLoadWarnings: ['1 stored root_page_ids no longer visible'],
|
||||
};
|
||||
const { lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={5000}
|
||||
currentCrawlMode="all_accessible"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Notion pages visible to integration "Design Workspace"');
|
||||
expect(frame).toContain('5000-page cap reached - some pages not shown');
|
||||
expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save');
|
||||
expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)');
|
||||
expect(frame).toContain(' [ ] Marketing');
|
||||
expect(frame).not.toContain('Search ready: -');
|
||||
expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit');
|
||||
});
|
||||
|
||||
it('renders partial discovery warnings without stale-root save suffix', () => {
|
||||
const initialState = {
|
||||
...state(),
|
||||
preLoadWarnings: ['Notion search stopped early: rate limit after first page'],
|
||||
};
|
||||
const { lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('Notion search stopped early: rate limit after first page');
|
||||
expect(frame).not.toContain(
|
||||
'Notion search stopped early: rate limit after first page - they will be removed if you save',
|
||||
);
|
||||
});
|
||||
|
||||
it('renders checked parents and locked descendants with the locked design glyphs', () => {
|
||||
const initialState = {
|
||||
...state(),
|
||||
checked: new Set([IDS.engineering]),
|
||||
expanded: new Set([IDS.engineering]),
|
||||
};
|
||||
const { lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('▸ [×] Engineering Docs ▾');
|
||||
expect(frame).toContain(' [~] Architecture');
|
||||
});
|
||||
|
||||
it('supports keyboard selection, all_accessible confirmation, and save callback', async () => {
|
||||
const onExit = vi.fn();
|
||||
const { stdin, lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={state('all_accessible')}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="all_accessible"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
stdin.write(' ');
|
||||
await waitForInkInput();
|
||||
expect(lastFrame()).toContain('[×] Engineering Docs');
|
||||
|
||||
stdin.write('s');
|
||||
await waitForInkInput();
|
||||
expect(normalizeFrameWrap(lastFrame())).toContain(
|
||||
'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back',
|
||||
);
|
||||
|
||||
stdin.write('y');
|
||||
await waitForInkInput();
|
||||
expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] });
|
||||
});
|
||||
|
||||
it('removes transient hints after their expiry time', async () => {
|
||||
vi.useFakeTimers();
|
||||
const onExit = vi.fn();
|
||||
const { stdin, lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={state()}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
stdin.write('s');
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
});
|
||||
expect(lastFrame()).toContain('Select at least one page or press q to quit');
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(2500);
|
||||
});
|
||||
expect(lastFrame()).not.toContain('Select at least one page or press q to quit');
|
||||
expect(onExit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('renders row-window overflow indicators when the visible list is clipped', async () => {
|
||||
const onExit = vi.fn();
|
||||
const initialState = buildInitialState({
|
||||
tree: buildPickerTree(manyPages()),
|
||||
existingRootPageIds: [],
|
||||
currentCrawlMode: 'selected_roots',
|
||||
});
|
||||
initialState.expanded = new Set([IDS.engineering]);
|
||||
const { stdin, lastFrame } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={initialState}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={13}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(lastFrame()).toContain('↓ 4 more');
|
||||
|
||||
stdin.write('\u001B[B');
|
||||
stdin.write('\u001B[B');
|
||||
stdin.write('\u001B[B');
|
||||
stdin.write('\u001B[B');
|
||||
await waitForInkInput();
|
||||
|
||||
const frame = lastFrame() ?? '';
|
||||
expect(frame).toContain('↑ ');
|
||||
expect(frame).toContain('↓ ');
|
||||
expect(onExit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns quit without saving', async () => {
|
||||
const onExit = vi.fn();
|
||||
const { stdin } = renderInkTest(
|
||||
<NotionPickerApp
|
||||
initialState={state()}
|
||||
connectionId="notion-main"
|
||||
workspaceLabel="Design Workspace"
|
||||
cappedAtCount={null}
|
||||
currentCrawlMode="selected_roots"
|
||||
terminalRows={24}
|
||||
terminalWidth={100}
|
||||
onExit={onExit}
|
||||
/>,
|
||||
);
|
||||
|
||||
stdin.write('q');
|
||||
await waitForInkInput();
|
||||
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderNotionPickerTui', () => {
|
||||
it('returns the app result from the Ink runtime', async () => {
|
||||
const io = {
|
||||
stdin: { isTTY: true, setRawMode: vi.fn() },
|
||||
stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() },
|
||||
stderr: { write: vi.fn() },
|
||||
};
|
||||
const renderInk = vi.fn((_tree: ReactNode, _options: NotionPickerInkRenderOptions) => fakeInkInstance());
|
||||
|
||||
await expect(
|
||||
renderNotionPickerTui(
|
||||
{
|
||||
initialState: state(),
|
||||
connectionId: 'notion-main',
|
||||
workspaceLabel: 'Design Workspace',
|
||||
cappedAtCount: null,
|
||||
currentCrawlMode: 'selected_roots',
|
||||
},
|
||||
io,
|
||||
{ renderInk },
|
||||
),
|
||||
).resolves.toEqual({ kind: 'quit' });
|
||||
expect(renderInk).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('sanitizes render errors and tells the user to use no-input mode', async () => {
|
||||
expect(sanitizeNotionPickerTuiError(new Error('token=secret https://api.notion.com/v1/search'))).toBe(
|
||||
'[redacted] [redacted-url]',
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to quit with a scripted-mode hint when Ink cannot initialize', async () => {
|
||||
let stderr = '';
|
||||
const io = {
|
||||
stdin: { isTTY: false, setRawMode: vi.fn() },
|
||||
stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() },
|
||||
stderr: {
|
||||
write(chunk: string) {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
renderNotionPickerTui(
|
||||
{
|
||||
initialState: state(),
|
||||
connectionId: 'notion-main',
|
||||
workspaceLabel: 'Design Workspace',
|
||||
cappedAtCount: null,
|
||||
currentCrawlMode: 'selected_roots',
|
||||
},
|
||||
io,
|
||||
{
|
||||
renderInk: vi.fn(() => {
|
||||
throw new Error('token=secret');
|
||||
}),
|
||||
},
|
||||
),
|
||||
).resolves.toEqual({ kind: 'quit' });
|
||||
expect(stderr).toContain('Use --no-input --root-page-id <UUID> for scripted mode');
|
||||
expect(stderr).not.toContain('secret');
|
||||
});
|
||||
});
|
||||
338
packages/cli/src/commands/connection-notion-tui.tsx
Normal file
338
packages/cli/src/commands/connection-notion-tui.tsx
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
/* @jsxImportSource react */
|
||||
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
|
||||
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import {
|
||||
filterTree,
|
||||
flattenSelection,
|
||||
isAncestorChecked,
|
||||
reducer,
|
||||
visibleNodeIds,
|
||||
type PickerCommand,
|
||||
type PickerState,
|
||||
} from './connection-notion-tree.js';
|
||||
import type { KloCliIo } from '../index.js';
|
||||
|
||||
const COLOR_THEME = {
|
||||
text: 'white',
|
||||
muted: 'gray',
|
||||
active: 'cyan',
|
||||
warning: 'yellow',
|
||||
} as const;
|
||||
|
||||
const NO_COLOR_THEME = {
|
||||
text: 'white',
|
||||
muted: 'white',
|
||||
active: 'white',
|
||||
warning: 'white',
|
||||
} as const;
|
||||
|
||||
type NotionPickerTheme = Record<keyof typeof COLOR_THEME, string>;
|
||||
|
||||
export interface NotionPickerTuiIo extends KloCliIo {
|
||||
stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void };
|
||||
stdout: KloCliIo['stdout'] & { isTTY?: boolean; columns?: number; rows?: number };
|
||||
}
|
||||
|
||||
interface InkKey {
|
||||
leftArrow?: boolean;
|
||||
rightArrow?: boolean;
|
||||
upArrow?: boolean;
|
||||
downArrow?: boolean;
|
||||
return?: boolean;
|
||||
escape?: boolean;
|
||||
ctrl?: boolean;
|
||||
backspace?: boolean;
|
||||
delete?: boolean;
|
||||
}
|
||||
|
||||
export type PickerRenderResult = { kind: 'save'; rootPageIds: string[] } | { kind: 'quit' };
|
||||
|
||||
export interface PickerRenderInput {
|
||||
initialState: PickerState;
|
||||
connectionId: string;
|
||||
workspaceLabel: string;
|
||||
cappedAtCount: number | null;
|
||||
currentCrawlMode: 'all_accessible' | 'selected_roots';
|
||||
}
|
||||
|
||||
interface NotionPickerAppProps extends PickerRenderInput {
|
||||
terminalRows?: number;
|
||||
terminalWidth?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
onExit(result: PickerRenderResult): void;
|
||||
}
|
||||
|
||||
export interface NotionPickerInkInstance {
|
||||
rerender(tree: ReactNode): void;
|
||||
unmount(): void;
|
||||
waitUntilExit(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface NotionPickerInkRenderOptions {
|
||||
stdin?: NotionPickerTuiIo['stdin'];
|
||||
stdout: NotionPickerTuiIo['stdout'];
|
||||
stderr: NotionPickerTuiIo['stderr'];
|
||||
exitOnCtrlC: boolean;
|
||||
patchConsole: boolean;
|
||||
maxFps: number;
|
||||
alternateScreen: boolean;
|
||||
}
|
||||
|
||||
function resolveTheme(env: NodeJS.ProcessEnv = process.env): NotionPickerTheme {
|
||||
return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME;
|
||||
}
|
||||
|
||||
export function resolveNotionPickerWidth(columns: number | undefined): number {
|
||||
const resolvedColumns = columns ?? 100;
|
||||
return Math.max(60, Math.min(120, resolvedColumns - 4));
|
||||
}
|
||||
|
||||
function staleWarningText(warning: string): string {
|
||||
return warning.includes('stored root_page_ids no longer visible')
|
||||
? `${warning} - they will be removed if you save`
|
||||
: warning;
|
||||
}
|
||||
|
||||
function selectedPageCountText(count: number): string {
|
||||
return `${count} selected ${count === 1 ? 'page' : 'pages'}`;
|
||||
}
|
||||
|
||||
function rowMatchesSearch(state: PickerState, nodeId: string): boolean {
|
||||
const query = state.search.query.trim().toLocaleLowerCase();
|
||||
if (!query) {
|
||||
return false;
|
||||
}
|
||||
const node = state.byId.get(nodeId);
|
||||
if (!node) {
|
||||
return false;
|
||||
}
|
||||
return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query);
|
||||
}
|
||||
|
||||
export function sanitizeNotionPickerTuiError(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message
|
||||
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]')
|
||||
.replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]');
|
||||
}
|
||||
|
||||
export function windowOffset(count: number, selected: number, visible: number): number {
|
||||
if (count <= visible) return 0;
|
||||
return Math.max(0, Math.min(count - visible, selected - Math.floor(visible / 2)));
|
||||
}
|
||||
|
||||
export function windowItems<T>(items: T[], selected: number, visible: number): { items: T[]; offset: number } {
|
||||
const offset = windowOffset(items.length, selected, visible);
|
||||
return { items: items.slice(offset, offset + visible), offset };
|
||||
}
|
||||
|
||||
function truncateText(value: string, width: number): string {
|
||||
if (value.length <= width) return value;
|
||||
if (width <= 3) return value.slice(0, width);
|
||||
return `${value.slice(0, width - 3)}...`;
|
||||
}
|
||||
|
||||
export function notionPickerCommandForInkInput(
|
||||
input: string,
|
||||
key: InkKey,
|
||||
search: PickerState['search'],
|
||||
pendingConfirm: PickerState['pendingConfirm'],
|
||||
): PickerCommand | null {
|
||||
if (pendingConfirm) {
|
||||
if (input === 'y' || key.return) return 'save-confirm';
|
||||
if (input === 'n' || key.escape) return 'save-cancel';
|
||||
if (key.ctrl === true && input === 'c') return 'quit';
|
||||
return null;
|
||||
}
|
||||
if (search.editing) {
|
||||
if (key.escape) return 'search-cancel';
|
||||
if (key.return) return 'search-submit';
|
||||
if (key.backspace || key.delete) return 'search-backspace';
|
||||
if (key.downArrow) return 'cursor-down';
|
||||
if (key.upArrow) return 'cursor-up';
|
||||
if (input.length === 1 && input >= ' ' && input !== '\u007f') return { type: 'search-input', value: input };
|
||||
return null;
|
||||
}
|
||||
if (key.ctrl === true && input === 'c') return 'quit';
|
||||
if (key.upArrow) return 'cursor-up';
|
||||
if (key.downArrow) return 'cursor-down';
|
||||
if (key.leftArrow) return 'cursor-left';
|
||||
if (key.rightArrow) return 'cursor-right';
|
||||
if (key.return) return 'expand';
|
||||
if (input === ' ') return 'toggle-check';
|
||||
if (input === '/') return 'search-start';
|
||||
if (input === 'a') return 'select-all-visible';
|
||||
if (input === 'n') return 'select-none';
|
||||
if (input === 's') return 'save-request';
|
||||
if (input === 'q' || key.escape) return 'quit';
|
||||
return null;
|
||||
}
|
||||
|
||||
function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: NotionPickerTheme }): ReactNode {
|
||||
const node = props.state.byId.get(props.nodeId);
|
||||
if (!node) return null;
|
||||
const focused = props.state.cursorId === node.id;
|
||||
const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId);
|
||||
const checked = props.state.checked.has(node.id);
|
||||
const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]';
|
||||
const children =
|
||||
node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : '';
|
||||
const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`;
|
||||
const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text;
|
||||
const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length));
|
||||
const inverse = rowMatchesSearch(props.state, node.id);
|
||||
|
||||
return (
|
||||
<Text color={color} strikethrough={node.archived}>
|
||||
{prefix}
|
||||
<Text inverse={inverse}>{title}</Text>
|
||||
</Text>
|
||||
);
|
||||
}
|
||||
|
||||
export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
|
||||
const app = useApp();
|
||||
const [state, setState] = useState(props.initialState);
|
||||
const stateRef = useRef(state);
|
||||
const theme = useMemo(() => resolveTheme(props.env), [props.env]);
|
||||
const visibleIds = visibleNodeIds(state);
|
||||
const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId));
|
||||
const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8;
|
||||
const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows));
|
||||
const rows = windowItems(visibleIds, selectedIndex, visibleRows);
|
||||
const hiddenAbove = rows.offset;
|
||||
const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length);
|
||||
const searchMatchCount = filterTree(state).visibleIds.size;
|
||||
const width = resolveNotionPickerWidth(props.terminalWidth);
|
||||
const showSearch = state.search.editing || state.search.query.trim().length > 0;
|
||||
const selectedCount = flattenSelection(state.checked, state.byId).length;
|
||||
|
||||
stateRef.current = state;
|
||||
|
||||
useEffect(() => {
|
||||
const hint = state.transientHint;
|
||||
if (!hint) {
|
||||
return;
|
||||
}
|
||||
|
||||
const clearHint = () => {
|
||||
setState((current) => {
|
||||
const { next } = reducer(current, 'clear-transient-hint');
|
||||
stateRef.current = next;
|
||||
return next;
|
||||
});
|
||||
};
|
||||
const delay = hint.expiresAt - Date.now();
|
||||
if (delay <= 0) {
|
||||
clearHint();
|
||||
return;
|
||||
}
|
||||
|
||||
const timeout = setTimeout(clearHint, delay);
|
||||
|
||||
return () => clearTimeout(timeout);
|
||||
}, [state.transientHint?.expiresAt]);
|
||||
|
||||
useInput((input, key) => {
|
||||
const command = notionPickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm);
|
||||
if (!command) {
|
||||
return;
|
||||
}
|
||||
const { next, effect } = reducer(stateRef.current, command);
|
||||
stateRef.current = next;
|
||||
setState(next);
|
||||
if (effect === 'save') {
|
||||
props.onExit({ kind: 'save', rootPageIds: flattenSelection(next.checked, next.byId) });
|
||||
app.exit();
|
||||
return;
|
||||
}
|
||||
if (effect === 'quit-without-save') {
|
||||
props.onExit({ kind: 'quit' });
|
||||
app.exit();
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
<Text color={theme.active}>Notion pages visible to integration "{props.workspaceLabel}"</Text>
|
||||
{props.cappedAtCount ? <Text color={theme.warning}>{props.cappedAtCount}-page cap reached - some pages not shown</Text> : null}
|
||||
{state.preLoadWarnings.map((warning) => (
|
||||
<Text key={warning} color={theme.warning}>
|
||||
{staleWarningText(warning)}
|
||||
</Text>
|
||||
))}
|
||||
{showSearch ? (
|
||||
<Text color={theme.muted}>
|
||||
/ {state.search.query}
|
||||
{state.search.editing ? '█' : ''} ({searchMatchCount} matches)
|
||||
</Text>
|
||||
) : null}
|
||||
<Box flexDirection="column">
|
||||
{hiddenAbove > 0 ? <Text color={theme.muted}>↑ {hiddenAbove} more</Text> : null}
|
||||
{rows.items.map((nodeId) => (
|
||||
<PickerRow key={nodeId} state={state} nodeId={nodeId} width={width} theme={theme} />
|
||||
))}
|
||||
{hiddenBelow > 0 ? <Text color={theme.muted}>↓ {hiddenBelow} more</Text> : null}
|
||||
</Box>
|
||||
{state.pendingConfirm === 'mode-switch' ? (
|
||||
<Text color={theme.warning}>
|
||||
Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to{' '}
|
||||
{selectedPageCountText(selectedCount)}. [y] confirm [esc] back
|
||||
</Text>
|
||||
) : null}
|
||||
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
|
||||
<Text color={theme.muted}>space toggle · enter expand · / search · a all · n none · s save & exit · q quit</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): NotionPickerInkInstance {
|
||||
return renderInkRuntime(tree, {
|
||||
stdin: options.stdin as NodeJS.ReadStream | undefined,
|
||||
stdout: options.stdout as NodeJS.WriteStream,
|
||||
stderr: options.stderr as NodeJS.WriteStream,
|
||||
exitOnCtrlC: options.exitOnCtrlC,
|
||||
patchConsole: options.patchConsole,
|
||||
maxFps: options.maxFps,
|
||||
alternateScreen: options.alternateScreen,
|
||||
}) as NotionPickerInkInstance;
|
||||
}
|
||||
|
||||
export async function renderNotionPickerTui(
|
||||
input: PickerRenderInput,
|
||||
io: NotionPickerTuiIo,
|
||||
options: { renderInk?: (tree: ReactNode, options: NotionPickerInkRenderOptions) => NotionPickerInkInstance } = {},
|
||||
): Promise<PickerRenderResult> {
|
||||
let result: PickerRenderResult = { kind: 'quit' };
|
||||
let instance: NotionPickerInkInstance | null = null;
|
||||
try {
|
||||
instance = (options.renderInk ?? renderInk)(
|
||||
<NotionPickerApp
|
||||
{...input}
|
||||
terminalRows={(io.stdout as { rows?: number }).rows ?? process.stdout.rows ?? 24}
|
||||
terminalWidth={io.stdout.columns ?? process.stdout.columns}
|
||||
onExit={(next) => {
|
||||
result = next;
|
||||
instance?.unmount();
|
||||
}}
|
||||
/>,
|
||||
{
|
||||
stdin: io.stdin,
|
||||
stdout: io.stdout,
|
||||
stderr: io.stderr,
|
||||
exitOnCtrlC: false,
|
||||
patchConsole: false,
|
||||
maxFps: 30,
|
||||
alternateScreen: true,
|
||||
},
|
||||
);
|
||||
await instance.waitUntilExit();
|
||||
instance.unmount();
|
||||
return result;
|
||||
} catch (error) {
|
||||
io.stderr.write(
|
||||
`Notion picker requires a TTY. Use --no-input --root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
|
||||
);
|
||||
return { kind: 'quit' };
|
||||
}
|
||||
}
|
||||
466
packages/cli/src/commands/connection-notion.test.ts
Normal file
466
packages/cli/src/commands/connection-notion.test.ts
Normal file
|
|
@ -0,0 +1,466 @@
|
|||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
initKloProject,
|
||||
loadKloProject,
|
||||
serializeKloProjectConfig,
|
||||
type KloProjectConfig,
|
||||
} from '@klo/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
applyNotionPickerWriteback,
|
||||
discoverNotionPickerPages,
|
||||
notionPickerPageFromSearchResult,
|
||||
normalizeNotionPageId,
|
||||
resolveNotionWorkspaceLabel,
|
||||
runKloConnectionNotion,
|
||||
type NotionPickerApi,
|
||||
type PickerRenderInput,
|
||||
type PickerRenderResult,
|
||||
} from './connection-notion.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
|
||||
|
||||
const PAGE_IDS = {
|
||||
engineering: '11111111-1111-1111-1111-111111111111',
|
||||
architecture: '22222222-2222-2222-2222-222222222222',
|
||||
stale: '99999999-9999-9999-9999-999999999999',
|
||||
};
|
||||
|
||||
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
|
||||
return {
|
||||
object: 'page',
|
||||
id,
|
||||
archived: false,
|
||||
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
|
||||
properties: {
|
||||
title: {
|
||||
type: 'title',
|
||||
title: [{ plain_text: title }],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
|
||||
return {
|
||||
search: vi.fn(async (_filterValue, startCursor) => {
|
||||
if (startCursor === 'page-2') {
|
||||
return { results: pages.slice(2), hasMore: false, nextCursor: null };
|
||||
}
|
||||
return {
|
||||
results: pages.slice(0, 2),
|
||||
hasMore: pages.length > 2,
|
||||
nextCursor: pages.length > 2 ? 'page-2' : null,
|
||||
};
|
||||
}),
|
||||
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
|
||||
};
|
||||
}
|
||||
|
||||
describe('normalizeNotionPageId', () => {
|
||||
it('accepts dashed and compact UUIDs', () => {
|
||||
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
|
||||
'11111111-2222-3333-4444-555555555555',
|
||||
);
|
||||
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
|
||||
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runKloConnectionNotion', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-cli-notion-pick-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeProjectConfig(projectDir: string, config: KloProjectConfig): Promise<void> {
|
||||
const project = await loadKloProject({ projectDir });
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(config),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
'seed test config',
|
||||
);
|
||||
}
|
||||
|
||||
it('rejects unsafe connection ids before loading a project', async () => {
|
||||
const io = makeIo();
|
||||
const loadProject = vi.fn(async () => {
|
||||
throw new Error('loadProject should not be called');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir: '/tmp/project',
|
||||
connectionId: '../evil',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{ loadProject },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(loadProject).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('Unsafe connection id: ../evil');
|
||||
});
|
||||
|
||||
it('writes selected root_page_ids while preserving every other Notion connection field', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'all_accessible',
|
||||
root_page_ids: ['99999999-9999-9999-9999-999999999999'],
|
||||
root_database_ids: ['database-1'],
|
||||
root_data_source_ids: ['data-source-1'],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
|
||||
unknown_future_field: 'keep-me',
|
||||
},
|
||||
},
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'non-interactive',
|
||||
rootPageIds: [
|
||||
'11111111-2222-3333-4444-555555555555',
|
||||
'66666666-7777-8888-9999-aaaaaaaaaaaa',
|
||||
],
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(yaml).toContain('crawl_mode: selected_roots');
|
||||
expect(yaml).toContain('root_page_ids:');
|
||||
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
|
||||
expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa');
|
||||
expect(yaml).toContain('root_database_ids:');
|
||||
expect(yaml).toContain('database-1');
|
||||
expect(yaml).toContain('root_data_source_ids:');
|
||||
expect(yaml).toContain('data-source-1');
|
||||
expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\'');
|
||||
expect(yaml).toContain('unknown_future_field: keep-me');
|
||||
expect(io.stdout()).toContain('Connection: notion-main');
|
||||
expect(io.stdout()).toContain('rootPageIds: 2');
|
||||
expect(io.stdout()).toContain('crawlMode: selected_roots');
|
||||
});
|
||||
|
||||
it('rejects empty writeback, missing connections, and non-Notion connections', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const project = await loadKloProject({ projectDir });
|
||||
|
||||
await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow(
|
||||
'connection notion pick requires at least one root page id',
|
||||
);
|
||||
await expect(
|
||||
applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']),
|
||||
).rejects.toThrow('Connection "missing" not found');
|
||||
await expect(
|
||||
applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']),
|
||||
).rejects.toThrow('Connection "warehouse" is not a Notion connection');
|
||||
});
|
||||
|
||||
it('extracts picker page inputs from Notion search results', () => {
|
||||
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
|
||||
.toEqual({
|
||||
id: PAGE_IDS.architecture,
|
||||
title: 'Architecture',
|
||||
archived: false,
|
||||
parentId: PAGE_IDS.engineering,
|
||||
});
|
||||
|
||||
expect(
|
||||
notionPickerPageFromSearchResult({
|
||||
object: 'page',
|
||||
id: PAGE_IDS.engineering.replaceAll('-', ''),
|
||||
archived: true,
|
||||
parent: { type: 'workspace', workspace: true },
|
||||
properties: {},
|
||||
}),
|
||||
).toEqual({
|
||||
id: PAGE_IDS.engineering,
|
||||
title: 'Untitled',
|
||||
archived: true,
|
||||
parentId: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('discovers visible pages up to the cap and reports cap state', async () => {
|
||||
const api = fakeNotionApi([
|
||||
notionPage(PAGE_IDS.engineering, 'Engineering'),
|
||||
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
|
||||
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
|
||||
]);
|
||||
|
||||
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
|
||||
pages: [
|
||||
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
|
||||
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
|
||||
],
|
||||
cappedAtCount: 2,
|
||||
warnings: [],
|
||||
});
|
||||
expect(api.search).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
|
||||
const api: NotionPickerApi = {
|
||||
search: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
|
||||
hasMore: true,
|
||||
nextCursor: 'cursor-2',
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('rate limit after first page')),
|
||||
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
|
||||
};
|
||||
|
||||
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
|
||||
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
|
||||
cappedAtCount: null,
|
||||
warnings: ['Notion search stopped early: rate limit after first page'],
|
||||
});
|
||||
});
|
||||
|
||||
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
|
||||
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
|
||||
await expect(
|
||||
resolveNotionWorkspaceLabel(
|
||||
{
|
||||
search: vi.fn(),
|
||||
retrieveBotUser: vi.fn(async () => {
|
||||
throw new Error('users.me unavailable');
|
||||
}),
|
||||
},
|
||||
'notion-main',
|
||||
),
|
||||
).resolves.toBe('notion-main');
|
||||
});
|
||||
|
||||
it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'all_accessible',
|
||||
root_page_ids: [PAGE_IDS.stale],
|
||||
root_database_ids: ['database-1'],
|
||||
root_data_source_ids: ['data-source-1'],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
const api = fakeNotionApi([
|
||||
notionPage(PAGE_IDS.engineering, 'Engineering'),
|
||||
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
|
||||
]);
|
||||
const renderPicker = vi.fn(async (input): Promise<PickerRenderResult> => {
|
||||
expect(input.connectionId).toBe('notion-main');
|
||||
expect(input.workspaceLabel).toBe('Design Workspace');
|
||||
expect(input.currentCrawlMode).toBe('all_accessible');
|
||||
expect(input.cappedAtCount).toBeNull();
|
||||
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
|
||||
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: { NOTION_TOKEN: 'ntn_test_token' },
|
||||
createNotionApi: vi.fn(() => api),
|
||||
renderPicker,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const yaml = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
expect(yaml).toContain('crawl_mode: selected_roots');
|
||||
expect(yaml).toContain(PAGE_IDS.engineering);
|
||||
expect(yaml).not.toContain(PAGE_IDS.stale);
|
||||
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
|
||||
expect(io.stdout()).toContain('Connection: notion-main');
|
||||
expect(io.stdout()).toContain('rootPageIds: 1');
|
||||
});
|
||||
|
||||
it('passes partial-discovery warnings into the TUI banner state', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: [PAGE_IDS.engineering],
|
||||
root_database_ids: [],
|
||||
root_data_source_ids: [],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
const api: NotionPickerApi = {
|
||||
search: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
|
||||
hasMore: true,
|
||||
nextCursor: 'cursor-2',
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('rate limit after first page')),
|
||||
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
|
||||
};
|
||||
let renderInput: PickerRenderInput | undefined;
|
||||
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
|
||||
renderInput = input;
|
||||
return { kind: 'quit' };
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: { NOTION_TOKEN: 'ntn_test_token' },
|
||||
createNotionApi: vi.fn(() => api),
|
||||
renderPicker,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(renderPicker).toHaveBeenCalledOnce();
|
||||
if (!renderInput) {
|
||||
throw new Error('renderPicker was not called');
|
||||
}
|
||||
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
|
||||
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
|
||||
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
|
||||
expect(io.stdout()).toContain('No changes saved.');
|
||||
});
|
||||
|
||||
it('quits interactive mode without writing when the TUI returns quit', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const initialized = await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeProjectConfig(projectDir, {
|
||||
...initialized.config,
|
||||
connections: {
|
||||
'notion-main': {
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: [PAGE_IDS.engineering],
|
||||
root_database_ids: [],
|
||||
root_data_source_ids: [],
|
||||
max_pages_per_run: 12,
|
||||
max_knowledge_creates_per_run: 2,
|
||||
max_knowledge_updates_per_run: 7,
|
||||
last_successful_cursor: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
const before = await readFile(join(projectDir, 'klo.yaml'), 'utf-8');
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKloConnectionNotion(
|
||||
{
|
||||
command: 'pick',
|
||||
projectDir,
|
||||
connectionId: 'notion-main',
|
||||
mode: 'interactive',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
env: { NOTION_TOKEN: 'ntn_test_token' },
|
||||
createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])),
|
||||
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
await expect(readFile(join(projectDir, 'klo.yaml'), 'utf-8')).resolves.toBe(before);
|
||||
expect(io.stdout()).toContain('No changes saved.');
|
||||
});
|
||||
});
|
||||
278
packages/cli/src/commands/connection-notion.ts
Normal file
278
packages/cli/src/commands/connection-notion.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
import { parseNotionConnectionConfig, resolveNotionAuthToken } from '@klo/context/connections';
|
||||
import { type NotionApi, type NotionBotInfo, NotionClient } from '@klo/context/ingest';
|
||||
import {
|
||||
type KloLocalProject,
|
||||
type KloProjectConnectionConfig,
|
||||
loadKloProject,
|
||||
serializeKloProjectConfig,
|
||||
} from '@klo/context/project';
|
||||
import type { KloCliIo } from '../index.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
|
||||
import {
|
||||
type NotionPickerTuiIo,
|
||||
type PickerRenderInput,
|
||||
type PickerRenderResult,
|
||||
renderNotionPickerTui,
|
||||
} from './connection-notion-tui.js';
|
||||
|
||||
profileMark('module:commands/connection-notion');
|
||||
|
||||
export type KloConnectionNotionArgs =
|
||||
| {
|
||||
command: 'pick';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
mode: 'interactive';
|
||||
}
|
||||
| {
|
||||
command: 'pick';
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
mode: 'non-interactive';
|
||||
rootPageIds: string[];
|
||||
};
|
||||
|
||||
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
|
||||
export type { PickerRenderInput, PickerRenderResult };
|
||||
|
||||
interface KloConnectionNotionDeps {
|
||||
env?: Record<string, string | undefined>;
|
||||
loadProject?: typeof loadKloProject;
|
||||
createNotionApi?: (authToken: string) => NotionPickerApi;
|
||||
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
|
||||
}
|
||||
|
||||
const NOTION_PICKER_PAGE_CAP = 5000;
|
||||
|
||||
function assertSafeConnectionId(connectionId: string): void {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
|
||||
throw new Error(`Unsafe connection id: ${connectionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeNotionPageId(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
|
||||
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
|
||||
throw new Error(`Invalid Notion page UUID: ${value}`);
|
||||
}
|
||||
const lower = compact.toLowerCase();
|
||||
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
|
||||
}
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
function extractTitleFromNotionPage(page: Record<string, unknown>): string {
|
||||
const properties = recordValue(page.properties);
|
||||
if (!properties) {
|
||||
return 'Untitled';
|
||||
}
|
||||
for (const property of Object.values(properties)) {
|
||||
const value = recordValue(property);
|
||||
if (!value || value.type !== 'title' || !Array.isArray(value.title)) {
|
||||
continue;
|
||||
}
|
||||
const text = value.title
|
||||
.map((part) => {
|
||||
const richText = recordValue(part);
|
||||
return typeof richText?.plain_text === 'string' ? richText.plain_text : '';
|
||||
})
|
||||
.join('')
|
||||
.trim();
|
||||
if (text.length > 0) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return 'Untitled';
|
||||
}
|
||||
|
||||
function extractParentPageId(page: Record<string, unknown>): string | null {
|
||||
const parent = recordValue(page.parent);
|
||||
if (!parent || parent.type !== 'page_id' || typeof parent.page_id !== 'string') {
|
||||
return null;
|
||||
}
|
||||
return normalizeNotionPageId(parent.page_id);
|
||||
}
|
||||
|
||||
export function notionPickerPageFromSearchResult(result: Record<string, unknown>): NotionPickerPageInput {
|
||||
const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : '';
|
||||
if (!id) {
|
||||
throw new Error('Notion page search result is missing id');
|
||||
}
|
||||
return {
|
||||
id,
|
||||
title: extractTitleFromNotionPage(result),
|
||||
archived: result.archived === true,
|
||||
parentId: extractParentPageId(result),
|
||||
};
|
||||
}
|
||||
|
||||
export async function discoverNotionPickerPages(
|
||||
api: NotionPickerApi,
|
||||
options: { cap?: number } = {},
|
||||
): Promise<{ pages: NotionPickerPageInput[]; cappedAtCount: number | null; warnings: string[] }> {
|
||||
const cap = options.cap ?? NOTION_PICKER_PAGE_CAP;
|
||||
const pages: NotionPickerPageInput[] = [];
|
||||
const warnings: string[] = [];
|
||||
let cursor: string | null | undefined = null;
|
||||
|
||||
while (pages.length < cap) {
|
||||
let response: Awaited<ReturnType<NotionPickerApi['search']>>;
|
||||
try {
|
||||
response = await api.search('page', cursor, Math.min(100, cap - pages.length));
|
||||
} catch (error) {
|
||||
if (pages.length === 0) {
|
||||
throw error;
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
warnings.push(`Notion search stopped early: ${message}`);
|
||||
return { pages, cappedAtCount: null, warnings };
|
||||
}
|
||||
|
||||
for (const result of response.results) {
|
||||
pages.push(notionPickerPageFromSearchResult(result));
|
||||
if (pages.length >= cap) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.hasMore || !response.nextCursor || pages.length >= cap) {
|
||||
return {
|
||||
pages,
|
||||
cappedAtCount: response.hasMore ? cap : null,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
cursor = response.nextCursor;
|
||||
}
|
||||
|
||||
return { pages, cappedAtCount: cap, warnings };
|
||||
}
|
||||
|
||||
export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connectionId: string): Promise<string> {
|
||||
try {
|
||||
const bot = (await api.retrieveBotUser()) as NotionBotInfo;
|
||||
const workspaceName = typeof bot.bot?.workspace_name === 'string' ? bot.bot.workspace_name.trim() : '';
|
||||
if (workspaceName.length > 0) {
|
||||
return workspaceName;
|
||||
}
|
||||
const name = typeof bot.name === 'string' ? bot.name.trim() : '';
|
||||
return name.length > 0 ? name : connectionId;
|
||||
} catch {
|
||||
return connectionId;
|
||||
}
|
||||
}
|
||||
|
||||
function notionConnection(project: KloLocalProject, connectionId: string): KloProjectConnectionConfig {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection) {
|
||||
throw new Error(`Connection "${connectionId}" not found`);
|
||||
}
|
||||
if (connection.driver !== 'notion') {
|
||||
throw new Error(`Connection "${connectionId}" is not a Notion connection`);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
export async function applyNotionPickerWriteback(
|
||||
project: KloLocalProject,
|
||||
connectionId: string,
|
||||
rootPageIds: string[],
|
||||
): Promise<void> {
|
||||
if (rootPageIds.length === 0) {
|
||||
throw new Error('connection notion pick requires at least one root page id');
|
||||
}
|
||||
|
||||
const existing = notionConnection(project, connectionId);
|
||||
const nextConfig = {
|
||||
...project.config,
|
||||
connections: {
|
||||
...project.config.connections,
|
||||
[connectionId]: {
|
||||
...existing,
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: rootPageIds,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await project.fileStore.writeFile(
|
||||
'klo.yaml',
|
||||
serializeKloProjectConfig(nextConfig),
|
||||
'klo',
|
||||
'klo@example.com',
|
||||
`Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function runKloConnectionNotion(
|
||||
args: KloConnectionNotionArgs,
|
||||
io: KloCliIo = process,
|
||||
deps: KloConnectionNotionDeps = {},
|
||||
): Promise<number> {
|
||||
try {
|
||||
assertSafeConnectionId(args.connectionId);
|
||||
const loadProject = deps.loadProject ?? loadKloProject;
|
||||
|
||||
if (args.mode === 'interactive') {
|
||||
const project = await loadProject({ projectDir: args.projectDir });
|
||||
const rawConnection = notionConnection(project, args.connectionId);
|
||||
const notion = parseNotionConnectionConfig(rawConnection);
|
||||
const authToken = await resolveNotionAuthToken(notion.auth_token_ref, { env: deps.env });
|
||||
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
|
||||
const discovery = await discoverNotionPickerPages(api);
|
||||
const tree = buildPickerTree(discovery.pages);
|
||||
const initialState = buildInitialState({
|
||||
tree,
|
||||
existingRootPageIds: notion.root_page_ids,
|
||||
currentCrawlMode: notion.crawl_mode,
|
||||
});
|
||||
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
|
||||
const renderState =
|
||||
preLoadWarnings.length > 0
|
||||
? {
|
||||
...initialState,
|
||||
preLoadWarnings,
|
||||
}
|
||||
: initialState;
|
||||
for (const warning of preLoadWarnings) {
|
||||
io.stderr.write(`${warning}\n`);
|
||||
}
|
||||
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
|
||||
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
|
||||
{
|
||||
initialState: renderState,
|
||||
connectionId: args.connectionId,
|
||||
workspaceLabel,
|
||||
cappedAtCount: discovery.cappedAtCount,
|
||||
currentCrawlMode: notion.crawl_mode,
|
||||
},
|
||||
io as NotionPickerTuiIo,
|
||||
);
|
||||
if (result.kind === 'quit') {
|
||||
io.stdout.write('No changes saved.\n');
|
||||
return 0;
|
||||
}
|
||||
await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds);
|
||||
io.stdout.write(`Connection: ${args.connectionId}\n`);
|
||||
io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`);
|
||||
io.stdout.write('crawlMode: selected_roots\n');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const project = await loadProject({ projectDir: args.projectDir });
|
||||
await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds);
|
||||
io.stdout.write(`Connection: ${args.connectionId}\n`);
|
||||
io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`);
|
||||
io.stdout.write('crawlMode: selected_roots\n');
|
||||
return 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
26
packages/cli/src/commands/demo-commands.test.ts
Normal file
26
packages/cli/src/commands/demo-commands.test.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { resolveDemoCommandOptions } from './demo-commands.js';
|
||||
|
||||
describe('resolveDemoCommandOptions', () => {
|
||||
it('lets parent --no-input override a child default from optsWithGlobals', () => {
|
||||
const rootCommand = {
|
||||
opts: () => ({}),
|
||||
};
|
||||
const setupCommand = {
|
||||
parent: rootCommand,
|
||||
opts: () => ({ input: false }),
|
||||
getOptionValueSource: (name: string) => (name === 'input' ? 'cli' : undefined),
|
||||
};
|
||||
const demoCommand = {
|
||||
parent: setupCommand,
|
||||
opts: () => ({ input: true, mode: 'seeded' }),
|
||||
optsWithGlobals: () => ({ input: true, mode: 'seeded' }),
|
||||
getOptionValueSource: (name: string) => (name === 'input' ? 'default' : name === 'mode' ? 'default' : undefined),
|
||||
};
|
||||
|
||||
expect(resolveDemoCommandOptions<{ input: boolean; mode: string }>(demoCommand)).toEqual({
|
||||
input: false,
|
||||
mode: 'seeded',
|
||||
});
|
||||
});
|
||||
});
|
||||
273
packages/cli/src/commands/demo-commands.ts
Normal file
273
packages/cli/src/commands/demo-commands.ts
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import {
|
||||
type CommandWithGlobalOptions,
|
||||
type KloCliCommandContext,
|
||||
resolveCommandProjectDirOverride,
|
||||
} from '../cli-program.js';
|
||||
import {
|
||||
type KloDemoArgs,
|
||||
type KloDemoInputMode,
|
||||
type KloDemoMode,
|
||||
type KloDemoOutputMode,
|
||||
} from '../demo.js';
|
||||
import { defaultDemoProjectDir } from '../demo-assets.js';
|
||||
import { resolveProjectDir } from '../project-dir.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/demo-commands');
|
||||
|
||||
interface DemoOptions {
|
||||
plain?: boolean;
|
||||
json?: boolean;
|
||||
input?: boolean;
|
||||
projectDir?: string;
|
||||
}
|
||||
|
||||
function demoOutputMode(options: { plain?: boolean; json?: boolean }): KloDemoOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (options.plain === true) {
|
||||
return 'plain';
|
||||
}
|
||||
return 'viz';
|
||||
}
|
||||
|
||||
function demoDoctorOutputMode(options: { json?: boolean }): 'plain' | 'json' {
|
||||
return options.json === true ? 'json' : 'plain';
|
||||
}
|
||||
|
||||
function demoInspectOutputMode(options: { plain?: boolean; json?: boolean }): KloDemoOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
function demoInputMode(options: { input?: boolean }): { inputMode?: KloDemoInputMode } {
|
||||
return options.input === false ? { inputMode: 'disabled' } : {};
|
||||
}
|
||||
|
||||
function demoProjectDir(options: { projectDir?: string }, command: CommandWithGlobalOptions): string {
|
||||
return resolveProjectDir(
|
||||
options.projectDir ?? resolveCommandProjectDirOverride(command),
|
||||
defaultDemoProjectDir(),
|
||||
);
|
||||
}
|
||||
|
||||
type CommandOptionSourceReader = {
|
||||
getOptionValueSource?: (name: string) => string | undefined;
|
||||
parent?: unknown;
|
||||
};
|
||||
|
||||
function inheritedOptionSource(command: CommandOptionSourceReader, key: string): string | undefined {
|
||||
let current = command.parent as (CommandOptionSourceReader & { opts?: () => Record<string, unknown> }) | undefined;
|
||||
while (current) {
|
||||
const source = current.getOptionValueSource?.(key);
|
||||
if (source !== undefined) {
|
||||
return source;
|
||||
}
|
||||
current = current.parent as (CommandOptionSourceReader & { opts?: () => Record<string, unknown> }) | undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function definedOptions(
|
||||
options: Record<string, unknown>,
|
||||
inherited: Record<string, unknown> = {},
|
||||
command?: CommandOptionSourceReader,
|
||||
): Record<string, unknown> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(options).filter(([key, value]) => {
|
||||
if (value === undefined) return false;
|
||||
if (key === 'input' && value === true && inherited.input === false) return false;
|
||||
if (
|
||||
key === 'mode' &&
|
||||
command?.getOptionValueSource?.(key) === 'default' &&
|
||||
inherited[key] !== undefined &&
|
||||
inherited[key] !== value &&
|
||||
inheritedOptionSource(command, key) === 'cli'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveDemoCommandOptions<T>(command: { opts: () => T; optsWithGlobals?: () => T; parent?: unknown }): T {
|
||||
const chain: Array<{ opts?: () => Record<string, unknown>; parent?: unknown }> = [];
|
||||
let current = command.parent as { opts?: () => Record<string, unknown>; parent?: unknown } | undefined;
|
||||
while (current) {
|
||||
chain.unshift(current);
|
||||
current = current.parent as { opts?: () => Record<string, unknown>; parent?: unknown } | undefined;
|
||||
}
|
||||
const inherited = Object.assign({}, ...chain.map((parent) => definedOptions(parent.opts?.() ?? {})));
|
||||
|
||||
if (command.optsWithGlobals) {
|
||||
const withGlobals = {
|
||||
...inherited,
|
||||
...definedOptions(command.optsWithGlobals() as Record<string, unknown>, inherited, command),
|
||||
};
|
||||
return {
|
||||
...withGlobals,
|
||||
...definedOptions(command.opts() as Record<string, unknown>, withGlobals, command),
|
||||
} as T;
|
||||
}
|
||||
|
||||
return { ...inherited, ...definedOptions(command.opts() as Record<string, unknown>, inherited, command) } as T;
|
||||
}
|
||||
|
||||
async function runDemoArgs(context: KloCliCommandContext, args: KloDemoArgs): Promise<void> {
|
||||
const runner = context.deps.demo ?? (await import('../demo.js')).runKloDemo;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerDemoCommands(
|
||||
program: Command,
|
||||
context: KloCliCommandContext,
|
||||
options: { description?: string } = {},
|
||||
): void {
|
||||
const demo = program
|
||||
.command('demo')
|
||||
.description(options.description ?? 'Run the pre-seeded KLO demo or a full LLM-backed demo')
|
||||
.addOption(
|
||||
new Option('--mode <mode>', 'Demo mode: seeded (default), replay, or full')
|
||||
.choices(['seeded', 'replay', 'full'])
|
||||
.default('seeded'),
|
||||
)
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.showHelpAfterError()
|
||||
.action(async (options: { mode: 'seeded' | 'replay' | 'full' } & DemoOptions, command) => {
|
||||
const resolvedOptions = resolveDemoCommandOptions<typeof options>(command);
|
||||
await runDemoArgs(context, {
|
||||
command: resolvedOptions.mode,
|
||||
projectDir: demoProjectDir(resolvedOptions, command),
|
||||
outputMode: demoOutputMode(resolvedOptions),
|
||||
...demoInputMode(resolvedOptions),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('init')
|
||||
.description('Initialize the packaged demo project')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.option('--force', 'Recreate an existing demo project', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { projectDir?: string; force?: boolean; input?: boolean } }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'init',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
force: options.force === true,
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('reset')
|
||||
.description('Reset the packaged demo project')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.option('--force', 'Recreate the demo project without prompting', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { projectDir?: string; force?: boolean; input?: boolean } }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'reset',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
force: options.force === true,
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('replay')
|
||||
.description('Replay the packaged demo memory-flow')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'replay',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('scan')
|
||||
.description('Run the packaged demo scan')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { projectDir?: string; input?: boolean } }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'scan',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('inspect')
|
||||
.description('Inspect packaged demo outputs')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'inspect',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoInspectOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('doctor')
|
||||
.description('Check packaged demo readiness')
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'doctor',
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoDoctorOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
demo
|
||||
.command('ingest')
|
||||
.description('Run packaged demo ingest')
|
||||
.addOption(
|
||||
new Option('--mode <mode>', 'Demo ingest mode: full or seeded')
|
||||
.choices(['full', 'seeded'])
|
||||
.default('full'),
|
||||
)
|
||||
.option('--project-dir <path>', 'Demo project directory')
|
||||
.addOption(new Option('--plain', 'Print plain text output instead of the visual demo').conflicts('json'))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts('plain'))
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (_options, command: { opts: () => { mode: KloDemoMode } & DemoOptions }) => {
|
||||
const options = resolveDemoCommandOptions(command);
|
||||
await runDemoArgs(context, {
|
||||
command: 'ingest',
|
||||
mode: options.mode,
|
||||
projectDir: demoProjectDir(options, command),
|
||||
outputMode: demoOutputMode(options),
|
||||
...demoInputMode(options),
|
||||
});
|
||||
});
|
||||
}
|
||||
53
packages/cli/src/commands/doctor-commands.ts
Normal file
53
packages/cli/src/commands/doctor-commands.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import type { Command } from '@commander-js/extra-typings';
|
||||
import { type CommandWithGlobalOptions, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloDoctorArgs } from '../doctor.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/doctor-commands');
|
||||
|
||||
function outputMode(options: { json?: boolean }): 'plain' | 'json' {
|
||||
return options.json === true ? 'json' : 'plain';
|
||||
}
|
||||
|
||||
function inputMode(options: { input?: boolean }): { inputMode?: 'disabled' } {
|
||||
return options.input === false ? { inputMode: 'disabled' } : {};
|
||||
}
|
||||
|
||||
async function runDoctorArgs(context: KloCliCommandContext, args: KloDoctorArgs): Promise<void> {
|
||||
const runner = context.deps.doctor ?? (await import('../doctor.js')).runKloDoctor;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerDoctorCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const doctor = program
|
||||
.command('doctor')
|
||||
.description('Check KLO setup, project, and demo readiness')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (options: { json?: boolean; input?: boolean }, command) => {
|
||||
await runDoctorArgs(context, {
|
||||
command: 'project',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
});
|
||||
});
|
||||
|
||||
doctor
|
||||
.command('setup')
|
||||
.description('Check KLO install, build, and local runtime readiness')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(
|
||||
async (
|
||||
_options: { json?: boolean; input?: boolean },
|
||||
command: CommandWithGlobalOptions,
|
||||
) => {
|
||||
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as {
|
||||
json?: boolean;
|
||||
input?: boolean;
|
||||
};
|
||||
await runDoctorArgs(context, { command: 'setup', outputMode: outputMode(options), ...inputMode(options) });
|
||||
},
|
||||
);
|
||||
}
|
||||
171
packages/cli/src/commands/ingest-commands.ts
Normal file
171
packages/cli/src/commands/ingest-commands.ts
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
import { resolve } from 'node:path';
|
||||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloCliDeps, KloCliIo } from '../index.js';
|
||||
import type { KloIngestArgs, KloIngestOutputMode } from '../ingest.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/ingest-commands');
|
||||
|
||||
interface IngestCommandOptions {
|
||||
runIngestWithProgress: (
|
||||
args: KloIngestArgs,
|
||||
io: KloCliIo,
|
||||
deps: KloCliDeps,
|
||||
defaultRunIngest: (args: KloIngestArgs, io: KloCliIo) => Promise<number>,
|
||||
) => Promise<number>;
|
||||
}
|
||||
|
||||
function outputMode(options: OutputModeOptions): KloIngestOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (options.viz === true) {
|
||||
return 'viz';
|
||||
}
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
function watchOutputMode(options: OutputModeOptions): KloIngestOutputMode {
|
||||
if (options.json === true) {
|
||||
return 'json';
|
||||
}
|
||||
if (options.plain === true) {
|
||||
return 'plain';
|
||||
}
|
||||
return 'viz';
|
||||
}
|
||||
|
||||
function inputMode(options: OutputModeOptions): Pick<KloIngestArgs, 'inputMode'> {
|
||||
return options.input === false ? { inputMode: 'disabled' } : {};
|
||||
}
|
||||
|
||||
async function runIngestArgs(
|
||||
context: KloCliCommandContext,
|
||||
args: KloIngestArgs,
|
||||
options: IngestCommandOptions,
|
||||
): Promise<void> {
|
||||
const { runKloIngest } = await import('../ingest.js');
|
||||
context.setExitCode(await options.runIngestWithProgress(args, context.io, context.deps, runKloIngest));
|
||||
}
|
||||
|
||||
export function registerIngestCommands(
|
||||
program: Command,
|
||||
context: KloCliCommandContext,
|
||||
commandOptions: IngestCommandOptions,
|
||||
): void {
|
||||
const ingest = program
|
||||
.command('ingest')
|
||||
.description('Run or inspect local ingest memory-flow output')
|
||||
.showHelpAfterError();
|
||||
|
||||
ingest.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('ingest', actionCommand);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('run')
|
||||
.description('Run local ingest for one configured connection and source adapter')
|
||||
.requiredOption('--connection-id <connectionId>', 'KLO connection id')
|
||||
.requiredOption('--adapter <adapter>', 'Ingest source adapter name')
|
||||
.option('--source-dir <path>', 'Directory containing source files')
|
||||
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
|
||||
.option('--debug-llm-request-file <path>', 'Write sanitized LLM request structure to a JSONL file')
|
||||
.option('--report-file <path>', 'Unsupported for ingest run; use ingest status/watch instead')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (options, command) => {
|
||||
if (options.reportFile) {
|
||||
throw new Error('--report-file is only supported for ingest status/watch');
|
||||
}
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
adapter: options.adapter,
|
||||
sourceDir: options.sourceDir ? resolve(options.sourceDir) : undefined,
|
||||
databaseIntrospectionUrl: options.databaseIntrospectionUrl || undefined,
|
||||
...(options.debugLlmRequestFile ? { debugLlmRequestFile: resolve(options.debugLlmRequestFile) } : {}),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('status')
|
||||
.description('Print status for the latest or selected stored local ingest run or report file')
|
||||
.argument('[runId]', 'Local ingest run id, report id, run id, or job id')
|
||||
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (runId: string | undefined, options, command) => {
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('watch')
|
||||
.description('Open the latest or selected stored ingest visual report')
|
||||
.argument('[runId]', 'Local ingest run id, report id, run id, or job id')
|
||||
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (runId: string | undefined, options, command) => {
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'watch',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
|
||||
outputMode: watchOutputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('replay')
|
||||
.description('Replay a stored ingest run or bundle report through memory-flow output')
|
||||
.argument('<runId>', 'Local ingest run id, report id, run id, or job id')
|
||||
.option('--report-file <path>', 'Bundle ingest report JSON file to render')
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json', 'viz']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain', 'viz']))
|
||||
.addOption(new Option('--viz', 'Render memory-flow TUI output').conflicts(['plain', 'json']))
|
||||
.option('--no-input', 'Disable interactive terminal input for visualization')
|
||||
.action(async (runId: string, options, command) => {
|
||||
await runIngestArgs(
|
||||
context,
|
||||
{
|
||||
command: 'replay',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
...(options.reportFile ? { reportFile: resolve(options.reportFile) } : {}),
|
||||
outputMode: outputMode(options),
|
||||
...inputMode(options),
|
||||
},
|
||||
commandOptions,
|
||||
);
|
||||
});
|
||||
}
|
||||
90
packages/cli/src/commands/knowledge-commands.ts
Normal file
90
packages/cli/src/commands/knowledge-commands.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import { collectOption, type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import { wikiWriteCommandSchema } from '../command-schemas.js';
|
||||
import type { KloKnowledgeArgs } from '../knowledge.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/knowledge-commands');
|
||||
|
||||
async function runKnowledgeArgs(context: KloCliCommandContext, args: KloKnowledgeArgs): Promise<void> {
|
||||
const runner = context.deps.knowledge ?? (await import('../knowledge.js')).runKloKnowledge;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerWikiCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const wiki = program
|
||||
.command('wiki')
|
||||
.description('List, read, search, or write local wiki pages')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
wiki
|
||||
.command('list')
|
||||
.description('List local wiki pages')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.action(async (options: { userId: string }, command) => {
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
userId: options.userId,
|
||||
});
|
||||
});
|
||||
|
||||
wiki
|
||||
.command('read')
|
||||
.description('Read one local wiki page')
|
||||
.argument('<key>', 'Wiki page key')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.action(async (key: string, options: { userId: string }, command) => {
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
key,
|
||||
userId: options.userId,
|
||||
});
|
||||
});
|
||||
|
||||
wiki
|
||||
.command('search')
|
||||
.description('Search local wiki pages')
|
||||
.argument('<query>', 'Search query')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.action(async (query: string, options: { userId: string }, command) => {
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'search',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
query,
|
||||
userId: options.userId,
|
||||
});
|
||||
});
|
||||
|
||||
wiki
|
||||
.command('write')
|
||||
.description('Write one local wiki page')
|
||||
.argument('<key>', 'Wiki page key')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.addOption(new Option('--scope <scope>', 'global or user').choices(['global', 'user']).default('global'))
|
||||
.requiredOption('--summary <summary>', 'Wiki summary')
|
||||
.requiredOption('--content <content>', 'Wiki content')
|
||||
.option('--tag <tag>', 'Wiki tag; repeatable', collectOption, [])
|
||||
.option('--ref <ref>', 'Wiki ref; repeatable', collectOption, [])
|
||||
.option('--sl-ref <ref>', 'Semantic-layer ref; repeatable', collectOption, [])
|
||||
.action(async (key: string, options, command) => {
|
||||
const args = wikiWriteCommandSchema.parse({
|
||||
command: 'write',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
key,
|
||||
scope: options.scope === 'user' ? 'USER' : 'GLOBAL',
|
||||
userId: options.userId,
|
||||
summary: options.summary,
|
||||
content: options.content,
|
||||
tags: options.tag,
|
||||
refs: options.ref,
|
||||
slRefs: options.slRef,
|
||||
});
|
||||
await runKnowledgeArgs(context, args);
|
||||
});
|
||||
}
|
||||
109
packages/cli/src/commands/public-ingest-commands.ts
Normal file
109
packages/cli/src/commands/public-ingest-commands.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { InvalidArgumentError, type Command } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import { publicIngestReadCommandSchema, publicIngestRunCommandSchema } from '../command-schemas.js';
|
||||
import type { KloPublicIngestArgs, KloPublicIngestInputMode } from '../public-ingest.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/public-ingest-commands');
|
||||
|
||||
interface PublicIngestOptions {
|
||||
all?: boolean;
|
||||
json?: boolean;
|
||||
input?: boolean;
|
||||
}
|
||||
|
||||
function inputMode(options: { input?: boolean }): KloPublicIngestInputMode {
|
||||
return options.input === false ? 'disabled' : 'auto';
|
||||
}
|
||||
|
||||
async function runPublicIngestArgs(context: KloCliCommandContext, args: KloPublicIngestArgs): Promise<void> {
|
||||
const runner = context.deps.publicIngest ?? (await import('../public-ingest.js')).runKloPublicIngest;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
function parsePublicIngestConnectionId(value: string): string {
|
||||
if (value === 'run') {
|
||||
throw new InvalidArgumentError('run is reserved; use klo dev ingest run for low-level adapter syntax');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function registerPublicIngestCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const ingest = program
|
||||
.command('ingest')
|
||||
.description('Build and refresh KLO context from configured sources')
|
||||
.usage('[options] [connectionId]')
|
||||
.argument('[connectionId]', 'Connection id to ingest', parsePublicIngestConnectionId)
|
||||
.option('--all', 'Ingest every eligible configured source', false)
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.addHelpText(
|
||||
'after',
|
||||
[
|
||||
'',
|
||||
'Examples:',
|
||||
' klo ingest <connectionId> [options]',
|
||||
' klo ingest --all [options]',
|
||||
' klo ingest status [runId] [options]',
|
||||
' klo ingest watch [runId] [options]',
|
||||
'',
|
||||
'Project directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.',
|
||||
'',
|
||||
].join('\n'),
|
||||
)
|
||||
.showHelpAfterError()
|
||||
.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('ingest', actionCommand);
|
||||
})
|
||||
.action(async (connectionId: string | undefined, _options: PublicIngestOptions, command) => {
|
||||
const options = command.opts();
|
||||
if (options.all === true && connectionId) {
|
||||
throw new Error('klo ingest accepts either --all or <connectionId>, not both');
|
||||
}
|
||||
const args = publicIngestRunCommandSchema.parse({
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(connectionId ? { targetConnectionId: connectionId } : {}),
|
||||
all: options.all === true,
|
||||
json: options.json === true,
|
||||
inputMode: inputMode(options),
|
||||
});
|
||||
await runPublicIngestArgs(context, args);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('status')
|
||||
.description('Print status for the latest or selected public ingest run')
|
||||
.argument('[runId]', 'Public ingest run id')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
|
||||
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
|
||||
const args = publicIngestReadCommandSchema.parse({
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
json: options.json === true,
|
||||
inputMode: inputMode(options),
|
||||
});
|
||||
await runPublicIngestArgs(context, args);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('watch')
|
||||
.description('Open the latest or selected public ingest visual report')
|
||||
.argument('[runId]', 'Public ingest run id')
|
||||
.option('--json', 'Print JSON output instead of the visual report', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
|
||||
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
|
||||
const args = publicIngestReadCommandSchema.parse({
|
||||
command: 'watch',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
json: options.json === true,
|
||||
inputMode: inputMode(options),
|
||||
});
|
||||
await runPublicIngestArgs(context, args);
|
||||
});
|
||||
}
|
||||
353
packages/cli/src/commands/scan-commands.ts
Normal file
353
packages/cli/src/commands/scan-commands.ts
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloScanArgs } from '../scan.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/scan-commands');
|
||||
|
||||
async function runScanArgs(context: KloCliCommandContext, args: KloScanArgs): Promise<void> {
|
||||
const runner = context.deps.scan ?? (await import('../scan.js')).runKloScan;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
type KloScanModeOption = Extract<KloScanArgs, { command: 'run' }>['mode'];
|
||||
|
||||
function parseScanModeOption(value: string): KloScanModeOption {
|
||||
if (value === 'structural' || value === 'enriched' || value === 'relationships') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed choices are structural, enriched, relationships');
|
||||
}
|
||||
|
||||
type KloRelationshipStatusOption = Extract<KloScanArgs, { command: 'relationships' }>['status'];
|
||||
type KloRelationshipFeedbackDecisionOption = Extract<KloScanArgs, { command: 'relationshipFeedback' }>['decision'];
|
||||
|
||||
function parseRelationshipStatusOption(value: string): KloRelationshipStatusOption {
|
||||
if (value === 'accepted' || value === 'review' || value === 'rejected' || value === 'skipped' || value === 'all') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed choices are accepted, review, rejected, skipped, all');
|
||||
}
|
||||
|
||||
function parseRelationshipFeedbackDecisionOption(value: string): KloRelationshipFeedbackDecisionOption {
|
||||
if (value === 'accepted' || value === 'rejected' || value === 'all') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed choices are accepted, rejected, all');
|
||||
}
|
||||
|
||||
function parseNonEmptyOption(value: string): string {
|
||||
if (value.trim().length === 0) {
|
||||
throw new InvalidArgumentError('must not be empty');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseRelationshipCalibrationThreshold(value: string): number {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
|
||||
return parsed;
|
||||
}
|
||||
throw new InvalidArgumentError('Allowed range is 0 through 1');
|
||||
}
|
||||
|
||||
function relationshipDecisionArgs(options: {
|
||||
accept?: string;
|
||||
reject?: string;
|
||||
reviewer?: string;
|
||||
note?: string;
|
||||
json?: boolean;
|
||||
}): Pick<
|
||||
Extract<KloScanArgs, { command: 'relationshipDecision' }>,
|
||||
'candidateId' | 'decision' | 'reviewer' | 'note' | 'json'
|
||||
> | null {
|
||||
const decisionCount = [options.accept !== undefined, options.reject !== undefined].filter(Boolean).length;
|
||||
if (decisionCount > 1) {
|
||||
throw new Error('Only one relationship review decision option can be used: --accept and --reject conflict');
|
||||
}
|
||||
if (options.accept !== undefined) {
|
||||
return {
|
||||
candidateId: options.accept,
|
||||
decision: 'accepted',
|
||||
reviewer: options.reviewer ?? 'klo',
|
||||
note: options.note ?? null,
|
||||
json: options.json === true,
|
||||
};
|
||||
}
|
||||
if (options.reject !== undefined) {
|
||||
return {
|
||||
candidateId: options.reject,
|
||||
decision: 'rejected',
|
||||
reviewer: options.reviewer ?? 'klo',
|
||||
note: options.note ?? null,
|
||||
json: options.json === true,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectRelationshipCandidateOption(value: string, previous: string[]): string[] {
|
||||
return [...previous, parseNonEmptyOption(value)];
|
||||
}
|
||||
|
||||
export function registerScanCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const scan = program
|
||||
.command('scan')
|
||||
.description('Run or inspect standalone connection scans')
|
||||
.argument('[connectionId]', 'KLO connection id to scan')
|
||||
.option(
|
||||
'--mode <mode>',
|
||||
'Scan mode: structural, enriched, relationships (default: structural)',
|
||||
parseScanModeOption,
|
||||
)
|
||||
.option('--dry-run', 'Run without writing scan results', false)
|
||||
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
)
|
||||
.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('scan', actionCommand);
|
||||
})
|
||||
.action(async (connectionId: string | undefined, options, command) => {
|
||||
if (!connectionId) {
|
||||
scan.outputHelp();
|
||||
context.io.stderr.write('klo dev scan requires <connectionId> or a subcommand\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
const mode = options.mode ?? 'structural';
|
||||
await runScanArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
mode,
|
||||
detectRelationships: mode === 'relationships',
|
||||
dryRun: options.dryRun === true,
|
||||
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('status')
|
||||
.description('Print status for a local scan run')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, _options: unknown, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('report')
|
||||
.description('Print a local scan report')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.option('--json', 'Print the raw scan report JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'report',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationships')
|
||||
.description('Print relationship artifacts for a local scan run')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.option(
|
||||
'--status <status>',
|
||||
'Relationship status: accepted, review, rejected, skipped, all',
|
||||
parseRelationshipStatusOption,
|
||||
'review',
|
||||
)
|
||||
.option('--limit <count>', 'Maximum relationships to print per status', parsePositiveIntegerOption, 25)
|
||||
.addOption(
|
||||
new Option('--accept <candidateId>', 'Record a reviewer accepted decision for a relationship candidate')
|
||||
.argParser(parseNonEmptyOption)
|
||||
.conflicts('reject'),
|
||||
)
|
||||
.addOption(
|
||||
new Option('--reject <candidateId>', 'Record a reviewer rejected decision for a relationship candidate')
|
||||
.argParser(parseNonEmptyOption)
|
||||
.conflicts('accept'),
|
||||
)
|
||||
.option('--note <text>', 'Attach a note when recording a relationship review decision')
|
||||
.option('--reviewer <name>', 'Reviewer name for a relationship review decision')
|
||||
.option('--json', 'Print relationship artifacts as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, options, command) => {
|
||||
const decision = relationshipDecisionArgs(options);
|
||||
if (decision) {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipDecision',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
candidateId: decision.candidateId,
|
||||
decision: decision.decision,
|
||||
reviewer: decision.reviewer,
|
||||
note: decision.note,
|
||||
json: decision.json,
|
||||
});
|
||||
return;
|
||||
}
|
||||
await runScanArgs(context, {
|
||||
command: 'relationships',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
status: options.status,
|
||||
json: options.json === true,
|
||||
limit: options.limit,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-apply')
|
||||
.description('Apply accepted relationship review decisions as manual manifest joins')
|
||||
.argument('<runId>', 'Local scan run id')
|
||||
.option('--all-accepted', 'Apply all accepted relationship review decisions for the scan run', false)
|
||||
.option(
|
||||
'--candidate <candidateId>',
|
||||
'Apply one accepted relationship review decision',
|
||||
collectRelationshipCandidateOption,
|
||||
[],
|
||||
)
|
||||
.option('--dry-run', 'Preview relationships that would be written without rewriting manifest shards', false)
|
||||
.option('--json', 'Print the apply result as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (runId: string, options, command) => {
|
||||
const parentOptions = command.parent?.opts() as { dryRun?: boolean } | undefined;
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipApply',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
runId,
|
||||
applyAllAccepted: options.allAccepted === true,
|
||||
candidateIds: options.candidate,
|
||||
dryRun: options.dryRun === true || parentOptions?.dryRun === true,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-feedback')
|
||||
.description('Export persisted relationship review decisions as calibration labels')
|
||||
.option('--connection <connectionId>', 'Only export labels for one KLO connection')
|
||||
.option(
|
||||
'--decision <decision>',
|
||||
'Relationship feedback decision: accepted, rejected, all',
|
||||
parseRelationshipFeedbackDecisionOption,
|
||||
'all',
|
||||
)
|
||||
.addOption(new Option('--json', 'Print the export as JSON').default(false).conflicts('jsonl'))
|
||||
.addOption(new Option('--jsonl', 'Print labels as newline-delimited JSON').default(false).conflicts('json'))
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipFeedback',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connection ?? null,
|
||||
decision: options.decision,
|
||||
json: options.json === true,
|
||||
jsonl: options.jsonl === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-calibration')
|
||||
.description('Summarize relationship feedback labels against current score thresholds')
|
||||
.option('--connection <connectionId>', 'Only calibrate labels for one KLO connection')
|
||||
.option(
|
||||
'--decision <decision>',
|
||||
'Relationship feedback decision: accepted, rejected, all',
|
||||
parseRelationshipFeedbackDecisionOption,
|
||||
'all',
|
||||
)
|
||||
.option(
|
||||
'--accept-threshold <value>',
|
||||
'Score threshold treated as predicted accepted',
|
||||
parseRelationshipCalibrationThreshold,
|
||||
0.85,
|
||||
)
|
||||
.option(
|
||||
'--review-threshold <value>',
|
||||
'Score threshold treated as predicted review',
|
||||
parseRelationshipCalibrationThreshold,
|
||||
0.55,
|
||||
)
|
||||
.option('--json', 'Print the calibration report as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipCalibration',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connection ?? null,
|
||||
decision: options.decision,
|
||||
acceptThreshold: options.acceptThreshold,
|
||||
reviewThreshold: options.reviewThreshold,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
scan
|
||||
.command('relationship-thresholds')
|
||||
.description('Evaluate relationship feedback labels for offline threshold advice')
|
||||
.option('--connection <connectionId>', 'Only evaluate labels for one KLO connection')
|
||||
.option(
|
||||
'--min-total-labels <count>',
|
||||
'Minimum scored labels before advice can be ready',
|
||||
parsePositiveIntegerOption,
|
||||
20,
|
||||
)
|
||||
.option(
|
||||
'--min-accepted-labels <count>',
|
||||
'Minimum accepted labels before advice can be ready',
|
||||
parsePositiveIntegerOption,
|
||||
5,
|
||||
)
|
||||
.option(
|
||||
'--min-rejected-labels <count>',
|
||||
'Minimum rejected labels before advice can be ready',
|
||||
parsePositiveIntegerOption,
|
||||
5,
|
||||
)
|
||||
.option('--json', 'Print the threshold advice report as JSON', false)
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\n--project-dir is inherited from `klo dev scan` (default: KLO_PROJECT_DIR or current working directory).\n',
|
||||
)
|
||||
.action(async (options, command) => {
|
||||
await runScanArgs(context, {
|
||||
command: 'relationshipThresholds',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connection ?? null,
|
||||
minTotalLabels: options.minTotalLabels,
|
||||
minAcceptedLabels: options.minAcceptedLabels,
|
||||
minRejectedLabels: options.minRejectedLabels,
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
}
|
||||
47
packages/cli/src/commands/serve-commands.ts
Normal file
47
packages/cli/src/commands/serve-commands.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
|
||||
import { type KloCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloServeArgs } from '../serve.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/serve-commands');
|
||||
|
||||
function parseMcp(value: string): 'stdio' {
|
||||
if (value === 'stdio') {
|
||||
return 'stdio';
|
||||
}
|
||||
throw new InvalidArgumentError('Only stdio is supported in this phase');
|
||||
}
|
||||
|
||||
export function registerServeCommands(program: Command, context: KloCliCommandContext): void {
|
||||
program
|
||||
.command('serve')
|
||||
.description('Run standalone KLO services such as MCP stdio')
|
||||
.requiredOption('--mcp <mode>', 'MCP transport mode', parseMcp)
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
.option('--semantic-compute', 'Enable semantic-layer compute', false)
|
||||
.option('--semantic-compute-url <url>', 'HTTP semantic-layer compute URL')
|
||||
.option('--database-introspection-url <url>', 'Daemon URL for live-database introspection')
|
||||
.option('--execute-queries', 'Allow semantic-layer query execution', false)
|
||||
.option('--memory-capture', 'Enable memory capture', false)
|
||||
.option('--memory-model <model>', 'Memory capture model')
|
||||
.showHelpAfterError()
|
||||
.action(async (options, command): Promise<void> => {
|
||||
const semanticCompute = options.semanticCompute === true || Boolean(options.semanticComputeUrl);
|
||||
if (options.executeQueries === true && !semanticCompute) {
|
||||
throw new Error('--execute-queries requires --semantic-compute');
|
||||
}
|
||||
const args: KloServeArgs = {
|
||||
mcp: options.mcp,
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
userId: options.userId,
|
||||
semanticCompute,
|
||||
semanticComputeUrl: options.semanticComputeUrl,
|
||||
databaseIntrospectionUrl: options.databaseIntrospectionUrl,
|
||||
executeQueries: options.executeQueries === true,
|
||||
memoryCapture: options.memoryCapture === true,
|
||||
memoryModel: options.memoryModel,
|
||||
};
|
||||
const runner = context.deps.serveStdio ?? (await import('../serve.js')).runKloServeStdio;
|
||||
context.setExitCode(await runner(args));
|
||||
});
|
||||
}
|
||||
517
packages/cli/src/commands/setup-commands.ts
Normal file
517
packages/cli/src/commands/setup-commands.ts
Normal file
|
|
@ -0,0 +1,517 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KloSetupDatabaseDriver } from '../setup-databases.js';
|
||||
import type { KloSetupSourceType } from '../setup-sources.js';
|
||||
import { registerDemoCommands } from './demo-commands.js';
|
||||
|
||||
async function runSetupArgs(
|
||||
context: KloCliCommandContext,
|
||||
args: Parameters<NonNullable<typeof context.deps.setup>>[0],
|
||||
) {
|
||||
const runner = context.deps.setup ?? (await import('../setup.js')).runKloSetup;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
function positiveInteger(value: string): number {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`Expected a positive integer, received ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
|
||||
if (value === 'openai' || value === 'sentence-transformers') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function databaseDriver(value: string): KloSetupDatabaseDriver {
|
||||
if (
|
||||
value === 'sqlite' ||
|
||||
value === 'postgres' ||
|
||||
value === 'mysql' ||
|
||||
value === 'clickhouse' ||
|
||||
value === 'sqlserver' ||
|
||||
value === 'bigquery' ||
|
||||
value === 'snowflake'
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function sourceType(value: string): KloSetupSourceType {
|
||||
if (
|
||||
value === 'dbt' ||
|
||||
value === 'metricflow' ||
|
||||
value === 'metabase' ||
|
||||
value === 'looker' ||
|
||||
value === 'lookml' ||
|
||||
value === 'notion'
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function agentScope(value: string): 'project' | 'global' {
|
||||
if (value === 'project' || value === 'global') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function agentInstallMode(value: string): 'cli' | 'mcp' | 'both' {
|
||||
if (value === 'cli' || value === 'mcp' || value === 'both') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
}
|
||||
|
||||
function positiveNumber(value: string): number {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new InvalidArgumentError(`Expected a positive integer, received ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function optionWasSpecified(command: Command, optionName: string): boolean {
|
||||
const commandWithSources = command as Command & {
|
||||
getOptionValueSource?: (name: string) => string | undefined;
|
||||
getOptionValueSourceWithGlobals?: (name: string) => string | undefined;
|
||||
};
|
||||
const source =
|
||||
commandWithSources.getOptionValueSourceWithGlobals?.(optionName) ??
|
||||
commandWithSources.getOptionValueSource?.(optionName);
|
||||
return source !== undefined && source !== 'default';
|
||||
}
|
||||
|
||||
function shouldShowSetupEntryMenu(
|
||||
options: {
|
||||
new?: boolean;
|
||||
existing?: boolean;
|
||||
agents?: boolean;
|
||||
target?: string;
|
||||
global?: boolean;
|
||||
project?: boolean;
|
||||
skipAgents?: boolean;
|
||||
yes?: boolean;
|
||||
input?: boolean;
|
||||
anthropicApiKeyEnv?: string;
|
||||
anthropicApiKeyFile?: string;
|
||||
anthropicModel?: string;
|
||||
skipLlm?: boolean;
|
||||
embeddingBackend?: string;
|
||||
embeddingApiKeyEnv?: string;
|
||||
embeddingApiKeyFile?: string;
|
||||
skipEmbeddings?: boolean;
|
||||
database?: KloSetupDatabaseDriver[];
|
||||
databaseConnectionId?: string[];
|
||||
newDatabaseConnectionId?: string;
|
||||
databaseUrl?: string;
|
||||
databaseSchema?: string[];
|
||||
enableHistoricSql?: boolean;
|
||||
disableHistoricSql?: boolean;
|
||||
historicSqlWindowDays?: number;
|
||||
historicSqlMinCalls?: number;
|
||||
historicSqlServiceAccountPattern?: string[];
|
||||
historicSqlRedactionPattern?: string[];
|
||||
skipDatabases?: boolean;
|
||||
source?: KloSetupSourceType;
|
||||
sourceConnectionId?: string;
|
||||
sourcePath?: string;
|
||||
sourceGitUrl?: string;
|
||||
sourceBranch?: string;
|
||||
sourceSubpath?: string;
|
||||
sourceAuthTokenRef?: string;
|
||||
sourceUrl?: string;
|
||||
sourceApiKeyRef?: string;
|
||||
sourceClientId?: string;
|
||||
sourceClientSecretRef?: string;
|
||||
sourceWarehouseConnectionId?: string;
|
||||
sourceProjectName?: string;
|
||||
sourceProfilesPath?: string;
|
||||
sourceTarget?: string;
|
||||
metabaseDatabaseId?: number;
|
||||
notionCrawlMode?: string;
|
||||
notionRootPageId?: string[];
|
||||
skipInitialSourceIngest?: boolean;
|
||||
skipSources?: boolean;
|
||||
},
|
||||
command: Command,
|
||||
): boolean {
|
||||
if (options.database && options.database.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.databaseConnectionId && options.databaseConnectionId.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.databaseSchema && options.databaseSchema.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.historicSqlServiceAccountPattern && options.historicSqlServiceAccountPattern.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.historicSqlRedactionPattern && options.historicSqlRedactionPattern.length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (options.notionRootPageId && options.notionRootPageId.length > 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ![
|
||||
'new',
|
||||
'existing',
|
||||
'agents',
|
||||
'target',
|
||||
'global',
|
||||
'project',
|
||||
'skipAgents',
|
||||
'yes',
|
||||
'input',
|
||||
'anthropicApiKeyEnv',
|
||||
'anthropicApiKeyFile',
|
||||
'anthropicModel',
|
||||
'skipLlm',
|
||||
'embeddingBackend',
|
||||
'embeddingApiKeyEnv',
|
||||
'embeddingApiKeyFile',
|
||||
'skipEmbeddings',
|
||||
'newDatabaseConnectionId',
|
||||
'databaseUrl',
|
||||
'enableHistoricSql',
|
||||
'disableHistoricSql',
|
||||
'historicSqlWindowDays',
|
||||
'historicSqlMinCalls',
|
||||
'skipDatabases',
|
||||
'source',
|
||||
'sourceConnectionId',
|
||||
'sourcePath',
|
||||
'sourceGitUrl',
|
||||
'sourceBranch',
|
||||
'sourceSubpath',
|
||||
'sourceAuthTokenRef',
|
||||
'sourceUrl',
|
||||
'sourceApiKeyRef',
|
||||
'sourceClientId',
|
||||
'sourceClientSecretRef',
|
||||
'sourceWarehouseConnectionId',
|
||||
'sourceProjectName',
|
||||
'sourceProfilesPath',
|
||||
'sourceTarget',
|
||||
'metabaseDatabaseId',
|
||||
'notionCrawlMode',
|
||||
'skipInitialSourceIngest',
|
||||
'skipSources',
|
||||
].some((optionName) => optionWasSpecified(command, optionName));
|
||||
}
|
||||
|
||||
export function registerSetupCommands(program: Command, context: KloCliCommandContext): void {
|
||||
const setup = program
|
||||
.command('setup')
|
||||
.description('Set up or resume a local KLO project')
|
||||
.option('--project-dir <path>', 'KLO project directory')
|
||||
.option('--new', 'Create a new KLO project before setup', false)
|
||||
.option('--existing', 'Use an existing KLO project', false)
|
||||
.option('--agents', 'Install agent integration only', false)
|
||||
.addOption(
|
||||
new Option('--target <target>', 'Agent target').choices([
|
||||
'claude-code',
|
||||
'codex',
|
||||
'cursor',
|
||||
'opencode',
|
||||
'universal',
|
||||
]),
|
||||
)
|
||||
.addOption(new Option('--agent-scope <scope>', 'Agent install scope').argParser(agentScope).default('project'))
|
||||
.option('--project', 'Install agent integration into the project scope', false)
|
||||
.option('--global', 'Install agent integration into the global target scope', false)
|
||||
.addOption(
|
||||
new Option('--agent-install-mode <mode>', 'Agent install mode').argParser(agentInstallMode).default('cli'),
|
||||
)
|
||||
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
|
||||
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
|
||||
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
|
||||
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
|
||||
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
|
||||
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
|
||||
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
|
||||
.option('--embedding-api-key-file <path>', 'File containing the embedding provider API key')
|
||||
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
|
||||
.option(
|
||||
'--database <driver>',
|
||||
'Database driver to configure; repeatable',
|
||||
(value, previous: KloSetupDatabaseDriver[]) => {
|
||||
return [...previous, databaseDriver(value)];
|
||||
},
|
||||
[] as KloSetupDatabaseDriver[],
|
||||
)
|
||||
.option(
|
||||
'--database-connection-id <id>',
|
||||
'Existing selected connection id or new connection id',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--new-database-connection-id <id>', 'Connection id for one new database connection', (value) => {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
|
||||
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
|
||||
}
|
||||
return value;
|
||||
})
|
||||
.option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection')
|
||||
.option(
|
||||
'--database-schema <schema>',
|
||||
'Database schema to include; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false)
|
||||
.option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false)
|
||||
.option('--historic-sql-window-days <number>', 'Historic SQL query-history window', positiveInteger)
|
||||
.option(
|
||||
'--historic-sql-min-calls <number>',
|
||||
'Postgres Historic SQL pg_stat_statements minimum calls floor',
|
||||
positiveInteger,
|
||||
)
|
||||
.option(
|
||||
'--historic-sql-service-account-pattern <pattern>',
|
||||
'Historic SQL service-account regex; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option(
|
||||
'--historic-sql-redaction-pattern <pattern>',
|
||||
'Historic SQL SQL-literal redaction regex; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--skip-databases', 'Leave database setup incomplete; KLO cannot work until a primary source is added', false)
|
||||
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType))
|
||||
.option('--source-connection-id <id>', 'Connection id for source setup')
|
||||
.option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML')
|
||||
.option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML')
|
||||
.option('--source-branch <branch>', 'Git branch for source setup')
|
||||
.option('--source-subpath <path>', 'Repo subpath for source setup')
|
||||
.option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth')
|
||||
.option('--source-url <url>', 'Source service URL for Metabase or Looker')
|
||||
.option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion')
|
||||
.option('--source-client-id <id>', 'Looker client id')
|
||||
.option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref')
|
||||
.option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id')
|
||||
.option('--source-project-name <name>', 'dbt project name override')
|
||||
.option('--source-profiles-path <path>', 'dbt profiles path')
|
||||
.option('--source-target <target>', 'dbt target or source-specific mapping target')
|
||||
.option('--metabase-database-id <id>', 'Metabase database id to map', positiveNumber)
|
||||
.addOption(
|
||||
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']),
|
||||
)
|
||||
.option(
|
||||
'--notion-root-page-id <id>',
|
||||
'Notion root page id; repeatable',
|
||||
(value, previous: string[]) => [...previous, value],
|
||||
[],
|
||||
)
|
||||
.option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false)
|
||||
.option('--skip-sources', 'Mark optional source setup complete with no sources', false)
|
||||
.showHelpAfterError();
|
||||
|
||||
setup.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('setup', actionCommand);
|
||||
});
|
||||
|
||||
setup.action(async (options, command) => {
|
||||
if (options.anthropicApiKeyEnv && options.anthropicApiKeyFile) {
|
||||
context.io.stderr.write(
|
||||
'Choose only one Anthropic credential source: --anthropic-api-key-env or --anthropic-api-key-file.\n',
|
||||
);
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.embeddingApiKeyEnv && options.embeddingApiKeyFile) {
|
||||
context.io.stderr.write(
|
||||
'Choose only one embedding credential source: --embedding-api-key-env or --embedding-api-key-file.\n',
|
||||
);
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.enableHistoricSql && options.disableHistoricSql) {
|
||||
context.io.stderr.write(
|
||||
'Choose only one Historic SQL action: --enable-historic-sql or --disable-historic-sql.\n',
|
||||
);
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.sourcePath && options.sourceGitUrl) {
|
||||
context.io.stderr.write('Choose only one source location: --source-path or --source-git-url.\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
if (options.skipSources && options.source) {
|
||||
context.io.stderr.write('Choose either --source or --skip-sources.\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
|
||||
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
|
||||
await runSetupArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
mode,
|
||||
agents: options.agents === true,
|
||||
...(options.target ? { target: options.target } : {}),
|
||||
agentScope: resolvedAgentScope,
|
||||
agentInstallMode: options.agentInstallMode,
|
||||
skipAgents: options.skipAgents === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
yes: options.yes === true,
|
||||
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
|
||||
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
|
||||
...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}),
|
||||
skipLlm: options.skipLlm === true,
|
||||
...(options.embeddingBackend ? { embeddingBackend: options.embeddingBackend } : {}),
|
||||
...(options.embeddingApiKeyEnv ? { embeddingApiKeyEnv: options.embeddingApiKeyEnv } : {}),
|
||||
...(options.embeddingApiKeyFile ? { embeddingApiKeyFile: options.embeddingApiKeyFile } : {}),
|
||||
skipEmbeddings: options.skipEmbeddings === true,
|
||||
...(options.database.length > 0 ? { databaseDrivers: options.database } : {}),
|
||||
...(options.databaseConnectionId.length > 0 ? { databaseConnectionIds: options.databaseConnectionId } : {}),
|
||||
...(options.newDatabaseConnectionId ? { databaseConnectionId: options.newDatabaseConnectionId } : {}),
|
||||
...(options.databaseUrl ? { databaseUrl: options.databaseUrl } : {}),
|
||||
databaseSchemas: options.databaseSchema,
|
||||
...(options.enableHistoricSql ? { enableHistoricSql: true } : {}),
|
||||
...(options.disableHistoricSql ? { disableHistoricSql: true } : {}),
|
||||
...(options.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: options.historicSqlWindowDays } : {}),
|
||||
...(options.historicSqlMinCalls !== undefined ? { historicSqlMinCalls: options.historicSqlMinCalls } : {}),
|
||||
...(options.historicSqlServiceAccountPattern.length > 0
|
||||
? { historicSqlServiceAccountPatterns: options.historicSqlServiceAccountPattern }
|
||||
: {}),
|
||||
...(options.historicSqlRedactionPattern.length > 0
|
||||
? { historicSqlRedactionPatterns: options.historicSqlRedactionPattern }
|
||||
: {}),
|
||||
skipDatabases: options.skipDatabases === true,
|
||||
...(options.source ? { source: options.source } : {}),
|
||||
...(options.sourceConnectionId ? { sourceConnectionId: options.sourceConnectionId } : {}),
|
||||
...(options.sourcePath ? { sourcePath: options.sourcePath } : {}),
|
||||
...(options.sourceGitUrl ? { sourceGitUrl: options.sourceGitUrl } : {}),
|
||||
...(options.sourceBranch ? { sourceBranch: options.sourceBranch } : {}),
|
||||
...(options.sourceSubpath ? { sourceSubpath: options.sourceSubpath } : {}),
|
||||
...(options.sourceAuthTokenRef ? { sourceAuthTokenRef: options.sourceAuthTokenRef } : {}),
|
||||
...(options.sourceUrl ? { sourceUrl: options.sourceUrl } : {}),
|
||||
...(options.sourceApiKeyRef ? { sourceApiKeyRef: options.sourceApiKeyRef } : {}),
|
||||
...(options.sourceClientId ? { sourceClientId: options.sourceClientId } : {}),
|
||||
...(options.sourceClientSecretRef ? { sourceClientSecretRef: options.sourceClientSecretRef } : {}),
|
||||
...(options.sourceWarehouseConnectionId
|
||||
? { sourceWarehouseConnectionId: options.sourceWarehouseConnectionId }
|
||||
: {}),
|
||||
...(options.sourceProjectName ? { sourceProjectName: options.sourceProjectName } : {}),
|
||||
...(options.sourceProfilesPath ? { sourceProfilesPath: options.sourceProfilesPath } : {}),
|
||||
...(options.sourceTarget ? { sourceTarget: options.sourceTarget } : {}),
|
||||
...(options.metabaseDatabaseId !== undefined ? { metabaseDatabaseId: options.metabaseDatabaseId } : {}),
|
||||
...(options.notionCrawlMode ? { notionCrawlMode: options.notionCrawlMode } : {}),
|
||||
...(options.notionRootPageId.length > 0 ? { notionRootPageIds: options.notionRootPageId } : {}),
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: options.skipSources === true,
|
||||
showEntryMenu: shouldShowSetupEntryMenu(options, command),
|
||||
});
|
||||
});
|
||||
|
||||
registerDemoCommands(setup, context, { description: 'Run the packaged KLO demo from setup' });
|
||||
|
||||
const setupContext = setup.command('context').description('Build, inspect, and recover setup-managed KLO context');
|
||||
|
||||
function setupContextInputMode(command: {
|
||||
optsWithGlobals?: () => unknown;
|
||||
opts?: () => unknown;
|
||||
}): 'auto' | 'disabled' {
|
||||
const options = command.optsWithGlobals?.() as { input?: boolean } | undefined;
|
||||
return options?.input === false ? 'disabled' : 'auto';
|
||||
}
|
||||
|
||||
setupContext
|
||||
.command('build')
|
||||
.description('Build agent-ready KLO context for setup')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (options: { input?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-build',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
inputMode: options.input === false ? 'disabled' : setupContextInputMode(command),
|
||||
});
|
||||
});
|
||||
|
||||
setupContext
|
||||
.command('watch')
|
||||
.description('Watch a setup-managed context build')
|
||||
.argument('[runId]', 'Setup context build run id')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.action(async (runId: string | undefined, options: { input?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-watch',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
inputMode: options.input === false ? 'disabled' : setupContextInputMode(command),
|
||||
});
|
||||
});
|
||||
|
||||
setupContext
|
||||
.command('status')
|
||||
.description('Print setup-managed context build status')
|
||||
.argument('[runId]', 'Setup context build run id')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (runId: string | undefined, options: { json?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
|
||||
setupContext
|
||||
.command('stop')
|
||||
.description('Request a pause for a setup-managed context build')
|
||||
.argument('[runId]', 'Setup context build run id')
|
||||
.option('--force', 'Request the pause without an interactive confirmation', false)
|
||||
.action(async (runId: string | undefined, _options: { force?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'context-stop',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
...(runId ? { runId } : {}),
|
||||
});
|
||||
});
|
||||
|
||||
setup
|
||||
.command('remove')
|
||||
.description('Remove setup-managed local integrations')
|
||||
.option('--agents', 'Remove setup-managed agent integration files', false)
|
||||
.action(async (options: { agents?: boolean }, command) => {
|
||||
const parentOptions = command.parent?.opts() as { agents?: boolean } | undefined;
|
||||
if (options.agents !== true && parentOptions?.agents !== true) {
|
||||
context.io.stderr.write('Choose what to remove: --agents.\n');
|
||||
context.setExitCode(1);
|
||||
return;
|
||||
}
|
||||
await runSetupArgs(context, {
|
||||
command: 'remove-agents',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
});
|
||||
});
|
||||
|
||||
setup
|
||||
.command('status')
|
||||
.description('Show setup readiness for the resolved KLO project')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (options: { json?: boolean }, command) => {
|
||||
await runSetupArgs(context, {
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: options.json === true,
|
||||
});
|
||||
});
|
||||
}
|
||||
148
packages/cli/src/commands/sl-commands.ts
Normal file
148
packages/cli/src/commands/sl-commands.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
|
||||
import {
|
||||
collectOption,
|
||||
type KloCliCommandContext,
|
||||
parsePositiveIntegerOption,
|
||||
resolveCommandProjectDir,
|
||||
} from '../cli-program.js';
|
||||
import { slQueryCommandSchema } from '../command-schemas.js';
|
||||
import type { KloSlArgs } from '../sl.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/sl-commands');
|
||||
|
||||
function parseOrderBy(value: string): string | { field: string; direction?: string } {
|
||||
const [field, direction] = value.split(':');
|
||||
if (!field) {
|
||||
throw new InvalidArgumentError('requires a field');
|
||||
}
|
||||
if (!direction) {
|
||||
return field;
|
||||
}
|
||||
if (direction !== 'asc' && direction !== 'desc') {
|
||||
throw new InvalidArgumentError('direction must be asc or desc');
|
||||
}
|
||||
return { field, direction };
|
||||
}
|
||||
|
||||
function collectOrderBy(
|
||||
value: string,
|
||||
previous: Array<string | { field: string; direction?: string }> = [],
|
||||
): Array<string | { field: string; direction?: string }> {
|
||||
return [...previous, parseOrderBy(value)];
|
||||
}
|
||||
|
||||
async function runSlArgs(context: KloCliCommandContext, args: KloSlArgs): Promise<void> {
|
||||
const runner = context.deps.sl ?? (await import('../sl.js')).runKloSl;
|
||||
context.setExitCode(await runner(args, context.io));
|
||||
}
|
||||
|
||||
export function registerSlCommands(program: Command, context: KloCliCommandContext, commandName = 'sl'): void {
|
||||
const sl = program
|
||||
.command(commandName)
|
||||
.description('List, read, validate, query, or write local semantic-layer sources')
|
||||
.showHelpAfterError()
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nProject directory defaults to KLO_PROJECT_DIR when set, otherwise the current working directory.\n',
|
||||
);
|
||||
|
||||
sl.command('list')
|
||||
.description('List semantic-layer sources')
|
||||
.option('--connection-id <id>', 'KLO connection id')
|
||||
.addOption(
|
||||
new Option('--output <mode>', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
|
||||
'pretty',
|
||||
'plain',
|
||||
'json',
|
||||
]),
|
||||
)
|
||||
.option('--json', 'Shortcut for --output=json (overrides --output)', false)
|
||||
.action(async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'list',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
output: options.output,
|
||||
json: options.json,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('read')
|
||||
.description('Read a semantic-layer source')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.requiredOption('--connection-id <id>', 'KLO connection id')
|
||||
.action(async (sourceName: string, options: { connectionId: string }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
sourceName,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('validate')
|
||||
.description('Validate a semantic-layer source')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.requiredOption('--connection-id <id>', 'KLO connection id')
|
||||
.action(async (sourceName: string, options: { connectionId: string }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'validate',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
sourceName,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('write')
|
||||
.description('Write a semantic-layer source')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.requiredOption('--connection-id <id>', 'KLO connection id')
|
||||
.requiredOption('--yaml <yaml>', 'Semantic-layer source YAML')
|
||||
.action(async (sourceName: string, options: { connectionId: string; yaml: string }, command) => {
|
||||
await runSlArgs(context, {
|
||||
command: 'write',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
sourceName,
|
||||
yaml: options.yaml,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('query')
|
||||
.description('Compile or execute a semantic-layer query')
|
||||
.option('--connection-id <id>', 'KLO connection id')
|
||||
.option('--measure <measure>', 'Measure to query; repeatable', collectOption, [])
|
||||
.option('--dimension <dimension>', 'Dimension to include; repeatable', collectOption, [])
|
||||
.option('--filter <filter>', 'Filter expression; repeatable', collectOption, [])
|
||||
.option('--segment <segment>', 'Segment to include; repeatable', collectOption, [])
|
||||
.option('--order-by <field[:direction]>', 'Order field, optionally suffixed with :asc or :desc', collectOrderBy, [])
|
||||
.option('--limit <n>', 'Query limit', parsePositiveIntegerOption)
|
||||
.option('--include-empty', 'Include empty rows', false)
|
||||
.addOption(new Option('--format <format>', 'json or sql').choices(['json', 'sql']).default('json'))
|
||||
.option('--execute', 'Execute the compiled query', false)
|
||||
.option('--max-rows <n>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
|
||||
.action(async (options, command) => {
|
||||
if (options.measure.length === 0) {
|
||||
throw new Error('sl query requires at least one --measure');
|
||||
}
|
||||
const args = slQueryCommandSchema.parse({
|
||||
command: 'query',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: options.connectionId,
|
||||
query: {
|
||||
measures: options.measure,
|
||||
dimensions: options.dimension,
|
||||
...(options.filter.length > 0 ? { filters: options.filter } : {}),
|
||||
...(options.segment.length > 0 ? { segments: options.segment } : {}),
|
||||
...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}),
|
||||
...(options.limit !== undefined ? { limit: options.limit } : {}),
|
||||
...(options.includeEmpty === true ? { include_empty: true } : {}),
|
||||
},
|
||||
format: options.format,
|
||||
execute: options.execute === true,
|
||||
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
|
||||
});
|
||||
await runSlArgs(context, args);
|
||||
});
|
||||
}
|
||||
23
packages/cli/src/commands/status-commands.ts
Normal file
23
packages/cli/src/commands/status-commands.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import type { Command } from '@commander-js/extra-typings';
|
||||
import type { KloCliCommandContext } from '../cli-program.js';
|
||||
import { resolveCommandProjectDir } from '../cli-program.js';
|
||||
|
||||
export function registerStatusCommands(program: Command, context: KloCliCommandContext): void {
|
||||
program
|
||||
.command('status')
|
||||
.description('Show current KLO project setup status')
|
||||
.option('--json', 'Print JSON output', false)
|
||||
.action(async (options: { json?: boolean }, command) => {
|
||||
const runner = context.deps.setup ?? (await import('../setup.js')).runKloSetup;
|
||||
context.setExitCode(
|
||||
await runner(
|
||||
{
|
||||
command: 'status',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
json: options.json === true,
|
||||
},
|
||||
context.io,
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue