mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-07 07:55:13 +02:00
fix(cli): align Notion setup credential to --source-auth-token-ref (#236)
Notion's setup path read --source-api-key-ref while writing the auth_token_ref config field, so --source-auth-token-ref was silently dropped. Align Notion to the flag=field convention every other connector follows: it now reads --source-auth-token-ref, and --source-api-key-ref becomes Metabase-only. Also add validation rejecting any credential-ref flag not applicable to the chosen --source, with a pointer to the correct flag, closing the silent-drop class for all connectors. Update CLI-reference docs, the ktx skill Notion example, and tests. Fixes KLO-724.
This commit is contained in:
parent
8ebc4ce107
commit
637891f030
5 changed files with 137 additions and 12 deletions
|
|
@ -160,9 +160,9 @@ sources. This is equivalent to passing `--skip-sources` in scripted setup.
|
||||||
| `--source-git-url <url>` | Git URL for dbt, MetricFlow, or LookML |
|
| `--source-git-url <url>` | Git URL for dbt, MetricFlow, or LookML |
|
||||||
| `--source-branch <branch>` | Git branch for context-source setup |
|
| `--source-branch <branch>` | Git branch for context-source setup |
|
||||||
| `--source-subpath <path>` | Repo subpath for context-source setup |
|
| `--source-subpath <path>` | Repo subpath for context-source setup |
|
||||||
| `--source-auth-token-ref <ref>` | `env:` or `file:` credential reference for source repo auth |
|
| `--source-auth-token-ref <ref>` | `env:` or `file:` credential reference for source repo auth or Notion integration token |
|
||||||
| `--source-url <url>` | Source service URL for Metabase or Looker |
|
| `--source-url <url>` | Source service URL for Metabase or Looker |
|
||||||
| `--source-api-key-ref <ref>` | `env:` or `file:` API key reference for Metabase or Notion |
|
| `--source-api-key-ref <ref>` | `env:` or `file:` API key reference for Metabase |
|
||||||
| `--source-client-id <id>` | Looker client id |
|
| `--source-client-id <id>` | Looker client id |
|
||||||
| `--source-client-secret-ref <ref>` | `env:` or `file:` Looker client secret reference |
|
| `--source-client-secret-ref <ref>` | `env:` or `file:` Looker client secret reference |
|
||||||
| `--source-warehouse-connection-id <id>` | Warehouse connection id used for context-source mapping |
|
| `--source-warehouse-connection-id <id>` | Warehouse connection id used for context-source mapping |
|
||||||
|
|
@ -221,6 +221,14 @@ ktx setup \
|
||||||
--source-warehouse-connection-id warehouse \
|
--source-warehouse-connection-id warehouse \
|
||||||
--metabase-database-id 1
|
--metabase-database-id 1
|
||||||
|
|
||||||
|
# Add a Notion source that crawls selected root pages
|
||||||
|
ktx setup \
|
||||||
|
--source notion \
|
||||||
|
--source-connection-id notion-main \
|
||||||
|
--source-auth-token-ref env:NOTION_TOKEN \
|
||||||
|
--notion-crawl-mode selected_roots \
|
||||||
|
--notion-root-page-id abc123def456
|
||||||
|
|
||||||
# Install project-scoped agent integration for Codex
|
# Install project-scoped agent integration for Codex
|
||||||
ktx setup --agents --target codex
|
ktx setup --agents --target codex
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -308,9 +308,14 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
||||||
.addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
|
.addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
|
||||||
.addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
|
.addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
|
||||||
.addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
|
.addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
|
||||||
.addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth').hideHelp())
|
.addOption(
|
||||||
|
new Option(
|
||||||
|
'--source-auth-token-ref <ref>',
|
||||||
|
'env: or file: credential ref for source repo auth or Notion integration token',
|
||||||
|
).hideHelp(),
|
||||||
|
)
|
||||||
.addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
|
.addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
|
||||||
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion').hideHelp())
|
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase').hideHelp())
|
||||||
.addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
|
.addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
|
||||||
.addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
|
.addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
|
||||||
.addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())
|
.addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())
|
||||||
|
|
|
||||||
|
|
@ -217,6 +217,39 @@ function credentialRef(value: string | undefined, label: string): string {
|
||||||
return ref;
|
return ref;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SourceCredentialFlag = {
|
||||||
|
field: 'sourceAuthTokenRef' | 'sourceApiKeyRef' | 'sourceClientSecretRef';
|
||||||
|
flag: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Each connector reads exactly one credential ref; the flag name mirrors the
|
||||||
|
// ktx.yaml field it writes (auth_token_ref / api_key_ref / client_secret_ref).
|
||||||
|
const SOURCE_CREDENTIAL_FLAG: Record<KtxSetupSourceType, SourceCredentialFlag> = {
|
||||||
|
dbt: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||||
|
metricflow: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||||
|
lookml: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||||
|
notion: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||||
|
metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
||||||
|
looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
||||||
|
};
|
||||||
|
|
||||||
|
const ALL_SOURCE_CREDENTIAL_FLAGS: SourceCredentialFlag[] = [
|
||||||
|
{ field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||||
|
{ field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
||||||
|
{ field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
||||||
|
];
|
||||||
|
|
||||||
|
// Reject a credential ref flag the chosen source does not read, so a wrong flag
|
||||||
|
// fails loudly instead of being silently dropped (KLO-724).
|
||||||
|
function assertSourceCredentialFlags(source: KtxSetupSourceType, args: KtxSetupSourcesArgs): void {
|
||||||
|
const allowed = SOURCE_CREDENTIAL_FLAG[source];
|
||||||
|
for (const { field, flag } of ALL_SOURCE_CREDENTIAL_FLAGS) {
|
||||||
|
if (args[field] && field !== allowed.field) {
|
||||||
|
throw new Error(`${flag} does not apply to --source ${source}; use ${allowed.flag}.`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function chooseSourceCredentialRef(input: {
|
async function chooseSourceCredentialRef(input: {
|
||||||
prompts: KtxSetupSourcesPromptAdapter;
|
prompts: KtxSetupSourcesPromptAdapter;
|
||||||
projectDir: string;
|
projectDir: string;
|
||||||
|
|
@ -515,7 +548,7 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
driver: 'notion',
|
driver: 'notion',
|
||||||
auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'),
|
auth_token_ref: credentialRef(args.sourceAuthTokenRef, 'Notion token ref'),
|
||||||
crawl_mode: crawlMode,
|
crawl_mode: crawlMode,
|
||||||
...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
|
...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
|
||||||
root_database_ids: [],
|
root_database_ids: [],
|
||||||
|
|
@ -1295,10 +1328,10 @@ async function promptForInteractiveSource(
|
||||||
label: 'Notion integration token',
|
label: 'Notion integration token',
|
||||||
envName: 'NOTION_TOKEN',
|
envName: 'NOTION_TOKEN',
|
||||||
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
||||||
existingRef: currentState.sourceApiKeyRef,
|
existingRef: currentState.sourceAuthTokenRef,
|
||||||
});
|
});
|
||||||
if (ref === 'back') return 'back';
|
if (ref === 'back') return 'back';
|
||||||
currentState.sourceApiKeyRef = ref;
|
currentState.sourceAuthTokenRef = ref;
|
||||||
return 'next';
|
return 'next';
|
||||||
},
|
},
|
||||||
async (currentState) => {
|
async (currentState) => {
|
||||||
|
|
@ -1326,7 +1359,7 @@ async function promptForInteractiveSource(
|
||||||
connectionId,
|
connectionId,
|
||||||
connection: {
|
connection: {
|
||||||
driver: 'notion',
|
driver: 'notion',
|
||||||
auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'),
|
auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'),
|
||||||
crawl_mode: 'selected_roots',
|
crawl_mode: 'selected_roots',
|
||||||
root_page_ids: currentState.notionRootPageIds ?? [],
|
root_page_ids: currentState.notionRootPageIds ?? [],
|
||||||
root_database_ids: [],
|
root_database_ids: [],
|
||||||
|
|
@ -1516,7 +1549,7 @@ function sourceArgsFromExistingConnection(input: {
|
||||||
return sourceArgs;
|
return sourceArgs;
|
||||||
}
|
}
|
||||||
|
|
||||||
sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref);
|
sourceArgs.sourceAuthTokenRef = stringField(input.connection.auth_token_ref);
|
||||||
sourceArgs.notionCrawlMode =
|
sourceArgs.notionCrawlMode =
|
||||||
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
||||||
if (Array.isArray(input.connection.root_page_ids)) {
|
if (Array.isArray(input.connection.root_page_ids)) {
|
||||||
|
|
@ -1817,6 +1850,10 @@ export async function runKtxSetupSourcesStep(
|
||||||
return { status: 'skipped', projectDir: args.projectDir };
|
return { status: 'skipped', projectDir: args.projectDir };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (args.source) {
|
||||||
|
assertSourceCredentialFlags(args.source, args);
|
||||||
|
}
|
||||||
|
|
||||||
const prompts = deps.prompts ?? createPromptAdapter();
|
const prompts = deps.prompts ?? createPromptAdapter();
|
||||||
const project = await loadKtxProject({ projectDir: args.projectDir });
|
const project = await loadKtxProject({ projectDir: args.projectDir });
|
||||||
if (!hasPrimarySource(project.config)) {
|
if (!hasPrimarySource(project.config)) {
|
||||||
|
|
|
||||||
|
|
@ -260,7 +260,7 @@ describe('setup sources step', () => {
|
||||||
inputMode: 'disabled',
|
inputMode: 'disabled',
|
||||||
source: 'notion',
|
source: 'notion',
|
||||||
sourceConnectionId: 'notion-main',
|
sourceConnectionId: 'notion-main',
|
||||||
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
sourceAuthTokenRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||||
notionCrawlMode: 'selected_roots',
|
notionCrawlMode: 'selected_roots',
|
||||||
notionRootPageIds: ['page-1'],
|
notionRootPageIds: ['page-1'],
|
||||||
runInitialSourceIngest: false,
|
runInitialSourceIngest: false,
|
||||||
|
|
@ -281,6 +281,81 @@ describe('setup sources step', () => {
|
||||||
expect((await readConfig()).connections['notion-main']?.last_successful_cursor).toBeUndefined();
|
expect((await readConfig()).connections['notion-main']?.last_successful_cursor).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rejects --source-api-key-ref for Notion and points at --source-auth-token-ref', async () => {
|
||||||
|
await addPrimarySource();
|
||||||
|
const io = makeIo();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runKtxSetupSourcesStep(
|
||||||
|
{
|
||||||
|
projectDir,
|
||||||
|
inputMode: 'disabled',
|
||||||
|
source: 'notion',
|
||||||
|
sourceConnectionId: 'notion-main',
|
||||||
|
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||||
|
notionCrawlMode: 'selected_roots',
|
||||||
|
notionRootPageIds: ['page-1'],
|
||||||
|
runInitialSourceIngest: false,
|
||||||
|
skipSources: false,
|
||||||
|
},
|
||||||
|
io.io,
|
||||||
|
{},
|
||||||
|
),
|
||||||
|
).resolves.toEqual({ status: 'failed', projectDir });
|
||||||
|
|
||||||
|
expect(io.stderr()).toContain('--source-api-key-ref does not apply to --source notion; use --source-auth-token-ref.');
|
||||||
|
expect((await readConfig()).connections['notion-main']).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects --source-auth-token-ref for Metabase and points at --source-api-key-ref', async () => {
|
||||||
|
await addPrimarySource();
|
||||||
|
const io = makeIo();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runKtxSetupSourcesStep(
|
||||||
|
{
|
||||||
|
projectDir,
|
||||||
|
inputMode: 'disabled',
|
||||||
|
source: 'metabase',
|
||||||
|
sourceConnectionId: 'prod_metabase',
|
||||||
|
sourceUrl: 'https://metabase.example.com',
|
||||||
|
sourceAuthTokenRef: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||||
|
sourceWarehouseConnectionId: 'warehouse',
|
||||||
|
metabaseDatabaseId: 1,
|
||||||
|
runInitialSourceIngest: false,
|
||||||
|
skipSources: false,
|
||||||
|
},
|
||||||
|
io.io,
|
||||||
|
{},
|
||||||
|
),
|
||||||
|
).resolves.toEqual({ status: 'failed', projectDir });
|
||||||
|
|
||||||
|
expect(io.stderr()).toContain('--source-auth-token-ref does not apply to --source metabase; use --source-api-key-ref.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects --source-client-secret-ref for dbt and points at --source-auth-token-ref', async () => {
|
||||||
|
await addPrimarySource();
|
||||||
|
const io = makeIo();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runKtxSetupSourcesStep(
|
||||||
|
{
|
||||||
|
projectDir,
|
||||||
|
inputMode: 'disabled',
|
||||||
|
source: 'dbt',
|
||||||
|
sourceConnectionId: 'dbt-main',
|
||||||
|
sourceClientSecretRef: 'env:DBT_SECRET', // pragma: allowlist secret
|
||||||
|
runInitialSourceIngest: false,
|
||||||
|
skipSources: false,
|
||||||
|
},
|
||||||
|
io.io,
|
||||||
|
{},
|
||||||
|
),
|
||||||
|
).resolves.toEqual({ status: 'failed', projectDir });
|
||||||
|
|
||||||
|
expect(io.stderr()).toContain('--source-client-secret-ref does not apply to --source dbt; use --source-auth-token-ref.');
|
||||||
|
});
|
||||||
|
|
||||||
it('accepts former ingest subcommand names as interactive source connection ids', async () => {
|
it('accepts former ingest subcommand names as interactive source connection ids', async () => {
|
||||||
await addPrimarySource();
|
await addPrimarySource();
|
||||||
const io = makeIo();
|
const io = makeIo();
|
||||||
|
|
@ -323,7 +398,7 @@ describe('setup sources step', () => {
|
||||||
inputMode: 'disabled',
|
inputMode: 'disabled',
|
||||||
source: 'notion',
|
source: 'notion',
|
||||||
sourceConnectionId: 'notion-main',
|
sourceConnectionId: 'notion-main',
|
||||||
sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
sourceAuthTokenRef: 'env:NOTION_TOKEN', // pragma: allowlist secret
|
||||||
notionCrawlMode: 'all_accessible',
|
notionCrawlMode: 'all_accessible',
|
||||||
notionRootPageIds: ['page-1'],
|
notionRootPageIds: ['page-1'],
|
||||||
runInitialSourceIngest: false,
|
runInitialSourceIngest: false,
|
||||||
|
|
|
||||||
|
|
@ -138,7 +138,7 @@ ktx setup --no-input --yes --skip-databases --skip-llm --skip-embeddings \
|
||||||
# Notion
|
# Notion
|
||||||
ktx setup --no-input --yes --skip-databases --skip-llm --skip-embeddings \
|
ktx setup --no-input --yes --skip-databases --skip-llm --skip-embeddings \
|
||||||
--source notion --source-connection-id <id> \
|
--source notion --source-connection-id <id> \
|
||||||
--source-api-key-ref env:NOTION_TOKEN \
|
--source-auth-token-ref env:NOTION_TOKEN \
|
||||||
--notion-crawl-mode selected_roots --notion-root-page-id <page-id>
|
--notion-crawl-mode selected_roots --notion-root-page-id <page-id>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue